<commit_before>##############################################################################
# Copyright (c) 2013-2017, Lawrence Livermore National Security, LLC.
# Produced at the Lawrence Livermore National Laboratory.
#
# This file is part of Spack.
# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved.
# LLNL-CODE-647188
#
# For details, see https://github.com/spack/spack
# Please also see the NOTICE and LICENSE files for our notice and the LGPL.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License (as
# published by the Free Software Foundation) version 2.1, February 1999.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and
# conditions of the GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
##############################################################################
from spack import *
import os
import sys
import glob

class Mumps(Package):
    """MUMPS: a MUltifrontal Massively Parallel sparse direct Solver"""

    homepage = "http://mumps.enseeiht.fr"
    url      = "http://mumps.enseeiht.fr/MUMPS_5.0.1.tar.gz"

    version('5.1.1', 'f15c6b5dd8c71b1241004cd19818259d')
    version('5.0.2', '591bcb2c205dcb0283872608cdf04927')
    # Alternate location if main server is down.
    # version('5.0.1', 'b477573fdcc87babe861f62316833db0', url='http://pkgs.fedoraproject.org/repo/pkgs/MUMPS/MUMPS_5.0.1.tar.gz/md5/b477573fdcc87babe861f62316833db0/MUMPS_5.0.1.tar.gz')
    version('5.0.1', 'b477573fdcc87babe861f62316833db0')

    variant('mpi', default=True,
            description='Compile MUMPS with MPI support')
    variant('scotch', default=False,
            description='Activate Scotch as a possible ordering library')
    variant('ptscotch', default=False,
            description='Activate PT-Scotch as a possible ordering library')
    variant('metis', default=False,
            description='Activate Metis as a possible ordering library')
    variant('parmetis', default=False,
            description='Activate Parmetis as a possible ordering library')
    variant('double', default=True,
            description='Activate the compilation of dmumps')
    variant('float', default=True,
            description='Activate the compilation of smumps')
    variant('complex', default=True,
            description='Activate the compilation of cmumps and/or zmumps')
    variant('int64', default=False,
            description='Use int64_t/integer*8 as default index type')
    variant('shared', default=True, description='Build shared libraries')

    depends_on('scotch + esmumps', when='~ptscotch+scotch')
    depends_on('scotch + esmumps ~ metis + mpi', when='+ptscotch')
    depends_on('metis@5:', when='+metis')
    depends_on('parmetis', when="+parmetis")
    depends_on('blas')
    depends_on('lapack')
    depends_on('scalapack', when='+mpi')
    depends_on('mpi', when='+mpi')

    patch('mumps-5.0.2-spectrum-mpi-xl.patch', when='@5.0.2%xl^spectrum-mpi')
    patch('mumps-5.0.2-spectrum-mpi-xl.patch', when='@5.0.2%xl_r^spectrum-mpi')
    patch('mumps-5.1.1-spectrum-mpi-xl.patch', when='@5.1.1%xl^spectrum-mpi')
    patch('mumps-5.1.1-spectrum-mpi-xl.patch', when='@5.1.1%xl_r^spectrum-mpi')

    # this function is not a patch function because in case scalapack
    # is needed it uses self.spec['scalapack'].fc_link set by the
    # setup_dependent_environment in scalapck. This happen after patch
    # end before install
    # def patch(self):
    def write_makefile_inc(self):
        if ('+parmetis' in self.spec or '+ptscotch' in self.spec) and (
                '+mpi' not in self.spec):
            raise RuntimeError(
                'You cannot use the variants parmetis or ptscotch without mpi')

        lapack_blas = (self.spec['lapack'].libs +
                       self.spec['blas'].libs)
        makefile_conf = ["LIBBLAS = %s" % lapack_blas.ld_flags]

        orderings = ['-Dpord']

        if '+ptscotch' in self.spec or '+scotch' in self.spec:
            makefile_conf.extend([
                "ISCOTCH = -I%s" % self.spec['scotch'].prefix.include,
                "LSCOTCH = {0}".format(self.spec['scotch'].libs.ld_flags)
            ])

            orderings.append('-Dscotch')
            if '+ptscotch' in self.spec:
                orderings.append('-Dptscotch')

        if '+parmetis' in self.spec and '+metis' in self.spec:
            makefile_conf.extend([
                "IMETIS = -I%s" % self.spec['parmetis'].prefix.include,
                "LMETIS = -L%s -l%s -L%s -l%s" % (
                    self.spec['parmetis'].prefix.lib, 'parmetis',
                    self.spec['metis'].prefix.lib, 'metis')
            ])

            orderings.append('-Dparmetis')
        elif '+metis' in self.spec:
            makefile_conf.extend([
                "IMETIS = -I%s" % self.spec['metis'].prefix.include,
                "LMETIS = -L%s -l%s" % (self.spec['metis'].prefix.lib, 'metis')
            ])

            orderings.append('-Dmetis')

        makefile_conf.append("ORDERINGSF = %s" % (' '.join(orderings)))

        # when building shared libs need -fPIC, otherwise
        # /usr/bin/ld: graph.o: relocation R_X86_64_32 against `.rodata.str1.1'
        # can not be used when making a shared object; recompile with -fPIC
        fpic = self.compiler.pic_flag if '+shared' in self.spec else ''
        # TODO: test this part, it needs a full blas, scalapack and
        # partitionning environment with 64bit integers

        if '+int64' in self.spec:
            if self.compiler.name == "xl" or self.compiler.name == "xl_r":
                makefile_conf.extend(
                    ['OPTF    = -O3',
                     'OPTL    = %s -O3' % fpic,
                     'OPTC    = %s -O3-DINTSIZE64' % fpic])
            else:
                makefile_conf.extend(
                    # the fortran compilation flags most probably are
                    # working only for intel and gnu compilers this is
                    # perhaps something the compiler should provide
                    ['OPTF    = %s -O  -DALLOW_NON_INIT %s' % (fpic, '-fdefault-integer-8' if self.compiler.name == "gcc" else '-i8'),  # noqa
                     'OPTL    = %s -O ' % fpic,
                     'OPTC    = %s -O -DINTSIZE64' % fpic])
        else:
            if self.compiler.name == "xl" or self.compiler.name == "xl_r":
                makefile_conf.extend(
                    ['OPTF    = -O3 -qfixed',
                     'OPTL    = %s -O3' % fpic,
                     'OPTC    = %s -O3' % fpic])
            else:
                makefile_conf.extend(
                    ['OPTF    = %s -O  -DALLOW_NON_INIT' % fpic,
                     'OPTL    = %s -O ' % fpic,
                     'OPTC    = %s -O ' % fpic])

        if '+mpi' in self.spec:
            scalapack = self.spec['scalapack'].libs
            makefile_conf.extend(
                ['CC = {0}'.format(self.spec['mpi'].mpicc),
                 'FC = {0}'.format(self.spec['mpi'].mpifc),
                 "SCALAP = %s" % scalapack.ld_flags,
                 "MUMPS_TYPE = par"])
            if (self.spec.satisfies('%xl_r' or '%xl')) and self.spec.satisfies('^spectrum-mpi'):  # noqa
                makefile_conf.extend(
                    ['FL = {0}'.format(self.spec['mpi'].mpicc)])
            else:
                makefile_conf.extend(
                    ['FL = {0}'.format(self.spec['mpi'].mpifc)])
        else:
            makefile_conf.extend(
                ["CC = cc",
                 "FC = fc",
                 "FL = fc",
                 "MUMPS_TYPE = seq"])

        # TODO: change the value to the correct one according to the
        # compiler possible values are -DAdd_, -DAdd__ and/or -DUPPER
        if self.compiler.name == 'intel' or self.compiler.name == 'pgi':
            # Intel & PGI Fortran compiler provides the main() function so
            # C examples linked with the Fortran compiler require a
            # hack defined by _DMAIN_COMP (see examples/c_example.c)
            makefile_conf.append("CDEFS   = -DAdd_ -DMAIN_COMP")
        else:
            if self.compiler.name != "xl" and self.compiler.name != "xl_r":
                makefile_conf.append("CDEFS   = -DAdd_")

        if '+shared' in self.spec:
            if sys.platform == 'darwin':
                # Building dylibs with mpif90 causes segfaults on 10.8 and
                # 10.10. Use gfortran. (Homebrew)
                makefile_conf.extend([
                    'LIBEXT=.dylib',
                    'AR=%s -dynamiclib -Wl,-install_name -Wl,%s/$(notdir $@) -undefined dynamic_lookup -o ' % (os.environ['FC'], prefix.lib),  # noqa
                    'RANLIB=echo'
                ])
            else:
                makefile_conf.extend([
                    'LIBEXT=.so',
                    'AR=$(FL) -shared -Wl,-soname -Wl,%s/$(notdir $@) -o' % prefix.lib,  # noqa
                    'RANLIB=echo'
                ])

                if self.compiler.name == 'xl' or self.compiler.name == 'xl_r':
                    makefile_conf.extend([
                        'SAR=/bin/xlc -shared -Wl,-soname -Wl,%s/$(notdir $@) -o' % prefix.lib  # noqa
                    ])
        else:
            makefile_conf.extend([
                'LIBEXT  = .a',
                'AR = ar vr',
                'RANLIB = ranlib'
            ])

        makefile_inc_template = join_path(
            os.path.dirname(self.module.__file__), 'Makefile.inc')
        with open(makefile_inc_template, "r") as fh:
            makefile_conf.extend(fh.read().split('\n'))

        with working_dir('.'):
            with open("Makefile.inc", "w") as fh:
                makefile_inc = '\n'.join(makefile_conf)
                fh.write(makefile_inc)

    def install(self, spec, prefix):
        make_libs = []

        # the choice to compile ?examples is to have kind of a sanity
        # check on the libraries generated.
        if '+float' in spec:
            make_libs.append('sexamples')
            if '+complex' in spec:
                make_libs.append('cexamples')

        if '+double' in spec:
            make_libs.append('dexamples')
            if '+complex' in spec:
                make_libs.append('zexamples')

        self.write_makefile_inc()

        # Build fails in parallel
        make(*make_libs, parallel=False)

        install_tree('lib', prefix.lib)
        install_tree('include', prefix.include)

        if '~mpi' in spec:
            lib_dsuffix = '.dylib' if sys.platform == 'darwin' else '.so'
            lib_suffix = lib_dsuffix if '+shared' in spec else '.a'
            install('libseq/libmpiseq%s' % lib_suffix, prefix.lib)
            for f in glob.glob(join_path('libseq', '*.h')):
                install(f, prefix.include)

        # FIXME: extend the tests to mpirun -np 2 when build with MPI
        # FIXME: use something like numdiff to compare output files
        with working_dir('examples'):
            if '+float' in spec:
                os.system('./ssimpletest < input_simpletest_real')
                if '+complex' in spec:
                    os.system('./csimpletest < input_simpletest_real')
            if '+double' in spec:
                os.system('./dsimpletest < input_simpletest_real')
                if '+complex' in spec:
                    os.system('./zsimpletest < input_simpletest_cmplx')
<commit_msg>mumps: add the libs property (#7264)<commit_after>##############################################################################
# Copyright (c) 2013-2017, Lawrence Livermore National Security, LLC.
# Produced at the Lawrence Livermore National Laboratory.
#
# This file is part of Spack.
# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved.
# LLNL-CODE-647188
#
# For details, see https://github.com/spack/spack
# Please also see the NOTICE and LICENSE files for our notice and the LGPL.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License (as
# published by the Free Software Foundation) version 2.1, February 1999.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and
# conditions of the GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
##############################################################################
from spack import *
import os
import sys
import glob

class Mumps(Package):
    """MUMPS: a MUltifrontal Massively Parallel sparse direct Solver"""

    homepage = "http://mumps.enseeiht.fr"
    url      = "http://mumps.enseeiht.fr/MUMPS_5.0.1.tar.gz"

    version('5.1.1', 'f15c6b5dd8c71b1241004cd19818259d')
    version('5.0.2', '591bcb2c205dcb0283872608cdf04927')
    # Alternate location if main server is down.
    # version('5.0.1', 'b477573fdcc87babe861f62316833db0', url='http://pkgs.fedoraproject.org/repo/pkgs/MUMPS/MUMPS_5.0.1.tar.gz/md5/b477573fdcc87babe861f62316833db0/MUMPS_5.0.1.tar.gz')
    version('5.0.1', 'b477573fdcc87babe861f62316833db0')

    variant('mpi', default=True,
            description='Compile MUMPS with MPI support')
    variant('scotch', default=False,
            description='Activate Scotch as a possible ordering library')
    variant('ptscotch', default=False,
            description='Activate PT-Scotch as a possible ordering library')
    variant('metis', default=False,
            description='Activate Metis as a possible ordering library')
    variant('parmetis', default=False,
            description='Activate Parmetis as a possible ordering library')
    variant('double', default=True,
            description='Activate the compilation of dmumps')
    variant('float', default=True,
            description='Activate the compilation of smumps')
    variant('complex', default=True,
            description='Activate the compilation of cmumps and/or zmumps')
    variant('int64', default=False,
            description='Use int64_t/integer*8 as default index type')
    variant('shared', default=True, description='Build shared libraries')

    depends_on('scotch + esmumps', when='~ptscotch+scotch')
    depends_on('scotch + esmumps ~ metis + mpi', when='+ptscotch')
    depends_on('metis@5:', when='+metis')
    depends_on('parmetis', when="+parmetis")
    depends_on('blas')
    depends_on('lapack')
    depends_on('scalapack', when='+mpi')
    depends_on('mpi', when='+mpi')

    patch('mumps-5.0.2-spectrum-mpi-xl.patch', when='@5.0.2%xl^spectrum-mpi')
    patch('mumps-5.0.2-spectrum-mpi-xl.patch', when='@5.0.2%xl_r^spectrum-mpi')
    patch('mumps-5.1.1-spectrum-mpi-xl.patch', when='@5.1.1%xl^spectrum-mpi')
    patch('mumps-5.1.1-spectrum-mpi-xl.patch', when='@5.1.1%xl_r^spectrum-mpi')

    # this function is not a patch function because in case scalapack
    # is needed it uses self.spec['scalapack'].fc_link set by the
    # setup_dependent_environment in scalapck. This happen after patch
    # end before install
    # def patch(self):
    def write_makefile_inc(self):
        if ('+parmetis' in self.spec or '+ptscotch' in self.spec) and (
                '+mpi' not in self.spec):
            raise RuntimeError(
                'You cannot use the variants parmetis or ptscotch without mpi')

        lapack_blas = (self.spec['lapack'].libs +
                       self.spec['blas'].libs)
        makefile_conf = ["LIBBLAS = %s" % lapack_blas.ld_flags]

        orderings = ['-Dpord']

        if '+ptscotch' in self.spec or '+scotch' in self.spec:
            makefile_conf.extend([
                "ISCOTCH = -I%s" % self.spec['scotch'].prefix.include,
                "LSCOTCH = {0}".format(self.spec['scotch'].libs.ld_flags)
            ])

            orderings.append('-Dscotch')
            if '+ptscotch' in self.spec:
                orderings.append('-Dptscotch')

        if '+parmetis' in self.spec and '+metis' in self.spec:
            makefile_conf.extend([
                "IMETIS = -I%s" % self.spec['parmetis'].prefix.include,
                "LMETIS = -L%s -l%s -L%s -l%s" % (
                    self.spec['parmetis'].prefix.lib, 'parmetis',
                    self.spec['metis'].prefix.lib, 'metis')
            ])

            orderings.append('-Dparmetis')
        elif '+metis' in self.spec:
            makefile_conf.extend([
                "IMETIS = -I%s" % self.spec['metis'].prefix.include,
                "LMETIS = -L%s -l%s" % (self.spec['metis'].prefix.lib, 'metis')
            ])

            orderings.append('-Dmetis')

        makefile_conf.append("ORDERINGSF = %s" % (' '.join(orderings)))

        # when building shared libs need -fPIC, otherwise
        # /usr/bin/ld: graph.o: relocation R_X86_64_32 against `.rodata.str1.1'
        # can not be used when making a shared object; recompile with -fPIC
        fpic = self.compiler.pic_flag if '+shared' in self.spec else ''
        # TODO: test this part, it needs a full blas, scalapack and
        # partitionning environment with 64bit integers

        if '+int64' in self.spec:
            if self.compiler.name == "xl" or self.compiler.name == "xl_r":
                makefile_conf.extend(
                    ['OPTF    = -O3',
                     'OPTL    = %s -O3' % fpic,
                     'OPTC    = %s -O3-DINTSIZE64' % fpic])
            else:
                makefile_conf.extend(
                    # the fortran compilation flags most probably are
                    # working only for intel and gnu compilers this is
                    # perhaps something the compiler should provide
                    ['OPTF    = %s -O  -DALLOW_NON_INIT %s' % (fpic, '-fdefault-integer-8' if self.compiler.name == "gcc" else '-i8'),  # noqa
                     'OPTL    = %s -O ' % fpic,
                     'OPTC    = %s -O -DINTSIZE64' % fpic])
        else:
            if self.compiler.name == "xl" or self.compiler.name == "xl_r":
                makefile_conf.extend(
                    ['OPTF    = -O3 -qfixed',
                     'OPTL    = %s -O3' % fpic,
                     'OPTC    = %s -O3' % fpic])
            else:
                makefile_conf.extend(
                    ['OPTF    = %s -O  -DALLOW_NON_INIT' % fpic,
                     'OPTL    = %s -O ' % fpic,
                     'OPTC    = %s -O ' % fpic])

        if '+mpi' in self.spec:
            scalapack = self.spec['scalapack'].libs
            makefile_conf.extend(
                ['CC = {0}'.format(self.spec['mpi'].mpicc),
                 'FC = {0}'.format(self.spec['mpi'].mpifc),
                 "SCALAP = %s" % scalapack.ld_flags,
                 "MUMPS_TYPE = par"])
            if (self.spec.satisfies('%xl_r' or '%xl')) and self.spec.satisfies('^spectrum-mpi'):  # noqa
                makefile_conf.extend(
                    ['FL = {0}'.format(self.spec['mpi'].mpicc)])
            else:
                makefile_conf.extend(
                    ['FL = {0}'.format(self.spec['mpi'].mpifc)])
        else:
            makefile_conf.extend(
                ["CC = cc",
                 "FC = fc",
                 "FL = fc",
                 "MUMPS_TYPE = seq"])

        # TODO: change the value to the correct one according to the
        # compiler possible values are -DAdd_, -DAdd__ and/or -DUPPER
        if self.compiler.name == 'intel' or self.compiler.name == 'pgi':
            # Intel & PGI Fortran compiler provides the main() function so
            # C examples linked with the Fortran compiler require a
            # hack defined by _DMAIN_COMP (see examples/c_example.c)
            makefile_conf.append("CDEFS   = -DAdd_ -DMAIN_COMP")
        else:
            if self.compiler.name != "xl" and self.compiler.name != "xl_r":
                makefile_conf.append("CDEFS   = -DAdd_")

        if '+shared' in self.spec:
            if sys.platform == 'darwin':
                # Building dylibs with mpif90 causes segfaults on 10.8 and
                # 10.10. Use gfortran. (Homebrew)
                makefile_conf.extend([
                    'LIBEXT=.dylib',
                    'AR=%s -dynamiclib -Wl,-install_name -Wl,%s/$(notdir $@) -undefined dynamic_lookup -o ' % (os.environ['FC'], prefix.lib),  # noqa
                    'RANLIB=echo'
                ])
            else:
                makefile_conf.extend([
                    'LIBEXT=.so',
                    'AR=$(FL) -shared -Wl,-soname -Wl,%s/$(notdir $@) -o' % prefix.lib,  # noqa
                    'RANLIB=echo'
                ])

                if self.compiler.name == 'xl' or self.compiler.name == 'xl_r':
                    makefile_conf.extend([
                        'SAR=/bin/xlc -shared -Wl,-soname -Wl,%s/$(notdir $@) -o' % prefix.lib  # noqa
                    ])
        else:
            makefile_conf.extend([
                'LIBEXT  = .a',
                'AR = ar vr',
                'RANLIB = ranlib'
            ])

        makefile_inc_template = join_path(
            os.path.dirname(self.module.__file__), 'Makefile.inc')
        with open(makefile_inc_template, "r") as fh:
            makefile_conf.extend(fh.read().split('\n'))

        with working_dir('.'):
            with open("Makefile.inc", "w") as fh:
                makefile_inc = '\n'.join(makefile_conf)
                fh.write(makefile_inc)

    def install(self, spec, prefix):
        make_libs = []

        # the choice to compile ?examples is to have kind of a sanity
        # check on the libraries generated.
        if '+float' in spec:
            make_libs.append('sexamples')
            if '+complex' in spec:
                make_libs.append('cexamples')

        if '+double' in spec:
            make_libs.append('dexamples')
            if '+complex' in spec:
                make_libs.append('zexamples')

        self.write_makefile_inc()

        # Build fails in parallel
        make(*make_libs, parallel=False)

        install_tree('lib', prefix.lib)
        install_tree('include', prefix.include)

        if '~mpi' in spec:
            lib_dsuffix = '.dylib' if sys.platform == 'darwin' else '.so'
            lib_suffix = lib_dsuffix if '+shared' in spec else '.a'
            install('libseq/libmpiseq%s' % lib_suffix, prefix.lib)
            for f in glob.glob(join_path('libseq', '*.h')):
                install(f, prefix.include)

        # FIXME: extend the tests to mpirun -np 2 when build with MPI
        # FIXME: use something like numdiff to compare output files
        with working_dir('examples'):
            if '+float' in spec:
                os.system('./ssimpletest < input_simpletest_real')
                if '+complex' in spec:
                    os.system('./csimpletest < input_simpletest_real')
            if '+double' in spec:
                os.system('./dsimpletest < input_simpletest_real')
                if '+complex' in spec:
                    os.system('./zsimpletest < input_simpletest_cmplx')

    @property
    def libs(self):
        component_libs = ['*mumps*', 'pord']
        return find_libraries(['lib' + comp for comp in component_libs],
                              root=self.prefix.lib,
                              shared=('+shared' in self.spec),
                              recursive=False) or None
<|endoftext|>


<commit_before>import html2text
from defusedxml import ElementTree

from dojo.models import Finding, Endpoint

class NexposeParser(object):
    """
    The objective of this class is to parse Nexpose's XML 2.0 Report.

    TODO: Handle errors.
    TODO: Test nexpose output version. Handle what happens if the parser doesn't support it.
    TODO: Test cases.

    @param xml_filepath A proper xml generated by nexpose
    """

    def get_scan_types(self):
        return ["Nexpose Scan"]

    def get_label_for_scan_types(self, scan_type):
        return scan_type  # no custom label for now

    def get_description_for_scan_types(self, scan_type):
        return "Use the full XML export template from Nexpose."

    def get_findings(self, xml_output, test):
        tree = ElementTree.parse(xml_output)
        vuln_definitions = self.get_vuln_definitions(tree)
        return self.get_items(tree, vuln_definitions, test)

    def parse_html_type(self, node):
        """
        Parse XML element of type HtmlType

        @return ret A string containing the parsed element
        """
        ret = ""
        tag = node.tag.lower()

        if tag == 'containerblockelement':

            if len(list(node)) > 0:
                for child in list(node):
                    ret += self.parse_html_type(child)
            else:
                if node.text:
                    ret += "
" + str(node.text).strip()
                if node.tail:
                    ret += str(node.tail).strip() + "
"
                else:
                    ret += "
"
        if tag == 'listitem':
            if len(list(node)) > 0:
                for child in list(node):
                    ret += self.parse_html_type(child)
            else:
                if node.text:
                    ret += "
" + str(node.text).strip() + "
"
        if tag == 'orderedlist':
            i = 1
            for item in list(node):
                ret += "
" + str(i) + " " + self.parse_html_type(item) + "
"
                i += 1
        if tag == 'paragraph':
            if len(list(node)) > 0:
                for child in list(node):
                    ret += self.parse_html_type(child)
            else:
                if node.text:
                    ret += "
" + node.text.strip()
                if node.tail:
                    ret += str(node.tail).strip() + "
"
                else:
                    ret += "
"
        if tag == 'unorderedlist':
            for item in list(node):
                unorderedlist = self.parse_html_type(item)
                if unorderedlist not in ret:
                    ret += "* " + unorderedlist
        if tag == 'urllink':
            if node.text:
                ret += str(node.text).strip() + " "
            last = ""

            for attr in node.attrib:
                if last != "":
                    if node.get(attr) != node.get(last):
                        ret += str(node.get(attr)) + " "
                last = attr

        return ret

    def parse_tests_type(self, node, vulnsDefinitions):
        """
        Parse XML element of type TestsType

        @return vulns A list of vulnerabilities according to vulnsDefinitions
        """
        vulns = list()

        for tests in node.findall('tests'):
            for test in tests.findall('test'):
                if test.get('id') in vulnsDefinitions and (
                        test.get('status') in ['vulnerable-exploited', 'vulnerable-version', 'vulnerable-potential']):
                    vuln = vulnsDefinitions[test.get('id').lower()]
                    for desc in test.getchildren():
                        if 'pluginOutput' in vuln:
                            vuln['pluginOutput'] += "\n\n" + \
                                self.parse_html_type(desc)
                        else:
                            vuln['pluginOutput'] = self.parse_html_type(desc)
                    vulns.append(vuln)

        return vulns

    def get_vuln_definitions(self, tree):
        """
        @returns vulns A dict of Vulnerability Definitions
        """
        vulns = dict()
        url_index = 0
        for vulnsDef in tree.findall('VulnerabilityDefinitions'):
            for vulnDef in vulnsDef.findall('vulnerability'):
                vid = vulnDef.get('id').lower()
                severity_chk = int(vulnDef.get('severity'))
                if severity_chk >= 9:
                    sev = 'Critical'
                elif severity_chk >= 7:
                    sev = 'High'
                elif severity_chk >= 4:
                    sev = 'Medium'
                elif 0 < severity_chk < 4:
                    sev = 'Low'
                else:
                    sev = 'Info'
                vuln = {
                    'desc': "",
                    'name': vulnDef.get('title'),
                    'vector': vulnDef.get('cvssVector'),  # this is CVSS v2
                    'refs': dict(),
                    'resolution': "",
                    'severity': sev,
                    'tags': list()
                }
                for item in vulnDef.getchildren():
                    if item.tag == 'description':
                        for htmlType in item.getchildren():
                            vuln['desc'] += self.parse_html_type(htmlType)

                    elif item.tag == 'exploits':
                        for exploit in item.getchildren():
                            vuln['refs'][exploit.get('title')] = str(exploit.get('title')).strip() + ' ' + \
                                                                 str(exploit.get('link')).strip()

                    elif item.tag == 'references':
                        for ref in item.getchildren():
                            if 'URL' in ref.get('source'):
                                vuln['refs'][ref.get('source') + str(url_index)] = str(ref.text).strip()
                                url_index += 1
                            else:
                                vuln['refs'][ref.get('source')] = str(ref.text).strip()

                    elif item.tag == 'solution':
                        for htmlType in item.getchildren():
                            vuln['resolution'] += self.parse_html_type(htmlType)

                    # there is currently no method to register tags in vulns
                    elif item.tag == 'tags':
                        for tag in item.getchildren():
                            vuln['tags'].append(tag.text.lower())

                vulns[vid] = vuln
        return vulns

    def get_items(self, tree, vulns, test):
        hosts = list()
        for nodes in tree.findall('nodes'):
            for node in nodes.findall('node'):
                host = dict()
                host['name'] = node.get('address')
                host['hostnames'] = set()
                host['os'] = ""
                host['services'] = list()
                host['vulns'] = self.parse_tests_type(node, vulns)

                for names in node.findall('names'):
                    for name in names.findall('name'):
                        host['hostnames'].add(name.text)

                for endpoints in node.findall('endpoints'):
                    for endpoint in endpoints.findall('endpoint'):
                        svc = {
                            'protocol': endpoint.get('protocol'),
                            'port': int(endpoint.get('port')),
                            'status': endpoint.get('status'),
                        }
                        for services in endpoint.findall('services'):
                            for service in services.findall('service'):
                                svc['name'] = service.get('name')
                                svc['vulns'] = self.parse_tests_type(service, vulns)

                                for configs in service.findall('configurations'):
                                    for config in configs.findall('config'):
                                        if "banner" in config.get('name'):
                                            svc['version'] = config.get('name')

                        host['services'].append(svc)

                hosts.append(host)

        dupes = {}

        for host in hosts:
            # manage findings by node only
            for vuln in host['vulns']:
                dupe_key = vuln['severity'] + vuln['name']

                find = self.findings(dupe_key, dupes, test, vuln)

                endpoint = Endpoint(host=host['name'])
                find.unsaved_endpoints.append(endpoint)
                find.unsaved_tags = vuln['tags']

            # manage findings by service
            for service in host['services']:
                for vuln in service['vulns']:
                    dupe_key = vuln['severity'] + vuln['name']

                    find = self.findings(dupe_key, dupes, test, vuln)

                    endpoint = Endpoint(host=host['name'], port=service['port'], protocol=service['name'].lower())
                    find.unsaved_endpoints.append(endpoint)
                    find.unsaved_tags = vuln['tags']

        return list(dupes.values())

    @staticmethod
    def findings(dupe_key, dupes, test, vuln):
        """

        """
        if dupe_key in dupes:
            find = dupes[dupe_key]
            dupe_text = html2text.html2text(vuln['pluginOutput'])
            if dupe_text not in find.description:
                find.description += "\n\n" + dupe_text
        else:
            find = Finding(title=vuln['name'],
                           description=html2text.html2text(
                               vuln['desc'].strip()) + "\n\n" + html2text.html2text(vuln['pluginOutput'].strip()),
                           severity=vuln['severity'],
                           numerical_severity=Finding.get_numerical_severity(vuln['severity']),
                           mitigation=html2text.html2text(vuln['resolution']),
                           impact=vuln['vector'],
                           test=test,
                           active=False,
                           verified=False,
                           false_p=False,
                           duplicate=False,
                           out_of_scope=False,
                           mitigated=None,
                           dynamic_finding=True)
            # build references
            refs = ''
            for ref in vuln['refs']:
                if ref.startswith('BID'):
                    refs += f" * [{vuln['refs'][ref]}](https://www.securityfocus.com/bid/{vuln['refs'][ref]})"
                elif ref.startswith('CA'):
                    refs += f" * [{vuln['refs'][ref]}](https://www.cert.org/advisories/{vuln['refs'][ref]}.html)"
                elif ref.startswith('CERT-VN'):
                    refs += f" * [{vuln['refs'][ref]}](https://www.kb.cert.org/vuls/id/{vuln['refs'][ref]}.html)"
                elif ref.startswith('CVE'):
                    refs += f" * [{vuln['refs'][ref]}](https://cve.mitre.org/cgi-bin/cvename.cgi?name={vuln['refs'][ref]})"
                if ref.startswith('DEBIAN'):
                    refs += f" * [{vuln['refs'][ref]}](https://security-tracker.debian.org/tracker/{vuln['refs'][ref]})"
                elif ref.startswith('URL'):
                    refs += f" * URL: {vuln['refs'][ref]}"
                if ref.startswith('XF'):
                    refs += f" * [{vuln['refs'][ref]}](https://exchange.xforce.ibmcloud.com/vulnerabilities/{vuln['refs'][ref]})"
                else:
                    refs += f" * {ref}: {vuln['refs'][ref]}"
                refs += "\n"
            find.references = refs
            # update CVE
            if "CVE" in vuln['refs']:
                find.cve = vuln['refs']['CVE']
            find.unsaved_endpoints = list()
            dupes[dupe_key] = find
        return find
<commit_msg>Fix: Ref URL format (#4166)<commit_after>import html2text
from defusedxml import ElementTree

from dojo.models import Finding, Endpoint

class NexposeParser(object):
    """
    The objective of this class is to parse Nexpose's XML 2.0 Report.

    TODO: Handle errors.
    TODO: Test nexpose output version. Handle what happens if the parser doesn't support it.
    TODO: Test cases.

    @param xml_filepath A proper xml generated by nexpose
    """

    def get_scan_types(self):
        return ["Nexpose Scan"]

    def get_label_for_scan_types(self, scan_type):
        return scan_type  # no custom label for now

    def get_description_for_scan_types(self, scan_type):
        return "Use the full XML export template from Nexpose."

    def get_findings(self, xml_output, test):
        tree = ElementTree.parse(xml_output)
        vuln_definitions = self.get_vuln_definitions(tree)
        return self.get_items(tree, vuln_definitions, test)

    def parse_html_type(self, node):
        """
        Parse XML element of type HtmlType

        @return ret A string containing the parsed element
        """
        ret = ""
        tag = node.tag.lower()

        if tag == 'containerblockelement':

            if len(list(node)) > 0:
                for child in list(node):
                    ret += self.parse_html_type(child)
            else:
                if node.text:
                    ret += "
" + str(node.text).strip()
                if node.tail:
                    ret += str(node.tail).strip() + "
"
                else:
                    ret += "
"
        if tag == 'listitem':
            if len(list(node)) > 0:
                for child in list(node):
                    ret += self.parse_html_type(child)
            else:
                if node.text:
                    ret += "
" + str(node.text).strip() + "
"
        if tag == 'orderedlist':
            i = 1
            for item in list(node):
                ret += "
" + str(i) + " " + self.parse_html_type(item) + "
"
                i += 1
        if tag == 'paragraph':
            if len(list(node)) > 0:
                for child in list(node):
                    ret += self.parse_html_type(child)
            else:
                if node.text:
                    ret += "
" + node.text.strip()
                if node.tail:
                    ret += str(node.tail).strip() + "
"
                else:
                    ret += "
"
        if tag == 'unorderedlist':
            for item in list(node):
                unorderedlist = self.parse_html_type(item)
                if unorderedlist not in ret:
                    ret += "* " + unorderedlist
        if tag == 'urllink':
            if node.text:
                ret += str(node.text).strip() + " "
            last = ""

            for attr in node.attrib:
                if last != "":
                    if node.get(attr) != node.get(last):
                        ret += str(node.get(attr)) + " "
                last = attr

        return ret

    def parse_tests_type(self, node, vulnsDefinitions):
        """
        Parse XML element of type TestsType

        @return vulns A list of vulnerabilities according to vulnsDefinitions
        """
        vulns = list()

        for tests in node.findall('tests'):
            for test in tests.findall('test'):
                if test.get('id') in vulnsDefinitions and (
                        test.get('status') in ['vulnerable-exploited', 'vulnerable-version', 'vulnerable-potential']):
                    vuln = vulnsDefinitions[test.get('id').lower()]
                    for desc in test.getchildren():
                        if 'pluginOutput' in vuln:
                            vuln['pluginOutput'] += "\n\n" + \
                                self.parse_html_type(desc)
                        else:
                            vuln['pluginOutput'] = self.parse_html_type(desc)
                    vulns.append(vuln)

        return vulns

    def get_vuln_definitions(self, tree):
        """
        @returns vulns A dict of Vulnerability Definitions
        """
        vulns = dict()
        url_index = 0
        for vulnsDef in tree.findall('VulnerabilityDefinitions'):
            for vulnDef in vulnsDef.findall('vulnerability'):
                vid = vulnDef.get('id').lower()
                severity_chk = int(vulnDef.get('severity'))
                if severity_chk >= 9:
                    sev = 'Critical'
                elif severity_chk >= 7:
                    sev = 'High'
                elif severity_chk >= 4:
                    sev = 'Medium'
                elif 0 < severity_chk < 4:
                    sev = 'Low'
                else:
                    sev = 'Info'
                vuln = {
                    'desc': "",
                    'name': vulnDef.get('title'),
                    'vector': vulnDef.get('cvssVector'),  # this is CVSS v2
                    'refs': dict(),
                    'resolution': "",
                    'severity': sev,
                    'tags': list()
                }
                for item in vulnDef.getchildren():
                    if item.tag == 'description':
                        for htmlType in item.getchildren():
                            vuln['desc'] += self.parse_html_type(htmlType)

                    elif item.tag == 'exploits':
                        for exploit in item.getchildren():
                            vuln['refs'][exploit.get('title')] = str(exploit.get('title')).strip() + ' ' + \
                                                                 str(exploit.get('link')).strip()

                    elif item.tag == 'references':
                        for ref in item.getchildren():
                            if 'URL' in ref.get('source'):
                                vuln['refs'][ref.get('source') + str(url_index)] = str(ref.text).strip()
                                url_index += 1
                            else:
                                vuln['refs'][ref.get('source')] = str(ref.text).strip()

                    elif item.tag == 'solution':
                        for htmlType in item.getchildren():
                            vuln['resolution'] += self.parse_html_type(htmlType)

                    # there is currently no method to register tags in vulns
                    elif item.tag == 'tags':
                        for tag in item.getchildren():
                            vuln['tags'].append(tag.text.lower())

                vulns[vid] = vuln
        return vulns

    def get_items(self, tree, vulns, test):
        hosts = list()
        for nodes in tree.findall('nodes'):
            for node in nodes.findall('node'):
                host = dict()
                host['name'] = node.get('address')
                host['hostnames'] = set()
                host['os'] = ""
                host['services'] = list()
                host['vulns'] = self.parse_tests_type(node, vulns)

                for names in node.findall('names'):
                    for name in names.findall('name'):
                        host['hostnames'].add(name.text)

                for endpoints in node.findall('endpoints'):
                    for endpoint in endpoints.findall('endpoint'):
                        svc = {
                            'protocol': endpoint.get('protocol'),
                            'port': int(endpoint.get('port')),
                            'status': endpoint.get('status'),
                        }
                        for services in endpoint.findall('services'):
                            for service in services.findall('service'):
                                svc['name'] = service.get('name')
                                svc['vulns'] = self.parse_tests_type(service, vulns)

                                for configs in service.findall('configurations'):
                                    for config in configs.findall('config'):
                                        if "banner" in config.get('name'):
                                            svc['version'] = config.get('name')

                        host['services'].append(svc)

                hosts.append(host)

        dupes = {}

        for host in hosts:
            # manage findings by node only
            for vuln in host['vulns']:
                dupe_key = vuln['severity'] + vuln['name']

                find = self.findings(dupe_key, dupes, test, vuln)

                endpoint = Endpoint(host=host['name'])
                find.unsaved_endpoints.append(endpoint)
                find.unsaved_tags = vuln['tags']

            # manage findings by service
            for service in host['services']:
                for vuln in service['vulns']:
                    dupe_key = vuln['severity'] + vuln['name']

                    find = self.findings(dupe_key, dupes, test, vuln)

                    endpoint = Endpoint(host=host['name'], port=service['port'], protocol=service['name'].lower())
                    find.unsaved_endpoints.append(endpoint)
                    find.unsaved_tags = vuln['tags']

        return list(dupes.values())

    @staticmethod
    def findings(dupe_key, dupes, test, vuln):
        """

        """
        if dupe_key in dupes:
            find = dupes[dupe_key]
            dupe_text = html2text.html2text(vuln['pluginOutput'])
            if dupe_text not in find.description:
                find.description += "\n\n" + dupe_text
        else:
            find = Finding(title=vuln['name'],
                           description=html2text.html2text(
                               vuln['desc'].strip()) + "\n\n" + html2text.html2text(vuln['pluginOutput'].strip()),
                           severity=vuln['severity'],
                           numerical_severity=Finding.get_numerical_severity(vuln['severity']),
                           mitigation=html2text.html2text(vuln['resolution']),
                           impact=vuln['vector'],
                           test=test,
                           active=False,
                           verified=False,
                           false_p=False,
                           duplicate=False,
                           out_of_scope=False,
                           mitigated=None,
                           dynamic_finding=True)
            # build references
            refs = ''
            for ref in vuln['refs']:
                if ref.startswith('BID'):
                    refs += f" * [{vuln['refs'][ref]}](https://www.securityfocus.com/bid/{vuln['refs'][ref]})"
                elif ref.startswith('CA'):
                    refs += f" * [{vuln['refs'][ref]}](https://www.cert.org/advisories/{vuln['refs'][ref]}.html)"
                elif ref.startswith('CERT-VN'):
                    refs += f" * [{vuln['refs'][ref]}](https://www.kb.cert.org/vuls/id/{vuln['refs'][ref]}.html)"
                elif ref.startswith('CVE'):
                    refs += f" * [{vuln['refs'][ref]}](https://cve.mitre.org/cgi-bin/cvename.cgi?name={vuln['refs'][ref]})"
                elif ref.startswith('DEBIAN'):
                    refs += f" * [{vuln['refs'][ref]}](https://security-tracker.debian.org/tracker/{vuln['refs'][ref]})"
                elif ref.startswith('XF'):
                    refs += f" * [{vuln['refs'][ref]}](https://exchange.xforce.ibmcloud.com/vulnerabilities/{vuln['refs'][ref]})"
                elif ref.startswith('URL'):
                    refs += f" * URL: {vuln['refs'][ref]}"
                else:
                    refs += f" * {ref}: {vuln['refs'][ref]}"
                refs += "\n"
            find.references = refs
            # update CVE
            if "CVE" in vuln['refs']:
                find.cve = vuln['refs']['CVE']
            find.unsaved_endpoints = list()
            dupes[dupe_key] = find
        return find
<|endoftext|>


<commit_before>##############################################################################
# Copyright (c) 2013-2017, Lawrence Livermore National Security, LLC.
# Produced at the Lawrence Livermore National Laboratory.
#
# This file is part of Spack.
# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved.
# LLNL-CODE-647188
#
# For details, see https://github.com/llnl/spack
# Please also see the NOTICE and LICENSE files for our notice and the LGPL.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License (as
# published by the Free Software Foundation) version 2.1, February 1999.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and
# conditions of the GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
##############################################################################

import os
from spack import *

class Nauty(AutotoolsPackage):
    """nauty and Traces are programs for computing automorphism groups of
    graphsq and digraphs"""
    homepage = "http://pallini.di.uniroma1.it/index.html"
    url      = "http://pallini.di.uniroma1.it/nauty26r7.tar.gz"

    version('2.6r7', 'b2b18e03ea7698db3fbe06c5d76ad8fe')

    # Debian/ Fedora patches for @2.6r7:
    urls_for_patches = {
        '@2.6r7': [
            # Debian patch to fix the gt_numorbits declaration
            ('https://src.fedoraproject.org/rpms/nauty/raw/0f07d01caf84e9d30cb06b11af4860dd3837636a/f/nauty-fix-gt_numorbits.patch', 'a6e1ef4897aabd67c104fd1d78bcc334'),  # noqa: E50
            # Debian patch to add explicit extern declarations where needed
            ('https://src.fedoraproject.org/rpms/nauty/raw/0f07d01caf84e9d30cb06b11af4860dd3837636a/f/nauty-fix-include-extern.patch', '741034dec2d2f8b418b6e186aa3eb50f'),  # noqa: E50
            # Debian patch to use zlib instead of invoking zcat through a pipe
            ('https://src.fedoraproject.org/rpms/nauty/raw/0f07d01caf84e9d30cb06b11af4860dd3837636a/f/nauty-zlib-blisstog.patch', '667e1ce341f2506482ad30afd04f17e3'),  # noqa: E50
            # Debian patch to improve usage and help information
            ('https://src.fedoraproject.org/rpms/nauty/raw/0f07d01caf84e9d30cb06b11af4860dd3837636a/f/nauty-help2man.patch', '4202e6d83362daa2c4c4ab0788e11ac5'),  # noqa: E50
            # Debian patch to add libtool support for building a shared library
            ('https://src.fedoraproject.org/rpms/nauty/raw/0f07d01caf84e9d30cb06b11af4860dd3837636a/f/nauty-autotoolization.patch', 'ea75f19c8a980c4d6d4e07223785c751'),  # noqa: E50
            # Debian patch to canonicalize header file usage
            ('https://src.fedoraproject.org/rpms/nauty/raw/0f07d01caf84e9d30cb06b11af4860dd3837636a/f/nauty-includes.patch', 'c6ce4209d1381fb5489ed552ef35d7dc'),  # noqa: E50
            # Debian patch to prefix "nauty-" to the names of the generic tools
            ('https://src.fedoraproject.org/rpms/nauty/raw/0f07d01caf84e9d30cb06b11af4860dd3837636a/f/nauty-tool-prefix.patch', 'e89d87b4450adc5d0009ce11438dc975'),  # noqa: E50
            # Fedora patch to detect availability of the popcnt
            # instruction at runtime
            ('https://src.fedoraproject.org/rpms/nauty/raw/0f07d01caf84e9d30cb06b11af4860dd3837636a/f/nauty-popcnt.patch', '8a32d31a7150c8f5f21ccb1f6dc857b1')  # noqa: E50
        ]
    }
    # Iterate over patches
    for condition, urls in urls_for_patches.items():
        for url, md5 in urls:
            patch(url, when=condition, level=1, md5=md5)

    depends_on('m4',  type='build', when='@2.6r7')
    depends_on('autoconf',  type='build', when='@2.6r7')
    depends_on('automake',  type='build', when='@2.6r7')
    depends_on('libtool',  type='build', when='@2.6r7')
    depends_on('pkg-config',  type='build')
    depends_on('zlib')

    @property
    def force_autoreconf(self):
        return self.spec.satisfies('@2.6r7')

    def url_for_version(self, version):
        url = "http://pallini.di.uniroma1.it/nauty{0}.tar.gz"
        return url.format(version.joined)

    def patch(self):
        os.remove('makefile')
        ver = str(self.version.dotted).replace('r', '.')
        if self.spec.satisfies('@2.6r7'):
            filter_file('@INJECTVER@', ver, "configure.ac")
<commit_msg>Added missing dependencies to nauty package (#5348)<commit_after>##############################################################################
# Copyright (c) 2013-2017, Lawrence Livermore National Security, LLC.
# Produced at the Lawrence Livermore National Laboratory.
#
# This file is part of Spack.
# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved.
# LLNL-CODE-647188
#
# For details, see https://github.com/llnl/spack
# Please also see the NOTICE and LICENSE files for our notice and the LGPL.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License (as
# published by the Free Software Foundation) version 2.1, February 1999.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and
# conditions of the GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
##############################################################################

import os
from spack import *

class Nauty(AutotoolsPackage):
    """nauty and Traces are programs for computing automorphism groups of
    graphsq and digraphs"""
    homepage = "http://pallini.di.uniroma1.it/index.html"
    url      = "http://pallini.di.uniroma1.it/nauty26r7.tar.gz"

    version('2.6r7', 'b2b18e03ea7698db3fbe06c5d76ad8fe')

    # Debian/ Fedora patches for @2.6r7:
    urls_for_patches = {
        '@2.6r7': [
            # Debian patch to fix the gt_numorbits declaration
            ('https://src.fedoraproject.org/rpms/nauty/raw/0f07d01caf84e9d30cb06b11af4860dd3837636a/f/nauty-fix-gt_numorbits.patch', 'a6e1ef4897aabd67c104fd1d78bcc334'),  # noqa: E50
            # Debian patch to add explicit extern declarations where needed
            ('https://src.fedoraproject.org/rpms/nauty/raw/0f07d01caf84e9d30cb06b11af4860dd3837636a/f/nauty-fix-include-extern.patch', '741034dec2d2f8b418b6e186aa3eb50f'),  # noqa: E50
            # Debian patch to use zlib instead of invoking zcat through a pipe
            ('https://src.fedoraproject.org/rpms/nauty/raw/0f07d01caf84e9d30cb06b11af4860dd3837636a/f/nauty-zlib-blisstog.patch', '667e1ce341f2506482ad30afd04f17e3'),  # noqa: E50
            # Debian patch to improve usage and help information
            ('https://src.fedoraproject.org/rpms/nauty/raw/0f07d01caf84e9d30cb06b11af4860dd3837636a/f/nauty-help2man.patch', '4202e6d83362daa2c4c4ab0788e11ac5'),  # noqa: E50
            # Debian patch to add libtool support for building a shared library
            ('https://src.fedoraproject.org/rpms/nauty/raw/0f07d01caf84e9d30cb06b11af4860dd3837636a/f/nauty-autotoolization.patch', 'ea75f19c8a980c4d6d4e07223785c751'),  # noqa: E50
            # Debian patch to canonicalize header file usage
            ('https://src.fedoraproject.org/rpms/nauty/raw/0f07d01caf84e9d30cb06b11af4860dd3837636a/f/nauty-includes.patch', 'c6ce4209d1381fb5489ed552ef35d7dc'),  # noqa: E50
            # Debian patch to prefix "nauty-" to the names of the generic tools
            ('https://src.fedoraproject.org/rpms/nauty/raw/0f07d01caf84e9d30cb06b11af4860dd3837636a/f/nauty-tool-prefix.patch', 'e89d87b4450adc5d0009ce11438dc975'),  # noqa: E50
            # Fedora patch to detect availability of the popcnt
            # instruction at runtime
            ('https://src.fedoraproject.org/rpms/nauty/raw/0f07d01caf84e9d30cb06b11af4860dd3837636a/f/nauty-popcnt.patch', '8a32d31a7150c8f5f21ccb1f6dc857b1')  # noqa: E50
        ]
    }
    # Iterate over patches
    for condition, urls in urls_for_patches.items():
        for url, md5 in urls:
            patch(url, when=condition, level=1, md5=md5)

    depends_on('m4',  type='build', when='@2.6r7')
    depends_on('autoconf',  type='build', when='@2.6r7')
    depends_on('automake',  type='build', when='@2.6r7')
    depends_on('libtool',  type='build', when='@2.6r7')
    depends_on('pkg-config',  type='build')
    depends_on('help2man', type='build')
    depends_on('zlib')
    depends_on('gmp')

    @property
    def force_autoreconf(self):
        return self.spec.satisfies('@2.6r7')

    def url_for_version(self, version):
        url = "http://pallini.di.uniroma1.it/nauty{0}.tar.gz"
        return url.format(version.joined)

    def patch(self):
        os.remove('makefile')
        ver = str(self.version.dotted).replace('r', '.')
        if self.spec.satisfies('@2.6r7'):
            filter_file('@INJECTVER@', ver, "configure.ac")
<|endoftext|>


<commit_before>"""
mbed SDK
Copyright (c) 2011-2013 ARM Limited

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""

CORE_LABELS = {
    "ARM7TDMI-S": "ARM7",
    "Cortex-M0" : "M0",
    "Cortex-M0+": "M0P",
    "Cortex-M3" : "M3",
    "Cortex-M4" : "M4",
    "Cortex-M4F" : "M4F"
}

import os
import shutil

class Target:
    def __init__(self):
        # ARM Core
        self.core = None

        # Is the disk provided by the interface chip of this board virtual?
        self.is_disk_virtual = False

        # list of toolchains that are supported by the mbed SDK for this target
        self.supported_toolchains = None

        # list of extra specific labels
        self.extra_labels = []

        self.name = self.__class__.__name__

    def program_cycle_s(self):
        return 4 if self.is_disk_virtual else 1.5

    def get_labels(self):
        return [self.name, CORE_LABELS[self.core]] + self.extra_labels

    def init_hooks(self, hook, toolchain_name):
        pass

class LPC2368(Target):
    def __init__(self):
        Target.__init__(self)

        self.core = "ARM7TDMI-S"

        self.extra_labels = ['NXP', 'LPC23XX']

        self.supported_toolchains = ["ARM","GCC_ARM","GCC_CR"]

class LPC1768(Target):
    def __init__(self):
        Target.__init__(self)

        self.core = "Cortex-M3"

        self.extra_labels = ['NXP', 'LPC176X']

        self.supported_toolchains = ["ARM", "uARM", "GCC_ARM", "GCC_CS", "GCC_CR", "IAR"]

class LPC11U24(Target):
    def __init__(self):
        Target.__init__(self)

        self.core = "Cortex-M0"

        self.extra_labels = ['NXP', 'LPC11UXX', 'LPC11U24_401']

        self.supported_toolchains = ["ARM", "uARM", "GCC_ARM"]

class LPC11U24_301(Target):
    def __init__(self):
        Target.__init__(self)

        self.core = "Cortex-M0"

        self.extra_labels = ['NXP', 'LPC11UXX']

        self.supported_toolchains = ["ARM", "uARM", "GCC_ARM"]

class KL05Z(Target):
    def __init__(self):
        Target.__init__(self)

        self.core = "Cortex-M0+"

        self.extra_labels = ['Freescale']

        self.supported_toolchains = ["ARM"]

        self.is_disk_virtual = True

class KL25Z(Target):
    def __init__(self):
        Target.__init__(self)

        self.core = "Cortex-M0+"

        self.extra_labels = ['Freescale']

        self.supported_toolchains = ["ARM", "GCC_CW_EWL", "GCC_CW_NEWLIB", "GCC_ARM"]

        self.is_disk_virtual = True

class KL46Z(Target):
    def __init__(self):
        Target.__init__(self)

        self.core = "Cortex-M0+"

        self.extra_labels = ['Freescale']

        self.supported_toolchains = ["GCC_ARM", "ARM"]

        self.is_disk_virtual = True

class LPC812(Target):
    def __init__(self):
        Target.__init__(self)

        self.core = "Cortex-M0+"

        self.extra_labels = ['NXP', 'LPC81X']

        self.supported_toolchains = ["uARM"]

        self.is_disk_virtual = True

class LPC810(Target):
    def __init__(self):
        Target.__init__(self)

        self.core = "Cortex-M0+"

        self.extra_labels = ['NXP', 'LPC81X']

        self.supported_toolchains = ["uARM"]

        self.is_disk_virtual = True

class LPC4088(Target):
    def __init__(self):
        Target.__init__(self)

        self.core = "Cortex-M4F"

        self.extra_labels = ['NXP', 'LPC408X']

        self.supported_toolchains = ["ARM", "GCC_CR", "GCC_ARM"]

    def init_hooks(self, hook, toolchain_name):
        if toolchain_name in ['ARM_STD', 'ARM_MICRO']:
            hook.hook_add_binary("post", self.binary_hook)
            hook.hook_cmdline_linker(self.cmdline_hook)

    @staticmethod
    def cmdline_hook(toolchain, cmdline):
        return cmdline + ["--any_placement=first_fit"]

    @staticmethod
    def binary_hook(t_self, elf, binf):
        if not os.path.isdir(binf):
            # Regular binary file, nothing to do
            return
        outbin = open(binf + ".temp", "wb")
        partf = open(os.path.join(binf, "ER_IROM1"), "rb")
        # Pad the fist part (internal flash) with 0xFF to 512k
        data = partf.read()
        outbin.write(data)
        outbin.write('\xFF' * (512*1024 - len(data)))
        partf.close()
        # Read and append the second part (external flash) in chunks of fixed size
        chunksize = 128 * 1024
        partf = open(os.path.join(binf, "ER_IROM2"), "rb")
        while True:
            data = partf.read(chunksize)
            outbin.write(data)
            if len(data) < chunksize:
                break
        partf.close()
        outbin.close()
        # Remove the directory with the binary parts and rename the temporary
        # file to 'binf'
        shutil.rmtree(binf, True)
        os.rename(binf + '.temp', binf)
        t_self.debug("Generated custom binary file (internal flash + SPIFI)")

class LPC4330_M4(Target):
    def __init__(self):
        Target.__init__(self)

        self.core = "Cortex-M4F"

        self.extra_labels = ['NXP', 'LPC43XX']

        self.supported_toolchains = ["ARM", "GCC_CR", "IAR", "GCC_ARM"]

class LPC4330_M0(Target):
    def __init__(self):
        Target.__init__(self)

        self.core = "Cortex-M0"

        self.extra_labels = ['NXP', 'LPC43XX']

        self.supported_toolchains = ["ARM", "GCC_CR", "IAR"]

class LPC1800(Target):
    def __init__(self):
        Target.__init__(self)

        self.core = "Cortex-M3"

        self.extra_labels = ['NXP', 'LPC43XX']

        self.supported_toolchains = ["ARM", "GCC_CR", "IAR"]

class STM32F407(Target):
    def __init__(self):
        Target.__init__(self)

        self.core = "Cortex-M4F"

        self.extra_labels = ['STM', 'STM32F4XX']

        self.supported_toolchains = ["ARM", "GCC_ARM"]

class NUCLEO_F103RB(Target):
    def __init__(self):
        Target.__init__(self)

        self.core = "Cortex-M3"

        self.extra_labels = ['STM', 'STM32F10X', 'STM32F103RB']

        self.supported_toolchains = ["ARM", "uARM", "GCC_ARM"]

class MBED_MCU(Target):
    def __init__(self):
        Target.__init__(self)

        self.core = "Cortex-M0+"

        self.extra_labels = ['ARM']

        self.supported_toolchains = ["ARM"]

class LPC1347(Target):
    def __init__(self):
        Target.__init__(self)

        self.core = "Cortex-M3"

        self.extra_labels = ['NXP', 'LPC13XX']

        self.supported_toolchains = ["ARM", "GCC_ARM"]

class LPC1114(Target):
    def __init__(self):
        Target.__init__(self)

        self.core = "Cortex-M0"

        self.extra_labels = ['NXP', 'LPC11XX_11CXX', 'LPC11XX']

        self.supported_toolchains = ["ARM", "uARM", "GCC_ARM"]

class LPC11C24(Target):
    def __init__(self):
        Target.__init__(self)

        self.core = "Cortex-M0"

        self.extra_labels = ['NXP', 'LPC11XX_11CXX', 'LPC11CXX']

        self.supported_toolchains = ["ARM", "uARM", "GCC_ARM"]

class LPC11U35_401(Target):
    def __init__(self):
        Target.__init__(self)

        self.core = "Cortex-M0"

        self.extra_labels = ['NXP', 'LPC11UXX']

        self.supported_toolchains = ["ARM", "uARM", "GCC_ARM"]

class nRF51822(Target):
    def __init__(self):
        Target.__init__(self)

        self.core = "Cortex-M0"

        self.extra_labels = ["NORDIC"]

        self.supported_toolchains = ["ARM"]

# Get a single instance for each target
TARGETS = [
    LPC2368(),
    LPC1768(),
    LPC11U24(),
    LPC11U24_301(),
    KL05Z(),
    KL25Z(),
    KL46Z(),
    LPC812(),
    LPC810(),
    LPC4088(),
    LPC4330_M4(),
    STM32F407(),
    NUCLEO_F103RB(),
    MBED_MCU(),
    LPC1347(),
    LPC1114(),
    LPC11C24(),
    LPC11U35_401(),
    nRF51822()
]

# Map each target name to its unique instance
TARGET_MAP = {}
for t in TARGETS:
    TARGET_MAP[t.name] = t

TARGET_NAMES = TARGET_MAP.keys()

# Some targets with different name have the same exporters
EXPORT_MAP = {}
<commit_msg>Fix defined labels for Cortex-M4F<commit_after>"""
mbed SDK
Copyright (c) 2011-2013 ARM Limited

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""

CORE_LABELS = {
    "ARM7TDMI-S": "ARM7",
    "Cortex-M0" : "M0",
    "Cortex-M0+": "M0P",
    "Cortex-M3" : "M3",
    "Cortex-M4" : "M4",
    "Cortex-M4F" : "M4"
}

import os
import shutil

class Target:
    def __init__(self):
        # ARM Core
        self.core = None

        # Is the disk provided by the interface chip of this board virtual?
        self.is_disk_virtual = False

        # list of toolchains that are supported by the mbed SDK for this target
        self.supported_toolchains = None

        # list of extra specific labels
        self.extra_labels = []

        self.name = self.__class__.__name__

    def program_cycle_s(self):
        return 4 if self.is_disk_virtual else 1.5

    def get_labels(self):
        return [self.name, CORE_LABELS[self.core]] + self.extra_labels

    def init_hooks(self, hook, toolchain_name):
        pass

class LPC2368(Target):
    def __init__(self):
        Target.__init__(self)

        self.core = "ARM7TDMI-S"

        self.extra_labels = ['NXP', 'LPC23XX']

        self.supported_toolchains = ["ARM","GCC_ARM","GCC_CR"]

class LPC1768(Target):
    def __init__(self):
        Target.__init__(self)

        self.core = "Cortex-M3"

        self.extra_labels = ['NXP', 'LPC176X']

        self.supported_toolchains = ["ARM", "uARM", "GCC_ARM", "GCC_CS", "GCC_CR", "IAR"]

class LPC11U24(Target):
    def __init__(self):
        Target.__init__(self)

        self.core = "Cortex-M0"

        self.extra_labels = ['NXP', 'LPC11UXX', 'LPC11U24_401']

        self.supported_toolchains = ["ARM", "uARM", "GCC_ARM"]

class LPC11U24_301(Target):
    def __init__(self):
        Target.__init__(self)

        self.core = "Cortex-M0"

        self.extra_labels = ['NXP', 'LPC11UXX']

        self.supported_toolchains = ["ARM", "uARM", "GCC_ARM"]

class KL05Z(Target):
    def __init__(self):
        Target.__init__(self)

        self.core = "Cortex-M0+"

        self.extra_labels = ['Freescale']

        self.supported_toolchains = ["ARM"]

        self.is_disk_virtual = True

class KL25Z(Target):
    def __init__(self):
        Target.__init__(self)

        self.core = "Cortex-M0+"

        self.extra_labels = ['Freescale']

        self.supported_toolchains = ["ARM", "GCC_CW_EWL", "GCC_CW_NEWLIB", "GCC_ARM"]

        self.is_disk_virtual = True

class KL46Z(Target):
    def __init__(self):
        Target.__init__(self)

        self.core = "Cortex-M0+"

        self.extra_labels = ['Freescale']

        self.supported_toolchains = ["GCC_ARM", "ARM"]

        self.is_disk_virtual = True

class LPC812(Target):
    def __init__(self):
        Target.__init__(self)

        self.core = "Cortex-M0+"

        self.extra_labels = ['NXP', 'LPC81X']

        self.supported_toolchains = ["uARM"]

        self.is_disk_virtual = True

class LPC810(Target):
    def __init__(self):
        Target.__init__(self)

        self.core = "Cortex-M0+"

        self.extra_labels = ['NXP', 'LPC81X']

        self.supported_toolchains = ["uARM"]

        self.is_disk_virtual = True

class LPC4088(Target):
    def __init__(self):
        Target.__init__(self)

        self.core = "Cortex-M4F"

        self.extra_labels = ['NXP', 'LPC408X']

        self.supported_toolchains = ["ARM", "GCC_CR", "GCC_ARM"]

    def init_hooks(self, hook, toolchain_name):
        if toolchain_name in ['ARM_STD', 'ARM_MICRO']:
            hook.hook_add_binary("post", self.binary_hook)
            hook.hook_cmdline_linker(self.cmdline_hook)

    @staticmethod
    def cmdline_hook(toolchain, cmdline):
        return cmdline + ["--any_placement=first_fit"]

    @staticmethod
    def binary_hook(t_self, elf, binf):
        if not os.path.isdir(binf):
            # Regular binary file, nothing to do
            return
        outbin = open(binf + ".temp", "wb")
        partf = open(os.path.join(binf, "ER_IROM1"), "rb")
        # Pad the fist part (internal flash) with 0xFF to 512k
        data = partf.read()
        outbin.write(data)
        outbin.write('\xFF' * (512*1024 - len(data)))
        partf.close()
        # Read and append the second part (external flash) in chunks of fixed size
        chunksize = 128 * 1024
        partf = open(os.path.join(binf, "ER_IROM2"), "rb")
        while True:
            data = partf.read(chunksize)
            outbin.write(data)
            if len(data) < chunksize:
                break
        partf.close()
        outbin.close()
        # Remove the directory with the binary parts and rename the temporary
        # file to 'binf'
        shutil.rmtree(binf, True)
        os.rename(binf + '.temp', binf)
        t_self.debug("Generated custom binary file (internal flash + SPIFI)")

class LPC4330_M4(Target):
    def __init__(self):
        Target.__init__(self)

        self.core = "Cortex-M4F"

        self.extra_labels = ['NXP', 'LPC43XX']

        self.supported_toolchains = ["ARM", "GCC_CR", "IAR", "GCC_ARM"]

class LPC4330_M0(Target):
    def __init__(self):
        Target.__init__(self)

        self.core = "Cortex-M0"

        self.extra_labels = ['NXP', 'LPC43XX']

        self.supported_toolchains = ["ARM", "GCC_CR", "IAR"]

class LPC1800(Target):
    def __init__(self):
        Target.__init__(self)

        self.core = "Cortex-M3"

        self.extra_labels = ['NXP', 'LPC43XX']

        self.supported_toolchains = ["ARM", "GCC_CR", "IAR"]

class STM32F407(Target):
    def __init__(self):
        Target.__init__(self)

        self.core = "Cortex-M4F"

        self.extra_labels = ['STM', 'STM32F4XX']

        self.supported_toolchains = ["ARM", "GCC_ARM"]

class NUCLEO_F103RB(Target):
    def __init__(self):
        Target.__init__(self)

        self.core = "Cortex-M3"

        self.extra_labels = ['STM', 'STM32F10X', 'STM32F103RB']

        self.supported_toolchains = ["ARM", "uARM", "GCC_ARM"]

class MBED_MCU(Target):
    def __init__(self):
        Target.__init__(self)

        self.core = "Cortex-M0+"

        self.extra_labels = ['ARM']

        self.supported_toolchains = ["ARM"]

class LPC1347(Target):
    def __init__(self):
        Target.__init__(self)

        self.core = "Cortex-M3"

        self.extra_labels = ['NXP', 'LPC13XX']

        self.supported_toolchains = ["ARM", "GCC_ARM"]

class LPC1114(Target):
    def __init__(self):
        Target.__init__(self)

        self.core = "Cortex-M0"

        self.extra_labels = ['NXP', 'LPC11XX_11CXX', 'LPC11XX']

        self.supported_toolchains = ["ARM", "uARM", "GCC_ARM"]

class LPC11C24(Target):
    def __init__(self):
        Target.__init__(self)

        self.core = "Cortex-M0"

        self.extra_labels = ['NXP', 'LPC11XX_11CXX', 'LPC11CXX']

        self.supported_toolchains = ["ARM", "uARM", "GCC_ARM"]

class LPC11U35_401(Target):
    def __init__(self):
        Target.__init__(self)

        self.core = "Cortex-M0"

        self.extra_labels = ['NXP', 'LPC11UXX']

        self.supported_toolchains = ["ARM", "uARM", "GCC_ARM"]

class nRF51822(Target):
    def __init__(self):
        Target.__init__(self)

        self.core = "Cortex-M0"

        self.extra_labels = ["NORDIC"]

        self.supported_toolchains = ["ARM"]

# Get a single instance for each target
TARGETS = [
    LPC2368(),
    LPC1768(),
    LPC11U24(),
    LPC11U24_301(),
    KL05Z(),
    KL25Z(),
    KL46Z(),
    LPC812(),
    LPC810(),
    LPC4088(),
    LPC4330_M4(),
    STM32F407(),
    NUCLEO_F103RB(),
    MBED_MCU(),
    LPC1347(),
    LPC1114(),
    LPC11C24(),
    LPC11U35_401(),
    nRF51822()
]

# Map each target name to its unique instance
TARGET_MAP = {}
for t in TARGETS:
    TARGET_MAP[t.name] = t

TARGET_NAMES = TARGET_MAP.keys()

# Some targets with different name have the same exporters
EXPORT_MAP = {}
<|endoftext|>


<commit_before>#!/usr/bin/env python2

from argparse import ArgumentParser
from sys import stdout

from protocolBase import PushjetProtocolBase
from twisted.internet import reactor, protocol, endpoints
from twisted.protocols import basic
from twisted.python import log

class PushjetTCPBase(basic.LineOnlyReceiver, PushjetProtocolBase):
    magic_start = "\002"
    magic_end   = "\003"

    def __init__(self):
        PushjetProtocolBase.__init__(self, args.api, args.pub)
        self.onMessage = self.lineReceived
        self.lineReceived = self.onClientMessage

    def sendMessage(self, message):
        self.sendLine(self.magic_start + message + self.magic_end)

if __name__ == '__main__':
    parser = ArgumentParser(description='Pushjet websocket server')

    parser.add_argument('--port', '-p', default='7171', type=int,
                        help='the port the server should bind to (default: 8181)')
    parser.add_argument('--api', '-a', default='https://api.pushjet.io', type=str, metavar='SRV',
                        help='the api server url (default: https://api.pushjet.io)')
    parser.add_argument('--pub', '-z', default='ipc:///tmp/pushjet-publisher.ipc', type=str,
                        help='the publisher uri for receiving messages (default: ipc:///tmp/pushjet-publisher.ipc)')
    parser.add_argument('--log', '-l', default=None, type=str, metavar='PATH', dest='logfile',
                        help='log file path')
    parser.add_argument('--quiet', '-q', action='store_true')

    args = parser.parse_args()

    if not args.quiet:
        log.startLogging(stdout)
    if args.logfile:
        log.startLogging(open(args.logfile, 'a'))
        print("Started logging to file %s" % args.logfile)

    factory = protocol.Factory()
    factory.protocol = PushjetTCPBase

    endpoints.serverFromString(reactor, str("tcp:%i" % args.port)).listen(factory)
    reactor.run()
<commit_msg>TCP Server<commit_after>#!/usr/bin/env python2

from argparse import ArgumentParser
from sys import stdout

from protocolBase import PushjetProtocolBase
from twisted.internet import reactor, protocol
from twisted.python import log

class PushjetTCPBase(protocol.Protocol, PushjetProtocolBase):
    magic_start = '\002'
    magic_end   = '\003'
    uuid_len    = 36
    _buffer     = b''

    def __init__(self):
        PushjetProtocolBase.__init__(self, args.api, args.pub)

    def dataReceived(self, data):
        self._buffer += data
        if len(self._buffer) >= self.uuid_len:
            frame = self._buffer[:self.uuid_len]
            self._buffer = self._buffer[self.uuid_len:]
            print frame
            self.onClientMessage(frame)

    def sendMessage(self, message):
        self.transport.writeSequence((self.magic_start, message, self.magic_end))

class PushjetTCPBaseFactory(protocol.Factory):
    protocol = PushjetTCPBase

    def __init__(self, reactor):
        self.reactor = reactor

if __name__ == '__main__':
    parser = ArgumentParser(description='Pushjet websocket server')

    parser.add_argument('--port', '-p', default='7171', type=int,
                        help='the port the server should bind to (default: 8181)')
    parser.add_argument('--api', '-a', default='https://api.pushjet.io', type=str, metavar='SRV',
                        help='the api server url (default: https://api.pushjet.io)')
    parser.add_argument('--pub', '-z', default='ipc:///tmp/pushjet-publisher.ipc', type=str,
                        help='the publisher uri for receiving messages (default: ipc:///tmp/pushjet-publisher.ipc)')
    parser.add_argument('--log', '-l', default=None, type=str, metavar='PATH', dest='logfile',
                        help='log file path')
    parser.add_argument('--quiet', '-q', action='store_true')

    args = parser.parse_args()

    if not args.quiet:
        log.startLogging(stdout)
    if args.logfile:
        log.startLogging(open(args.logfile, 'a'))
        print("Started logging to file %s" % args.logfile)

    reactor.listenTCP(args.port, PushjetTCPBaseFactory(reactor))
    reactor.run()
<|endoftext|>


<commit_before># Licensed to the StackStorm, Inc ('StackStorm') under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import sys
import json
import argparse
from oslo_config import cfg

from st2common import log as logging
from st2actions import config
from st2common.runners.base_action import Action
from st2common.runners.utils import get_logger_for_python_runner_action
from st2common.runners.utils import get_action_class_instance
from st2common.util import loader as action_loader
from st2common.util.config_loader import ContentPackConfigLoader
from st2common.constants.action import ACTION_OUTPUT_RESULT_DELIMITER
from st2common.constants.keyvalue import SYSTEM_SCOPE
from st2common.constants.runners import PYTHON_RUNNER_INVALID_ACTION_STATUS_EXIT_CODE
from st2common.database_setup import db_setup

__all__ = [
    'PythonActionWrapper',
    'ActionService'
]

LOG = logging.getLogger(__name__)

INVALID_STATUS_ERROR_MESSAGE = """
If this is an existing action which returns a tuple with two items, it needs to be updated to
either:

1. Return a list instead of a tuple
2. Return a tuple where a first items is a status flag - (True, ('item1', 'item2'))

For more information, please see: https://docs.stackstorm.com/upgrade_notes.html#st2-v1-6
""".strip()

class ActionService(object):
    """
    Instance of this class is passed to the action instance and exposes "public"
    methods which can be called by the action.
    """

    def __init__(self, action_wrapper):
        self._action_wrapper = action_wrapper
        self._datastore_service = None

    @property
    def datastore_service(self):
        # Late import to avoid very expensive in-direct import (~1 second) when this function is
        # not called / used
        from st2common.services.datastore import DatastoreService

        if not self._datastore_service:
            logger = get_logger_for_python_runner_action(action_name=action_wrapper._class_name)
            self._datastore_service = DatastoreService(logger=logger,
                                                       pack_name=self._action_wrapper._pack,
                                                       class_name=self._action_wrapper._class_name,
                                                       api_username='action_service')
        return self._datastore_service

    ##################################
    # Methods for datastore management
    ##################################

    def list_values(self, local=True, prefix=None):
        return self.datastore_service.list_values(local, prefix)

    def get_value(self, name, local=True, scope=SYSTEM_SCOPE, decrypt=False):
        return self.datastore_service.get_value(name, local, scope=scope, decrypt=decrypt)

    def set_value(self, name, value, ttl=None, local=True, scope=SYSTEM_SCOPE, encrypt=False):
        return self.datastore_service.set_value(name, value, ttl, local, scope=scope,
                                                 encrypt=encrypt)

    def delete_value(self, name, local=True, scope=SYSTEM_SCOPE):
        return self.datastore_service.delete_value(name, local)

class PythonActionWrapper(object):
    def __init__(self, pack, file_path, parameters=None, user=None, parent_args=None):
        """
        :param pack: Name of the pack this action belongs to.
        :type pack: ``str``

        :param file_path: Path to the action module.
        :type file_path: ``str``

        :param parameters: action parameters.
        :type parameters: ``dict`` or ``None``

        :param user: Name of the user who triggered this action execution.
        :type user: ``str``

        :param parent_args: Command line arguments passed to the parent process.
        :type parse_args: ``list``
        """

        self._pack = pack
        self._file_path = file_path
        self._parameters = parameters or {}
        self._user = user
        self._parent_args = parent_args or []
        self._class_name = None
        self._logger = logging.getLogger('PythonActionWrapper')

        try:
            config.parse_args(args=self._parent_args)
        except Exception:
            pass

        db_setup()

        # Note: We can only set a default user value if one is not provided after parsing the
        # config
        if not self._user:
            self._user = cfg.CONF.system_user.user

    def run(self):
        action = self._get_action_instance()
        output = action.run(**self._parameters)

        if isinstance(output, tuple) and len(output) == 2:
            # run() method returned status and data - (status, data)
            action_status = output[0]
            action_result = output[1]
        else:
            # run() method returned only data, no status (pre StackStorm v1.6)
            action_status = None
            action_result = output

        action_output = {
            'result': action_result,
            'status': None
        }

        if action_status is not None and not isinstance(action_status, bool):
            sys.stderr.write('Status returned from the action run() method must either be '
                             'True or False, got: %s\n' % (action_status))
            sys.stderr.write(INVALID_STATUS_ERROR_MESSAGE)
            sys.exit(PYTHON_RUNNER_INVALID_ACTION_STATUS_EXIT_CODE)

        if action_status is not None and isinstance(action_status, bool):
            action_output['status'] = action_status

        try:
            print_output = json.dumps(action_output)
        except Exception:
            print_output = str(action_output)

        # Print output to stdout so the parent can capture it
        sys.stdout.write(ACTION_OUTPUT_RESULT_DELIMITER)
        sys.stdout.write(print_output + '\n')
        sys.stdout.write(ACTION_OUTPUT_RESULT_DELIMITER)

    def _get_action_instance(self):
        actions_cls = action_loader.register_plugin(Action, self._file_path)
        action_cls = actions_cls[0] if actions_cls and len(actions_cls) > 0 else None

        if not action_cls:
            raise Exception('File "%s" has no action or the file doesn\'t exist.' %
                            (self._file_path))

        config_loader = ContentPackConfigLoader(pack_name=self._pack, user=self._user)
        config = config_loader.get_config()

        if config:
            LOG.info('Found config for action "%s"' % (self._file_path))
        else:
            LOG.info('No config found for action "%s"' % (self._file_path))
            config = None

        action_service = ActionService(action_wrapper=self)
        action_instance = get_action_class_instance(action_cls=action_cls,
                                                    config=config,
                                                    action_service=action_service)
        return action_instance

if __name__ == '__main__':
    parser = argparse.ArgumentParser(description='Python action runner process wrapper')
    parser.add_argument('--pack', required=True,
                        help='Name of the pack this action belongs to')
    parser.add_argument('--file-path', required=True,
                        help='Path to the action module')
    parser.add_argument('--parameters', required=False,
                        help='Serialized action parameters')
    parser.add_argument('--user', required=False,
                        help='User who triggered the action execution')
    parser.add_argument('--parent-args', required=False,
                        help='Command line arguments passed to the parent process')
    args = parser.parse_args()

    parameters = args.parameters
    parameters = json.loads(parameters) if parameters else {}
    user = args.user
    parent_args = json.loads(args.parent_args) if args.parent_args else []

    assert isinstance(parent_args, list)
    obj = PythonActionWrapper(pack=args.pack,
                              file_path=args.file_path,
                              parameters=parameters,
                              user=user,
                              parent_args=parent_args)

    obj.run()
<commit_msg>Fix lint.<commit_after># Licensed to the StackStorm, Inc ('StackStorm') under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import sys
import json
import argparse
from oslo_config import cfg

from st2common import log as logging
from st2actions import config
from st2common.runners.base_action import Action
from st2common.runners.utils import get_logger_for_python_runner_action
from st2common.runners.utils import get_action_class_instance
from st2common.util import loader as action_loader
from st2common.util.config_loader import ContentPackConfigLoader
from st2common.constants.action import ACTION_OUTPUT_RESULT_DELIMITER
from st2common.constants.keyvalue import SYSTEM_SCOPE
from st2common.constants.runners import PYTHON_RUNNER_INVALID_ACTION_STATUS_EXIT_CODE
from st2common.database_setup import db_setup

__all__ = [
    'PythonActionWrapper',
    'ActionService'
]

LOG = logging.getLogger(__name__)

INVALID_STATUS_ERROR_MESSAGE = """
If this is an existing action which returns a tuple with two items, it needs to be updated to
either:

1. Return a list instead of a tuple
2. Return a tuple where a first items is a status flag - (True, ('item1', 'item2'))

For more information, please see: https://docs.stackstorm.com/upgrade_notes.html#st2-v1-6
""".strip()

class ActionService(object):
    """
    Instance of this class is passed to the action instance and exposes "public"
    methods which can be called by the action.
    """

    def __init__(self, action_wrapper):
        self._action_wrapper = action_wrapper
        self._datastore_service = None

    @property
    def datastore_service(self):
        # Late import to avoid very expensive in-direct import (~1 second) when this function is
        # not called / used
        from st2common.services.datastore import DatastoreService

        if not self._datastore_service:
            action_name = self._action_wrapper._class_name
            logger = get_logger_for_python_runner_action(action_name=action_name)
            self._datastore_service = DatastoreService(logger=logger,
                                                       pack_name=self._action_wrapper._pack,
                                                       class_name=self._action_wrapper._class_name,
                                                       api_username='action_service')
        return self._datastore_service

    ##################################
    # Methods for datastore management
    ##################################

    def list_values(self, local=True, prefix=None):
        return self.datastore_service.list_values(local, prefix)

    def get_value(self, name, local=True, scope=SYSTEM_SCOPE, decrypt=False):
        return self.datastore_service.get_value(name, local, scope=scope, decrypt=decrypt)

    def set_value(self, name, value, ttl=None, local=True, scope=SYSTEM_SCOPE, encrypt=False):
        return self.datastore_service.set_value(name, value, ttl, local, scope=scope,
                                                encrypt=encrypt)

    def delete_value(self, name, local=True, scope=SYSTEM_SCOPE):
        return self.datastore_service.delete_value(name, local)

class PythonActionWrapper(object):
    def __init__(self, pack, file_path, parameters=None, user=None, parent_args=None):
        """
        :param pack: Name of the pack this action belongs to.
        :type pack: ``str``

        :param file_path: Path to the action module.
        :type file_path: ``str``

        :param parameters: action parameters.
        :type parameters: ``dict`` or ``None``

        :param user: Name of the user who triggered this action execution.
        :type user: ``str``

        :param parent_args: Command line arguments passed to the parent process.
        :type parse_args: ``list``
        """

        self._pack = pack
        self._file_path = file_path
        self._parameters = parameters or {}
        self._user = user
        self._parent_args = parent_args or []
        self._class_name = None
        self._logger = logging.getLogger('PythonActionWrapper')

        try:
            config.parse_args(args=self._parent_args)
        except Exception:
            pass

        db_setup()

        # Note: We can only set a default user value if one is not provided after parsing the
        # config
        if not self._user:
            self._user = cfg.CONF.system_user.user

    def run(self):
        action = self._get_action_instance()
        output = action.run(**self._parameters)

        if isinstance(output, tuple) and len(output) == 2:
            # run() method returned status and data - (status, data)
            action_status = output[0]
            action_result = output[1]
        else:
            # run() method returned only data, no status (pre StackStorm v1.6)
            action_status = None
            action_result = output

        action_output = {
            'result': action_result,
            'status': None
        }

        if action_status is not None and not isinstance(action_status, bool):
            sys.stderr.write('Status returned from the action run() method must either be '
                             'True or False, got: %s\n' % (action_status))
            sys.stderr.write(INVALID_STATUS_ERROR_MESSAGE)
            sys.exit(PYTHON_RUNNER_INVALID_ACTION_STATUS_EXIT_CODE)

        if action_status is not None and isinstance(action_status, bool):
            action_output['status'] = action_status

        try:
            print_output = json.dumps(action_output)
        except Exception:
            print_output = str(action_output)

        # Print output to stdout so the parent can capture it
        sys.stdout.write(ACTION_OUTPUT_RESULT_DELIMITER)
        sys.stdout.write(print_output + '\n')
        sys.stdout.write(ACTION_OUTPUT_RESULT_DELIMITER)

    def _get_action_instance(self):
        actions_cls = action_loader.register_plugin(Action, self._file_path)
        action_cls = actions_cls[0] if actions_cls and len(actions_cls) > 0 else None

        if not action_cls:
            raise Exception('File "%s" has no action or the file doesn\'t exist.' %
                            (self._file_path))

        config_loader = ContentPackConfigLoader(pack_name=self._pack, user=self._user)
        config = config_loader.get_config()

        if config:
            LOG.info('Found config for action "%s"' % (self._file_path))
        else:
            LOG.info('No config found for action "%s"' % (self._file_path))
            config = None

        action_service = ActionService(action_wrapper=self)
        action_instance = get_action_class_instance(action_cls=action_cls,
                                                    config=config,
                                                    action_service=action_service)
        return action_instance

if __name__ == '__main__':
    parser = argparse.ArgumentParser(description='Python action runner process wrapper')
    parser.add_argument('--pack', required=True,
                        help='Name of the pack this action belongs to')
    parser.add_argument('--file-path', required=True,
                        help='Path to the action module')
    parser.add_argument('--parameters', required=False,
                        help='Serialized action parameters')
    parser.add_argument('--user', required=False,
                        help='User who triggered the action execution')
    parser.add_argument('--parent-args', required=False,
                        help='Command line arguments passed to the parent process')
    args = parser.parse_args()

    parameters = args.parameters
    parameters = json.loads(parameters) if parameters else {}
    user = args.user
    parent_args = json.loads(args.parent_args) if args.parent_args else []

    assert isinstance(parent_args, list)
    obj = PythonActionWrapper(pack=args.pack,
                              file_path=args.file_path,
                              parameters=parameters,
                              user=user,
                              parent_args=parent_args)

    obj.run()
<|endoftext|>