<commit_before>#! /usr/bin/python
# -*- coding: utf8 -*-

from __future__ import absolute_import, division, print_function

import os
import logging

from tldp.utils import which, firstfoundfile
from tldp.utils import arg_isexecutable, isexecutable
from tldp.utils import arg_isreadablefile, isreadablefile

from tldp.doctypes.common import BaseDoctype, SignatureChecker, depends

logger = logging.getLogger(__name__)

def rngfile_finder():
    l = ['/usr/share/xml/docbook/schema/rng/5.0/docbook.rng',
         ]
    return firstfoundfile(l)

def xslchunk_finder():
    l = ['/usr/share/xml/docbook/stylesheet/nwalsh5/current/html/chunk.xsl',
         '/usr/share/xml/docbook/stylesheet/docbook-xsl-ns/html/chunk.xsl',
         'http://docbook.sourceforge.net/release/xsl/current/html/chunk.xsl',
         ]
    return firstfoundfile(l)

def xslsingle_finder():
    l = ['/usr/share/xml/docbook/stylesheet/nwalsh5/current/html/docbook.xsl',
         '/usr/share/xml/docbook/stylesheet/docbook-xsl-ns/html/docbook.xsl',
         'http://docbook.sourceforge.net/release/xsl/current/html/docbook.xsl',
         ]
    return firstfoundfile(l)

def xslprint_finder():
    l = ['/usr/share/xml/docbook/stylesheet/nwalsh5/current/fo/docbook.xsl',
         '/usr/share/xml/docbook/stylesheet/docbook-xsl-ns/fo/docbook.xsl',
         'http://docbook.sourceforge.net/release/xsl/current/fo/docbook.xsl',
         ]
    return firstfoundfile(l)

class Docbook5XML(BaseDoctype, SignatureChecker):
    formatname = 'DocBook XML 5.x'
    extensions = ['.xml']
    signatures = ['-//OASIS//DTD DocBook V5.0/EN',
                  'http://docbook.org/ns/docbook', ]

    required = {'docbook5xml_xsltproc': isexecutable,
                'docbook5xml_xmllint': isexecutable,
                'docbook5xml_html2text': isexecutable,
                'docbook5xml_dblatex': isexecutable,
                'docbook5xml_fop': isexecutable,
                'docbook5xml_jing': isexecutable,
                'docbook5xml_rngfile': isreadablefile,
                'docbook5xml_xslprint': isreadablefile,
                'docbook5xml_xslchunk': isreadablefile,
                'docbook5xml_xslsingle': isreadablefile,
                }

    def chdir_output(self):
        os.chdir(self.output.dirname)
        return True

    @depends(chdir_output)
    def make_xincluded_source(self):
        s = '''"{config.docbook5xml_xmllint}" > "{output.validsource}" \\
                  --nonet \\
                  --noent \\
                  --xinclude \\
                  "{source.filename}"'''
        return self.shellscript(s)

    @depends(make_xincluded_source)
    def validate_source(self):
        '''consider lxml.etree and other validators'''
        s = '''"{config.docbook5xml_jing}" \\
                  "{config.docbook5xml_rngfile}" \\
                  "{output.validsource}"'''
        return self.shellscript(s)

    @depends(make_xincluded_source)
    def copy_static_resources(self):
        source = list()
        for d in ('images', 'resources'):
            fullpath = os.path.join(self.source.dirname, d)
            fullpath = os.path.abspath(fullpath)
            if os.path.isdir(fullpath):
                source.append('"' + fullpath + '"')
            if not source:
                logger.debug("%s no images or resources to copy",
                             self.source.stem)
                return True
            s = 'rsync --archive --verbose %s ./' % (' '.join(source))
        return self.shellscript(s)

    @depends(copy_static_resources)
    def make_name_htmls(self):
        '''create a single page HTML output'''
        s = '''"{config.docbook5xml_xsltproc}" > "{output.name_htmls}" \\
                  --nonet \\
                  --stringparam admon.graphics.path images/ \\
                  --stringparam base.dir . \\
                  "{config.docbook5xml_xslsingle}" \\
                  "{output.validsource}"'''
        return self.shellscript(s)

    @depends(make_name_htmls)
    def make_name_txt(self):
        '''create text output'''
        s = '''"{config.docbook5xml_html2text}" > "{output.name_txt}" \\
                  -style pretty \\
                  -nobs \\
                  "{output.name_htmls}"'''
        return self.shellscript(s)

    @depends(copy_static_resources)
    def make_fo(self):
        '''generate the Formatting Objects intermediate output'''
        s = '''"{config.docbook5xml_xsltproc}" > "{output.name_fo}" \\
                  "{config.docbook5xml_xslprint}" \\
                  "{output.validsource}"'''
        self.removals.append(self.output.name_fo)
        return self.shellscript(s)

    # -- this is conditionally built--see logic in make_name_pdf() below
    # @depends(make_fo)
    def make_pdf_with_fop(self):
        '''use FOP to create a PDF'''
        s = '''"{config.docbook5xml_fop}" \\
                  -fo "{output.name_fo}" \\
                  -pdf "{output.name_pdf}"'''
        return self.shellscript(s)

    # -- this is conditionally built--see logic in make_name_pdf() below
    # @depends(chdir_output)
    def make_pdf_with_dblatex(self):
        '''use dblatex (fallback) to create a PDF'''
        s = '''"{config.docbook5xml_dblatex}" \\
                  -F xml \\
                  -t pdf \\
                  -o "{output.name_pdf}" \\
                  "{output.validsource}"'''
        return self.shellscript(s)

    @depends(make_fo)
    def make_name_pdf(self):
        stem = self.source.stem
        classname = self.__class__.__name__
        logger.info("%s calling method %s.%s",
                    stem, classname, 'make_pdf_with_fop')
        if self.make_pdf_with_fop():
            return True
        logger.error("%s %s failed creating PDF, falling back to dblatex...",
                     stem, self.config.docbook5xml_fop)
        logger.info("%s calling method %s.%s",
                    stem, classname, 'make_pdf_with_dblatex')
        return self.make_pdf_with_dblatex()

    @depends(make_name_htmls)
    def make_html(self):
        '''create chunked HTML output'''
        s = '''"{config.docbook5xml_xsltproc}" \\
                  --nonet \\
                  --stringparam admon.graphics.path images/ \\
                  --stringparam base.dir . \\
                  "{config.docbook5xml_xslchunk}" \\
                  "{output.validsource}"'''
        return self.shellscript(s)

    @depends(make_html)
    def make_name_html(self):
        '''rename DocBook XSL's index.html to LDP standard STEM.html'''
        s = 'mv -v --no-clobber -- "{output.name_indexhtml}" "{output.name_html}"'
        return self.shellscript(s)

    @depends(make_name_html)
    def make_name_indexhtml(self):
        '''create final index.html symlink'''
        s = 'ln -svr -- "{output.name_html}" "{output.name_indexhtml}"'
        return self.shellscript(s)

    @depends(make_name_indexhtml, make_name_pdf)
    def remove_xincluded_source(self):
        '''remove the xincluded source file'''
        s = 'rm --verbose -- "{output.validsource}"'
        return self.shellscript(s)

    @classmethod
    def argparse(cls, p):
        descrip = 'executables for %s' % (cls.formatname,)
        g = p.add_argument_group(title=cls.__name__, description=descrip)
        g.add_argument('--docbook5xml-xslchunk', type=arg_isreadablefile,
                       default=xslchunk_finder(),
                       help='full path to LDP HTML chunker XSL [%(default)s]')
        g.add_argument('--docbook5xml-xslsingle', type=arg_isreadablefile,
                       default=xslsingle_finder(),
                       help='full path to LDP HTML single-page XSL [%(default)s]')
        g.add_argument('--docbook5xml-xslprint', type=arg_isreadablefile,
                       default=xslprint_finder(),
                       help='full path to LDP FO print XSL [%(default)s]')

        g.add_argument('--docbook5xml-rngfile', type=arg_isreadablefile,
                       default=rngfile_finder(),
                       help='full path to docbook.rng [%(default)s]')
        g.add_argument('--docbook5xml-xmllint', type=arg_isexecutable,
                       default=which('xmllint'),
                       help='full path to xmllint [%(default)s]')
        g.add_argument('--docbook5xml-xsltproc', type=arg_isexecutable,
                       default=which('xsltproc'),
                       help='full path to xsltproc [%(default)s]')
        g.add_argument('--docbook5xml-html2text', type=arg_isexecutable,
                       default=which('html2text'),
                       help='full path to html2text [%(default)s]')
        g.add_argument('--docbook5xml-fop', type=arg_isexecutable,
                       default=which('fop'),
                       help='full path to fop [%(default)s]')
        g.add_argument('--docbook5xml-dblatex', type=arg_isexecutable,
                       default=which('dblatex'),
                       help='full path to dblatex [%(default)s]')
        g.add_argument('--docbook5xml-jing', type=arg_isexecutable,
                       default=which('jing'),
                       help='full path to jing [%(default)s]')

#
# -- end of file
<commit_msg>tldp/doctypes/docbook4xml.py<commit_after>#! /usr/bin/python
# -*- coding: utf8 -*-

from __future__ import absolute_import, division, print_function

import os
import logging

from tldp.utils import which, firstfoundfile
from tldp.utils import arg_isexecutable, isexecutable
from tldp.utils import arg_isreadablefile, isreadablefile

from tldp.doctypes.common import BaseDoctype, SignatureChecker, depends

logger = logging.getLogger(__name__)

def rngfile_finder():
    l = ['/usr/share/xml/docbook/schema/rng/5.0/docbook.rng',
         ]
    return firstfoundfile(l)

def xslchunk_finder():
    l = ['/usr/share/xml/docbook/stylesheet/nwalsh5/current/html/chunk.xsl',
         '/usr/share/xml/docbook/stylesheet/docbook-xsl-ns/html/chunk.xsl',
         'http://docbook.sourceforge.net/release/xsl/current/html/chunk.xsl',
         ]
    return firstfoundfile(l)

def xslsingle_finder():
    l = ['/usr/share/xml/docbook/stylesheet/nwalsh5/current/html/docbook.xsl',
         '/usr/share/xml/docbook/stylesheet/docbook-xsl-ns/html/docbook.xsl',
         'http://docbook.sourceforge.net/release/xsl/current/html/docbook.xsl',
         ]
    return firstfoundfile(l)

def xslprint_finder():
    l = ['/usr/share/xml/docbook/stylesheet/nwalsh5/current/fo/docbook.xsl',
         '/usr/share/xml/docbook/stylesheet/docbook-xsl-ns/fo/docbook.xsl',
         'http://docbook.sourceforge.net/release/xsl/current/fo/docbook.xsl',
         ]
    return firstfoundfile(l)

class Docbook5XML(BaseDoctype, SignatureChecker):
    formatname = 'DocBook XML 5.x'
    extensions = ['.xml']
    signatures = ['-//OASIS//DTD DocBook V5.0/EN',
                  'http://docbook.org/ns/docbook', ]

    required = {'docbook5xml_xsltproc': isexecutable,
                'docbook5xml_xmllint': isexecutable,
                'docbook5xml_html2text': isexecutable,
                'docbook5xml_dblatex': isexecutable,
                'docbook5xml_fop': isexecutable,
                'docbook5xml_jing': isexecutable,
                'docbook5xml_rngfile': isreadablefile,
                'docbook5xml_xslprint': isreadablefile,
                'docbook5xml_xslchunk': isreadablefile,
                'docbook5xml_xslsingle': isreadablefile,
                }

    def chdir_output(self):
        os.chdir(self.output.dirname)
        return True

    @depends(chdir_output)
    def make_xincluded_source(self):
        s = '''"{config.docbook5xml_xmllint}" > "{output.validsource}" \\
                  --nonet \\
                  --noent \\
                  --xinclude \\
                  "{source.filename}"'''
        return self.shellscript(s)

    @depends(make_xincluded_source)
    def validate_source(self):
        '''consider lxml.etree and other validators'''
        s = '''"{config.docbook5xml_jing}" \\
                  "{config.docbook5xml_rngfile}" \\
                  "{output.validsource}"'''
        return self.shellscript(s)

    @depends(make_xincluded_source)
    def copy_static_resources(self):
        source = list()
        for d in ('images', 'resources'):
            fullpath = os.path.join(self.source.dirname, d)
            fullpath = os.path.abspath(fullpath)
            if os.path.isdir(fullpath):
                source.append('"' + fullpath + '"')
            if not source:
                logger.debug("%s no images or resources to copy",
                             self.source.stem)
                return True
            s = 'rsync --archive --verbose %s ./' % (' '.join(source))
        return self.shellscript(s)

    @depends(copy_static_resources)
    def make_name_htmls(self):
        '''create a single page HTML output'''
        s = '''"{config.docbook5xml_xsltproc}" > "{output.name_htmls}" \\
                  --nonet \\
                  --stringparam admon.graphics.path images/ \\
                  --stringparam base.dir . \\
                  "{config.docbook5xml_xslsingle}" \\
                  "{output.validsource}"'''
        return self.shellscript(s)

    @depends(make_name_htmls)
    def make_name_txt(self):
        '''create text output'''
        s = '''"{config.docbook5xml_html2text}" > "{output.name_txt}" \\
                  -style pretty \\
                  -nobs \\
                  "{output.name_htmls}"'''
        return self.shellscript(s)

    @depends(copy_static_resources)
    def make_fo(self):
        '''generate the Formatting Objects intermediate output'''
        s = '''"{config.docbook5xml_xsltproc}" > "{output.name_fo}" \\
                  "{config.docbook5xml_xslprint}" \\
                  "{output.validsource}"'''
        self.removals.append(self.output.name_fo)
        return self.shellscript(s)

    # -- this is conditionally built--see logic in make_name_pdf() below
    # @depends(make_fo)
    def make_pdf_with_fop(self):
        '''use FOP to create a PDF'''
        s = '''"{config.docbook5xml_fop}" \\
                  -fo "{output.name_fo}" \\
                  -pdf "{output.name_pdf}"'''
        return self.shellscript(s)

    # -- this is conditionally built--see logic in make_name_pdf() below
    # @depends(chdir_output)
    def make_pdf_with_dblatex(self):
        '''use dblatex (fallback) to create a PDF'''
        s = '''"{config.docbook5xml_dblatex}" \\
                  -F xml \\
                  -t pdf \\
                  -o "{output.name_pdf}" \\
                  "{output.validsource}"'''
        return self.shellscript(s)

    @depends(make_fo)
    def make_name_pdf(self):
        stem = self.source.stem
        classname = self.__class__.__name__
        logger.info("%s calling method %s.%s",
                    stem, classname, 'make_pdf_with_fop')
        if self.make_pdf_with_fop():
            return True
        logger.error("%s %s failed creating PDF, falling back to dblatex...",
                     stem, self.config.docbook5xml_fop)
        logger.info("%s calling method %s.%s",
                    stem, classname, 'make_pdf_with_dblatex')
        return self.make_pdf_with_dblatex()

    @depends(make_name_htmls)
    def make_html(self):
        '''create chunked HTML output'''
        s = '''"{config.docbook5xml_xsltproc}" \\
                  --nonet \\
                  --stringparam admon.graphics.path images/ \\
                  --stringparam base.dir . \\
                  "{config.docbook5xml_xslchunk}" \\
                  "{output.validsource}"'''
        return self.shellscript(s)

    @depends(make_html)
    def make_name_html(self):
        '''rename DocBook XSL's index.html to LDP standard STEM.html'''
        s = 'mv -v --no-clobber -- "{output.name_indexhtml}" "{output.name_html}"'
        return self.shellscript(s)

    @depends(make_name_html)
    def make_name_indexhtml(self):
        '''create final index.html symlink'''
        s = 'ln -svr -- "{output.name_html}" "{output.name_indexhtml}"'
        return self.shellscript(s)

    @depends(make_name_htmls, make_html, make_fo, make_name_pdf)
    def remove_xincluded_source(self):
        '''remove the xincluded source file'''
        s = 'rm --verbose -- "{output.validsource}"'
        return self.shellscript(s)

    @classmethod
    def argparse(cls, p):
        descrip = 'executables for %s' % (cls.formatname,)
        g = p.add_argument_group(title=cls.__name__, description=descrip)
        g.add_argument('--docbook5xml-xslchunk', type=arg_isreadablefile,
                       default=xslchunk_finder(),
                       help='full path to LDP HTML chunker XSL [%(default)s]')
        g.add_argument('--docbook5xml-xslsingle', type=arg_isreadablefile,
                       default=xslsingle_finder(),
                       help='full path to LDP HTML single-page XSL [%(default)s]')
        g.add_argument('--docbook5xml-xslprint', type=arg_isreadablefile,
                       default=xslprint_finder(),
                       help='full path to LDP FO print XSL [%(default)s]')

        g.add_argument('--docbook5xml-rngfile', type=arg_isreadablefile,
                       default=rngfile_finder(),
                       help='full path to docbook.rng [%(default)s]')
        g.add_argument('--docbook5xml-xmllint', type=arg_isexecutable,
                       default=which('xmllint'),
                       help='full path to xmllint [%(default)s]')
        g.add_argument('--docbook5xml-xsltproc', type=arg_isexecutable,
                       default=which('xsltproc'),
                       help='full path to xsltproc [%(default)s]')
        g.add_argument('--docbook5xml-html2text', type=arg_isexecutable,
                       default=which('html2text'),
                       help='full path to html2text [%(default)s]')
        g.add_argument('--docbook5xml-fop', type=arg_isexecutable,
                       default=which('fop'),
                       help='full path to fop [%(default)s]')
        g.add_argument('--docbook5xml-dblatex', type=arg_isexecutable,
                       default=which('dblatex'),
                       help='full path to dblatex [%(default)s]')
        g.add_argument('--docbook5xml-jing', type=arg_isexecutable,
                       default=which('jing'),
                       help='full path to jing [%(default)s]')

#
# -- end of file
<|endoftext|>


<commit_before># Copyright (c) 2015 by the parties listed in the AUTHORS file.
# All rights reserved.  Use of this source code is governed by
# a BSD-style license that can be found in the LICENSE file.

from mpi4py import MPI

import unittest

import numpy as np

import healpy as hp

import quaternionarray as qa

import toast

class OpPointingPlanck(toast.Operator):
    """
    Operator which generates healpix pointing

    Args:
        nside (int): NSIDE resolution for Healpix maps.

    """

    def __init__(self, nside=1024, mode='I', detweights=None, RIMO=None, highmem=False):
        self._nside = nside
        self._mode = mode
        self._detweights = detweights
        self._highmem = highmem

        if RIMO is None:
            raise ValueError('You must specify which RIMO to use')

        self.RIMO = RIMO # The Reduced Instrument Model contains the necessary detector parameters

        # We call the parent class constructor, which currently does nothing
        super().__init__()

    @property
    def rimo(self):
        return self.RIMO

    @property
    def nside(self):
        return self._nside

    def exec(self, data):
        # the two-level pytoast communicator
        comm = data.comm
        # the global communicator
        cworld = comm.comm_world
        # the communicator within the group
        cgroup = comm.comm_group
        # the communicator with all processes with
        # the same rank within their group
        crank = comm.comm_rank

        xaxis = np.array([1,0,0], dtype=np.float64)
        yaxis = np.array([0,1,0], dtype=np.float64)
        zaxis = np.array([0,0,1], dtype=np.float64)
        nullquat = np.array([0,0,0,1], dtype=np.float64)

        for obs in data.obs:
            tod = obs['tod']

            for det in tod.local_dets:
                pdata, pflags = tod.read_pntg(detector=det, local_start=0, n=tod.local_samples)

                pdata = pdata.reshape(-1,4).copy()
                pdata[ pflags != 0 ] = nullquat
                vec_dir = qa.rotate( pdata, zaxis ).T.copy()

                pixels = hp.vec2pix(self._nside, *vec_dir, nest=True)
                pixels[ pflags != 0 ] = -1

                epsilon = self.RIMO[ det ].epsilon
                eta = (1 - epsilon) / (1 + epsilon)

                dweight = 1.0
                if self._detweights is not None:
                    dweight = self._detweights[det]

                if self._mode == 'I':
                    weights = np.ones(tod.local_samples, dtype=np.float64)
                    weights *= dweight
                    tod.write_pmat(detector=det, local_start=0, pixels=pixels, weights=weights)
                elif self._mode == 'IQU':
                    vec_orient = qa.rotate( pdata, xaxis ).T.copy()

                    ypa = vec_orient[0]*vec_dir[1] - vec_orient[1]*vec_dir[0]
                    #xpa = -vec_orient[0]*vec_dir[2]*vec_dir[0] - vec_orient[1]*vec_dir[2]*vec_dir[1] + vec_orient[2]*(vec_dir[0]**2 + vec_dir[1]**2)
                    xpa = -vec_dir[2]*(vec_orient[0]*vec_dir[0] - vec_orient[1]*vec_dir[1]) + vec_orient[2]*(vec_dir[0]**2 + vec_dir[1]**2)

                    psi = np.arctan2( ypa, xpa )

                    Ival = dweight * np.ones( tod.local_samples )
                    Qval = dweight * eta * np.cos( 2 * psi )
                    Uval = dweight * eta * np.sin( 2 * psi )

                    weights = np.ravel(np.column_stack((Ival, Qval, Uval)))

                    tod.write_pmat(detector=det, local_start=0, pixels=pixels, weights=weights)
                else:
                    raise RuntimeError("invalid mode for Planck Pointing")

                if not self._highmem:
                    tod.clear_pntg(detector=det)

        return

<commit_msg>Fix typo in optimization.<commit_after># Copyright (c) 2015 by the parties listed in the AUTHORS file.
# All rights reserved.  Use of this source code is governed by
# a BSD-style license that can be found in the LICENSE file.

from mpi4py import MPI

import unittest

import numpy as np

import healpy as hp

import quaternionarray as qa

import toast

class OpPointingPlanck(toast.Operator):
    """
    Operator which generates healpix pointing

    Args:
        nside (int): NSIDE resolution for Healpix maps.

    """

    def __init__(self, nside=1024, mode='I', detweights=None, RIMO=None, highmem=False):
        self._nside = nside
        self._mode = mode
        self._detweights = detweights
        self._highmem = highmem

        if RIMO is None:
            raise ValueError('You must specify which RIMO to use')

        self.RIMO = RIMO # The Reduced Instrument Model contains the necessary detector parameters

        # We call the parent class constructor, which currently does nothing
        super().__init__()

    @property
    def rimo(self):
        return self.RIMO

    @property
    def nside(self):
        return self._nside

    def exec(self, data):
        # the two-level pytoast communicator
        comm = data.comm
        # the global communicator
        cworld = comm.comm_world
        # the communicator within the group
        cgroup = comm.comm_group
        # the communicator with all processes with
        # the same rank within their group
        crank = comm.comm_rank

        xaxis = np.array([1,0,0], dtype=np.float64)
        yaxis = np.array([0,1,0], dtype=np.float64)
        zaxis = np.array([0,0,1], dtype=np.float64)
        nullquat = np.array([0,0,0,1], dtype=np.float64)

        for obs in data.obs:
            tod = obs['tod']

            for det in tod.local_dets:
                pdata, pflags = tod.read_pntg(detector=det, local_start=0, n=tod.local_samples)

                pdata = pdata.reshape(-1,4).copy()
                pdata[ pflags != 0 ] = nullquat
                vec_dir = qa.rotate( pdata, zaxis ).T.copy()

                pixels = hp.vec2pix(self._nside, *vec_dir, nest=True)
                pixels[ pflags != 0 ] = -1

                epsilon = self.RIMO[ det ].epsilon
                eta = (1 - epsilon) / (1 + epsilon)

                dweight = 1.0
                if self._detweights is not None:
                    dweight = self._detweights[det]

                if self._mode == 'I':
                    weights = np.ones(tod.local_samples, dtype=np.float64)
                    weights *= dweight
                    tod.write_pmat(detector=det, local_start=0, pixels=pixels, weights=weights)
                elif self._mode == 'IQU':
                    vec_orient = qa.rotate( pdata, xaxis ).T.copy()

                    ypa = vec_orient[0]*vec_dir[1] - vec_orient[1]*vec_dir[0]
                    #xpa = -vec_orient[0]*vec_dir[2]*vec_dir[0] - vec_orient[1]*vec_dir[2]*vec_dir[1] + vec_orient[2]*(vec_dir[0]**2 + vec_dir[1]**2)
                    xpa = -vec_dir[2]*(vec_orient[0]*vec_dir[0] + vec_orient[1]*vec_dir[1]) + vec_orient[2]*(vec_dir[0]**2 + vec_dir[1]**2)

                    psi = np.arctan2( ypa, xpa )

                    Ival = dweight * np.ones( tod.local_samples )
                    Qval = dweight * eta * np.cos( 2 * psi )
                    Uval = dweight * eta * np.sin( 2 * psi )

                    weights = np.ravel(np.column_stack((Ival, Qval, Uval)))

                    tod.write_pmat(detector=det, local_start=0, pixels=pixels, weights=weights)
                else:
                    raise RuntimeError("invalid mode for Planck Pointing")

                if not self._highmem:
                    tod.clear_pntg(detector=det)

        return

<|endoftext|>


<commit_before>import os
from jsonobject import JsonObject
import yaml
from corehq.apps.userreports.specs import TypeProperty
from corehq.apps.app_manager.models import Application
from corehq.util.quickcache import quickcache
from dimagi.utils.decorators.memoized import memoized

class AbtSupervisorExpressionSpec(JsonObject):
    type = TypeProperty('abt_supervisor')

    @property
    @memoized
    def _flag_specs(self):
        """
        Return a dict where keys are form xmlns and values are lists of FlagSpecs
        """
        path = os.path.join(os.path.dirname(__file__), 'flagspecs.yaml')
        with open(path) as f:
            return yaml.load(f)

    @classmethod
    def _get_val(cls, item, path):
        """
        Return the answer in the given submitted form (item) to the question specified by path.
        Return empty tuple if no answer was given to the given question.
        """
        if path:
            try:
                v = item['form']
                for key in path:
                    v = v[key]
                return v
            except KeyError:
                return ()

    @classmethod
    def _question_answered(cls, value):
        """
        Return true if the given value indicates that an answer was provided for its question.
        """
        return value != ()

    @classmethod
    def _raise_for_any_answer(cls, danger_value):
        """
        Return true if the given danger_value indicates that any question answer should raise the flag.
        """
        return danger_value == []

    @classmethod
    @quickcache(['app_id', 'xmlns'])
    def _get_questions(cls, app_id, xmlns):
        form = Application.get(app_id).get_form_by_xmlns(xmlns)
        return {
            q['value']: q for q in form.get_questions([], include_groups=True)
        }

    @classmethod
    def _get_question_options(cls, item, question_path):
        """
        Return a list of option values for the given question path and item
        (which is a dict representation of an XFormInstance)
        """
        questions = cls._get_questions(item['app_id'], item['xmlns'])
        question = questions.get('/data/' + "/".join(question_path), {})
        options = [o['value'] for o in question.get("options", [])]
        return options

    @classmethod
    def _get_unchecked(cls, xform_instance, question_path, answer, ignore=None):
        """
        Return the unchecked options in the given question.
        Do not return any which appear in the option ignore parameter.

        answer should be a string
        ignore should be a list of strings.
        """
        answer = answer or ""
        options = set(cls._get_question_options(xform_instance, question_path))
        checked = set(answer.split(" "))
        unchecked = options - checked
        ret = unchecked - set(ignore)
        return list(ret)

    @classmethod
    def _get_comments(cls, item, spec):
        """
        Return the comments for the question specified in the spec.
        If the spec does not contain a `comment` field, then the `question`
        field is used to build the path to the comment question.
        """
        comments_question = spec.get('comment', False)
        question_path = spec["question"]

        if not comments_question:
            parts = question_path[-1].split("_")
            parts.insert(1, "comments")
            comments_question = question_path[:-1] + ["_".join(parts)]

        comments = cls._get_val(item, comments_question)
        return comments if comments != () else ""

    @classmethod
    def _get_warning(cls, spec, item):
        country = cls._get_val(item, ["location_data", "country"])
        default = spec.get("warning", "")
        if country in ["Senegal", "Benin", "Mali", "Madagascar"]:
            return spec.get("french", default)
        return default

    @classmethod
    def _get_inspector_names(cls, item):
        repeat_items = cls._get_val(item, ['employee_group', 'employee'])
        if repeat_items == ():
            return ""
        if type(repeat_items) != list:
            repeat_items = [repeat_items]
        repeat_items = [{'form': x} for x in repeat_items]

        names = []
        for i in repeat_items:
            for q in ['other_abt_employee_name', 'abt_employee_name', 'other_non-abt_employee_name']:
                name = cls._get_val(i, ['abt_emp_list', q])
                if name:
                    names.append(name)
                    break

        return ", ".join(names)

    def __call__(self, item, context=None):
        """
        Given a document (item), return a list of documents representing each
        of the flagged questions.
        """
        names = self._get_inspector_names(item)
        docs = []
        for spec in self._flag_specs.get(item['xmlns'], []):

            if spec.get("base_path", False):
                repeat_items = self._get_val(item, spec['base_path'])
                if repeat_items == ():
                    repeat_items = []
                if type(repeat_items) != list:
                    # bases will be a dict if the repeat only happened once.
                    repeat_items = [repeat_items]
                # We have to add the 'form' key here so that _get_val works correctly.
                repeat_items = [{'form': x} for x in repeat_items]
            else:
                repeat_items = [item]

            # Iterate over the repeat items, or the single submission
            for partial in repeat_items:

                form_value = self._get_val(partial, spec['question'])
                warning_type = spec.get("warning_type", None)

                if warning_type == "unchecked" and form_value:
                    # Don't raise flag if no answer given
                    ignore = spec.get("ignore", [])
                    unchecked = self._get_unchecked(
                        item,
                        spec.get('base_path', []) + spec['question'],
                        form_value,
                        ignore
                    )
                    if unchecked:
                        # Raise a flag because there are unchecked answers.
                        docs.append({
                            'flag': spec['question'][-1],
                            'warning': self._get_warning(spec, item).format(msg=", ".join(unchecked)),
                            'comments': self._get_comments(partial, spec),
                            'names': names,
                        })

                if warning_type == "q3_special" and form_value:
                    # One of the questions doesn't follow the same format as the
                    # others, hence this special case.
                    missing_items = ""
                    if form_value == "only_license":
                        missing_items = "cell"
                    if form_value == "only_cell":
                        missing_items = "license"
                    if form_value == "none":
                        missing_items = "cell, license"
                    if missing_items:
                        docs.append({
                            'flag': spec['question'][-1],
                            'warning': self._get_warning(spec, item).format(msg=missing_items),
                            'comments': self._get_comments(partial, spec),
                            'names': names,
                        })

                else:
                    danger_value = spec.get('answer', [])
                    if form_value == danger_value or (
                        self._question_answered(form_value) and
                        self._raise_for_any_answer(danger_value)
                    ):
                        docs.append({
                            'flag': spec['question'][-1],
                            'warning': self._get_warning(spec, item).format(
                                msg=self._get_val(partial, spec.get('warning_question', None)) or ""
                            ),
                            'comments': self._get_comments(partial, spec),
                            'names': names,
                        })

        return docs

def abt_supervisor_expression(spec, context):
    wrapped = AbtSupervisorExpressionSpec.wrap(spec)
    return wrapped
<commit_msg>fix bug<commit_after>import os
from jsonobject import JsonObject
import yaml
from corehq.apps.userreports.specs import TypeProperty
from corehq.apps.app_manager.models import Application
from corehq.util.quickcache import quickcache
from dimagi.utils.decorators.memoized import memoized

class AbtSupervisorExpressionSpec(JsonObject):
    type = TypeProperty('abt_supervisor')

    @property
    @memoized
    def _flag_specs(self):
        """
        Return a dict where keys are form xmlns and values are lists of FlagSpecs
        """
        path = os.path.join(os.path.dirname(__file__), 'flagspecs.yaml')
        with open(path) as f:
            return yaml.load(f)

    @classmethod
    def _get_val(cls, item, path):
        """
        Return the answer in the given submitted form (item) to the question specified by path.
        Return empty tuple if no answer was given to the given question.
        """
        if path:
            try:
                v = item['form']
                for key in path:
                    v = v[key]
                return v
            except KeyError:
                return ()

    @classmethod
    def _question_answered(cls, value):
        """
        Return true if the given value indicates that an answer was provided for its question.
        """
        return value != ()

    @classmethod
    def _raise_for_any_answer(cls, danger_value):
        """
        Return true if the given danger_value indicates that any question answer should raise the flag.
        """
        return danger_value == []

    @classmethod
    @quickcache(['app_id', 'xmlns'])
    def _get_questions(cls, app_id, xmlns):
        form = Application.get(app_id).get_form_by_xmlns(xmlns)
        return {
            q['value']: q for q in form.get_questions([], include_groups=True)
        }

    @classmethod
    def _get_question_options(cls, item, question_path):
        """
        Return a list of option values for the given question path and item
        (which is a dict representation of an XFormInstance)
        """
        questions = cls._get_questions(item['app_id'], item['xmlns'])
        question = questions.get('/data/' + "/".join(question_path), {})
        options = [o['value'] for o in question.get("options", [])]
        return options

    @classmethod
    def _get_unchecked(cls, xform_instance, question_path, answer, ignore=None):
        """
        Return the unchecked options in the given question.
        Do not return any which appear in the option ignore parameter.

        answer should be a string
        ignore should be a list of strings.
        """
        answer = answer or ""
        options = set(cls._get_question_options(xform_instance, question_path))
        checked = set(answer.split(" "))
        unchecked = options - checked
        ret = unchecked - set(ignore)
        return list(ret)

    @classmethod
    def _get_comments(cls, item, spec):
        """
        Return the comments for the question specified in the spec.
        If the spec does not contain a `comment` field, then the `question`
        field is used to build the path to the comment question.
        """
        comments_question = spec.get('comment', False)
        question_path = spec["question"]

        if not comments_question:
            parts = question_path[-1].split("_")
            parts.insert(1, "comments")
            comments_question = question_path[:-1] + ["_".join(parts)]

        comments = cls._get_val(item, comments_question)
        return comments if comments != () else ""

    @classmethod
    def _get_warning(cls, spec, item):
        country = cls._get_val(item, ["location_data", "country"])
        default = spec.get("warning", "")
        if country in ["Senegal", "Benin", "Mali", "Madagascar"]:
            return spec.get("french", default)
        return default

    @classmethod
    def _get_inspector_names(cls, item):
        repeat_items = cls._get_val(item, ['employee_group', 'employee'])
        if repeat_items == ():
            return ""
        if type(repeat_items) != list:
            repeat_items = [repeat_items]
        repeat_items = [{'form': x} for x in repeat_items]

        names = []
        for i in repeat_items:
            for q in ['other_abt_employee_name', 'abt_employee_name', 'other_non-abt_employee_name']:
                name = cls._get_val(i, ['abt_emp_list', q])
                if name:
                    names.append(name)
                    break

        return ", ".join(names)

    def __call__(self, item, context=None):
        """
        Given a document (item), return a list of documents representing each
        of the flagged questions.
        """
        names = self._get_inspector_names(item)
        docs = []
        for spec in self._flag_specs.get(item['xmlns'], []):

            if spec.get("base_path", False):
                repeat_items = self._get_val(item, spec['base_path'])
                if repeat_items == ():
                    repeat_items = []
                if type(repeat_items) != list:
                    # bases will be a dict if the repeat only happened once.
                    repeat_items = [repeat_items]
                # We have to add the 'form' key here so that _get_val works correctly.
                repeat_items = [{'form': x} for x in repeat_items]
            else:
                repeat_items = [item]

            # Iterate over the repeat items, or the single submission
            for partial in repeat_items:

                form_value = self._get_val(partial, spec['question'])
                warning_type = spec.get("warning_type", None)

                if warning_type == "unchecked" and form_value:
                    # Don't raise flag if no answer given
                    ignore = spec.get("ignore", [])
                    unchecked = self._get_unchecked(
                        item,
                        spec.get('base_path', []) + spec['question'],
                        form_value,
                        ignore
                    )
                    if unchecked:
                        # Raise a flag because there are unchecked answers.
                        docs.append({
                            'flag': spec['question'][-1],
                            'warning': self._get_warning(spec, item).format(msg=", ".join(unchecked)),
                            'comments': self._get_comments(partial, spec),
                            'names': names,
                        })

                elif warning_type == "q3_special" and form_value:
                    # One of the questions doesn't follow the same format as the
                    # others, hence this special case.
                    missing_items = ""
                    if form_value == "only_license":
                        missing_items = "cell"
                    if form_value == "only_cell":
                        missing_items = "license"
                    if form_value == "none":
                        missing_items = "cell, license"
                    if missing_items:
                        docs.append({
                            'flag': spec['question'][-1],
                            'warning': self._get_warning(spec, item).format(msg=missing_items),
                            'comments': self._get_comments(partial, spec),
                            'names': names,
                        })

                else:
                    danger_value = spec.get('answer', [])
                    if form_value == danger_value or (
                        self._question_answered(form_value) and
                        self._raise_for_any_answer(danger_value)
                    ):
                        docs.append({
                            'flag': spec['question'][-1],
                            'warning': self._get_warning(spec, item).format(
                                msg=self._get_val(partial, spec.get('warning_question', None)) or ""
                            ),
                            'comments': self._get_comments(partial, spec),
                            'names': names,
                        })

        return docs

def abt_supervisor_expression(spec, context):
    wrapped = AbtSupervisorExpressionSpec.wrap(spec)
    return wrapped
<|endoftext|>


<commit_before># coding: utf-8
# ###################################################
# Copyright (C) 2008 The Zero-Projekt team
# http://zero-projekt.net
# info@zero-projekt.net
# This file is part of Zero "Was vom Morgen blieb"
#
# The Zero-Projekt codebase is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the
# Free Software Foundation, Inc.,
# 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
# ###################################################

""" a tool for FIFEdit to test global light """

from fife import fife
from fife.extensions import pychan
import fife.extensions.pychan.widgets as widgets
from fife.extensions.pychan.tools import callbackWithArguments as cbwa

from fife.extensions.fife_timer import Timer

import scripts
import scripts.plugin as plugin
from scripts.events import *
from scripts.gui.action import Action

import random

DEFAULT_GLOBAL_LIGHT = {
	"R"	:	1.0,
	"G"	:	1.0,
	"B"	:	1.0,
	"A" :	1.0,
}
DEFAULT_LIGHT_ID = "LightEdit"

class LightEdit(plugin.Plugin):
	""" The B{LightEdit} module is a plugin for FIFedit and allows to change the
	global light value

		FEATURES:
			- enable FIFE lighting renderer
			- generate random light values
			- test lightsetups by manipulating the color channels
			- reset to default
	"""
	def __init__(self):
		self.active = False

		self._renderer = None
		self._camera = None
		self._enabled = False
		self._light = False

		self.map_loaded = False

		self._color = {}
		self._color.update(DEFAULT_GLOBAL_LIGHT)

		random.seed()

		if "LightRenderer" in dir(fife):
			self._renderer_available = True
		else:
			self._renderer_available = False

	def _reset(self):
		""" 	resets all dynamic vars	"""
		pass

	def enable(self):
		""" plugin method """
		if not self._renderer_available:
			self._enabled = False
			return
		if self._enabled is True:
			return

		self._editor = scripts.editor.getEditor()
		self.engine = self._editor.getEngine()

		self._showAction = Action(unicode(self.getName(),"utf-8"), checkable=True)
		scripts.gui.action.activated.connect(self.toggle_gui, sender=self._showAction)

		self._editor._tools_menu.addAction(self._showAction)

		events.postMapShown.connect(self.update_renderer)
		events.onMapChanged.connect(self.update_renderer)

		self._reset()
		self.create_gui()

	def disable(self):
		""" plugin method """
		if self._enabled is False:
			return

		self._reset()
		self.container.hide()
		self.removeAllChildren()

		self._editor._tools_menu.removeAction(self._showAction)

		events.postMapShown.disconnect(self.update_renderer)
		events.onMapChanged.disconnect(self.update_renderer)

	def isEnabled(self):
		""" plugin method """
		return self._enabled;

	def getName(self):
		""" plugin method """
		return "Light editor"

	def create_gui(self):
		""" create gui container and setup callbacks """
		self.container = pychan.loadXML('gui/lightedit.xml')
		self.container.mapEvents({
			"enable_global_light"	:	self.toggle_light,
			"random_global_light"	:	self.random_color,
			"reset_global_light"	:	self.reset_global_light,

			"increase_R"			:	cbwa(self.increase_color, r=True),
			"decrease_R"			:	cbwa(self.decrease_color, r=True),
			"value_R/mouseWheelMovedUp"			:	cbwa(self.increase_color, step=0.2, r=True),
			"value_R/mouseWheelMovedDown"		:	cbwa(self.decrease_color, step=0.2, r=True),

			"increase_G"			:	cbwa(self.increase_color, g=True),
			"decrease_G"			:	cbwa(self.decrease_color, g=True),
			"value_G/mouseWheelMovedUp"			:	cbwa(self.increase_color, step=0.2, g=True),
			"value_G/mouseWheelMovedDown"		:	cbwa(self.decrease_color, step=0.2, g=True),

			"increase_B"			:	cbwa(self.increase_color, b=True),
			"decrease_B"			:	cbwa(self.decrease_color, b=True),
			"value_B/mouseWheelMovedUp"			:	cbwa(self.increase_color, step=0.2, b=True),
			"value_B/mouseWheelMovedDown"		:	cbwa(self.decrease_color, step=0.2, b=True),

			"increase_A"			:	cbwa(self.increase_color, a=True),
			"decrease_A"			:	cbwa(self.decrease_color, a=True),
			"value_A/mouseWheelMovedUp"			:	cbwa(self.increase_color, step=0.2, a=True),
			"value_A/mouseWheelMovedDown"		:	cbwa(self.decrease_color, step=0.2, a=True),
		})
		self._widgets = {
			"enable_global_light"	:	self.container.findChild(name="enable_global_light"),
			"random_global_light"	:	self.container.findChild(name="random_global_light"),
			"reset_global_light"	:	self.container.findChild(name="reset_global_light"),

			"value_R"				:	self.container.findChild(name="value_R"),
			"value_G"				:	self.container.findChild(name="value_G"),
			"value_B"				:	self.container.findChild(name="value_B"),
			"value_A"				:	self.container.findChild(name="value_A"),
		}

	def toggle_gui(self):
		""" show / hide the gui """
		if self.active:
			self.active = False
			if self.container.isVisible() or self.container.isDocked():
				self.container.setDocked(False)
				self.container.hide()
			self._showAction.setChecked(False)
		else:
			self.active = True
			self._showAction.setChecked(True)
			self.container.show()

	def toggle_light(self):
		""" toggle light on / off """
		if not self._renderer:
			self._widgets['enable_global_light']._setToggled(False)
			return

		if self._light:
			self._light = False
			self.reset_global_light()
			self._renderer.setEnabled(False)
		else:
			self._light = True
			self._renderer.setEnabled(True)

	def update_renderer(self):
		""" sets current camera and renderer
			bound to FIFedit core (updated on map change)
		"""
		self._camera = self._editor.getActiveMapView().getCamera()
		self._renderer = fife.LightRenderer.getInstance(self._camera)

	def update_gui(self):
		""" update gui widgets according to plugin data """
		self._widgets["value_R"].text = unicode(str(self._color["R"]))
		self._widgets["value_G"].text = unicode(str(self._color["G"]))
		self._widgets["value_B"].text = unicode(str(self._color["B"]))
		self._widgets["value_A"].text = unicode(str(self._color["A"]))

	def reset_global_light(self):
		""" reset global light to default values (1.0) """
		if not self._renderer: return

		self._color.update(DEFAULT_GLOBAL_LIGHT)
		self.update_gui()
		self.set_global_light()

	def increase_color(self, step=0.1, r=None, g=None, b=None, a=None):
		"""	increase a given color value by step value

		@type	step	float
		@param	step	the step for changing the color channel
		@type	r		bool
		@param	r		flag to alter red color value
		@type	g		bool
		@param	g		flag to alter green color value
		@type	b		bool
		@param	b		flag to alter blue color value
		@type	a		bool
		@type	a		flag to alter alpha channel value (no effect atm)
		"""
		if r:
			self._color["R"] += step
		if g:
			self._color["G"] += step
		if b:
			self._color["B"] += step
		if a:
			self._color["A"] += step

		self.update_gui()
		self.set_global_light()

	def decrease_color(self, step=0.1, r=None, g=None, b=None, a=None):
		"""	decrease a given color value by step value

		@type	step	float
		@param	step	the step for changing the color channel
		@type	r		bool
		@param	r		flag to alter red color value
		@type	g		bool
		@param	g		flag to alter green color value
		@type	b		bool
		@param	b		flag to alter blue color value
		@type	a		bool
		@type	a		flag to alter alpha channel value (no effect atm)
		"""
		if r:
			self._color["R"] -= step
		if g:
			self._color["G"] -= step
		if b:
			self._color["B"] -= step
		if a:
			self._color["A"] -= step

		self.update_gui()
		self.set_global_light()

	def random_color(self):
		""" generate random values for color channels """
		if not self._renderer: return

		self._color["R"] = random.uniform(0,2)
		self._color["G"] = random.uniform(0,2)
		self._color["B"] = random.uniform(0,2)
		self._color["A"] = random.uniform(0,2)

		self.update_gui()
		self.set_global_light()

	def set_global_light(self):
		""" update the global light with the current set colors """
		if not self._renderer: return

		self._renderer.removeAll(DEFAULT_LIGHT_ID)
		self._renderer.setglobalLight(
			DEFAULT_LIGHT_ID,
			1,
			self._color["R"],
			self._color["G"],
			self._color["B"],
			self._color["A"]
		)

<commit_msg>*Editor plugin removed from trunk because it belongs to the light branch<commit_after><|endoftext|>


<commit_before>##########################################################################
# Copyright (c) 2017, ETH Zurich.
# All rights reserved.
#
# This file is distributed under the terms in the attached LICENSE file.
# If you do not find this file, copies can be found by writing to:
# ETH Zurich D-INFK, Haldeneggsteig 4, CH-8092 Zurich. Attn: Systems Group.
##########################################################################

import re, datetime
import debug, tests
import subprocess
import os
import socket, struct, fcntl
import thread
from common import TestCommon, TimeoutError
from results import RowResults, PassFailResult

TEST_TIMEOUT = datetime.timedelta(minutes=8)

mac = {'babybel1': 130587495626,
       'babybel2': 130587510022,
       'babybel3': 130587512798,
       'babybel4': 130589790232,
       'ziger2': 65817495764,
       'ziger1': 116527143012, }

# Fallback if gethostip does not work
ip = {'babybel1': 174982272,
       'babybel2': 174982270,
       'babybel3': 174982271,
       'ziger2': 174982183,
       'ziger1': 174982183, }

class DevifTests(TestCommon):

    def __init__(self, options):
        super(DevifTests, self).__init__(options)

    def get_module_name(self):
        return "devif_test"

    def boot(self, *args):
        super(DevifTests, self).boot(*args)
        self.set_timeout(TEST_TIMEOUT)

    def get_decimal_ip(self, hostname):
        try:
            iphex = subprocess.check_output('gethostip -x %s' % hostname, shell=True)
            return '%d' % int(iphex, 16)
        except:
            return ip[hostname.split('-')[0]]

    def get_local_mac(self, ifname):
        s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
        info = fcntl.ioctl(s.fileno(), 0x8927,  struct.pack('256s', ifname[:15]))
        hexmac = ''.join(['%02x' % ord(char) for char in info[18:24]])
        return '%d' % int(hexmac, 16)

    def get_modules(self, build, machine):
        self.machine = machine.name
        modules = super(DevifTests, self).get_modules(build, machine)
        modules.add_module("e10k", ["auto", "function=0"])
        modules.add_module("net_sockets_server", ["nospawn"])
        modules.add_module("sfn5122f", ["auto", "function=0"])
        modules.add_module("devif_idc", ["core=1"])

        hostname = '%s.in.barrelfish.org' % subprocess.check_output('hostname -s', shell=True).rstrip()
        src_ip = self.get_decimal_ip(hostname)

        if 'ziger2' in machine.name:
            dst_ip = self.get_decimal_ip('%s-sf.in.barrelfish.org' % machine.name)
        else:
            dst_ip = self.get_decimal_ip('%s-e10k.in.barrelfish.org' % machine.name)

        modules.add_module(self.get_module_name(), ["core=2", self.OP, src_ip, dst_ip, self.CARD])
        return modules

    def get_finish_string(self):
        return "SUCCESS"

    def thread_func (self, dummy, dummy2):
        s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
        while True:
            s.sendto("Data Data Data", (self.ip, 7))

    def start_loop(self):
        self.thread = thread.start_new_thread(self.thread_func, (self, 0))

    def process_line(self, line):
        m = re.match(r'# IP Addr (\d+\.\d+\.\d+\.\d+)', line)
        if m:
            self.start_loop()
            self.ip = m.group(1)

    def process_data(self, testdir, rawiter):
        for line in rawiter:
            if "SUCCESS" in line:
                return PassFailResult(True)

        return PassFailResult(False)

@tests.add_test
class DevifNetTxSF(DevifTests):
    ''' Devif Net TX Test'''
    name = "devif_nettx_sf"
    OP = "net_tx"
    CARD = "sfn5122f"

@tests.add_test
class DevifNetTxE10k(DevifTests):
    ''' Devif Net TX Test'''
    name = "devif_nettx_e10k"
    OP = "net_tx"
    CARD = "e10k"

@tests.add_test
class DevifNetRxSF(DevifTests):
    ''' Devif Net RX Test'''
    name = "devif_netrx_sf"
    OP = "net_rx"
    CARD = "sfn5122f"

@tests.add_test
class DevifNetRxE10k(DevifTests):
    ''' Devif Net RX Test'''
    name = "devif_netrx_e10k"
    OP = "net_rx"
    CARD = "e10k"

@tests.add_test
class DevifIdcTest(DevifTests):
    ''' Devif IDC Test'''
    name = "devif_idc_test"
    OP = "idc"
    CARD = "none"

@tests.add_test
class DevifDebug(DevifTests):
    ''' Devif Debug Backend Test'''
    name = "devif_debug"

    def get_modules(self, build, machine):
        self.machine = machine.name
        modules = super(DevifTests, self).get_modules(build, machine)
        modules.add_module("devif_idc", ["core=1"])
        modules.add_module("devif_debug_test")

        return modules

@tests.add_test
class DevifUDP(DevifTests):
    ''' Devif UDP Backend Test'''
    name = "devif_udp"
    data = ("Data Data Data Data")

    def get_module_name(self):
        return "devif_udp"

    def get_modules(self, build, machine):
        self.machine = machine.name
        modules = super(DevifTests, self).get_modules(build, machine)
        modules.add_module("net_sockets_server", ["nospawn"])
        hostname = '%s.in.barrelfish.org' % subprocess.check_output('hostname -s', shell=True).rstrip()
        dst_ip = self.get_decimal_ip(hostname)
        dst_mac = self.get_local_mac('eno2')

        if 'ziger2' in machine.name:
            src_ip = self.get_decimal_ip('%s-sf.in.barrelfish.org' % machine.name)
            modules.add_module("sfn5122f", ["auto", "function=0"])
            self.cardname = "sfn5122f"
        else:
            modules.add_module("e10k", ["auto", "function=0"])
            src_ip = self.get_decimal_ip('%s-e10k.in.barrelfish.org' % machine.name)
            self.cardname = "e10k"

        src_mac =  mac[machine.name]

        modules.add_module(self.get_module_name(), ["core=2", dst_ip, dst_mac, 20000, 20000, self.cardname])
        return modules

    def thread_func (self, dummy, dummy2):
        s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
        while True:
            s.sendto(self.data, (self.ip, 20000))

    def start_loop(self):
        self.thread = thread.start_new_thread(self.thread_func, (self, 0))

    def process_line(self, line):
        m = re.match(r'# IP Addr (\d+\.\d+\.\d+\.\d+)', line)
        if m:
            self.ip = m.group(1)

        m1 = re.match(r'Testing receiving UDP packets', line)
        if m1:
            self.start_loop()

    def process_data(self, testdir, rawiter):
        for line in rawiter:
            if "SUCCESS" in line:
                return PassFailResult(True)

        return PassFailResult(False)

#@tests.add_test
#class DevifUPDecho(DevifUDP):
#    ''' Devif Debug Backend Test'''
#    name = "devif_udp_echo"
#
#    def get_module_name(self):
#        return "devif_echo"

<commit_msg>harness: devif test load only e10k driver on babybel2<commit_after>##########################################################################
# Copyright (c) 2017, ETH Zurich.
# All rights reserved.
#
# This file is distributed under the terms in the attached LICENSE file.
# If you do not find this file, copies can be found by writing to:
# ETH Zurich D-INFK, Haldeneggsteig 4, CH-8092 Zurich. Attn: Systems Group.
##########################################################################

import re, datetime
import debug, tests
import subprocess
import os
import socket, struct, fcntl
import thread
from common import TestCommon, TimeoutError
from results import RowResults, PassFailResult

TEST_TIMEOUT = datetime.timedelta(minutes=8)

mac = {'babybel1': 130587495626,
       'babybel2': 130587510022,
       'babybel3': 130587512798,
       'babybel4': 130589790232,
       'ziger2': 65817495764,
       'ziger1': 116527143012, }

# Fallback if gethostip does not work
ip = {'babybel1': 174982272,
       'babybel2': 174982270,
       'babybel3': 174982271,
       'ziger2': 174982183,
       'ziger1': 174982183, }

class DevifTests(TestCommon):

    def __init__(self, options):
        super(DevifTests, self).__init__(options)

    def get_module_name(self):
        return "devif_test"

    def boot(self, *args):
        super(DevifTests, self).boot(*args)
        self.set_timeout(TEST_TIMEOUT)

    def get_decimal_ip(self, hostname):
        try:
            iphex = subprocess.check_output('gethostip -x %s' % hostname, shell=True)
            return '%d' % int(iphex, 16)
        except:
            return ip[hostname.split('-')[0]]

    def get_local_mac(self, ifname):
        s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
        info = fcntl.ioctl(s.fileno(), 0x8927,  struct.pack('256s', ifname[:15]))
        hexmac = ''.join(['%02x' % ord(char) for char in info[18:24]])
        return '%d' % int(hexmac, 16)

    def get_modules(self, build, machine):
        self.machine = machine.name
        modules = super(DevifTests, self).get_modules(build, machine)
        modules.add_module("net_sockets_server", ["nospawn"])
        modules.add_module("devif_idc", ["core=1"])
        modules.add_module("e10k", ["auto", "function=0"])

        hostname = '%s.in.barrelfish.org' % subprocess.check_output('hostname -s', shell=True).rstrip()
        src_ip = self.get_decimal_ip(hostname)

        if 'ziger2' in machine.name:
            modules.add_module("sfn5122f", ["auto", "function=0"])
            dst_ip = self.get_decimal_ip('%s-sf.in.barrelfish.org' % machine.name)
        else:
            dst_ip = self.get_decimal_ip('%s-e10k.in.barrelfish.org' % machine.name)

        modules.add_module(self.get_module_name(), ["core=2", self.OP, src_ip, dst_ip, self.CARD])
        return modules

    def get_finish_string(self):
        return "SUCCESS"

    def thread_func (self, dummy, dummy2):
        s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
        while True:
            s.sendto("Data Data Data", (self.ip, 7))

    def start_loop(self):
        self.thread = thread.start_new_thread(self.thread_func, (self, 0))

    def process_line(self, line):
        m = re.match(r'# IP Addr (\d+\.\d+\.\d+\.\d+)', line)
        if m:
            self.start_loop()
            self.ip = m.group(1)

    def process_data(self, testdir, rawiter):
        for line in rawiter:
            if "SUCCESS" in line:
                return PassFailResult(True)

        return PassFailResult(False)

@tests.add_test
class DevifNetTxSF(DevifTests):
    ''' Devif Net TX Test'''
    name = "devif_nettx_sf"
    OP = "net_tx"
    CARD = "sfn5122f"

@tests.add_test
class DevifNetTxE10k(DevifTests):
    ''' Devif Net TX Test'''
    name = "devif_nettx_e10k"
    OP = "net_tx"
    CARD = "e10k"

@tests.add_test
class DevifNetRxSF(DevifTests):
    ''' Devif Net RX Test'''
    name = "devif_netrx_sf"
    OP = "net_rx"
    CARD = "sfn5122f"

@tests.add_test
class DevifNetRxE10k(DevifTests):
    ''' Devif Net RX Test'''
    name = "devif_netrx_e10k"
    OP = "net_rx"
    CARD = "e10k"

@tests.add_test
class DevifIdcTest(DevifTests):
    ''' Devif IDC Test'''
    name = "devif_idc_test"
    OP = "idc"
    CARD = "none"

@tests.add_test
class DevifDebug(DevifTests):
    ''' Devif Debug Backend Test'''
    name = "devif_debug"

    def get_modules(self, build, machine):
        self.machine = machine.name
        modules = super(DevifTests, self).get_modules(build, machine)
        modules.add_module("devif_idc", ["core=1"])
        modules.add_module("devif_debug_test")

        return modules

@tests.add_test
class DevifUDP(DevifTests):
    ''' Devif UDP Backend Test'''
    name = "devif_udp"
    data = ("Data Data Data Data")

    def get_module_name(self):
        return "devif_udp"

    def get_modules(self, build, machine):
        self.machine = machine.name
        modules = super(DevifTests, self).get_modules(build, machine)
        modules.add_module("net_sockets_server", ["nospawn"])
        hostname = '%s.in.barrelfish.org' % subprocess.check_output('hostname -s', shell=True).rstrip()
        dst_ip = self.get_decimal_ip(hostname)
        dst_mac = self.get_local_mac('eno2')

        if 'ziger2' in machine.name:
            src_ip = self.get_decimal_ip('%s-sf.in.barrelfish.org' % machine.name)
            modules.add_module("sfn5122f", ["auto", "function=0"])
            self.cardname = "sfn5122f"
        else:
            modules.add_module("e10k", ["auto", "function=0"])
            src_ip = self.get_decimal_ip('%s-e10k.in.barrelfish.org' % machine.name)
            self.cardname = "e10k"

        src_mac =  mac[machine.name]

        modules.add_module(self.get_module_name(), ["core=2", dst_ip, dst_mac, 20000, 20000, self.cardname])
        return modules

    def thread_func (self, dummy, dummy2):
        s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
        while True:
            s.sendto(self.data, (self.ip, 20000))

    def start_loop(self):
        self.thread = thread.start_new_thread(self.thread_func, (self, 0))

    def process_line(self, line):
        m = re.match(r'# IP Addr (\d+\.\d+\.\d+\.\d+)', line)
        if m:
            self.ip = m.group(1)

        m1 = re.match(r'Testing receiving UDP packets', line)
        if m1:
            self.start_loop()

    def process_data(self, testdir, rawiter):
        for line in rawiter:
            if "SUCCESS" in line:
                return PassFailResult(True)

        return PassFailResult(False)

#@tests.add_test
#class DevifUPDecho(DevifUDP):
#    ''' Devif Debug Backend Test'''
#    name = "devif_udp_echo"
#
#    def get_module_name(self):
#        return "devif_echo"

<|endoftext|>


<commit_before>'''
    @author: ArcGIS for Water Utilities
    @contact: ArcGISTeamUtilities@esri.com
    @company: Esri
    @version: 1.1
    @description: Used to append content from a feature service
    @requirements: Python 2.7.x, ArcGIS 10.2.1
    @copyright: Esri, 2014
'''
import gc
import os
import sys
import arcpy

from arcresthelper import featureservicetools
from arcresthelper import common
def trace():
    """
        trace finds the line, the filename
        and error message and returns it
        to the user
    """
    import traceback, inspect
    tb = sys.exc_info()[2]
    tbinfo = traceback.format_tb(tb)[0]
    filename = inspect.getfile(inspect.currentframe())
    # script name + line number
    line = tbinfo.split(", ")[1]
    # Get Python syntax error
    #
    synerror = traceback.format_exc().splitlines()[-1]
    return line, filename, synerror

def outputPrinter(message,typeOfMessage='message'):
    if typeOfMessage == "error":
        arcpy.AddError(message=message)
    elif typeOfMessage == "warning":
        arcpy.AddWarning(message=message)
    else:
        arcpy.AddMessage(message=message)

    print message
def main():

    fsId = None
    groupLayer = None
    layerMap = None
    matchEntireName = None
    projection = None
    scratchGDB = None
    scratchLayer = None
    fst = None
    rows = None
    fieldList = None
    layerToServiceLayer = None
    matches = False
    try:

        proxy_port = None
        proxy_url = None

        securityinfo = {}
        securityinfo['security_type'] = 'ArcGIS'#LDAP, NTLM, OAuth, Portal, PKI

        securityinfo['proxy_url'] = proxy_url
        securityinfo['proxy_port'] = proxy_port
        securityinfo['referer_url'] = None
        securityinfo['token_url'] = None
        securityinfo['certificatefile'] = None
        securityinfo['keyfile'] = None
        securityinfo['client_id'] = None
        securityinfo['secret_id'] = None

        fsId = arcpy.GetParameterAsText(1)
        groupLayer = arcpy.GetParameterAsText(0)
        layerMap = arcpy.GetParameterAsText(2)
        matchEntireName = arcpy.GetParameterAsText(3)
        projection = arcpy.GetParameterAsText(4)
        lowerCaseFieldNames = argv[5]

        if str(lowerCaseFieldNames).upper() == 'TRUE':
            lowerCaseFieldNames = True
        else:
            lowerCaseFieldNames = False
        if projection is not None and projection != '#' and projection != '':
            #outputPrinter(message="Projecting %s" % str(projection))
            pass
        else:
            projection = None
            #outputPrinter(message="No Projection defined")
        arcpy.SetParameterAsText(6, "true")

        scratchGDB = arcpy.env.scratchWorkspace
        scratchLayName = "tempAppGrpFS"
        scratchLayer = os.path.join(scratchGDB,scratchLayName)

        groupLayer = arcpy.mapping.Layer(groupLayer)
        fst = featureservicetools.featureservicetools(securityinfo)
        if fst.valid:
            #outputPrinter(message="Security handler created")

            fs = fst.GetFeatureService(itemId=fsId,returnURLOnly=False)

            if not fs is None:
                #Get a cursor to the layer map recordset
                rows = arcpy.SearchCursor(layerMap)

                #Get the fields
                fieldList = arcpy.ListFields(layerMap)

                #initialize the Translation Dictionary
                layerToServiceLayer = {}

                #Loop through each input row and add it to the conversion dict
                for row in rows:
                    layerToServiceLayer[row.getValue(fieldList[1].name)] = row.getValue(fieldList[2].name)
                    del row
                if groupLayer.isGroupLayer:
                    for lyr in groupLayer:
                        for key, value in layerToServiceLayer.iteritems():
                            if matchEntireName =='true' and key == lyr.name:
                                matches = True
                            elif matchEntireName =='false' and key in lyr.name:
                                matches = True
                            else:
                                matches = False
                            if matches:
                                if arcpy.Exists(lyr.name) == True:
                                    result =  arcpy.GetCount_management(lyr.name)
                                    count = int(result.getOutput(0))
                                    if count > 0:
                                        layerNameFull = groupLayer.name + '\\' + lyr.name

                                        if projection is not None:
                                            outputPrinter(message="Projecting %s" % lyr.name)
                                            arcpy.Project_management(layerNameFull,
                                                                    scratchLayer,
                                                                    projection,
                                                                    "",
                                                                    "",
                                                                    "NO_PRESERVE_SHAPE",
                                                                    "")

                                        else:
                                            outputPrinter(message="Copying %s" % lyr.name)
                                            arcpy.FeatureClassToFeatureClass_conversion(layerNameFull,scratchGDB,scratchLayName)

                                        desc = arcpy.Describe(scratchLayer)
                                        if desc.shapeType == 'Polygon':
                                            outputPrinter(message="Densifying %s" % lyr.name)
                                            arcpy.Densify_edit(scratchLayer, "ANGLE", "33 Unknown", "0.33 Unknown", "4")
                                        syncLayer(fst, fs, scratchLayer, value, lyr.name,lowerCaseFieldNames)

                                    else:
                                        outputPrinter (message="%s does not contain any features, skipping" % lyr.name)
                                else:
                                    outputPrinter (message="%s does not exist, skipping" % lyr.name)
                                break

                else:
                    outputPrinter (message="Group layer is not a group layer", typeOfMessage='error')
            else:
                outputPrinter(message="Feature Service with id %s was not found" % fsId, typeOfMessage='error')
                arcpy.SetParameterAsText(6, "false")
        else:
            outputPrinter(fst.message,typeOfMessage='error')
            arcpy.SetParameterAsText(6, "false")

    except arcpy.ExecuteError:
        line, filename, synerror = trace()
        outputPrinter(message="error on line: %s" % line,typeOfMessage='error')
        outputPrinter(message="error in file name: %s" % filename,typeOfMessage='error')
        outputPrinter(message="with error message: %s" % synerror,typeOfMessage='error')
        outputPrinter(message="ArcPy Error Message: %s" % arcpy.GetMessages(2),typeOfMessage='error')
        arcpy.SetParameterAsText(6, "false")
    except (common.ArcRestHelperError),e:
        outputPrinter(message=e,typeOfMessage='error')
        arcpy.SetParameterAsText(6, "false")
    except:
        line, filename, synerror = trace()
        outputPrinter(message="error on line: %s" % line,typeOfMessage='error')
        outputPrinter(message="error in file name: %s" % filename,typeOfMessage='error')
        outputPrinter(message="with error message: %s" % synerror,typeOfMessage='error')
        arcpy.SetParameterAsText(6, "false")
    finally:
        if scratchLayer is not None:
            if arcpy.Exists(scratchLayer):
                arcpy.Delete_management(scratchLayer)

        fsId = None
        groupLayer = None
        layerMap = None
        matchEntireName = None
        projection = None
        scratchGDB = None
        scratchLayer = None

        fst = None
        rows = None
        fieldList = None
        layerToServiceLayer = None

        del fsId
        del groupLayer
        del layerMap
        del matchEntireName
        del projection
        del scratchGDB
        del scratchLayer
        del fst
        del rows
        del fieldList
        del layerToServiceLayer

        gc.collect()
def syncLayer(fst, fs, layer, layerName, displayName, lowerCaseFieldNames):

    outputPrinter (message="Attemping to sync %s to %s" % (displayName,layerName))
    fl = fst.GetLayerFromFeatureService(fs=fs,layerName=layerName,returnURLOnly=False)
    if not fl is None:
        results = fl.addFeatures(fc=layer,lowerCaseFieldNames=lowerCaseFieldNames)

        if 'error' in results:
            outputPrinter(message="Error in response from server:  %s" % results['error'],typeOfMessage='error')
            arcpy.SetParameterAsText(5, "false")
        else:
            if results['addResults'] is not None:
                outputPrinter (message="%s features added to %s" % (len(results['addResults']),layerName) )
            else:
                outputPrinter (message="0 features added to %s" % layerName)
    else:
        outputPrinter(message="Layer %s was not found, please check your credentials and layer name" % layerName,typeOfMessage='error')
        arcpy.SetParameterAsText(5, "false")

if __name__ == "__main__":
    main()
<commit_msg>resolved invalid arg<commit_after>'''
    @author: ArcGIS for Water Utilities
    @contact: ArcGISTeamUtilities@esri.com
    @company: Esri
    @version: 1.1
    @description: Used to append content from a feature service
    @requirements: Python 2.7.x, ArcGIS 10.2.1
    @copyright: Esri, 2014
'''
import gc
import os
import sys
import arcpy

from arcresthelper import featureservicetools
from arcresthelper import common
def trace():
    """
        trace finds the line, the filename
        and error message and returns it
        to the user
    """
    import traceback, inspect
    tb = sys.exc_info()[2]
    tbinfo = traceback.format_tb(tb)[0]
    filename = inspect.getfile(inspect.currentframe())
    # script name + line number
    line = tbinfo.split(", ")[1]
    # Get Python syntax error
    #
    synerror = traceback.format_exc().splitlines()[-1]
    return line, filename, synerror

def outputPrinter(message,typeOfMessage='message'):
    if typeOfMessage == "error":
        arcpy.AddError(message=message)
    elif typeOfMessage == "warning":
        arcpy.AddWarning(message=message)
    else:
        arcpy.AddMessage(message=message)

    print message
def main():

    fsId = None
    groupLayer = None
    layerMap = None
    matchEntireName = None
    projection = None
    scratchGDB = None
    scratchLayer = None
    fst = None
    rows = None
    fieldList = None
    layerToServiceLayer = None
    matches = False
    try:

        proxy_port = None
        proxy_url = None

        securityinfo = {}
        securityinfo['security_type'] = 'ArcGIS'#LDAP, NTLM, OAuth, Portal, PKI

        securityinfo['proxy_url'] = proxy_url
        securityinfo['proxy_port'] = proxy_port
        securityinfo['referer_url'] = None
        securityinfo['token_url'] = None
        securityinfo['certificatefile'] = None
        securityinfo['keyfile'] = None
        securityinfo['client_id'] = None
        securityinfo['secret_id'] = None

        fsId = arcpy.GetParameterAsText(1)
        groupLayer = arcpy.GetParameterAsText(0)
        layerMap = arcpy.GetParameterAsText(2)
        matchEntireName = arcpy.GetParameterAsText(3)
        projection = arcpy.GetParameterAsText(4)
        lowerCaseFieldNames = arcpy.GetParameterAsText(5)

        if str(lowerCaseFieldNames).upper() == 'TRUE':
            lowerCaseFieldNames = True
        else:
            lowerCaseFieldNames = False
        if projection is not None and projection != '#' and projection != '':
            #outputPrinter(message="Projecting %s" % str(projection))
            pass
        else:
            projection = None
            #outputPrinter(message="No Projection defined")
        arcpy.SetParameterAsText(6, "true")

        scratchGDB = arcpy.env.scratchWorkspace
        scratchLayName = "tempAppGrpFS"
        scratchLayer = os.path.join(scratchGDB,scratchLayName)

        groupLayer = arcpy.mapping.Layer(groupLayer)
        fst = featureservicetools.featureservicetools(securityinfo)
        if fst.valid:
            #outputPrinter(message="Security handler created")

            fs = fst.GetFeatureService(itemId=fsId,returnURLOnly=False)

            if not fs is None:
                #Get a cursor to the layer map recordset
                rows = arcpy.SearchCursor(layerMap)

                #Get the fields
                fieldList = arcpy.ListFields(layerMap)

                #initialize the Translation Dictionary
                layerToServiceLayer = {}

                #Loop through each input row and add it to the conversion dict
                for row in rows:
                    layerToServiceLayer[row.getValue(fieldList[1].name)] = row.getValue(fieldList[2].name)
                    del row
                if groupLayer.isGroupLayer:
                    for lyr in groupLayer:
                        for key, value in layerToServiceLayer.iteritems():
                            if matchEntireName =='true' and key == lyr.name:
                                matches = True
                            elif matchEntireName =='false' and key in lyr.name:
                                matches = True
                            else:
                                matches = False
                            if matches:
                                if arcpy.Exists(lyr.name) == True:
                                    result =  arcpy.GetCount_management(lyr.name)
                                    count = int(result.getOutput(0))
                                    if count > 0:
                                        layerNameFull = groupLayer.name + '\\' + lyr.name

                                        if projection is not None:
                                            outputPrinter(message="Projecting %s" % lyr.name)
                                            arcpy.Project_management(layerNameFull,
                                                                    scratchLayer,
                                                                    projection,
                                                                    "",
                                                                    "",
                                                                    "NO_PRESERVE_SHAPE",
                                                                    "")

                                        else:
                                            outputPrinter(message="Copying %s" % lyr.name)
                                            arcpy.FeatureClassToFeatureClass_conversion(layerNameFull,scratchGDB,scratchLayName)

                                        desc = arcpy.Describe(scratchLayer)
                                        if desc.shapeType == 'Polygon':
                                            outputPrinter(message="Densifying %s" % lyr.name)
                                            arcpy.Densify_edit(scratchLayer, "ANGLE", "33 Unknown", "0.33 Unknown", "4")
                                        syncLayer(fst, fs, scratchLayer, value, lyr.name,lowerCaseFieldNames)

                                    else:
                                        outputPrinter (message="%s does not contain any features, skipping" % lyr.name)
                                else:
                                    outputPrinter (message="%s does not exist, skipping" % lyr.name)
                                break

                else:
                    outputPrinter (message="Group layer is not a group layer", typeOfMessage='error')
            else:
                outputPrinter(message="Feature Service with id %s was not found" % fsId, typeOfMessage='error')
                arcpy.SetParameterAsText(6, "false")
        else:
            outputPrinter(fst.message,typeOfMessage='error')
            arcpy.SetParameterAsText(6, "false")

    except arcpy.ExecuteError:
        line, filename, synerror = trace()
        outputPrinter(message="error on line: %s" % line,typeOfMessage='error')
        outputPrinter(message="error in file name: %s" % filename,typeOfMessage='error')
        outputPrinter(message="with error message: %s" % synerror,typeOfMessage='error')
        outputPrinter(message="ArcPy Error Message: %s" % arcpy.GetMessages(2),typeOfMessage='error')
        arcpy.SetParameterAsText(6, "false")
    except (common.ArcRestHelperError),e:
        outputPrinter(message=e,typeOfMessage='error')
        arcpy.SetParameterAsText(6, "false")
    except:
        line, filename, synerror = trace()
        outputPrinter(message="error on line: %s" % line,typeOfMessage='error')
        outputPrinter(message="error in file name: %s" % filename,typeOfMessage='error')
        outputPrinter(message="with error message: %s" % synerror,typeOfMessage='error')
        arcpy.SetParameterAsText(6, "false")
    finally:
        if scratchLayer is not None:
            if arcpy.Exists(scratchLayer):
                arcpy.Delete_management(scratchLayer)

        fsId = None
        groupLayer = None
        layerMap = None
        matchEntireName = None
        projection = None
        scratchGDB = None
        scratchLayer = None

        fst = None
        rows = None
        fieldList = None
        layerToServiceLayer = None

        del fsId
        del groupLayer
        del layerMap
        del matchEntireName
        del projection
        del scratchGDB
        del scratchLayer
        del fst
        del rows
        del fieldList
        del layerToServiceLayer

        gc.collect()
def syncLayer(fst, fs, layer, layerName, displayName, lowerCaseFieldNames):

    outputPrinter (message="Attemping to sync %s to %s" % (displayName,layerName))
    fl = fst.GetLayerFromFeatureService(fs=fs,layerName=layerName,returnURLOnly=False)
    if not fl is None:
        results = fl.addFeatures(fc=layer,lowerCaseFieldNames=lowerCaseFieldNames)

        if 'error' in results:
            outputPrinter(message="Error in response from server:  %s" % results['error'],typeOfMessage='error')
            arcpy.SetParameterAsText(5, "false")
        else:
            if results['addResults'] is not None:
                outputPrinter (message="%s features added to %s" % (len(results['addResults']),layerName) )
            else:
                outputPrinter (message="0 features added to %s" % layerName)
    else:
        outputPrinter(message="Layer %s was not found, please check your credentials and layer name" % layerName,typeOfMessage='error')
        arcpy.SetParameterAsText(5, "false")

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before>from totalimpact.providers import provider
from totalimpact.providers.provider import Provider, ProviderContentMalformedError

import simplejson, urllib, os

import logging
logger = logging.getLogger('ti.providers.mendeley')

class Mendeley(Provider):

    example_id = ("doi", "10.1371/journal.pcbi.1000361")

    url = "http://www.mendeley.com"
    descr = " A research management tool for desktop and web."
    uuid_from_title_template = 'http://api.mendeley.com/oapi/documents/search/"%s"/?consumer_key=' + os.environ["MENDELEY_KEY"]
    metrics_from_uuid_template = "http://api.mendeley.com/oapi/documents/details/%s?consumer_key=" + os.environ["MENDELEY_KEY"]

    static_meta_dict = {
        "readers": {
            "display_name": "readers",
            "provider": "Mendeley",
            "provider_url": "http://www.mendeley.com/",
            "description": "The number of readers who have added the article to their libraries",
            "icon": "http://www.mendeley.com/favicon.ico",
        },
        "groups": {
            "display_name": "groups",
            "provider": "Mendeley",
            "provider_url": "http://www.mendeley.com/",
            "description": "The number of groups who have added the article to their libraries",
            "icon": "http://www.mendeley.com/favicon.ico",
        },
        "discipline": {
            "display_name": "discipline, top 3 percentages",
            "provider": "Mendeley",
            "provider_url": "http://www.mendeley.com/",
            "description": "Percent of readers by discipline, for top three disciplines (csv, api only)",
            "icon": "http://www.mendeley.com/favicon.ico",
        },
        "career_stage": {
            "display_name": "career stage, top 3 percentages",
            "provider": "Mendeley",
            "provider_url": "http://www.mendeley.com/",
            "description": "Percent of readers by career stage, for top three career stages (csv, api only)",
            "icon": "http://www.mendeley.com/favicon.ico",
        },
        "country": {
            "display_name": "country, top 3 percentages",
            "provider": "Mendeley",
            "provider_url": "http://www.mendeley.com/",
            "description": "Percent of readers by country, for top three countries (csv, api only)",
            "icon": "http://www.mendeley.com/favicon.ico",
        }
    }

    def __init__(self):
        super(Mendeley, self).__init__()

    def is_relevant_alias(self, alias):
        (namespace, nid) = alias
        # right now restricted to doi because we check the title lookup matches doi
        ## to keep precision high.  Later could experiment with opening this up.
        relevant = (namespace=="doi")
        return(relevant)

    def _extract_metrics(self, page, status_code=200, id=None):
        if not "identifiers" in page:
            raise ProviderContentMalformedError()

        dict_of_keylists = {"mendeley:readers": ["stats", "readers"],
                            "mendeley:discipline": ["stats", "discipline"],
                            "mendeley:career_stage": ["stats", "status"],
                            "mendeley:country": ["stats", "country"],
                            "mendeley:groups" : ["groups"]}

        metrics_dict = provider._extract_from_json(page, dict_of_keylists)

        # get count of groups
        try:
            metrics_dict["mendeley:groups"] = len(metrics_dict["mendeley:groups"])
        except (TypeError, KeyError):
            # don't add null or zero metrics
            pass

        return metrics_dict

    def _extract_provenance_url(self, page, status_code=200, id=None):
        data = provider._load_json(page)
        try:
            provenance_url = data['mendeley_url']
        except KeyError:
            provenance_url = ""
        return provenance_url

    def _get_page(self, url):
        response = self.http_get(url)
        if response.status_code != 200:
            if response.status_code == 404:
                return None
            else:
                raise(self._get_error(response.status_code))
        return response.text

    def _get_uuid_lookup_page(self, title):
        uuid_from_title_url = self.uuid_from_title_template % title
        page = self._get_page(uuid_from_title_url)
        if not "documents" in page:
            raise ProviderContentMalformedError()
        return page

    def _get_metrics_lookup_page(self, uuid):
        metrics_from_uuid_url = self.metrics_from_uuid_template %uuid
        page = self._get_page(metrics_from_uuid_url)
        if not "identifiers" in page:
            raise ProviderContentMalformedError()
        return page

    def _get_uuid_from_title(self, aliases_dict, page):
        doi = aliases_dict["doi"][0]
        data = provider._load_json(page)
        for mendeley_record in data["documents"]:
            if mendeley_record["doi"] == doi:
                uuid = mendeley_record["uuid"]
                # our job here is done
                return uuid
            else:
                if mendeley_record["year"] == aliases_dict["biblio"][0]["year"]:

                    # check if author name in common. if not, yell, but continue anyway
                    first_mendeley_surname = mendeley_record["authors"][0]["surname"]
                    has_matching_authors = first_mendeley_surname in aliases_dict["biblio"][0]["authors"]
                    if not has_matching_authors:
                        logger.warning("Mendeley: NO MATCHING AUTHORS between %s and %s", %(
                            first_mendeley_surname, aliases_dict["biblio"][0]["authors"]))

                    uuid = mendeley_record["uuid"]
                    return uuid
        return None

    def _get_metrics_and_drilldown_from_uuid(self, page):
        metrics_dict = self._extract_metrics(page)
        metrics_and_drilldown = {}
        for metric_name in metrics_dict:
            drilldown_url = self._extract_provenance_url(page)
            metrics_and_drilldown[metric_name] = (metrics_dict[metric_name], drilldown_url)
        return metrics_and_drilldown

    # default method; providers can override
    def metrics(self,
            aliases,
            provider_url_template=None, # ignore this because multiple url steps
            cache_enabled=True):

        # Only lookup metrics for items with appropriate ids
        from totalimpact.models import ItemFactory
        aliases_dict = ItemFactory.alias_dict_from_tuples(aliases)

        if (not "biblio" in aliases_dict) or (not "doi" in aliases_dict):
            return {}

        page = self._get_uuid_lookup_page(aliases_dict["biblio"][0]["title"])
        if not page:
            return {}
        uuid = self._get_uuid_from_title(aliases_dict, page)
        if not uuid:
            logger.info("Mendeley: couldn't find uuid for %s" %(aliases_dict["biblio"][0]["title"]))
            return {}

        logger.info("Mendeley: uuid for %s is %s" %(aliases_dict["biblio"][0]["title"], uuid))
        page = self._get_metrics_lookup_page(uuid)
        if not page:
            return {}
        metrics_and_drilldown = self._get_metrics_and_drilldown_from_uuid(page)

        return metrics_and_drilldown

<commit_msg>fix typo<commit_after>from totalimpact.providers import provider
from totalimpact.providers.provider import Provider, ProviderContentMalformedError

import simplejson, urllib, os

import logging
logger = logging.getLogger('ti.providers.mendeley')

class Mendeley(Provider):

    example_id = ("doi", "10.1371/journal.pcbi.1000361")

    url = "http://www.mendeley.com"
    descr = " A research management tool for desktop and web."
    uuid_from_title_template = 'http://api.mendeley.com/oapi/documents/search/"%s"/?consumer_key=' + os.environ["MENDELEY_KEY"]
    metrics_from_uuid_template = "http://api.mendeley.com/oapi/documents/details/%s?consumer_key=" + os.environ["MENDELEY_KEY"]

    static_meta_dict = {
        "readers": {
            "display_name": "readers",
            "provider": "Mendeley",
            "provider_url": "http://www.mendeley.com/",
            "description": "The number of readers who have added the article to their libraries",
            "icon": "http://www.mendeley.com/favicon.ico",
        },
        "groups": {
            "display_name": "groups",
            "provider": "Mendeley",
            "provider_url": "http://www.mendeley.com/",
            "description": "The number of groups who have added the article to their libraries",
            "icon": "http://www.mendeley.com/favicon.ico",
        },
        "discipline": {
            "display_name": "discipline, top 3 percentages",
            "provider": "Mendeley",
            "provider_url": "http://www.mendeley.com/",
            "description": "Percent of readers by discipline, for top three disciplines (csv, api only)",
            "icon": "http://www.mendeley.com/favicon.ico",
        },
        "career_stage": {
            "display_name": "career stage, top 3 percentages",
            "provider": "Mendeley",
            "provider_url": "http://www.mendeley.com/",
            "description": "Percent of readers by career stage, for top three career stages (csv, api only)",
            "icon": "http://www.mendeley.com/favicon.ico",
        },
        "country": {
            "display_name": "country, top 3 percentages",
            "provider": "Mendeley",
            "provider_url": "http://www.mendeley.com/",
            "description": "Percent of readers by country, for top three countries (csv, api only)",
            "icon": "http://www.mendeley.com/favicon.ico",
        }
    }

    def __init__(self):
        super(Mendeley, self).__init__()

    def is_relevant_alias(self, alias):
        (namespace, nid) = alias
        # right now restricted to doi because we check the title lookup matches doi
        ## to keep precision high.  Later could experiment with opening this up.
        relevant = (namespace=="doi")
        return(relevant)

    def _extract_metrics(self, page, status_code=200, id=None):
        if not "identifiers" in page:
            raise ProviderContentMalformedError()

        dict_of_keylists = {"mendeley:readers": ["stats", "readers"],
                            "mendeley:discipline": ["stats", "discipline"],
                            "mendeley:career_stage": ["stats", "status"],
                            "mendeley:country": ["stats", "country"],
                            "mendeley:groups" : ["groups"]}

        metrics_dict = provider._extract_from_json(page, dict_of_keylists)

        # get count of groups
        try:
            metrics_dict["mendeley:groups"] = len(metrics_dict["mendeley:groups"])
        except (TypeError, KeyError):
            # don't add null or zero metrics
            pass

        return metrics_dict

    def _extract_provenance_url(self, page, status_code=200, id=None):
        data = provider._load_json(page)
        try:
            provenance_url = data['mendeley_url']
        except KeyError:
            provenance_url = ""
        return provenance_url

    def _get_page(self, url):
        response = self.http_get(url)
        if response.status_code != 200:
            if response.status_code == 404:
                return None
            else:
                raise(self._get_error(response.status_code))
        return response.text

    def _get_uuid_lookup_page(self, title):
        uuid_from_title_url = self.uuid_from_title_template % title
        page = self._get_page(uuid_from_title_url)
        if not "documents" in page:
            raise ProviderContentMalformedError()
        return page

    def _get_metrics_lookup_page(self, uuid):
        metrics_from_uuid_url = self.metrics_from_uuid_template %uuid
        page = self._get_page(metrics_from_uuid_url)
        if not "identifiers" in page:
            raise ProviderContentMalformedError()
        return page

    def _get_uuid_from_title(self, aliases_dict, page):
        doi = aliases_dict["doi"][0]
        data = provider._load_json(page)
        for mendeley_record in data["documents"]:
            if mendeley_record["doi"] == doi:
                uuid = mendeley_record["uuid"]
                # our job here is done
                return uuid
            else:
                if mendeley_record["year"] == aliases_dict["biblio"][0]["year"]:

                    # check if author name in common. if not, yell, but continue anyway
                    first_mendeley_surname = mendeley_record["authors"][0]["surname"]
                    has_matching_authors = first_mendeley_surname in aliases_dict["biblio"][0]["authors"]
                    if not has_matching_authors:
                        logger.warning("Mendeley: NO MATCHING AUTHORS between %s and %s" %(
                            first_mendeley_surname, aliases_dict["biblio"][0]["authors"]))

                    uuid = mendeley_record["uuid"]
                    return uuid
        return None

    def _get_metrics_and_drilldown_from_uuid(self, page):
        metrics_dict = self._extract_metrics(page)
        metrics_and_drilldown = {}
        for metric_name in metrics_dict:
            drilldown_url = self._extract_provenance_url(page)
            metrics_and_drilldown[metric_name] = (metrics_dict[metric_name], drilldown_url)
        return metrics_and_drilldown

    # default method; providers can override
    def metrics(self,
            aliases,
            provider_url_template=None, # ignore this because multiple url steps
            cache_enabled=True):

        # Only lookup metrics for items with appropriate ids
        from totalimpact.models import ItemFactory
        aliases_dict = ItemFactory.alias_dict_from_tuples(aliases)

        if (not "biblio" in aliases_dict) or (not "doi" in aliases_dict):
            return {}

        page = self._get_uuid_lookup_page(aliases_dict["biblio"][0]["title"])
        if not page:
            return {}
        uuid = self._get_uuid_from_title(aliases_dict, page)
        if not uuid:
            logger.info("Mendeley: couldn't find uuid for %s" %(aliases_dict["biblio"][0]["title"]))
            return {}

        logger.info("Mendeley: uuid for %s is %s" %(aliases_dict["biblio"][0]["title"], uuid))
        page = self._get_metrics_lookup_page(uuid)
        if not page:
            return {}
        metrics_and_drilldown = self._get_metrics_and_drilldown_from_uuid(page)

        return metrics_and_drilldown

<|endoftext|>