<commit_before># vim: tabstop=4 shiftwidth=4 softtabstop=4

# Copyright 2010 United States Government as represented by the
# Administrator of the National Aeronautics and Space Administration.
# All Rights Reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.
#
# Template library copied from bottle: http://bottlepy.org/
#
# Copyright (c) 2011, Marcel Hellkamp.
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.
#

import cgi
import re
import os
import functools
import time
import tokenize
from webob import Response

import keystone.logic.types.fault as fault

TEMPLATES = {}
DEBUG = False
TEMPLATE_PATH = ['./', './views/']

class BaseTemplate(object):
    """ Base class and minimal API for template adapters """
    extentions = ['tpl', 'html', 'thtml', 'stpl']
    settings = {}  #used in prepare()
    defaults = {}  #used in render()

    def __init__(self, source=None, name=None, lookup=[], encoding='utf8',
                 **settings):
        """ Create a new template.
        If the source parameter (str or buffer) is missing, the name argument
        is used to guess a template filename. Subclasses can assume that
        self.source and/or self.filename are set. Both are strings.
        The lookup, encoding and settings parameters are stored as instance
        variables.
        The lookup parameter stores a list containing directory paths.
        The encoding parameter should be used to decode byte strings or files.
        The settings parameter contains a dict for engine-specific settings.
        """
        self.name = name
        self.source = source.read() if hasattr(source, 'read') else source
        self.filename = source.filename if hasattr(source, 'filename') else None
        self.lookup = map(os.path.abspath, lookup)
        self.encoding = encoding
        self.settings = self.settings.copy() # Copy from class variable
        self.settings.update(settings) # Apply
        if not self.source and self.name:
            self.filename = self.search(self.name, self.lookup)
            if not self.filename:
                raise TemplateError('Template %s not found.' % repr(name))
        if not self.source and not self.filename:
            raise TemplateError('No template specified.')
        self.prepare(**self.settings)

    @classmethod
    def search(cls, name, lookup=[]):
        """ Search name in all directories specified in lookup.
        First without, then with common extensions. Return first hit. """
        if os.path.isfile(name):
            return name
        for spath in lookup:
            fname = os.path.join(spath, name)
            if os.path.isfile(fname):
                return fname
            for ext in cls.extentions:
                if os.path.isfile('%s.%s' % (fname, ext)):
                    return '%s.%s' % (fname, ext)

    @classmethod
    def global_config(cls, key, *args):
        ''' This reads or sets the global settings stored in class.settings. '''
        if args:
            cls.settings[key] = args[0]
        else:
            return cls.settings[key]

    def prepare(self, **options):
        """ Run preparations (parsing, caching, ...).
        It should be possible to call this again to refresh a template or to
        update settings.
        """
        raise NotImplementedError

    def render(self, **args):
        """ Render the template with the specified local variables and return
        a single byte or unicode string. If it is a byte string, the encoding
        must match self.encoding. This method must be thread-safe!
        """
        raise NotImplementedError

class SimpleTemplate(BaseTemplate):
    blocks = ('if', 'elif', 'else', 'try', 'except', 'finally', 'for', 'while', 'with', 'def', 'class')
    dedent_blocks = ('elif', 'else', 'except', 'finally')

    def prepare(self, escape_func=cgi.escape, noescape=False):
        self.cache = {}
        if self.source:
            self.code = self.translate(self.source)
            self.co = compile(self.code, '<string>', 'exec')
        else:
            self.code = self.translate(open(self.filename).read())
            self.co = compile(self.code, self.filename, 'exec')
        enc = self.encoding
        touni = functools.partial(unicode, encoding=self.encoding)
        self._str = lambda x: touni(x, enc)
        self._escape = lambda x: escape_func(touni(x))
        if noescape:
            self._str, self._escape = self._escape, self._str

    def translate(self, template):
        stack = [] # Current Code indentation
        lineno = 0 # Current line of code
        ptrbuffer = [] # Buffer for printable strings and token tuple instances
        codebuffer = [] # Buffer for generated python code
        touni = functools.partial(unicode, encoding=self.encoding)
        multiline = dedent = False

        def yield_tokens(line):
            for i, part in enumerate(re.split(r'\{\{(.*?)\}\}', line)):
                if i % 2:
                    if part.startswith('!'): yield 'RAW', part[1:]
                    else: yield 'CMD', part
                else: yield 'TXT', part

        def split_comment(codeline):
            """ Removes comments from a line of code. """
            line = codeline.splitlines()[0]
            try:
                tokens = list(tokenize.generate_tokens(iter(line).next))
            except tokenize.TokenError:
                return line.rsplit('#', 1) if '#' in line else (line, '')
            for token in tokens:
                if token[0] == tokenize.COMMENT:
                    start, end = token[2][1], token[3][1]
                    return codeline[:start] + codeline[end:], codeline[start:end]
            return line, ''

        def flush(): # Flush the ptrbuffer
            if not ptrbuffer: return
            cline = ''
            for line in ptrbuffer:
                for token, value in line:
                    if token == 'TXT':
                        cline += repr(value)
                    elif token == 'RAW':
                        cline += '_str(%s)' % value
                    elif token == 'CMD':
                        cline += '_escape(%s)' % value
                    cline += ', '
                cline = cline[:-2] + '\\\n'
            cline = cline[:-2]
            if cline[:-1].endswith('\\\\\\\\\\n'):
                cline = cline[:-7] + cline[-1] # 'nobr\\\\\n' --> 'nobr'
            cline = '_printlist([' + cline + '])'
            del ptrbuffer[:] # Do this before calling code() again
            code(cline)

        def code(stmt):
            for line in stmt.splitlines():
                codebuffer.append('  ' * len(stack) + line.strip())

        for line in template.splitlines(True):
            lineno += 1
            line = line if isinstance(line, unicode)\
                        else unicode(line, encoding=self.encoding)
            if lineno <= 2:
                m = re.search(r"%.*coding[:=]\s*([-\w\.]+)", line)
                if m: self.encoding = m.group(1)
                if m: line = line.replace('coding', 'coding (removed)')
            if line.strip()[:2].count('%') == 1:
                line = line.split('%', 1)[1].lstrip() # Full line following the %
                cline = split_comment(line)[0].strip()
                cmd = re.split(r'[^a-zA-Z0-9_]', cline)[0]
                flush() ##encodig (TODO: why?)
                if cmd in self.blocks or multiline:
                    cmd = multiline or cmd
                    dedent = cmd in self.dedent_blocks # "else:"
                    if dedent and not oneline and not multiline:
                        cmd = stack.pop()
                    code(line)
                    oneline = not cline.endswith(':') # "if 1: pass"
                    multiline = cmd if cline.endswith('\\') else False
                    if not oneline and not multiline:
                        stack.append(cmd)
                elif cmd == 'end' and stack:
                    code('#end(%s) %s' % (stack.pop(), line.strip()[3:]))
                elif cmd == 'include':
                    p = cline.split(None, 2)[1:]
                    if len(p) == 2:
                        code("_=_include(%s, _stdout, %s)" % (repr(p[0]), p[1]))
                    elif p:
                        code("_=_include(%s, _stdout)" % repr(p[0]))
                    else: # Empty %include -> reverse of %rebase
                        code("_printlist(_base)")
                elif cmd == 'rebase':
                    p = cline.split(None, 2)[1:]
                    if len(p) == 2:
                        code("globals()['_rebase']=(%s, dict(%s))" % (repr(p[0]), p[1]))
                    elif p:
                        code("globals()['_rebase']=(%s, {})" % repr(p[0]))
                else:
                    code(line)
            else: # Line starting with text (not '%') or '%%' (escaped)
                if line.strip().startswith('%%'):
                    line = line.replace('%%', '%', 1)
                ptrbuffer.append(yield_tokens(line))
        flush()
        return '\n'.join(codebuffer) + '\n'

    def subtemplate(self, _name, _stdout, **args):
        if _name not in self.cache:
            self.cache[_name] = self.__class__(name=_name, lookup=self.lookup)
        return self.cache[_name].execute(_stdout, **args)

    def execute(self, _stdout, **args):
        env = self.defaults.copy()
        env.update({'_stdout': _stdout, '_printlist': _stdout.extend,
               '_include': self.subtemplate, '_str': self._str,
               '_escape': self._escape})
        env.update(args)
        eval(self.co, env)
        if '_rebase' in env:
            subtpl, rargs = env['_rebase']
            subtpl = self.__class__(name=subtpl, lookup=self.lookup)
            rargs['_base'] = _stdout[:] #copy stdout
            del _stdout[:] # clear stdout
            return subtpl.execute(_stdout, **rargs)
        return env

    def render(self, **args):
        """ Render the template using keyword arguments as local variables. """
        stdout = []
        self.execute(stdout, **args)
        return ''.join(stdout)

def static_file(resp, req, filename, root, guessmime=True, mimetype=None, download=False):
    """ Opens a file in a safe way and returns a HTTPError object with status
        code 200, 305, 401 or 404. Sets Content-Type, Content-Length and
        Last-Modified header. Obeys If-Modified-Since header and HEAD requests.
    """
    root = os.path.abspath(root) + os.sep
    filename = os.path.abspath(os.path.join(root, filename.strip('/\\')))
    header = {}
    if not filename.startswith(root):
        #return HTTPError(403, "Access denied.")
        return ForbiddenFault("Access denied.")
    if not os.path.exists(filename) or not os.path.isfile(filename):
        #return HTTPError(404, "File does not exist.")
        return fault.ItemNotFoundFault("File does not exist.")
    if not os.access(filename, os.R_OK):
        #return HTTPError(403, "You do not have permission to access this file.")
        return ForbiddenFault("You do not have permission to access this file.")

    if not mimetype and guessmime:
        resp.content_type = mimetypes.guess_type(filename)[0]
    else:
        resp.content_type = mimetype if mimetype else 'text/plain'

    if download == True:
        download = os.path.basename(filename)
    if download:
        resp.content_disposition = 'attachment; filename="%s"' % download

    stats = os.stat(filename)
    lm = time.strftime("%a, %d %b %Y %H:%M:%S GMT", time.gmtime(stats.st_mtime))
    resp.last_modified = lm
    ims = req.environ.get('HTTP_IF_MODIFIED_SINCE')
    if ims:
        ims = ims.split(";")[0].strip() # IE sends "<date>; length=146"
        ims = parse_date(ims)
        if ims is not None and ims >= int(stats.st_mtime):
            resp.date = time.strftime("%a, %d %b %Y %H:%M:%S GMT", time.gmtime())
            return Response(body=None, status=304, headerlist=resp.headerlist)
    resp.content_length = stats.st_size
    if req.method == 'HEAD':
        return Response(body=None, status=200, headerlist=resp.headerlist)
    else:
        return Response(body=open(filename).read(), status=200, headerlist=resp.headerlist)

def template(tpl, template_adapter=SimpleTemplate, **kwargs):
    '''
    Get a rendered template as a string iterator.
    You can use a name, a filename or a template string as first parameter.
    '''
    if tpl not in TEMPLATES or DEBUG:
        settings = kwargs.get('template_settings', {})
        lookup = kwargs.get('template_lookup', TEMPLATE_PATH)
        if isinstance(tpl, template_adapter):
            TEMPLATES[tpl] = tpl
            if settings: TEMPLATES[tpl].prepare(**settings)
        elif "\n" in tpl or "{" in tpl or "%" in tpl or '$' in tpl:
            TEMPLATES[tpl] = template_adapter(source=tpl, lookup=lookup, **settings)
        else:
            TEMPLATES[tpl] = template_adapter(name=tpl, lookup=lookup, **settings)
    if not TEMPLATES[tpl]:
        abort(500, 'Template (%s) not found' % tpl)
    return TEMPLATES[tpl].render(**kwargs)

<commit_msg>Added missing imports and fixed a few pylint issues<commit_after># vim: tabstop=4 shiftwidth=4 softtabstop=4

# Copyright 2010 United States Government as represented by the
# Administrator of the National Aeronautics and Space Administration.
# All Rights Reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.
#
# Template library copied from bottle: http://bottlepy.org/
#
# Copyright (c) 2011, Marcel Hellkamp.
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.
#

import cgi
import re
import os
import functools
import time
import tokenize
import mimetypes
from webob import Response
from paste.util.template import TemplateError
from paste.util.datetimeutil import parse_date

import keystone.logic.types.fault as fault
from keystone.logic.types.fault import ForbiddenFault

TEMPLATES = {}
DEBUG = False
TEMPLATE_PATH = ['./', './views/']

class BaseTemplate(object):
    """ Base class and minimal API for template adapters """
    extentions = ['tpl', 'html', 'thtml', 'stpl']
    settings = {}  #used in prepare()
    defaults = {}  #used in render()

    def __init__(self, source=None, name=None, lookup=[], encoding='utf8',
                 **settings):
        """ Create a new template.
        If the source parameter (str or buffer) is missing, the name argument
        is used to guess a template filename. Subclasses can assume that
        self.source and/or self.filename are set. Both are strings.
        The lookup, encoding and settings parameters are stored as instance
        variables.
        The lookup parameter stores a list containing directory paths.
        The encoding parameter should be used to decode byte strings or files.
        The settings parameter contains a dict for engine-specific settings.
        """
        self.name = name
        self.source = source.read() if hasattr(source, 'read') else source
        self.filename = source.filename if hasattr(source, 'filename') else None
        self.lookup = map(os.path.abspath, lookup)
        self.encoding = encoding
        self.settings = self.settings.copy() # Copy from class variable
        self.settings.update(settings) # Apply
        if not self.source and self.name:
            self.filename = self.search(self.name, self.lookup)
            if not self.filename:
                raise TemplateError('Template %s not found.' % repr(name))
        if not self.source and not self.filename:
            raise TemplateError('No template specified.')
        self.prepare(**self.settings)

    @classmethod
    def search(cls, name, lookup=[]):
        """ Search name in all directories specified in lookup.
        First without, then with common extensions. Return first hit. """
        if os.path.isfile(name):
            return name
        for spath in lookup:
            fname = os.path.join(spath, name)
            if os.path.isfile(fname):
                return fname
            for ext in cls.extentions:
                if os.path.isfile('%s.%s' % (fname, ext)):
                    return '%s.%s' % (fname, ext)

    @classmethod
    def global_config(cls, key, *args):
        ''' This reads or sets the global settings stored in class.settings. '''
        if args:
            cls.settings[key] = args[0]
        else:
            return cls.settings[key]

    def prepare(self, **options):
        """ Run preparations (parsing, caching, ...).
        It should be possible to call this again to refresh a template or to
        update settings.
        """
        raise NotImplementedError

    def render(self, **args):
        """ Render the template with the specified local variables and return
        a single byte or unicode string. If it is a byte string, the encoding
        must match self.encoding. This method must be thread-safe!
        """
        raise NotImplementedError

class SimpleTemplate(BaseTemplate):
    blocks = ('if', 'elif', 'else', 'try', 'except', 'finally', 'for', 'while', 'with', 'def', 'class')
    dedent_blocks = ('elif', 'else', 'except', 'finally')

    def prepare(self, escape_func=cgi.escape, noescape=False):
        self.cache = {}
        if self.source:
            self.code = self.translate(self.source)
            self.co = compile(self.code, '<string>', 'exec')
        else:
            self.code = self.translate(open(self.filename).read())
            self.co = compile(self.code, self.filename, 'exec')
        enc = self.encoding
        touni = functools.partial(unicode, encoding=self.encoding)
        self._str = lambda x: touni(x, enc)
        self._escape = lambda x: escape_func(touni(x))
        if noescape:
            self._str, self._escape = self._escape, self._str

    def translate(self, template):
        stack = [] # Current Code indentation
        lineno = 0 # Current line of code
        ptrbuffer = [] # Buffer for printable strings and token tuple instances
        codebuffer = [] # Buffer for generated python code
        functools.partial(unicode, encoding=self.encoding)
        multiline = dedent = False

        def yield_tokens(line):
            for i, part in enumerate(re.split(r'\{\{(.*?)\}\}', line)):
                if i % 2:
                    if part.startswith('!'): yield 'RAW', part[1:]
                    else: yield 'CMD', part
                else: yield 'TXT', part

        def split_comment(codeline):
            """ Removes comments from a line of code. """
            line = codeline.splitlines()[0]
            try:
                tokens = list(tokenize.generate_tokens(iter(line).next))
            except tokenize.TokenError:
                return line.rsplit('#', 1) if '#' in line else (line, '')
            for token in tokens:
                if token[0] == tokenize.COMMENT:
                    start, end = token[2][1], token[3][1]
                    return codeline[:start] + codeline[end:], codeline[start:end]
            return line, ''

        def flush(): # Flush the ptrbuffer
            if not ptrbuffer: return
            cline = ''
            for line in ptrbuffer:
                for token, value in line:
                    if token == 'TXT':
                        cline += repr(value)
                    elif token == 'RAW':
                        cline += '_str(%s)' % value
                    elif token == 'CMD':
                        cline += '_escape(%s)' % value
                    cline += ', '
                cline = cline[:-2] + '\\\n'
            cline = cline[:-2]
            if cline[:-1].endswith('\\\\\\\\\\n'):
                cline = cline[:-7] + cline[-1] # 'nobr\\\\\n' --> 'nobr'
            cline = '_printlist([' + cline + '])'
            del ptrbuffer[:] # Do this before calling code() again
            code(cline)

        def code(stmt):
            for line in stmt.splitlines():
                codebuffer.append('  ' * len(stack) + line.strip())

        for line in template.splitlines(True):
            lineno += 1
            line = line if isinstance(line, unicode)\
                        else unicode(line, encoding=self.encoding)
            if lineno <= 2:
                m = re.search(r"%.*coding[:=]\s*([-\w\.]+)", line)
                if m: self.encoding = m.group(1)
                if m: line = line.replace('coding', 'coding (removed)')
            if line.strip()[:2].count('%') == 1:
                line = line.split('%', 1)[1].lstrip() # Full line following the %
                cline = split_comment(line)[0].strip()
                cmd = re.split(r'[^a-zA-Z0-9_]', cline)[0]
                flush() ##encodig (TODO: why?)
                if cmd in self.blocks or multiline:
                    cmd = multiline or cmd
                    dedent = cmd in self.dedent_blocks # "else:"
                    if dedent and not multiline:
                        cmd = stack.pop()
                    code(line)
                    oneline = not cline.endswith(':') # "if 1: pass"
                    multiline = cmd if cline.endswith('\\') else False
                    if not oneline and not multiline:
                        stack.append(cmd)
                elif cmd == 'end' and stack:
                    code('#end(%s) %s' % (stack.pop(), line.strip()[3:]))
                elif cmd == 'include':
                    p = cline.split(None, 2)[1:]
                    if len(p) == 2:
                        code("_=_include(%s, _stdout, %s)" % (repr(p[0]), p[1]))
                    elif p:
                        code("_=_include(%s, _stdout)" % repr(p[0]))
                    else: # Empty %include -> reverse of %rebase
                        code("_printlist(_base)")
                elif cmd == 'rebase':
                    p = cline.split(None, 2)[1:]
                    if len(p) == 2:
                        code("globals()['_rebase']=(%s, dict(%s))" % (repr(p[0]), p[1]))
                    elif p:
                        code("globals()['_rebase']=(%s, {})" % repr(p[0]))
                else:
                    code(line)
            else: # Line starting with text (not '%') or '%%' (escaped)
                if line.strip().startswith('%%'):
                    line = line.replace('%%', '%', 1)
                ptrbuffer.append(yield_tokens(line))
        flush()
        return '\n'.join(codebuffer) + '\n'

    def subtemplate(self, _name, _stdout, **args):
        if _name not in self.cache:
            self.cache[_name] = self.__class__(name=_name, lookup=self.lookup)
        return self.cache[_name].execute(_stdout, **args)

    def execute(self, _stdout, **args):
        env = self.defaults.copy()
        env.update({'_stdout': _stdout, '_printlist': _stdout.extend,
               '_include': self.subtemplate, '_str': self._str,
               '_escape': self._escape})
        env.update(args)
        eval(self.co, env)
        if '_rebase' in env:
            subtpl, rargs = env['_rebase']
            subtpl = self.__class__(name=subtpl, lookup=self.lookup)
            rargs['_base'] = _stdout[:] #copy stdout
            del _stdout[:] # clear stdout
            return subtpl.execute(_stdout, **rargs)
        return env

    def render(self, **args):
        """ Render the template using keyword arguments as local variables. """
        stdout = []
        self.execute(stdout, **args)
        return ''.join(stdout)

def static_file(resp, req, filename, root, guessmime=True, mimetype=None, download=False):
    """ Opens a file in a safe way and returns a HTTPError object with status
        code 200, 305, 401 or 404. Sets Content-Type, Content-Length and
        Last-Modified header. Obeys If-Modified-Since header and HEAD requests.
    """
    root = os.path.abspath(root) + os.sep
    filename = os.path.abspath(os.path.join(root, filename.strip('/\\')))
    if not filename.startswith(root):
        #return HTTPError(403, "Access denied.")
        return ForbiddenFault("Access denied.")
    if not os.path.exists(filename) or not os.path.isfile(filename):
        #return HTTPError(404, "File does not exist.")
        return fault.ItemNotFoundFault("File does not exist.")
    if not os.access(filename, os.R_OK):
        #return HTTPError(403, "You do not have permission to access this file.")
        return ForbiddenFault("You do not have permission to access this file.")

    if not mimetype and guessmime:
        resp.content_type = mimetypes.guess_type(filename)[0]
    else:
        resp.content_type = mimetype or 'text/plain'

    if download == True:
        download = os.path.basename(filename)
    if download:
        resp.content_disposition = 'attachment; filename="%s"' % download

    stats = os.stat(filename)
    lm = time.strftime("%a, %d %b %Y %H:%M:%S GMT", time.gmtime(stats.st_mtime))
    resp.last_modified = lm
    ims = req.environ.get('HTTP_IF_MODIFIED_SINCE')
    if ims:
        ims = ims.split(";")[0].strip() # IE sends "<date>; length=146"

        ims = parse_date(ims)
        if ims is not None and ims >= int(stats.st_mtime):
            resp.date = time.strftime("%a, %d %b %Y %H:%M:%S GMT", time.gmtime())
            return Response(body=None, status=304, headerlist=resp.headerlist)
    resp.content_length = stats.st_size
    if req.method == 'HEAD':
        return Response(body=None, status=200, headerlist=resp.headerlist)
    else:
        return Response(body=open(filename).read(), status=200, headerlist=resp.headerlist)

def template(tpl, template_adapter=SimpleTemplate, **kwargs):
    '''
    Get a rendered template as a string iterator.
    You can use a name, a filename or a template string as first parameter.
    '''
    if tpl not in TEMPLATES or DEBUG:
        settings = kwargs.get('template_settings', {})
        lookup = kwargs.get('template_lookup', TEMPLATE_PATH)
        if isinstance(tpl, template_adapter):
            TEMPLATES[tpl] = tpl
            if settings: TEMPLATES[tpl].prepare(**settings)
        elif "\n" in tpl or "{" in tpl or "%" in tpl or '$' in tpl:
            TEMPLATES[tpl] = template_adapter(source=tpl, lookup=lookup, **settings)
        else:
            TEMPLATES[tpl] = template_adapter(name=tpl, lookup=lookup, **settings)
    return TEMPLATES[tpl].render(**kwargs)

<|endoftext|>


<commit_before># *****************************************************************************
# Copyright (c) 2019 IBM Corporation and other Contributors.
#
# All rights reserved. This program and the accompanying materials
# are made available under the terms of the Eclipse Public License v1.0
# which accompanies this distribution, and is available at
# http://www.eclipse.org/legal/epl-v10.html
# *****************************************************************************

from datetime import datetime, timedelta
import pytest
import testUtils

@pytest.mark.skip(reason="API currently unavailable (503)")
class TestClientConnectivityStatus(testUtils.AbstractTest):

    # =========================================================================
    # Client Connectivity Staus Tests
    #
    # These all need to be rewritten -- hard to do while the API is broken tho :/
    # =========================================================================

    def testGetClientConnectionStates(self):
        response = self.appClient.registry.connectionStatus.getClientStates()
        assert response != None
        assert "results" in response

    def testGetConnectedClientConnectionStates(self):
        response = self.appClient.registry.connectionStatus.getConnectedClientStates()
        assert response != None
        assert "results" in response

    def testGetClientConnectionState(self):
        # gets the connection state of a particular client id, returns 404 if client id is not found
        response = self.appClient.registry.connectionStatus.getState("fakeId")
        assert response != None
        assert response["exception"]["properties"] == ["fakeId"]

    def testGetRecentClientConnectionStates(self):
        # checks for clients that have connected in the last two days
        iso8601Date = datetime.now() - timedelta(days=2)
        response = self.appClient.registry.connectionStatus.getRecentConnectionClientStates(iso8601Date.isoformat())
        assert response != None
        assert "results" in response
<commit_msg>Registry Client Connectivity Status Testing<commit_after># *****************************************************************************
# Copyright (c) 2019 IBM Corporation and other Contributors.
#
# All rights reserved. This program and the accompanying materials
# are made available under the terms of the Eclipse Public License v1.0
# which accompanies this distribution, and is available at
# http://www.eclipse.org/legal/epl-v10.html
# *****************************************************************************

from datetime import datetime, timedelta
import uuid
import pytest
import testUtils
from wiotp.sdk.exceptions import ApiException

class TestClientConnectivityStatus(testUtils.AbstractTest):

    # =========================================================================
    # Client Connectivity Staus Tests
    #
    # These all need to be rewritten -- hard to do while the API is broken tho :/
    # =========================================================================

    # def testGetClientConnectionStates(self):
    #    response = self.appClient.registry.connectionStatus.getClientStates()
    #   assert response != None
    #  assert "results" in response

    # def testGetConnectedClientConnectionStates(self):
    #   response = self.appClient.registry.connectionStatus.getConnectedClientStates()
    #  assert response != None
    # assert "results" in response

    # def testGetClientConnectionState(self):
    # gets the connection state of a particular client id, returns 404 if client id is not found
    #   response = self.appClient.registry.connectionStatus.getState("fakeId")
    #    assert response != None
    #  assert response["exception"]["properties"] == ["fakeId"]

    # def testGetRecentClientConnectionStates(self):
    # checks for clients that have connected in the last two days
    #   iso8601Date = datetime.now() - timedelta(days=2)
    #  response = self.appClient.registry.connectionStatus.getRecentConnectionClientStates(iso8601Date.isoformat())
    # assert response != None
    # assert "results" in response

    def testConnectionStatusContainsTrueValue(self):
        try:
            key = 15
            connectionStatusContains = self.appClient.registry.connectionStatus.__contains__(key)
            assert True
        except:
            assert False == True

    def testConnectionStatusContainsAPIException(self):
        with pytest.raises(ApiException) as e:
            key = "fff/kdk/f?"
            connectionStatusContains = self.appClient.registry.connectionStatus.__contains__(key)
            assert e

    def testConnectionStatusGetItemValue(self):
        try:
            key = 15
            connectionStatusGetItem = self.appClient.registry.connectionStatus.__getitem__(key)
            assert False == True
        except:
            assert True

    def testConnectionStatusIter(self):
        try:
            deviceIter = self.appClient.registry.connectionStatus.__iter__()
            assert True
        except:
            assert False == True

    def testConnectionStatusGetItemError(self):
        with pytest.raises(KeyError) as e:
            key = 1
            test = self.appClient.registry.connectionStatus.__getitem__(key)
            assert ("No status avaliable for client %s" % (key)) in str(e.value)

    def testConnectionStatusMissingDeviceError(self):
        with pytest.raises(KeyError) as e:
            key = "k"
            test = self.appClient.registry.connectionStatus.__missing__(key)
            assert ("No status avaliable for client %s" % (key)) in str(e.value)

    def testConnectionStatusSetItemError(self):
        with pytest.raises(Exception) as e:
            test = self.appClient.registry.connectionStatus.__setitem__("s", 1)
            assert "Unsupported operation" in str(e.value)

    def testConnectionStatusDelItemError(self):
        with pytest.raises(Exception) as e:
            message = self.appClient.registry.connectionStatus.__delitem__(1)
            assert "Unsupported operation" in str(e.value)

    def testConnectionStatusFindTypeIdValue(self):
        deviceTypeId = str(uuid.uuid4())
        try:
            deviceTypeTest = self.appClient.registry.connectionStatus.find(typeId=deviceTypeId)
            assert True  # It has successfully inputed the values into IterableClientStatusList
        except:
            assert False == True  # It has failed to input the values into IterableClientStatusList

    def testConnectionStatusFindDeviceIdValue(self):
        deviceTypeId = str(uuid.uuid4())
        try:
            deviceTypeTest = self.appClient.registry.connectionStatus.find(deviceId="Test")
            assert True  # It has successfully inputed the values into IterableClientStatusList
        except:
            assert False == True  # It has failed to input the values into IterableClientStatusList

    def testConnectionStatusFindConnectionStatusValues(self):
        deviceTypeId = str(uuid.uuid4())
        try:
            deviceTypeTest = self.appClient.registry.connectionStatus.find(connectionStatus="connected")
            assert True  # It has successfully inputed the values into IterableClientStatusList
        except:
            assert False == True  # It has failed to input the values into IterableClientStatusList

    def testConnectionStatusFindConnectedAfterValues(self):
        deviceTypeId = str(uuid.uuid4())
        try:
            deviceTypeTest = self.appClient.registry.connectionStatus.find(connectedAfter=100)
            assert True  # It has successfully inputed the values into IterableClientStatusList
        except:
            assert False == True  # It has failed to input the values into IterableClientStatusList

    def testConnectionStatusFindParametersValues(self):
        deviceTypeId = str(uuid.uuid4())
        try:
            deviceTypeTest = self.appClient.registry.connectionStatus.find(
                typeId=deviceTypeId, deviceId="Test", connectionStatus="connected", connectedAfter=100
            )
            assert True  # It has successfully inputed the values into IterableClientStatusList
        except:
            assert False == True  # It has failed to input the values into IterableClientStatusList
<|endoftext|>


<commit_before>"""
Show a Pareto curve plot for individuals in a given generation.
"""
from __future__ import division
from __future__ import print_function

import plotly.graph_objs as go

import cea.plots.supply_system
from cea.plots.variable_naming import NAMING, COLOR

__author__ = "Jimeno Fonseca"
__copyright__ = "Copyright 2019, Architecture and Building Systems - ETH Zurich"
__credits__ = ["Jimeno Fonseca"]
__license__ = "MIT"
__version__ = "0.1"
__maintainer__ = "Daren Thomas"
__email__ = "cea@arch.ethz.ch"
__status__ = "Production"

class DispatchCurveDistrictHeatingPlot(cea.plots.supply_system.SupplySystemPlotBase):
    """Show a pareto curve for a single generation"""
    name = "Dispatch curve heating plant"
    expected_parameters = {
        'system': 'plots-supply-system:system',
        'timeframe': 'plots-supply-system:timeframe',
        'scenario-name': 'general:scenario-name',
    }

    def __init__(self, project, parameters, cache):
        super(DispatchCurveDistrictHeatingPlot, self).__init__(project, parameters, cache)
        self.analysis_fields = [
            "Q_Storage_gen_directload_W",
            "Q_HP_Server_gen_directload_W",
            "Q_PVT_gen_directload_W",
            "Q_SC_ET_gen_directload_W",
            "Q_SC_FP_gen_directload_W",
            "Q_CHP_gen_directload_W",
            "Q_Furnace_dry_gen_directload_W",
            "Q_Furnace_wet_gen_directload_W",
            "Q_HP_Sew_gen_directload_W",
            "Q_HP_Lake_gen_directload_W",
            "Q_GHP_gen_directload_W",
            "Q_BaseBoiler_gen_directload_W",
            "Q_PeakBoiler_gen_directload_W",
            "Q_BackupBoiler_gen_directload_W",
        ]

        self.analysis_field_demand = ['Q_districtheating_sys_req_W']
        self.timeframe = self.parameters['timeframe']
        self.input_files = [(self.locator.get_optimization_slave_heating_activation_pattern,
                             [self.individual, self.generation])]

    @property
    def title(self):
        return "Dispatch curve for heating plant in system #%s (%s)" % (self.individual, self.timeframe)

    @property
    def output_path(self):
        return self.locator.get_timeseries_plots_file(
            'gen{generation}_ind{individual}dispatch_curve_heating_plant'.format(individual=self.individual,
                                                                           generation=self.generation),
            self.category_name)

    @property
    def layout(self):
        return dict(barmode='relative', yaxis=dict(title='Energy Generation [MWh]'))

    def calc_graph(self):
        # main data about technologies
        data = self.process_individual_dispatch_curve_heating()
        graph = []
        analysis_fields = self.remove_unused_fields(data, self.analysis_fields)
        for field in analysis_fields:
            y = (data[field].values) / 1E6  # into MW
            trace = go.Bar(x=data.index, y=y, name=NAMING[field],
                           marker=dict(color=COLOR[field]))
            graph.append(trace)

        # data about demand
        for field in self.analysis_field_demand:
            y = (data[field].values) / 1E6  # into MW
            trace = go.Scattergl(x=data.index, y=y, name=NAMING[field],
                               line=dict(width=1, color=COLOR[field]))

            graph.append(trace)

        return graph

def main():
    """Test this plot"""
    import cea.config
    import cea.plots.cache
    config = cea.config.Configuration()
    cache = cea.plots.cache.NullPlotCache()
    DispatchCurveDistrictHeatingPlot(config.project,
                                     {'scenario-name': config.scenario_name,
                                      'system': config.plots_supply_system.system,
                                      'timeframe': config.plots_supply_system.timeframe},
                                     cache).plot(auto_open=True)

if __name__ == '__main__':
    main()
<commit_msg>Update dispatch_curve_heating_plant.py<commit_after>"""
Show a Pareto curve plot for individuals in a given generation.
"""
from __future__ import division
from __future__ import print_function

import plotly.graph_objs as go

import cea.plots.supply_system
from cea.plots.variable_naming import NAMING, COLOR

__author__ = "Jimeno Fonseca"
__copyright__ = "Copyright 2019, Architecture and Building Systems - ETH Zurich"
__credits__ = ["Jimeno Fonseca"]
__license__ = "MIT"
__version__ = "0.1"
__maintainer__ = "Daren Thomas"
__email__ = "cea@arch.ethz.ch"
__status__ = "Production"

class DispatchCurveDistrictHeatingPlot(cea.plots.supply_system.SupplySystemPlotBase):
    """Show a pareto curve for a single generation"""
    name = "Dispatch curve heating plant"
    expected_parameters = {
        'system': 'plots-supply-system:system',
        'timeframe': 'plots-supply-system:timeframe',
        'scenario-name': 'general:scenario-name',
    }

    def __init__(self, project, parameters, cache):
        super(DispatchCurveDistrictHeatingPlot, self).__init__(project, parameters, cache)
        self.analysis_fields = [
            "Q_Storage_gen_directload_W",
            "Q_HP_Server_gen_directload_W",
            "Q_PVT_gen_directload_W",
            "Q_SC_ET_gen_directload_W",
            "Q_SC_FP_gen_directload_W",
            "Q_CHP_gen_directload_W",
            "Q_Furnace_dry_gen_directload_W",
            "Q_Furnace_wet_gen_directload_W",
            "Q_HP_Sew_gen_directload_W",
            "Q_HP_Lake_gen_directload_W",
            "Q_GHP_gen_directload_W",
            "Q_BaseBoiler_gen_directload_W",
            "Q_PeakBoiler_gen_directload_W",
            "Q_BackupBoiler_gen_directload_W",
        ]

        self.analysis_field_demand = ['Q_districtheating_sys_req_W']
        self.timeframe = self.parameters['timeframe']
        self.input_files = [(self.locator.get_optimization_slave_heating_activation_pattern,
                             [self.individual, self.generation])]

    @property
    def title(self):
        return "Dispatch curve for heating plant in system %s (%s)" % (self.system, self.timeframe)

    @property
    def output_path(self):
        return self.locator.get_timeseries_plots_file(
            '{system}_dispatch_curve_heating_plant'.format(system=self.system), self.category_name)

    @property
    def layout(self):
        return dict(barmode='relative', yaxis=dict(title='Energy Generation [MWh]'))

    def calc_graph(self):
        # main data about technologies
        data = self.process_individual_dispatch_curve_heating()
        graph = []
        analysis_fields = self.remove_unused_fields(data, self.analysis_fields)
        for field in analysis_fields:
            y = (data[field].values) / 1E6  # into MW
            trace = go.Bar(x=data.index, y=y, name=NAMING[field],
                           marker=dict(color=COLOR[field]))
            graph.append(trace)

        # data about demand
        for field in self.analysis_field_demand:
            y = (data[field].values) / 1E6  # into MW
            trace = go.Scattergl(x=data.index, y=y, name=NAMING[field],
                               line=dict(width=1, color=COLOR[field]))

            graph.append(trace)

        return graph

def main():
    """Test this plot"""
    import cea.config
    import cea.plots.cache
    config = cea.config.Configuration()
    cache = cea.plots.cache.NullPlotCache()
    DispatchCurveDistrictHeatingPlot(config.project,
                                     {'scenario-name': config.scenario_name,
                                      'system': config.plots_supply_system.system,
                                      'timeframe': config.plots_supply_system.timeframe},
                                     cache).plot(auto_open=True)

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>#!/usr/bin/python

"""
Simple helper script generating boilerplate code for a set of functions in order
to make them dynamically loaded and GObject-introspectable.

"""

from collections import namedtuple
import re
import sys
import os

FuncInfo = namedtuple("FuncInfo", ["name", "doc", "rtype", "args", "body"])
FUNC_SPEC_RE = re.compile(r'(?P<rtype>(const\s+)?\**\s*\w+\s*\**)'
                          r'\s*'
                          r'(?P<name>\w+)'
                          r'\s*\('
                          r'(?P<args>[\w*\s,]*)'
                          r'\)(;| \{)')

SIZE_CONST_DEF_RE = re.compile(r'^(?P<name_num>#define\s+\w+\s*\(\s*\d+\s+)(?P<unit>[kmgtpeKMGTPE]i?[bB])\s*\)\s*$')

KiB = 1024
MiB = 1024 * KiB
GiB = 1024 * MiB
TiB = 1024 * GiB
PiB = 1024 * TiB
EiB = 1024 * PiB
KB = 1000
MB = 1000 * KB
GB = 1000 * MB
TB = 1000 * GB
PB = 1000 * TB
EB = 1000 * PB

UNIT_MULTS = {"KiB": KiB, "MiB": MiB, "GiB": GiB, "TiB": TiB, "PiB": PiB, "EiB": EiB,
              "KB": KB, "MB": MB, "GB": GB, "TB": TB, "PB": PB, "EB": EB}

# overrides for function prefixes not matching the modules' names
MOD_FNAME_OVERRIDES = {"mdraid": "md"}

def expand_size_constants(definitions):
    """
    Expand macros that define size constants (e.g. '#define DEFAULT_PE_SIZE (4 MiB)').

    :param str definitions: lines from a source file potentially containing
                            definitions of size constants.
    :returns: definitions with size constants expanded

    """

    ret = ""
    for def_line in definitions.splitlines():
        match = SIZE_CONST_DEF_RE.match(def_line)
        if match:
            fields = match.groupdict()
            unit = fields["unit"]
            if unit in UNIT_MULTS:
                name_num = fields["name_num"]
                ret += (name_num + "* " + "%dULL" % UNIT_MULTS[unit] + ")\n")
            else:
                # unknown unit, cannot expand it, just leave it as it is
                ret += (def_line + "\n")
        else:
            ret += (def_line + "\n")

    return ret

def gather_defs_and_func_info(line_iter, includes):
    name = doc = rtype = args = ""
    defs = ""
    body = ""
    in_body = False
    in_doc = False
    in_skip = False
    for line in line_iter:
        if in_skip and line.strip().startswith("/* BpG-skip-end"):
            in_skip = False
        elif in_skip or line.strip().startswith("/* BpG-skip"):
            in_skip = True
        elif line.rstrip() == "}" and in_body:
            in_body = False
            # nothing more for this function
            break
        elif in_body:
            body += line
        elif line.strip().startswith("#include"):
            includes.append(line.strip()[8:])
        elif line.strip().startswith("/**"):
            doc += line
            in_doc = True
        elif line.strip().endswith("*/") and in_doc:
            doc += line
            in_doc = False
        else:
            match = FUNC_SPEC_RE.match(line.strip())
            if match:
                fields = match.groupdict()
                name = fields["name"]
                rtype = fields["rtype"]
                args = fields["args"]

                if line.strip().endswith("{"):
                    in_body = True
                else:
                    # nothing more for this function
                    break
            elif in_doc:
                doc += line
            else:
                defs += line
    else:
        return (expand_size_constants(defs), None)

    return (expand_size_constants(defs), FuncInfo(name, doc, rtype, args, body))

def process_file(fobj):
    includes = []

    line_iter = iter(fobj)
    items = list()

    defs, fn_info = gather_defs_and_func_info(line_iter, includes)
    while fn_info:
        if defs:
            items.append(defs)
        if fn_info:
            items.append(fn_info)
        defs, fn_info = gather_defs_and_func_info(line_iter, includes)

    if defs:
        # definitions after the last function
        items.append(defs)
    return includes, items

def get_arg_names(args):
    if not args.strip():
        return []

    typed_args = args.split(",")
    starred_names = (typed_arg.split()[-1] for typed_arg in typed_args)

    return [starred_name.strip("* ") for starred_name in starred_names]

def get_func_boilerplate(fn_info):
    call_args_str = ", ".join(get_arg_names(fn_info.args))

    if "int" in fn_info.rtype:
        default_ret = "0"
    elif "float" in fn_info.rtype:
        default_ret = "0.0"
    elif "bool" in fn_info.rtype:
        default_ret = "FALSE"
    elif fn_info.rtype.endswith("*"):
        # a pointer
        default_ret = "NULL"
    else:
        # enum or whatever
        default_ret = 0

    # first add the stub function doing nothing and just reporting error
    ret = ("{0.rtype} {0.name}_stub ({0.args}) {{\n" +
           "    g_critical (\"The function '{0.name}' called, but not implemented!\");\n" +
           "    g_set_error (error, BD_INIT_ERROR, BD_INIT_ERROR_NOT_IMPLEMENTED,\n"+
           "                \"The function '{0.name}' called, but not implemented!\");\n"
           "    return {1};\n"
           "}}\n\n").format(fn_info, default_ret)

    # then add a variable holding a reference to the dynamically loaded function
    # (if any) initialized to the stub
    ret += "{0.rtype} (*_{0.name}) ({0.args}) = {0.name}_stub;\n\n".format(fn_info)

    # then add a documented function calling the dynamically loaded one via the
    # reference
    ret += ("{0.doc}{0.rtype} {0.name} ({0.args}) {{\n" +
            "    return _{0.name} ({1});\n" +
            "}}\n\n\n").format(fn_info, call_args_str)

    return ret

def get_includes_str(includes):
    if not includes:
        return ""

    ret = ""
    for include in includes:
        ret += "#include%s\n" % include
    ret += "\n"

    return ret

def get_funcs_info(fn_infos, module_name):
    ret = "static const gchar const * const {0}_functions[] = {{\n".format(module_name)
    for info in fn_infos:
        ret += '    "{0.name}",\n'.format(info)
    ret += "    NULL};\n\n"

    ret += ("gchar const * const * get_{0}_functions (void) {{\n".format(module_name) +
            "    return {0}_functions;\n".format(module_name) +
            "}\n\n")

    ret += "static const guint8 {0}_num_functions = {1};\n\n".format(module_name, len(fn_infos))
    ret += ("guint8 get_{0}_num_functions (void) {{\n".format(module_name) +
            "    return {0}_num_functions;\n".format(module_name) +
            "}\n\n")

    return ret

def get_loading_func(fn_infos, module_name):
    # TODO: only error on functions provided by the plugin that fail to load
    # TODO: implement the 'gchar **errors' argument
    ret =  'gpointer load_{0}_from_plugin(const gchar *so_name) {{\n'.format(module_name)
    ret += '    void *handle = NULL;\n'
    ret += '    char *error = NULL;\n'
    ret += '    gboolean (*check_fn) (void) = NULL;\n'
    ret += '    gboolean (*init_fn) (void) = NULL;\n\n'

    ret += '    handle = dlopen(so_name, RTLD_LAZY);\n'
    ret += '    if (!handle) {\n'
    ret += '        g_warning("failed to load module {0}: %s", dlerror());\n'.format(module_name)
    ret += '        return NULL;\n'
    ret += '    }\n\n'

    ret += '    dlerror();\n'
    ret += '    * (void**) (&check_fn) = dlsym(handle, "bd_{0}_check_deps");\n'.format(MOD_FNAME_OVERRIDES.get(module_name, module_name))
    ret += '    if ((error = dlerror()) != NULL)\n'
    ret += '        g_debug("failed to load the check() function for {0}: %s", error);\n'.format(module_name)
    ret += '    if (!g_getenv ("LIBBLOCKDEV_SKIP_DEP_CHECKS") && check_fn && !check_fn()) {\n'
    ret += '        dlclose(handle);\n'
    ret += '        return NULL;\n'
    ret += '    }'
    ret += '    check_fn = NULL;\n\n'

    ret += '    dlerror();\n'
    ret += '    * (void**) (&init_fn) = dlsym(handle, "bd_{0}_init");\n'.format(MOD_FNAME_OVERRIDES.get(module_name, module_name))
    ret += '    if ((error = dlerror()) != NULL)\n'
    ret += '        g_debug("failed to load the init() function for {0}: %s", error);\n'.format(module_name)
    ret += '    if (init_fn && !init_fn()) {\n'
    ret += '        dlclose(handle);\n'
    ret += '        return NULL;\n'
    ret += '    }'
    ret += '    init_fn = NULL;\n\n'

    for info in fn_infos:
        # clear any previous error and load the function
        ret += '    dlerror();\n'
        ret += '    * (void**) (&_{0.name}) = dlsym(handle, "{0.name}");\n'.format(info)
        ret += '    if ((error = dlerror()) != NULL)\n'
        ret += '        g_warning("failed to load {0.name}: %s", error);\n\n'.format(info)

    ret += '    return handle;\n'
    ret += '}\n\n'

    return ret

def get_unloading_func(fn_infos, module_name):
    ret =  'gboolean unload_{0} (gpointer handle) {{\n'.format(module_name)
    ret += '    char *error = NULL;\n'
    ret += '    gboolean (*close_fn) (void) = NULL;\n\n'

    # revert the functions to stubs
    for info in fn_infos:
        ret += '    _{0.name} = {0.name}_stub;\n'.format(info)

    ret += '\n'
    ret += '    dlerror();\n'
    ret += '    * (void**) (&close_fn) = dlsym(handle, "bd_{0}_close");\n'.format(MOD_FNAME_OVERRIDES.get(module_name, module_name))
    ret += '    if (((error = dlerror()) != NULL) || !close_fn)\n'
    ret += '        g_debug("failed to load the close_plugin() function for {0}: %s", error);\n'.format(module_name)
    ret += '    if (close_fn) {\n'
    ret += '        close_fn();\n'
    ret += '    }\n\n'
    ret += '    return dlclose(handle) == 0;\n'
    ret += '}\n\n'

    return ret

def get_fn_code(fn_info):
    ret = ("{0.doc}{0.rtype} {0.name} ({0.args}) {{\n" +
            "    {0.body}" +
            "}}\n\n").format(fn_info)

    return ret

def get_fn_header(fn_info):
    return "{0.doc}{0.rtype} {0.name} ({0.args});\n\n".format(fn_info)

def generate_source_header(api_file, out_dir, skip_patterns=None):
    skip_patterns = skip_patterns or list()
    file_name = os.path.basename(api_file)
    mod_name, dot, ext = file_name.partition(".")
    if not dot or ext != "api":
        print("Invalid file given, needs to be in MODNAME.api format")
        return 1

    includes, items = process_file(open(api_file, "r"))
    filtered = list()
    for item in items:
        if isinstance(item, FuncInfo):
            if not any(re.search(pattern, item.name) for pattern in skip_patterns):
                filtered.append(item)
        elif not any(re.search(pattern, item) for pattern in skip_patterns):
            filtered.append(item)
    items = filtered

    nonapi_fn_infos = [item for item in items if isinstance(item, FuncInfo) and item.body]
    api_fn_infos = [item for item in items if isinstance(item, FuncInfo) and not item.body and item.doc]
    with open(os.path.join(out_dir, mod_name + ".c"), "w") as src_f:
        for info in nonapi_fn_infos:
            src_f.write(get_fn_code(info))
        src_f.write(get_funcs_info(api_fn_infos, mod_name))
        for info in api_fn_infos:
            src_f.write(get_func_boilerplate(info))
        src_f.write(get_loading_func(api_fn_infos, mod_name))
        src_f.write(get_unloading_func(api_fn_infos, mod_name))

    written_fns = set()
    with open(os.path.join(out_dir, mod_name + ".h"), "w") as hdr_f:
        hdr_f.write(get_includes_str(includes))
        for item in items:
            if isinstance(item, FuncInfo):
                if item.name not in written_fns:
                    hdr_f.write(get_fn_header(item))
                    written_fns.add(item.name)
            else:
                hdr_f.write(item)

    return 0

if __name__ == "__main__":
    if len(sys.argv) < 3:
        print("Needs a file name and output directory, exitting.")
        print("Usage: %s FILE_NAME OUTPUT_DIR [SKIP_PATTERNS]" % sys.argv[0])
        sys.exit(1)

    if not os.path.exists(sys.argv[1]):
        print("Input file '%s' doesn't exist" % sys.argv[1])
        sys.exit(1)

    skip_patterns = None
    if len(sys.argv) > 3:
        skip_patterns = sys.argv[3:]

    out_dir = sys.argv[2]
    if not os.path.exists (out_dir):
        os.makedirs(out_dir)

    status = generate_source_header(sys.argv[1], out_dir, skip_patterns)

    sys.exit(status)
<commit_msg>Fix duplicate 'const' in generated functions<commit_after>#!/usr/bin/python

"""
Simple helper script generating boilerplate code for a set of functions in order
to make them dynamically loaded and GObject-introspectable.

"""

from collections import namedtuple
import re
import sys
import os

FuncInfo = namedtuple("FuncInfo", ["name", "doc", "rtype", "args", "body"])
FUNC_SPEC_RE = re.compile(r'(?P<rtype>(const\s+)?\**\s*\w+\s*\**)'
                          r'\s*'
                          r'(?P<name>\w+)'
                          r'\s*\('
                          r'(?P<args>[\w*\s,]*)'
                          r'\)(;| \{)')

SIZE_CONST_DEF_RE = re.compile(r'^(?P<name_num>#define\s+\w+\s*\(\s*\d+\s+)(?P<unit>[kmgtpeKMGTPE]i?[bB])\s*\)\s*$')

KiB = 1024
MiB = 1024 * KiB
GiB = 1024 * MiB
TiB = 1024 * GiB
PiB = 1024 * TiB
EiB = 1024 * PiB
KB = 1000
MB = 1000 * KB
GB = 1000 * MB
TB = 1000 * GB
PB = 1000 * TB
EB = 1000 * PB

UNIT_MULTS = {"KiB": KiB, "MiB": MiB, "GiB": GiB, "TiB": TiB, "PiB": PiB, "EiB": EiB,
              "KB": KB, "MB": MB, "GB": GB, "TB": TB, "PB": PB, "EB": EB}

# overrides for function prefixes not matching the modules' names
MOD_FNAME_OVERRIDES = {"mdraid": "md"}

def expand_size_constants(definitions):
    """
    Expand macros that define size constants (e.g. '#define DEFAULT_PE_SIZE (4 MiB)').

    :param str definitions: lines from a source file potentially containing
                            definitions of size constants.
    :returns: definitions with size constants expanded

    """

    ret = ""
    for def_line in definitions.splitlines():
        match = SIZE_CONST_DEF_RE.match(def_line)
        if match:
            fields = match.groupdict()
            unit = fields["unit"]
            if unit in UNIT_MULTS:
                name_num = fields["name_num"]
                ret += (name_num + "* " + "%dULL" % UNIT_MULTS[unit] + ")\n")
            else:
                # unknown unit, cannot expand it, just leave it as it is
                ret += (def_line + "\n")
        else:
            ret += (def_line + "\n")

    return ret

def gather_defs_and_func_info(line_iter, includes):
    name = doc = rtype = args = ""
    defs = ""
    body = ""
    in_body = False
    in_doc = False
    in_skip = False
    for line in line_iter:
        if in_skip and line.strip().startswith("/* BpG-skip-end"):
            in_skip = False
        elif in_skip or line.strip().startswith("/* BpG-skip"):
            in_skip = True
        elif line.rstrip() == "}" and in_body:
            in_body = False
            # nothing more for this function
            break
        elif in_body:
            body += line
        elif line.strip().startswith("#include"):
            includes.append(line.strip()[8:])
        elif line.strip().startswith("/**"):
            doc += line
            in_doc = True
        elif line.strip().endswith("*/") and in_doc:
            doc += line
            in_doc = False
        else:
            match = FUNC_SPEC_RE.match(line.strip())
            if match:
                fields = match.groupdict()
                name = fields["name"]
                rtype = fields["rtype"]
                args = fields["args"]

                if line.strip().endswith("{"):
                    in_body = True
                else:
                    # nothing more for this function
                    break
            elif in_doc:
                doc += line
            else:
                defs += line
    else:
        return (expand_size_constants(defs), None)

    return (expand_size_constants(defs), FuncInfo(name, doc, rtype, args, body))

def process_file(fobj):
    includes = []

    line_iter = iter(fobj)
    items = list()

    defs, fn_info = gather_defs_and_func_info(line_iter, includes)
    while fn_info:
        if defs:
            items.append(defs)
        if fn_info:
            items.append(fn_info)
        defs, fn_info = gather_defs_and_func_info(line_iter, includes)

    if defs:
        # definitions after the last function
        items.append(defs)
    return includes, items

def get_arg_names(args):
    if not args.strip():
        return []

    typed_args = args.split(",")
    starred_names = (typed_arg.split()[-1] for typed_arg in typed_args)

    return [starred_name.strip("* ") for starred_name in starred_names]

def get_func_boilerplate(fn_info):
    call_args_str = ", ".join(get_arg_names(fn_info.args))

    if "int" in fn_info.rtype:
        default_ret = "0"
    elif "float" in fn_info.rtype:
        default_ret = "0.0"
    elif "bool" in fn_info.rtype:
        default_ret = "FALSE"
    elif fn_info.rtype.endswith("*"):
        # a pointer
        default_ret = "NULL"
    else:
        # enum or whatever
        default_ret = 0

    # first add the stub function doing nothing and just reporting error
    ret = ("{0.rtype} {0.name}_stub ({0.args}) {{\n" +
           "    g_critical (\"The function '{0.name}' called, but not implemented!\");\n" +
           "    g_set_error (error, BD_INIT_ERROR, BD_INIT_ERROR_NOT_IMPLEMENTED,\n"+
           "                \"The function '{0.name}' called, but not implemented!\");\n"
           "    return {1};\n"
           "}}\n\n").format(fn_info, default_ret)

    # then add a variable holding a reference to the dynamically loaded function
    # (if any) initialized to the stub
    ret += "{0.rtype} (*_{0.name}) ({0.args}) = {0.name}_stub;\n\n".format(fn_info)

    # then add a documented function calling the dynamically loaded one via the
    # reference
    ret += ("{0.doc}{0.rtype} {0.name} ({0.args}) {{\n" +
            "    return _{0.name} ({1});\n" +
            "}}\n\n\n").format(fn_info, call_args_str)

    return ret

def get_includes_str(includes):
    if not includes:
        return ""

    ret = ""
    for include in includes:
        ret += "#include%s\n" % include
    ret += "\n"

    return ret

def get_funcs_info(fn_infos, module_name):
    ret = "static const gchar* const {0}_functions[] = {{\n".format(module_name)
    for info in fn_infos:
        ret += '    "{0.name}",\n'.format(info)
    ret += "    NULL};\n\n"

    ret += ("const gchar* const* get_{0}_functions (void) {{\n".format(module_name) +
            "    return {0}_functions;\n".format(module_name) +
            "}\n\n")

    ret += "static const guint8 {0}_num_functions = {1};\n\n".format(module_name, len(fn_infos))
    ret += ("guint8 get_{0}_num_functions (void) {{\n".format(module_name) +
            "    return {0}_num_functions;\n".format(module_name) +
            "}\n\n")

    return ret

def get_loading_func(fn_infos, module_name):
    # TODO: only error on functions provided by the plugin that fail to load
    # TODO: implement the 'gchar **errors' argument
    ret =  'gpointer load_{0}_from_plugin(const gchar *so_name) {{\n'.format(module_name)
    ret += '    void *handle = NULL;\n'
    ret += '    char *error = NULL;\n'
    ret += '    gboolean (*check_fn) (void) = NULL;\n'
    ret += '    gboolean (*init_fn) (void) = NULL;\n\n'

    ret += '    handle = dlopen(so_name, RTLD_LAZY);\n'
    ret += '    if (!handle) {\n'
    ret += '        g_warning("failed to load module {0}: %s", dlerror());\n'.format(module_name)
    ret += '        return NULL;\n'
    ret += '    }\n\n'

    ret += '    dlerror();\n'
    ret += '    * (void**) (&check_fn) = dlsym(handle, "bd_{0}_check_deps");\n'.format(MOD_FNAME_OVERRIDES.get(module_name, module_name))
    ret += '    if ((error = dlerror()) != NULL)\n'
    ret += '        g_debug("failed to load the check() function for {0}: %s", error);\n'.format(module_name)
    ret += '    if (!g_getenv ("LIBBLOCKDEV_SKIP_DEP_CHECKS") && check_fn && !check_fn()) {\n'
    ret += '        dlclose(handle);\n'
    ret += '        return NULL;\n'
    ret += '    }'
    ret += '    check_fn = NULL;\n\n'

    ret += '    dlerror();\n'
    ret += '    * (void**) (&init_fn) = dlsym(handle, "bd_{0}_init");\n'.format(MOD_FNAME_OVERRIDES.get(module_name, module_name))
    ret += '    if ((error = dlerror()) != NULL)\n'
    ret += '        g_debug("failed to load the init() function for {0}: %s", error);\n'.format(module_name)
    ret += '    if (init_fn && !init_fn()) {\n'
    ret += '        dlclose(handle);\n'
    ret += '        return NULL;\n'
    ret += '    }'
    ret += '    init_fn = NULL;\n\n'

    for info in fn_infos:
        # clear any previous error and load the function
        ret += '    dlerror();\n'
        ret += '    * (void**) (&_{0.name}) = dlsym(handle, "{0.name}");\n'.format(info)
        ret += '    if ((error = dlerror()) != NULL)\n'
        ret += '        g_warning("failed to load {0.name}: %s", error);\n\n'.format(info)

    ret += '    return handle;\n'
    ret += '}\n\n'

    return ret

def get_unloading_func(fn_infos, module_name):
    ret =  'gboolean unload_{0} (gpointer handle) {{\n'.format(module_name)
    ret += '    char *error = NULL;\n'
    ret += '    gboolean (*close_fn) (void) = NULL;\n\n'

    # revert the functions to stubs
    for info in fn_infos:
        ret += '    _{0.name} = {0.name}_stub;\n'.format(info)

    ret += '\n'
    ret += '    dlerror();\n'
    ret += '    * (void**) (&close_fn) = dlsym(handle, "bd_{0}_close");\n'.format(MOD_FNAME_OVERRIDES.get(module_name, module_name))
    ret += '    if (((error = dlerror()) != NULL) || !close_fn)\n'
    ret += '        g_debug("failed to load the close_plugin() function for {0}: %s", error);\n'.format(module_name)
    ret += '    if (close_fn) {\n'
    ret += '        close_fn();\n'
    ret += '    }\n\n'
    ret += '    return dlclose(handle) == 0;\n'
    ret += '}\n\n'

    return ret

def get_fn_code(fn_info):
    ret = ("{0.doc}{0.rtype} {0.name} ({0.args}) {{\n" +
            "    {0.body}" +
            "}}\n\n").format(fn_info)

    return ret

def get_fn_header(fn_info):
    return "{0.doc}{0.rtype} {0.name} ({0.args});\n\n".format(fn_info)

def generate_source_header(api_file, out_dir, skip_patterns=None):
    skip_patterns = skip_patterns or list()
    file_name = os.path.basename(api_file)
    mod_name, dot, ext = file_name.partition(".")
    if not dot or ext != "api":
        print("Invalid file given, needs to be in MODNAME.api format")
        return 1

    includes, items = process_file(open(api_file, "r"))
    filtered = list()
    for item in items:
        if isinstance(item, FuncInfo):
            if not any(re.search(pattern, item.name) for pattern in skip_patterns):
                filtered.append(item)
        elif not any(re.search(pattern, item) for pattern in skip_patterns):
            filtered.append(item)
    items = filtered

    nonapi_fn_infos = [item for item in items if isinstance(item, FuncInfo) and item.body]
    api_fn_infos = [item for item in items if isinstance(item, FuncInfo) and not item.body and item.doc]
    with open(os.path.join(out_dir, mod_name + ".c"), "w") as src_f:
        for info in nonapi_fn_infos:
            src_f.write(get_fn_code(info))
        src_f.write(get_funcs_info(api_fn_infos, mod_name))
        for info in api_fn_infos:
            src_f.write(get_func_boilerplate(info))
        src_f.write(get_loading_func(api_fn_infos, mod_name))
        src_f.write(get_unloading_func(api_fn_infos, mod_name))

    written_fns = set()
    with open(os.path.join(out_dir, mod_name + ".h"), "w") as hdr_f:
        hdr_f.write(get_includes_str(includes))
        for item in items:
            if isinstance(item, FuncInfo):
                if item.name not in written_fns:
                    hdr_f.write(get_fn_header(item))
                    written_fns.add(item.name)
            else:
                hdr_f.write(item)

    return 0

if __name__ == "__main__":
    if len(sys.argv) < 3:
        print("Needs a file name and output directory, exitting.")
        print("Usage: %s FILE_NAME OUTPUT_DIR [SKIP_PATTERNS]" % sys.argv[0])
        sys.exit(1)

    if not os.path.exists(sys.argv[1]):
        print("Input file '%s' doesn't exist" % sys.argv[1])
        sys.exit(1)

    skip_patterns = None
    if len(sys.argv) > 3:
        skip_patterns = sys.argv[3:]

    out_dir = sys.argv[2]
    if not os.path.exists (out_dir):
        os.makedirs(out_dir)

    status = generate_source_header(sys.argv[1], out_dir, skip_patterns)

    sys.exit(status)
<|endoftext|>


<commit_before>"""
get_DMP_ID
~~~~~~~~~~

:Description: This module gets the information about Deidentified DMP ID associated with Identified DMP ID

"""

'''
Created on December 13, 2016
Description: This module gets the information about Deidentified DMP ID associated with Identified DMP ID
@author: Ronak H Shah
::Input::
listofid : List of identified ids
fileofmapping: csv file with identified and deidentified mapping
locationofbam: where the bam file will be found
::Output::
a tsv file with following header
SAMPLE_ID    D_SAMPLE_ID    SAMPLE_TYPE    GROUP_ID    BAM_LOCATION
'''

import argparse
import pandas as pd
import time
import os,sys
import coloredlogs
import logging
import glob
from collections import Counter

def main():
    parser = argparse.ArgumentParser(
        prog='get_DMP_ID.py',
        description='This module gets the information about Deidentified DMP ID associated with Identified DMP ID',
        usage='%(prog)s [options]')
    parser.add_argument(
        "-v",
        "--verbose",
        action="store_true",
        dest="verbose",
        default=True,
        help="make lots of noise [default]")
    parser.add_argument(
        "-i",
        "--ids",
        action="store",
        dest="idFilename",
        required=True,
        metavar='ids.txt',
        help="Location of the file containing identified sample ids")
    parser.add_argument(
        "-o",
        "--outputFile",
        action="store",
        dest="outFile",
        required=True,
        metavar='out.txt',
        help="Full path with name for the output file")
    parser.add_argument(
        "-r",
        "--refFile",
        action="store",
        dest="refFilename",
        required=True,
        metavar='refFile.txt',
        help="Location of the file containing identified and de-identified sample mapping")
    parser.add_argument(
        "-b",
        "--bamLocation",
        action="store",
        dest="bamLocation",
        required=True,
        metavar='/prefix/to/bam/file/location',
        help="Location where the deidentified bam files are")
    args = parser.parse_args()

    #Get the process is verbose or not
    verbose = args.verbose
    #Get where are we
    here = os.path.realpath('.')

    logger = logging.getLogger('get_DMP_ID')
    coloredlogs.install(level='DEBUG')

    if(os.path.isfile(args.idFilename)):
        if(verbose):
            logger.info("get_DMP_ID: %s is being Read...", args.idFilename)
        iDF = pd.read_table(args.idFilename,sep="\t",header=None,usecols=[0])
        iDF.columns = ["SAMPLE_ID"]
        if(verbose):
            logger.info("Identification DataFrame:\n%s",iDF.head(n=5))
    else:
        logging.error("get_DMP_ID: %s is not a file. We will exit. Please make sure you provide a valid sample id file before rerun.", args.idFilename)
        sys.exit(1)

    if(os.path.isfile(args.idFilename)):
        if(verbose):
            logger.info("get_DMP_ID: %s is being Read...", args.refFilename)
        dDF = pd.read_table(args.refFilename,sep=",",header=None,usecols=[0,1,2])
        dDF.columns = ["SAMPLE_ID","D_SAMPLE_ID","GROUP_ID"]
        if(verbose):
            logger.info("Mapping DataFrame:\n%s",dDF.head(n=5))
    else:
        logging.error("get_DMP_ID: %s is not a file. We will exit. Please make sure you provide a valid sample mapping file before rerun.", args.refFilename)
        sys.exit(1)

    if(os.path.isdir(args.bamLocation)):
        if(verbose):
            logger.info("get_DMP_ID: %s is a directory", args.bamLocation)
        else:
            pass
    else:
        logging.error("get_DMP_ID: %s is not a directory. We will exit. Please make sure you provide a valid bam location directory before rerun.", args.bamLocation)

    outDF = pd.DataFrame(
        columns=[
            "SAMPLE_ID",
            "D_SAMPLE_ID",
            "SAMPLE_TYPE",
            "GROUP_ID",
            "BAM_LOCATION"])

    for count,row in iDF.iterrows():
        iID = str(row.loc["SAMPLE_ID"])
        dDF_idx = dDF[dDF["SAMPLE_ID"] == iID].index.tolist()
        dID = dDF.loc[dDF_idx[0],"D_SAMPLE_ID"]
        gID = dDF.loc[dDF_idx[0],"GROUP_ID"]
        bamFile = glob.glob(args.bamLocation +"/" + dID +"*.bam")
        outDF.loc[count,["SAMPLE_ID",
            "D_SAMPLE_ID",
            "SAMPLE_TYPE",
            "GROUP_ID",
            "BAM_LOCATION"]] = [iID,dID,gID,bamFile]

    outDF.to_csv(args.outFile, sep='\t', index=False)
    if(verbose):
        logger.info("get_DMP_ID: Finished Mapping, Final data written in %s", args.outFile)

if __name__ == "__main__":
    start_time = time.time()
    main()
    end_time = time.time()
    totaltime = end_time - start_time
    logging.info("iCallSV:Elapsed time was %g seconds", totaltime)

<commit_msg>String Conversion<commit_after>"""
get_DMP_ID
~~~~~~~~~~

:Description: This module gets the information about Deidentified DMP ID associated with Identified DMP ID

"""

'''
Created on December 13, 2016
Description: This module gets the information about Deidentified DMP ID associated with Identified DMP ID
@author: Ronak H Shah
::Input::
listofid : List of identified ids
fileofmapping: csv file with identified and deidentified mapping
locationofbam: where the bam file will be found
::Output::
a tsv file with following header
SAMPLE_ID    D_SAMPLE_ID    SAMPLE_TYPE    GROUP_ID    BAM_LOCATION
'''

import argparse
import pandas as pd
import time
import os,sys
import coloredlogs
import logging
import glob
from collections import Counter

def main():
    parser = argparse.ArgumentParser(
        prog='get_DMP_ID.py',
        description='This module gets the information about Deidentified DMP ID associated with Identified DMP ID',
        usage='%(prog)s [options]')
    parser.add_argument(
        "-v",
        "--verbose",
        action="store_true",
        dest="verbose",
        default=True,
        help="make lots of noise [default]")
    parser.add_argument(
        "-i",
        "--ids",
        action="store",
        dest="idFilename",
        required=True,
        metavar='ids.txt',
        help="Location of the file containing identified sample ids")
    parser.add_argument(
        "-o",
        "--outputFile",
        action="store",
        dest="outFile",
        required=True,
        metavar='out.txt',
        help="Full path with name for the output file")
    parser.add_argument(
        "-r",
        "--refFile",
        action="store",
        dest="refFilename",
        required=True,
        metavar='refFile.txt',
        help="Location of the file containing identified and de-identified sample mapping")
    parser.add_argument(
        "-b",
        "--bamLocation",
        action="store",
        dest="bamLocation",
        required=True,
        metavar='/prefix/to/bam/file/location',
        help="Location where the deidentified bam files are")
    args = parser.parse_args()

    #Get the process is verbose or not
    verbose = args.verbose
    #Get where are we
    here = os.path.realpath('.')

    logger = logging.getLogger('get_DMP_ID')
    coloredlogs.install(level='DEBUG')

    if(os.path.isfile(args.idFilename)):
        if(verbose):
            logger.info("get_DMP_ID: %s is being Read...", args.idFilename)
        iDF = pd.read_table(args.idFilename,sep="\t",header=None,usecols=[0])
        iDF.columns = ["SAMPLE_ID"]
        if(verbose):
            logger.info("Identification DataFrame:\n%s",iDF.head(n=5))
    else:
        logging.error("get_DMP_ID: %s is not a file. We will exit. Please make sure you provide a valid sample id file before rerun.", args.idFilename)
        sys.exit(1)

    if(os.path.isfile(args.idFilename)):
        if(verbose):
            logger.info("get_DMP_ID: %s is being Read...", args.refFilename)
        dDF = pd.read_table(args.refFilename,sep=",",header=None,usecols=[0,1,2])
        dDF.columns = ["SAMPLE_ID","D_SAMPLE_ID","GROUP_ID"]
        if(verbose):
            logger.info("Mapping DataFrame:\n%s",dDF.head(n=5))
    else:
        logging.error("get_DMP_ID: %s is not a file. We will exit. Please make sure you provide a valid sample mapping file before rerun.", args.refFilename)
        sys.exit(1)

    if(os.path.isdir(args.bamLocation)):
        if(verbose):
            logger.info("get_DMP_ID: %s is a directory", args.bamLocation)
        else:
            pass
    else:
        logging.error("get_DMP_ID: %s is not a directory. We will exit. Please make sure you provide a valid bam location directory before rerun.", args.bamLocation)

    outDF = pd.DataFrame(
        columns=[
            "SAMPLE_ID",
            "D_SAMPLE_ID",
            "SAMPLE_TYPE",
            "GROUP_ID",
            "BAM_LOCATION"])

    for count,row in iDF.iterrows():
        iID = str(row.loc["SAMPLE_ID"])
        dDF_idx = dDF[dDF["SAMPLE_ID"] == iID].index.tolist()
        dID = dDF.loc[dDF_idx[0],"D_SAMPLE_ID"]
        gID = dDF.loc[dDF_idx[0],"GROUP_ID"]
        bamFile = glob.glob(args.bamLocation +"/" + str(dID) +"*.bam")
        outDF.loc[count,["SAMPLE_ID",
            "D_SAMPLE_ID",
            "SAMPLE_TYPE",
            "GROUP_ID",
            "BAM_LOCATION"]] = [str(iID),str(dID),str(gID),str(bamFile)]

    outDF.to_csv(args.outFile, sep='\t', index=False)
    if(verbose):
        logger.info("get_DMP_ID: Finished Mapping, Final data written in %s", args.outFile)

if __name__ == "__main__":
    start_time = time.time()
    main()
    end_time = time.time()
    totaltime = end_time - start_time
    logging.info("iCallSV:Elapsed time was %g seconds", totaltime)

<|endoftext|>


<commit_before>import re
import markdown
import logging
import traceback

from zephyr.lib.avatar  import gravatar_hash
from zephyr.lib.bugdown import codehilite

class Gravatar(markdown.inlinepatterns.Pattern):
    def handleMatch(self, match):
        img = markdown.util.etree.Element('img')
        img.set('class', 'message_body_gravatar img-rounded')
        img.set('src', 'https://secure.gravatar.com/avatar/%s?d=identicon&s=30'
            % (gravatar_hash(match.group('email')),))
        return img

class AutoLink(markdown.inlinepatterns.Pattern):
    def handleMatch(self, match):
        url = match.group('url')
        a = markdown.util.etree.Element('a')
        a.set('href', url)
        a.text = url
        return a

class Bugdown(markdown.Extension):
    def extendMarkdown(self, md, md_globals):
        for k in ('image_link', 'image_reference', 'automail', 'autolink'):
            del md.inlinePatterns[k]

        for k in ('hashheader', 'setextheader'):
            del md.parser.blockprocessors[k]

        md.inlinePatterns.add('gravatar', Gravatar(r'!gravatar\((?P<email>[^)]*)\)'), '_begin')

        # A link starts after whitespace and continues to the next whitespace.
        link_regex = r'(\s|\A)(?P<url>https?://[^\s]+)'
        md.inlinePatterns.add('autolink', AutoLink(link_regex), '_begin')

# We need to re-initialize the markdown engine every 30 messages
# due to some sort of performance leak in the markdown library.
MAX_MD_ENGINE_USES = 30

_md_engine = None
_use_count = 0

def _linkify(match):
    url = match.group('url')
    return ' [%s](%s) ' % (url, url)

def convert(md):
    """Convert Markdown to HTML, with Humbug-specific settings and hacks."""
    global _md_engine, _use_count

    if _md_engine is None:
        _md_engine = markdown.Markdown(
            safe_mode     = 'escape',
            output_format = 'xhtml',
            extensions    = ['fenced_code', 'nl2br',
                codehilite.makeExtension(configs=[('force_linenos', False)]),
                Bugdown()])

    try:
        html = _md_engine.convert(md)
    except:
        # FIXME: Do something more reasonable here!
        #
        # NB: For security, we must not print the bare Markdown input.
        # It could contain terminal control codes, which can do
        # surprisingly nasty things.

        html = '
[Humbug note: Sorry, we could not understand the formatting of your message]
'
        logging.getLogger('').error('Exception in Markdown parser: %sInput was: %s'
            % (traceback.format_exc(), repr(md)))

    _use_count += 1
    if _use_count >= MAX_MD_ENGINE_USES:
        _md_engine = None
        _use_count = 0

    return html
<commit_msg>bugdown: Start a URL with a word boundary, not a space<commit_after>import re
import markdown
import logging
import traceback

from zephyr.lib.avatar  import gravatar_hash
from zephyr.lib.bugdown import codehilite

class Gravatar(markdown.inlinepatterns.Pattern):
    def handleMatch(self, match):
        img = markdown.util.etree.Element('img')
        img.set('class', 'message_body_gravatar img-rounded')
        img.set('src', 'https://secure.gravatar.com/avatar/%s?d=identicon&s=30'
            % (gravatar_hash(match.group('email')),))
        return img

class AutoLink(markdown.inlinepatterns.Pattern):
    def handleMatch(self, match):
        url = match.group('url')
        a = markdown.util.etree.Element('a')
        a.set('href', url)
        a.text = url
        return a

class Bugdown(markdown.Extension):
    def extendMarkdown(self, md, md_globals):
        for k in ('image_link', 'image_reference', 'automail', 'autolink'):
            del md.inlinePatterns[k]

        for k in ('hashheader', 'setextheader'):
            del md.parser.blockprocessors[k]

        md.inlinePatterns.add('gravatar', Gravatar(r'!gravatar\((?P<email>[^)]*)\)'), '_begin')

        # A link starts after whitespace and continues to the next whitespace.
        link_regex = r'\b(?P<url>https?://[^\s]+)'
        md.inlinePatterns.add('autolink', AutoLink(link_regex), '_begin')

# We need to re-initialize the markdown engine every 30 messages
# due to some sort of performance leak in the markdown library.
MAX_MD_ENGINE_USES = 30

_md_engine = None
_use_count = 0

def _linkify(match):
    url = match.group('url')
    return ' [%s](%s) ' % (url, url)

def convert(md):
    """Convert Markdown to HTML, with Humbug-specific settings and hacks."""
    global _md_engine, _use_count

    if _md_engine is None:
        _md_engine = markdown.Markdown(
            safe_mode     = 'escape',
            output_format = 'xhtml',
            extensions    = ['fenced_code', 'nl2br',
                codehilite.makeExtension(configs=[('force_linenos', False)]),
                Bugdown()])

    try:
        html = _md_engine.convert(md)
    except:
        # FIXME: Do something more reasonable here!
        #
        # NB: For security, we must not print the bare Markdown input.
        # It could contain terminal control codes, which can do
        # surprisingly nasty things.

        html = '
[Humbug note: Sorry, we could not understand the formatting of your message]
'
        logging.getLogger('').error('Exception in Markdown parser: %sInput was: %s'
            % (traceback.format_exc(), repr(md)))

    _use_count += 1
    if _use_count >= MAX_MD_ENGINE_USES:
        _md_engine = None
        _use_count = 0

    return html
<|endoftext|>


<commit_before># Copyright 2009-2015 Eucalyptus Systems, Inc.
#
# Redistribution and use of this software in source and binary forms,
# with or without modification, are permitted provided that the following
# conditions are met:
#
#   Redistributions of source code must retain the above copyright notice,
#   this list of conditions and the following disclaimer.
#
#   Redistributions in binary form must reproduce the above copyright
#   notice, this list of conditions and the following disclaimer in the
#   documentation and/or other materials provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

import datetime
import getpass
import os.path
import stat
import struct
import sys
import tempfile

def build_progressbar_label_template(fnames):
    if len(fnames) == 0:
        return None
    elif len(fnames) == 1:
        return '{fname}'
    else:
        max_fname_len = max(len(os.path.basename(fname)) for fname in fnames)
        fmt_template = '{{fname:<{maxlen}}} ({{index:>{lenlen}}}/{total})'
        return fmt_template.format(maxlen=max_fname_len,
                                   lenlen=len(str(len(fnames))),
                                   total=len(fnames))

# pylint: disable=W0622
def mkdtemp_for_large_files(suffix='', prefix='tmp', dir=None):
    """
    Like tempfile.mkdtemp, but using /var/tmp as a last resort instead of /tmp.

    This is meant for utilities that create large files, as /tmp is often a
    ramdisk.
    """

    if dir is None:
        dir = (os.getenv('TMPDIR') or os.getenv('TEMP') or os.getenv('TMP') or
               '/var/tmp')
    return tempfile.mkdtemp(suffix=suffix, prefix=prefix, dir=dir)
# pylint: enable=W0622

def prompt_for_password():
    pass1 = getpass.getpass(prompt='New password: ')
    pass2 = getpass.getpass(prompt='Retype new password: ')
    if pass1 == pass2:
        return pass1
    else:
        print >> sys.stderr, 'error: passwords do not match'
        return prompt_for_password()

def strip_response_metadata(response_dict):
    useful_keys = [key for key in response_dict if key != 'ResponseMetadata']
    if len(useful_keys) == 1:
        return response_dict[useful_keys[0]] or {}
    else:
        return response_dict

def build_iam_policy(effect, resources, actions):
    policy = {'Statement': []}
    for resource in resources or []:
        sid = datetime.datetime.utcnow().strftime('Stmt%Y%m%d%H%M%S%f')
        statement = {'Sid': sid, 'Effect': effect, 'Action': actions,
                     'Resource': resource}
        policy['Statement'].append(statement)
    return policy

def get_filesize(filename):
    mode = os.stat(filename).st_mode
    if stat.S_ISBLK(mode):
        # os.path.getsize doesn't work on block devices, but we can use lseek
        # to figure it out
        block_fd = os.open(filename, os.O_RDONLY)
        try:
            return os.lseek(block_fd, 0, os.SEEK_END)
        finally:
            os.close(block_fd)
    elif any((stat.S_ISCHR(mode), stat.S_ISFIFO(mode), stat.S_ISSOCK(mode),
              stat.S_ISDIR(mode))):
        raise TypeError("'{0}' does not have a usable file size"
                        .format(filename))
    return os.path.getsize(filename)

def get_vmdk_image_size(filename):
    if get_filesize(filename) < 1024:
        raise ValueError('File {0} is to small to be a valid Stream'
                         ' Optimized VMDK'.format(filename))
    # see https://www.vmware.com/support/developer/vddk/vmdk_50_technote.pdf
    # for header/footer format
    with open(filename, 'rb') as disk:
        data = struct.unpack('<iiiqqqqiqqq?bbbbh433c', disk.read(512))
        if data[9] & 0xffffffffffffffff == 0:
            # move to 1024 bytes from the end and read footer
            disk.seek(-1024, 2)
            data = struct.unpack('<iiiqqqqiqqq?bbbbh433c', disk.read(512))

    # validate
    if 1447904331 != data[0]:
        raise ValueError('File {0} is not a Stream Optimized VMDK'
                         .format(filename))
    if data[2] & 0x10000 == 0:
        raise ValueError('File {0} does not contain compressed parts'
                         .format(filename))
    if data[2] & 0x20000 == 0:
        raise ValueError('File {0} does not have all data present'
                         .format(filename))
    if data[11]:
        raise ValueError('File {0} marked with unclean shutdown'
                         .format(filename))
    if data[16] != 1:
        raise ValueError('File {0} uses unsupported compression algorithm'
                         .format(filename))

    return 512 * data[3]

def check_dict_whitelist(dict_, err_context, whitelist=None):
    if not isinstance(dict_, dict):
        raise ValueError('{0} must be a dict'.format(err_context))
    if whitelist:
        differences = set(dict_.keys()) - set(whitelist)
        if differences:
            raise ValueError('unrecognized {0} argument(s): {1}'
                             .format(err_context, ', '.join(differences)))

def transform_dict(dict_, transformation_dict):
    transformed = {}
    for key, val in dict_.iteritems():
        if key in transformation_dict:
            transformed[transformation_dict[key]] = val
        else:
            transformed[key] = val
    return transformed

def add_fake_region_name(service):
    """
    If no name for a region is otherwise defined (i.e. service.region_name
    is None and the AWS_AUTH_REGION environment variable is not set),
    print a warning and add a fake region name so HmacV4Auth has something
    to work with.  This works because eucalyptus doesn't care what name
    one chooses for a region.

    Setups that use eucarc files against AWS will still be broken.

    This was added in euca2ools the 3.3 series and should be removed
    some time after that.
    """

    if service.region_name is None and not os.getenv('AWS_AUTH_REGION'):
        msg = ('region names are required by AWS and will be required '
               'for Eucalyptus in a future release; either use '
               'euca2ools.ini(5) files or set AWS_AUTH_REGION in the '
               'environment to provide one')
        print >> sys.stderr, 'warning:', msg
        service.region_name = 'undefined-{0}'.format(os.getpid())
        service.log.info('added fake region name %s', service.region_name)
<commit_msg>Kill off the "no region name" warning<commit_after># Copyright 2009-2015 Eucalyptus Systems, Inc.
#
# Redistribution and use of this software in source and binary forms,
# with or without modification, are permitted provided that the following
# conditions are met:
#
#   Redistributions of source code must retain the above copyright notice,
#   this list of conditions and the following disclaimer.
#
#   Redistributions in binary form must reproduce the above copyright
#   notice, this list of conditions and the following disclaimer in the
#   documentation and/or other materials provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

import datetime
import getpass
import os.path
import stat
import struct
import sys
import tempfile

def build_progressbar_label_template(fnames):
    if len(fnames) == 0:
        return None
    elif len(fnames) == 1:
        return '{fname}'
    else:
        max_fname_len = max(len(os.path.basename(fname)) for fname in fnames)
        fmt_template = '{{fname:<{maxlen}}} ({{index:>{lenlen}}}/{total})'
        return fmt_template.format(maxlen=max_fname_len,
                                   lenlen=len(str(len(fnames))),
                                   total=len(fnames))

# pylint: disable=W0622
def mkdtemp_for_large_files(suffix='', prefix='tmp', dir=None):
    """
    Like tempfile.mkdtemp, but using /var/tmp as a last resort instead of /tmp.

    This is meant for utilities that create large files, as /tmp is often a
    ramdisk.
    """

    if dir is None:
        dir = (os.getenv('TMPDIR') or os.getenv('TEMP') or os.getenv('TMP') or
               '/var/tmp')
    return tempfile.mkdtemp(suffix=suffix, prefix=prefix, dir=dir)
# pylint: enable=W0622

def prompt_for_password():
    pass1 = getpass.getpass(prompt='New password: ')
    pass2 = getpass.getpass(prompt='Retype new password: ')
    if pass1 == pass2:
        return pass1
    else:
        print >> sys.stderr, 'error: passwords do not match'
        return prompt_for_password()

def strip_response_metadata(response_dict):
    useful_keys = [key for key in response_dict if key != 'ResponseMetadata']
    if len(useful_keys) == 1:
        return response_dict[useful_keys[0]] or {}
    else:
        return response_dict

def build_iam_policy(effect, resources, actions):
    policy = {'Statement': []}
    for resource in resources or []:
        sid = datetime.datetime.utcnow().strftime('Stmt%Y%m%d%H%M%S%f')
        statement = {'Sid': sid, 'Effect': effect, 'Action': actions,
                     'Resource': resource}
        policy['Statement'].append(statement)
    return policy

def get_filesize(filename):
    mode = os.stat(filename).st_mode
    if stat.S_ISBLK(mode):
        # os.path.getsize doesn't work on block devices, but we can use lseek
        # to figure it out
        block_fd = os.open(filename, os.O_RDONLY)
        try:
            return os.lseek(block_fd, 0, os.SEEK_END)
        finally:
            os.close(block_fd)
    elif any((stat.S_ISCHR(mode), stat.S_ISFIFO(mode), stat.S_ISSOCK(mode),
              stat.S_ISDIR(mode))):
        raise TypeError("'{0}' does not have a usable file size"
                        .format(filename))
    return os.path.getsize(filename)

def get_vmdk_image_size(filename):
    if get_filesize(filename) < 1024:
        raise ValueError('File {0} is to small to be a valid Stream'
                         ' Optimized VMDK'.format(filename))
    # see https://www.vmware.com/support/developer/vddk/vmdk_50_technote.pdf
    # for header/footer format
    with open(filename, 'rb') as disk:
        data = struct.unpack('<iiiqqqqiqqq?bbbbh433c', disk.read(512))
        if data[9] & 0xffffffffffffffff == 0:
            # move to 1024 bytes from the end and read footer
            disk.seek(-1024, 2)
            data = struct.unpack('<iiiqqqqiqqq?bbbbh433c', disk.read(512))

    # validate
    if 1447904331 != data[0]:
        raise ValueError('File {0} is not a Stream Optimized VMDK'
                         .format(filename))
    if data[2] & 0x10000 == 0:
        raise ValueError('File {0} does not contain compressed parts'
                         .format(filename))
    if data[2] & 0x20000 == 0:
        raise ValueError('File {0} does not have all data present'
                         .format(filename))
    if data[11]:
        raise ValueError('File {0} marked with unclean shutdown'
                         .format(filename))
    if data[16] != 1:
        raise ValueError('File {0} uses unsupported compression algorithm'
                         .format(filename))

    return 512 * data[3]

def check_dict_whitelist(dict_, err_context, whitelist=None):
    if not isinstance(dict_, dict):
        raise ValueError('{0} must be a dict'.format(err_context))
    if whitelist:
        differences = set(dict_.keys()) - set(whitelist)
        if differences:
            raise ValueError('unrecognized {0} argument(s): {1}'
                             .format(err_context, ', '.join(differences)))

def transform_dict(dict_, transformation_dict):
    transformed = {}
    for key, val in dict_.iteritems():
        if key in transformation_dict:
            transformed[transformation_dict[key]] = val
        else:
            transformed[key] = val
    return transformed

def add_fake_region_name(service):
    """
    If no name for a region is otherwise defined (i.e. service.region_name
    is None and the AWS_AUTH_REGION environment variable is not set),
    log a warning and add a fake region name so HmacV4Auth has something
    to work with.  This works because eucalyptus doesn't care what name
    one chooses for a region.

    Setups that use eucarc files against AWS will still be broken.

    This was added in euca2ools the 3.3 series and should be removed
    some time after that.
    """

    if service.region_name is None and not os.getenv('AWS_AUTH_REGION'):
        service.region_name = 'undefined-{0}'.format(os.getpid())
        service.log.warn('added fake region name %s', service.region_name)
<|endoftext|>