<commit_before>#!/usr/bin/env python
# Copyright (c) 2005-2007 ActiveState Software Inc.

"""Perform tasks defined in a Makefile.py in the current or parent
directories.
"""

# The primary usage of this module is for command-line usage. The "mk"
# tool calls "main()" here.
#
#     from mklib import runner
#     runner.main()
#
# There is also a "mk()" function here for less command-line oriented
# running (it doesn't setup logging, it doesn't call sys.exit). Currently
# it is being used for creating doctests in the test area. Typical usage
# might be:
#
#     >>> from mklib import runner
#     >>> runner.setup_logging()
#     >>> runner.mk(['', 'some_task_name'])
#
#
# Notes on logging levels and verbosity
# -------------------------------------
#
# How loud `mk` is depends on these options (the last one given wins):
#     (none given)        default verbosity (logging.INFO level)
#     -v, --verbose       more verbose (logging.INFO-1 level)
#     -q, --quiet         less verbose (logging.WARN level)
#     -d, --debug         debugging output (logging.DEBUG level)
#
# Full tracebacks on errors are shown on the command-line with -d|--debug.
# '-v|--verbose' is useful for some commands that have normal and more
# verbose output modes (e.g. `mk --tasks` will show more information with
# '--verbose').

import os
from os.path import exists, dirname, abspath, join, basename
import sys
import re
from pprint import pprint
import glob
import logging
import optparse
import time
import traceback

from mklib.common import *
from mklib.makefile import find_makefile_path
from mklib.taskmaster import TaskMaster
import mklib

#---- internal support functions

# Recipe: regex_from_encoded_pattern (1.0)
def _regex_from_encoded_pattern(s):
    """'foo'    -> re.compile(re.escape('foo'))
       '/foo/'  -> re.compile('foo')
       '/foo/i' -> re.compile('foo', re.I)
    """
    if s.startswith('/') and s.rfind('/') != 0:
        # Parse it: /PATTERN/FLAGS
        idx = s.rfind('/')
        pattern, flags_str = s[1:idx], s[idx+1:]
        flag_from_char = {
            "i": re.IGNORECASE,
            "l": re.LOCALE,
            "s": re.DOTALL,
            "m": re.MULTILINE,
            "u": re.UNICODE,
        }
        flags = 0
        for char in flags_str:
            try:
                flags |= flag_from_char[char]
            except KeyError:
                raise ValueError("unsupported regex flag: '%s' in '%s' "
                                 "(must be one of '%s')"
                                 % (char, s, ''.join(flag_from_char.keys())))
        return re.compile(s[1:idx], flags)
    else: # not an encoded regex
        return re.compile(re.escape(s))

# Based on Recipe: pretty_logging (0.1)
class _MkLogFormatter(logging.Formatter):
    """mk-specific logging output.

    - lowercase logging level names
    - no "info" for normal info level
    - when logging from a task include the task name in the message:
      set the "task" attribute to the current task name.

    XXX The "task" stuff won't work for -j: multiple sync task
        building. Solution: could isolate via thread ID.
    """
    def get_fmt(self, record):
        fmt = "%(name)s: "
        task = getattr(record, "task", None)
        if task:
            fmt += "[%(task)s] "
        if record.levelno != logging.INFO:
            fmt += "%(levelname)s: "
        fmt += "%(message)s"
        return fmt
    def format(self, record):
        record.levelname = record.levelname.lower() # uppercase is ugly
        #XXX This is a non-threadsafe HACK. Really the base Formatter
        #    class should provide a hook accessor for the _fmt
        #    attribute. *Could* add a lock guard here (overkill?).
        _saved_fmt = self._fmt
        self._fmt = self.get_fmt(record)
        try:
            return logging.Formatter.format(self, record)
        finally:
            self._fmt = _saved_fmt

class _MkTaskFilter(logging.Filter):
    prefix = "mk.task."
    prefix_len = len("mk.task.")
    def filter(self, record):
        if record.name.startswith(self.prefix):
            record.task = record.name[self.prefix_len:]
            record.name = "mk"
        return 1

def setup_logging(stream=None):
    """Do logging setup:

    1. We want a prettier default format:
            do: level: ...
       Spacing. Lower case. Skip " level:" if INFO-level.

    2. We want the task name in there if this a "do.task.foo" logger:
            mk: [foo] level: ...
    """
    hdlr = logging.StreamHandler(stream)
    fmtr = _MkLogFormatter()
    hdlr.addFilter(_MkTaskFilter())
    hdlr.setFormatter(fmtr)
    logging.root.addHandler(hdlr)

def _optparse_zero_or_one_arg(option, opt_str, value, parser):
    """Add optparse option callback that will accept one or zero args.

    An arg is only consumed if (1) one is available and (2) it doesn't
    look like an option. Use this callback as follows:

        parser.add_option("-x", ..., action="callback",
            callback=_optparse_zero_or_one_arg, dest="blah")

    Specifying "dest" is necessary (getting auto-gleaning from the
    option strings is messy).

    After parsing, 'opts.blah' will be:
        None        option was not specified
        True        option was specified, no argument
        <string>    option was specified, the value is the argument string
    """
    value = True
    rargs = parser.rargs
    if parser.rargs:
        arg = parser.rargs[0]
        # Stop if we hit an arg like "--foo", "-a", "-fx", "--file=f",
        # etc.  Note that this also stops on "-3" or "-3.0", so if
        # your option takes numeric values, you will need to handle
        # this.
        if ((arg[:2] == "--" and len(arg) >= 2) or
            (arg[:1] == "-" and len(arg) > 1 and arg[1] != "-")):
            pass
        else:
            value = arg
            del parser.rargs[0]
    setattr(parser.values, option.dest, value)

#---- mainline

class _CleanHelpOptionParser(optparse.OptionParser):
    """Just get the OptionParser to not screw up the description when
    printing -h|--help output.

    TODO: Actually subclass optparse.IndentedHelpFormatter and pass that
    in instead.
    """
    def format_description(self, formatter):
        return self.get_description()

def mk(argv=sys.argv, doc=None):
    usage = "usage: %prog [-f makefile] {options} tasks..."
    version = "%prog "+mklib.__version__
    doc = doc or __doc__
    if not doc.endswith("\n"):
        doc += "\n"
    parser = _CleanHelpOptionParser(prog="mk", usage=usage,
                                    version=version,
                                    description=doc or __doc__)

    # Verbosity options.
    parser.add_option("-q", "--quiet", dest="log_level",
        action="store_const", const=logging.WARNING,
        help="quieter output")
    parser.add_option("-v", "--verbose", dest="log_level",
        action="store_const", const=logging.INFO-1,
        help="more verbose output")
    parser.add_option("-d", "--debug", dest="log_level",
        action="store_const", const=logging.DEBUG,
        help="verbose debugging output")
    parser.add_option("--start", action="store_true",
        help="create a new starter 'Makefile.py' in the current directory")

    # Other options (we're somewhat following make's and rake's leads).
    parser.add_option("-f", "--makefile", dest="makefile_path",
        metavar="PATH", help="use the given Makefile.py")
    parser.add_option("-c", "--config",
        dest="config_file_path_override", metavar="PATH",
        help="override the default config file")
    # `mk -Tv` provides dep info as per `rake -P`.
    parser.add_option("-T", "--tasks", action="callback",
        callback=_optparse_zero_or_one_arg, dest="list_tasks",
        help="list tasks with a description (optional arg is a filter "
             "string), use --verbose for more detail")
    parser.add_option("-n", "--dry-run", action="store_true",
        help="dry-run, don't *execute* any tasks (use with -v to "
             "trace how tasks are executed)")
    parser.add_option("-N", "--nosearch", action="store_true",
        help="do not search in parent dirs for 'Makefile.py'")
    parser.add_option("-k", "--keep-going", action="store_true",
        help="keep going as far as possible after an error")
    parser.add_option("-F", "--force", action="store_true",
        help="force redo of tasks, even if up-to-date")

    parser.set_defaults(log_level=logging.INFO,
                        config_file_path_override=None,
                        force=False, no_lookup=False,
                        start=False)
    opts, tasks = parser.parse_args(argv[1:])
    log.setLevel(opts.log_level)

    if opts.start:
        makefile_path = "Makefile.py"
        if exists(makefile_path):
            raise MkError("cannot create starter Makefile: `%s' exists"
                          % makefile_path)
        template_path = join(dirname(__file__), "Makefile.py.template")
        template = open(template_path, 'r').read()
        project_name = basename(os.getcwd()) or None
        if project_name:
            template = template.replace("PROJECT_NAME", project_name)
        fout = open(makefile_path, 'w')
        try:
            fout.write(template)
        finally:
            fout.close()
        log.info("`%s' starter created (Try `mk hello`.)", makefile_path)
        return

    if opts.config_file_path_override is not None \
       and not exists(opts.config_file_path_override):
        raise MkError("config file does not exist: `%s'"
                        % opts.config_file_path_override)
    makefile_path = opts.makefile_path \
        or find_makefile_path(allow_search=not opts.nosearch)
    master = TaskMaster(opts, makefile_path,
                        opts.config_file_path_override, force=opts.force)

    if opts.list_tasks is not None:
        if opts.list_tasks is True:
            filter = None
        elif opts.list_tasks.startswith('/'):
            filter = _regex_from_encoded_pattern(opts.list_tasks)
        else:
            filter = opts.list_tasks
        master.list_tasks(filter)
    else:
        before = time.time()
        master.make(*tasks)
        after = time.time()
        ##XXX Should use the state object to keep running total of *all*
        ##    tasks re-done. The top-level number here is useless.
        #log.info("%d tasks done in %.2fs.", master.num_tasks_done,
        #         after - before)

def main(argv=sys.argv, doc=None):
    """Main command-line entry point for 'mk' tool.

    "argv" (optional, default sys.argv) is the command line args array
    "doc" (optional) is a description for help output for the Makefile.py.
    """
    # Handle hidden 'mk --configurelib-path' option to ease deployment
    # of 'mk'. With this option and the following snippet, users need
    # only get 'mk' running (typically by putting the "mk/bin" dir on
    # their PATH) and they are good to go:
    #
    # try:
    #     import configurelib
    # except ImportError:
    #     configurelib_path = os.open("mk --configurelib-path").read().strip()
    #     sys.path.insert(0, configurelib_path)
    #     import configurelib
    #     del configurelib_path
    if "--configurelib-path" in argv:
        print join(dirname(dirname(abspath(__file__))))
        return 0

    setup_logging(sys.stdout)
    try:
        retval = mk(argv, doc)
    except KeyboardInterrupt:
        sys.exit(1)
    except SystemExit:
        raise
    except:
        exc_info = sys.exc_info()
        if hasattr(exc_info[0], "__name__"):
            exc_class, exc, tb = exc_info
            if issubclass(exc_class, MkUsageError):
                log.error(str(exc_info[1]))
            else:
                task = hasattr(exc, "mk_task") and exc.mk_task or None
                prefix = task and ("[%s] " % task.name) or ""

                exc_str = str(exc_info[1])
                sep = ('\n' in exc_str and '\n' or ' ')

                where_str = ""
                tb_path, tb_lineno, tb_func = traceback.extract_tb(tb)[-1][:3]
                if not tb_path.startswith(dirname(__file__)+os.sep):
                    in_str = (tb_func != "<module>"
                              and " in %s" % tb_func
                              or "")
                    where_str = "%s(%s:%s%s)" \
                                % (sep, tb_path, tb_lineno, in_str)

                log.error("%s%s%s", prefix, exc_str, where_str)
        else:  # string exception
            log.error(exc_info[0])
        if log.isEnabledFor(logging.INFO-1):
            print
            traceback.print_exception(*exc_info)
        sys.exit(1)
    else:
        sys.exit(retval)

if __name__ == "__main__":
    main()

<commit_msg>doc tweak, no functional change<commit_after>#!/usr/bin/env python
# Copyright (c) 2005-2007 ActiveState Software Inc.

"""Perform tasks defined in a Makefile.py in the current or parent
directories.
"""

# The primary usage of this module is for command-line usage. The "mk"
# tool calls "main()" here.
#
#     from mklib import runner
#     runner.main()
#
# There is also a "mk()" function here for less command-line oriented
# running (it doesn't setup logging, it doesn't call sys.exit). Currently
# it is being used for creating doctests in the test area. Typical usage
# might be:
#
#     >>> from mklib import runner
#     >>> runner.setup_logging()
#     >>> runner.mk(['', 'some_task_name'])
#
#
# Notes on logging levels and verbosity
# -------------------------------------
#
# How loud `mk` is depends on these options (the last one given wins):
#     (none given)        default verbosity (logging.INFO level)
#     -v, --verbose       more verbose (logging.INFO-1 level)
#     -q, --quiet         less verbose (logging.WARN level)
#     -d, --debug         debugging output (logging.DEBUG level)
#
# Full tracebacks on errors are shown on the command-line with -d|--debug.
# '-v|--verbose' is useful for some commands that have normal and more
# verbose output modes (e.g. `mk --tasks` will show more information with
# '--verbose').

import os
from os.path import exists, dirname, abspath, join, basename
import sys
import re
from pprint import pprint
import glob
import logging
import optparse
import time
import traceback

from mklib.common import *
from mklib.makefile import find_makefile_path
from mklib.taskmaster import TaskMaster
import mklib

#---- internal support functions

# Recipe: regex_from_encoded_pattern (1.0)
def _regex_from_encoded_pattern(s):
    """'foo'    -> re.compile(re.escape('foo'))
       '/foo/'  -> re.compile('foo')
       '/foo/i' -> re.compile('foo', re.I)
    """
    if s.startswith('/') and s.rfind('/') != 0:
        # Parse it: /PATTERN/FLAGS
        idx = s.rfind('/')
        pattern, flags_str = s[1:idx], s[idx+1:]
        flag_from_char = {
            "i": re.IGNORECASE,
            "l": re.LOCALE,
            "s": re.DOTALL,
            "m": re.MULTILINE,
            "u": re.UNICODE,
        }
        flags = 0
        for char in flags_str:
            try:
                flags |= flag_from_char[char]
            except KeyError:
                raise ValueError("unsupported regex flag: '%s' in '%s' "
                                 "(must be one of '%s')"
                                 % (char, s, ''.join(flag_from_char.keys())))
        return re.compile(s[1:idx], flags)
    else: # not an encoded regex
        return re.compile(re.escape(s))

# Based on Recipe: pretty_logging (0.1)
class _MkLogFormatter(logging.Formatter):
    """mk-specific logging output.

    - lowercase logging level names
    - no "info" for normal info level
    - when logging from a task include the task name in the message:
      set the "task" attribute to the current task name.

    XXX The "task" stuff won't work for -j: multiple sync task
        building. Solution: could isolate via thread ID.
    """
    def get_fmt(self, record):
        fmt = "%(name)s: "
        task = getattr(record, "task", None)
        if task:
            fmt += "[%(task)s] "
        if record.levelno != logging.INFO:
            fmt += "%(levelname)s: "
        fmt += "%(message)s"
        return fmt
    def format(self, record):
        record.levelname = record.levelname.lower() # uppercase is ugly
        #XXX This is a non-threadsafe HACK. Really the base Formatter
        #    class should provide a hook accessor for the _fmt
        #    attribute. *Could* add a lock guard here (overkill?).
        _saved_fmt = self._fmt
        self._fmt = self.get_fmt(record)
        try:
            return logging.Formatter.format(self, record)
        finally:
            self._fmt = _saved_fmt

class _MkTaskFilter(logging.Filter):
    prefix = "mk.task."
    prefix_len = len("mk.task.")
    def filter(self, record):
        if record.name.startswith(self.prefix):
            record.task = record.name[self.prefix_len:]
            record.name = "mk"
        return 1

def setup_logging(stream=None):
    """Do logging setup:

    1. We want a prettier default format:
            mk: level: ...
       Spacing. Lower case. Skip " level:" if INFO-level.

    2. We want the task name in there if this a "mk.task.foo" logger:
            mk: [foo] level: ...
    """
    hdlr = logging.StreamHandler(stream)
    fmtr = _MkLogFormatter()
    hdlr.addFilter(_MkTaskFilter())
    hdlr.setFormatter(fmtr)
    logging.root.addHandler(hdlr)

def _optparse_zero_or_one_arg(option, opt_str, value, parser):
    """Add optparse option callback that will accept one or zero args.

    An arg is only consumed if (1) one is available and (2) it doesn't
    look like an option. Use this callback as follows:

        parser.add_option("-x", ..., action="callback",
            callback=_optparse_zero_or_one_arg, dest="blah")

    Specifying "dest" is necessary (getting auto-gleaning from the
    option strings is messy).

    After parsing, 'opts.blah' will be:
        None        option was not specified
        True        option was specified, no argument
        <string>    option was specified, the value is the argument string
    """
    value = True
    rargs = parser.rargs
    if parser.rargs:
        arg = parser.rargs[0]
        # Stop if we hit an arg like "--foo", "-a", "-fx", "--file=f",
        # etc.  Note that this also stops on "-3" or "-3.0", so if
        # your option takes numeric values, you will need to handle
        # this.
        if ((arg[:2] == "--" and len(arg) >= 2) or
            (arg[:1] == "-" and len(arg) > 1 and arg[1] != "-")):
            pass
        else:
            value = arg
            del parser.rargs[0]
    setattr(parser.values, option.dest, value)

#---- mainline

class _CleanHelpOptionParser(optparse.OptionParser):
    """Just get the OptionParser to not screw up the description when
    printing -h|--help output.

    TODO: Actually subclass optparse.IndentedHelpFormatter and pass that
    in instead.
    """
    def format_description(self, formatter):
        return self.get_description()

def mk(argv=sys.argv, doc=None):
    usage = "usage: %prog [-f makefile] {options} tasks..."
    version = "%prog "+mklib.__version__
    doc = doc or __doc__
    if not doc.endswith("\n"):
        doc += "\n"
    parser = _CleanHelpOptionParser(prog="mk", usage=usage,
                                    version=version,
                                    description=doc or __doc__)

    # Verbosity options.
    parser.add_option("-q", "--quiet", dest="log_level",
        action="store_const", const=logging.WARNING,
        help="quieter output")
    parser.add_option("-v", "--verbose", dest="log_level",
        action="store_const", const=logging.INFO-1,
        help="more verbose output")
    parser.add_option("-d", "--debug", dest="log_level",
        action="store_const", const=logging.DEBUG,
        help="verbose debugging output")

    parser.add_option("--start", action="store_true",
        help="create a new starter 'Makefile.py' in the current directory")

    # Other options (we're somewhat following make's and rake's leads).
    parser.add_option("-f", "--makefile", dest="makefile_path",
        metavar="PATH", help="use the given Makefile.py")
    parser.add_option("-c", "--config",
        dest="config_file_path_override", metavar="PATH",
        help="override the default config file")
    # `mk -Tv` provides dep info as per `rake -P`.
    parser.add_option("-T", "--tasks", action="callback",
        callback=_optparse_zero_or_one_arg, dest="list_tasks",
        help="list tasks with a description (optional arg is a filter "
             "string), use --verbose for more detail")
    parser.add_option("-n", "--dry-run", action="store_true",
        help="dry-run, don't *execute* any tasks (use with -v to "
             "trace how tasks are executed)")
    parser.add_option("-N", "--nosearch", action="store_true",
        help="do not search in parent dirs for 'Makefile.py'")
    parser.add_option("-k", "--keep-going", action="store_true",
        help="keep going as far as possible after an error")
    parser.add_option("-F", "--force", action="store_true",
        help="force redo of tasks, even if up-to-date")

    parser.set_defaults(log_level=logging.INFO,
                        config_file_path_override=None,
                        force=False, no_lookup=False,
                        start=False)
    opts, tasks = parser.parse_args(argv[1:])
    log.setLevel(opts.log_level)

    if opts.start:
        makefile_path = "Makefile.py"
        if exists(makefile_path):
            raise MkError("cannot create starter Makefile: `%s' exists"
                          % makefile_path)
        template_path = join(dirname(__file__), "Makefile.py.template")
        template = open(template_path, 'r').read()
        project_name = basename(os.getcwd()) or None
        if project_name:
            template = template.replace("PROJECT_NAME", project_name)
        fout = open(makefile_path, 'w')
        try:
            fout.write(template)
        finally:
            fout.close()
        log.info("`%s' starter created (Try `mk hello`.)", makefile_path)
        return

    if opts.config_file_path_override is not None \
       and not exists(opts.config_file_path_override):
        raise MkError("config file does not exist: `%s'"
                        % opts.config_file_path_override)
    makefile_path = opts.makefile_path \
        or find_makefile_path(allow_search=not opts.nosearch)
    master = TaskMaster(opts, makefile_path,
                        opts.config_file_path_override, force=opts.force)

    if opts.list_tasks is not None:
        if opts.list_tasks is True:
            filter = None
        elif opts.list_tasks.startswith('/'):
            filter = _regex_from_encoded_pattern(opts.list_tasks)
        else:
            filter = opts.list_tasks
        master.list_tasks(filter)
    else:
        before = time.time()
        master.make(*tasks)
        after = time.time()
        ##XXX Should use the state object to keep running total of *all*
        ##    tasks re-done. The top-level number here is useless.
        #log.info("%d tasks done in %.2fs.", master.num_tasks_done,
        #         after - before)

def main(argv=sys.argv, doc=None):
    """Main command-line entry point for 'mk' tool.

    "argv" (optional, default sys.argv) is the command line args array
    "doc" (optional) is a description for help output for the Makefile.py.
    """
    # Handle hidden 'mk --configurelib-path' option to ease deployment
    # of 'mk'. With this option and the following snippet, users need
    # only get 'mk' running (typically by putting the "mk/bin" dir on
    # their PATH) and they are good to go:
    #
    # try:
    #     import configurelib
    # except ImportError:
    #     configurelib_path = os.open("mk --configurelib-path").read().strip()
    #     sys.path.insert(0, configurelib_path)
    #     import configurelib
    #     del configurelib_path
    if "--configurelib-path" in argv:
        print join(dirname(dirname(abspath(__file__))))
        return 0

    setup_logging(sys.stdout)
    try:
        retval = mk(argv, doc)
    except KeyboardInterrupt:
        sys.exit(1)
    except SystemExit:
        raise
    except:
        exc_info = sys.exc_info()
        if hasattr(exc_info[0], "__name__"):
            exc_class, exc, tb = exc_info
            if issubclass(exc_class, MkUsageError):
                log.error(str(exc_info[1]))
            else:
                task = hasattr(exc, "mk_task") and exc.mk_task or None
                prefix = task and ("[%s] " % task.name) or ""

                exc_str = str(exc_info[1])
                sep = ('\n' in exc_str and '\n' or ' ')

                where_str = ""
                tb_path, tb_lineno, tb_func = traceback.extract_tb(tb)[-1][:3]
                if not tb_path.startswith(dirname(__file__)+os.sep):
                    in_str = (tb_func != "<module>"
                              and " in %s" % tb_func
                              or "")
                    where_str = "%s(%s:%s%s)" \
                                % (sep, tb_path, tb_lineno, in_str)

                log.error("%s%s%s", prefix, exc_str, where_str)
        else:  # string exception
            log.error(exc_info[0])
        if log.isEnabledFor(logging.INFO-1):
            print
            traceback.print_exception(*exc_info)
        sys.exit(1)
    else:
        sys.exit(retval)

if __name__ == "__main__":
    main()

<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

## This file is part of Invenio.
## Copyright (C) 2011, 2012, 2013, 2014 CERN.
##
## Invenio is free software; you can redistribute it and/or
## modify it under the terms of the GNU General Public License as
## published by the Free Software Foundation; either version 2 of the
## License, or (at your option) any later version.
##
## Invenio is distributed in the hope that it will be useful, but
## WITHOUT ANY WARRANTY; without even the implied warranty of
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
## General Public License for more details.
##
## You should have received a copy of the GNU General Public License
## along with Invenio; if not, write to the Free Software Foundation, Inc.,
## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.

"""
Implement the Flask application wrapper.

The function :func:`~flask.url_for` in standard :class:`~flask.Flask`
application changes only protocol for secure sites from 'http' to 'https'.
However, in case the application uses non standard ports (e.g.
http://example.org:4000, https://example:4001) or even different urls (e.g.
http://www.example.org, https://secure.example.org) following
:class:`Flask` wrapper is necessary.

Configuration:

===================== ====================================================
`CFG_SITE_URL`        specify URL under which your installation will be
                      visible.  For example, use "http://your.site.com".
                      Do not leave trailing slash.
`CFG_SITE_SECURE_URL` specify secure URL under which your installation
                      secure pages such as login or registration will be
                      visible.  For example, use "https://your.site.com".
                      Do not leave trailing slash. If you don't plan on
                      using HTTPS, then you may leave this empty.
===================== ====================================================

Following example shows how to make http and https url scheme loos the
same::

>>> from flask import url_for
>>> from invenio.base.factory import create_app
>>> app = create_app()
>>> app.config['CFG_SITE_URL'] = 'http://localhost:4000'
>>> app.config['CFG_SITE_SECURE_URL'] = 'http://localhost:4000'
>>> ctx = app.test_request_context()
>>> ctx.push()
>>> url_for('search.search')
'/search'
>>> url_for('search.search', _external=True)
'http://localhost:4000/search'
>>> url_for('search.search', _external=True, _scheme='https')
'http://localhost:4000/search'

"""

from functools import wraps
from flask import Flask as FlaskBase, current_app
from werkzeug.utils import import_string
from werkzeug.local import LocalProxy

from .signals import before_handle_user_exception

def _decorate_url_adapter_build(wrapped):
    """Change behavior of :func:`flask.url_for` for http and https scheme."""
    @wraps(wrapped)
    def decorator(*args, **kwargs):
        """Overwrite blueprint static url.

        :note: Custom blueprint 'static' endpoint will **NOT** work!
        """
        args = list(args)
        if args[0].endswith('.static'):
            args[0] = 'static'

        url_scheme_prefixes = {
            'http': current_app.config['CFG_SITE_URL'],
            'https': current_app.config['CFG_SITE_SECURE_URL']
        }
        force_external = kwargs.get('force_external', True)
        url_scheme = getattr(wrapped.im_self, 'url_scheme', 'http')
        kwargs['force_external'] = False
        url = wrapped(*args, **kwargs)
        if force_external:
            url = url_scheme_prefixes.get(url_scheme) + url
        return url
    return decorator

class Flask(FlaskBase):  # pylint: disable=R0904

    """
    Overriding the default Flask class.

    For more information about :class:`Flask` class see the official
    documentation of :class:`flask.Flask`.
    """

    def create_url_adapter(self, request):
        """Create the URL adapter.

        Change behavior of default `create_url_adapter` method to allow
        advance url generation.
        """
        url_adapter = super(self.__class__, self).create_url_adapter(request)
        if url_adapter is not None and hasattr(url_adapter, 'build'):
            url_adapter.build = _decorate_url_adapter_build(url_adapter.build)
        return url_adapter

    def handle_user_exception(self, e):
        """Handle the user exception.

        Hook a before handle to it.
        """
        before_handle_user_exception.send(e)
        return super(Flask, self).handle_user_exception(e)

def lazy_import(name):
    """Lazy import of the given `name`.

    :see: :func:`werkzeug.utils.import_string` and
          :class:`werkzeug.local.LocalProxy`
    :param name: importable string
    :return: proxy to result of :func:`~werkzeug.utils.import_string`
    """
    return LocalProxy(lambda: import_string(name))
<commit_msg>docs: fix of sphinx warnings.<commit_after># -*- coding: utf-8 -*-

## This file is part of Invenio.
## Copyright (C) 2011, 2012, 2013, 2014 CERN.
##
## Invenio is free software; you can redistribute it and/or
## modify it under the terms of the GNU General Public License as
## published by the Free Software Foundation; either version 2 of the
## License, or (at your option) any later version.
##
## Invenio is distributed in the hope that it will be useful, but
## WITHOUT ANY WARRANTY; without even the implied warranty of
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
## General Public License for more details.
##
## You should have received a copy of the GNU General Public License
## along with Invenio; if not, write to the Free Software Foundation, Inc.,
## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.

"""
Implement the Flask application wrapper.

The function :func:`~flask.url_for` in standard :class:`~flask.Flask`
application changes only protocol for secure sites from 'http' to 'https'.
However, in case the application uses non standard ports (e.g.
http://example.org:4000, https://example:4001) or even different urls (e.g.
http://www.example.org, https://secure.example.org) following
:class:`Flask` wrapper is necessary.

Configuration:

+-----------------------+-----------------------------------------------------+
| `CFG_SITE_URL`        | specify URL under which your installation will be   |
|                       | visible.  For example, use "http://your.site.com".  |
|                       | Do not leave trailing slash.                        |
+-----------------------+-----------------------------------------------------+
| `CFG_SITE_SECURE_URL` | specify secure URL under which your installation    |
|                       | secure pages such as login or registration will be  |
|                       | visible.  For example, use "https://your.site.com". |
|                       | Do not leave trailing slash. If you don't plan on   |
|                       | using HTTPS, then you may leave this empty.         |
+-----------------------+-----------------------------------------------------+

Following example shows how to make http and https url scheme look the
same.

.. doctest::

    >>> from flask import url_for
    >>> from invenio.base.factory import create_app
    >>> app = create_app()
    >>> app.config['CFG_SITE_URL'] = 'http://localhost:4000'
    >>> app.config['CFG_SITE_SECURE_URL'] = 'http://localhost:4000'
    >>> ctx = app.test_request_context()
    >>> ctx.push()
    >>> url_for('search.search')
    '/search'
    >>> url_for('search.search', _external=True)
    'http://localhost:4000/search'
    >>> url_for('search.search', _external=True, _scheme='https')
    'http://localhost:4000/search'

"""

from functools import wraps
from flask import Flask as FlaskBase, current_app
from werkzeug.utils import import_string
from werkzeug.local import LocalProxy

from .signals import before_handle_user_exception

def _decorate_url_adapter_build(wrapped):
    """Change behavior of :func:`flask.url_for` for http and https scheme."""
    @wraps(wrapped)
    def decorator(*args, **kwargs):
        """Overwrite blueprint static url.

        :note: Custom blueprint 'static' endpoint will **NOT** work!
        """
        args = list(args)
        if args[0].endswith('.static'):
            args[0] = 'static'

        url_scheme_prefixes = {
            'http': current_app.config['CFG_SITE_URL'],
            'https': current_app.config['CFG_SITE_SECURE_URL']
        }
        force_external = kwargs.get('force_external', True)
        url_scheme = getattr(wrapped.im_self, 'url_scheme', 'http')
        kwargs['force_external'] = False
        url = wrapped(*args, **kwargs)
        if force_external:
            url = url_scheme_prefixes.get(url_scheme) + url
        return url
    return decorator

class Flask(FlaskBase):  # pylint: disable=R0904

    """
    Overriding the default Flask class.

    For more information about :class:`Flask` class see the official
    documentation of :class:`flask.Flask`.
    """

    def create_url_adapter(self, request):
        """Create the URL adapter.

        Change behavior of default `create_url_adapter` method to allow
        advance url generation.
        """
        url_adapter = super(self.__class__, self).create_url_adapter(request)
        if url_adapter is not None and hasattr(url_adapter, 'build'):
            url_adapter.build = _decorate_url_adapter_build(url_adapter.build)
        return url_adapter

    def handle_user_exception(self, e):
        """Handle the user exception.

        Hook a before handle to it.
        """
        before_handle_user_exception.send(e)
        return super(Flask, self).handle_user_exception(e)

def lazy_import(name):
    """Lazy import of the given `name`.

    :see: :func:`werkzeug.utils.import_string` and
          :class:`werkzeug.local.LocalProxy`
    :param name: importable string
    :return: proxy to result of :func:`~werkzeug.utils.import_string`
    """
    return LocalProxy(lambda: import_string(name))
<|endoftext|>


<commit_before>import os
import sys
import errno
import fcntl
import atexit
import signal
import logging
import tempfile

__version__ = "1.0.0"

class PidFileError(Exception):
    pass

class PidFileUnreadableError(PidFileError):
    pass

class PidFileAlreadyRunningError(PidFileError):
    pass

class PidFileAlreadyLockedError(PidFileError):
    pass

class PidFile(object):
    __slots__ = ("pid", "filename", "fh", "logger", "lock_pidfile", "chmod", "uid", "gid")

    def __init__(self, pidname=None, piddir=None, enforce_dotpid_postfix=True,
                 register_term_signal_handler=True, lock_pidfile=True,
                 chmod=0o644, uid=-1, gid=-1, force_tmpdir=False):
        self.logger = logging.getLogger("PidFile")
        if pidname is None:
            pidname = "%s.pid" % os.path.basename(sys.argv[0])
        if enforce_dotpid_postfix and not pidname.endswith(".pid"):
            pidname = "%s.pid" % pidname
        if piddir is None:
            if os.path.isdir("/var/run/") and force_tmpdir is False:
                piddir = "/var/run/"
            else:
                piddir = tempfile.gettempdir()

        if not os.path.isdir(piddir):
            os.makedirs(piddir)
        if not os.access(piddir, os.R_OK):
            raise IOError("Pid file directory '%s' cannot be read")
        if not os.access(piddir, os.W_OK):
            raise IOError("Pid file directory '%s' cannot be written to")

        self.lock_pidfile = lock_pidfile
        self.chmod = chmod
        self.uid = uid
        self.gid = gid
        self.filename = os.path.abspath(os.path.join(piddir, pidname))
        self.pid = os.getpid()

        if register_term_signal_handler:
            # Register TERM signal handler to make sure atexit runs on TERM signal
            def sigterm_noop_handler(*args, **kwargs):
                raise SystemExit(1)

            signal.signal(signal.SIGTERM, sigterm_noop_handler)

    def check(self):
        def __check(fh):
            try:
                fh.seek(0)
                pid_str = fh.read(16).split("\n", 1)[0].strip()
                if not pid_str:
                    return None
                pid = int(pid_str)
            except (IOError, ValueError) as exc:
                self.close(fh=fh)
                raise PidFileUnreadableError(exc)
            else:
                try:
                    os.kill(pid, 0)
                except OSError as exc:
                    if exc.errno == errno.ESRCH:
                        # this pid is not running
                        return None
                    self.close(fh=fh)
                    raise PidFileAlreadyRunningError(exc)
            self.close(fh=fh)
            raise PidFileAlreadyRunningError("Program already running with pid: %d" % pid)
        if self.fh is None:
            if os.path.isfile(self.filename):
                with open(self.filename, "r") as fh:
                    __check(fh)
        else:
            __check(self.fh)

    def create(self):
        self.fh = open(self.filename, 'a+')
        if self.lock_pidfile:
            try:
                fcntl.flock(self.fh.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB)
            except IOError as exc:
                self.close()
                raise PidFileAlreadyLockedError(exc)
        self.check()
        if self.chmod:
            os.fchmod(self.fh.fileno(), self.chmod)
        if self.uid >= 0 or self.gid >= 0:
            os.fchown(self.fh.fileno(), self.uid, self.gid)
        self.fh.seek(0)
        self.fh.truncate()
        # pidfile must consist of the pid and a newline character
        self.fh.write("%d\n" % self.pid)
        self.fh.flush()
        self.fh.seek(0)
        atexit.register(self.close)

    def close(self, fh=None):
        if not fh:
            fh = self.fh
        try:
            fh.close()
        except IOError as exc:
            # ignore error when file was already closed
            if exc.errno != errno.EBADF:
                raise
        finally:
            if os.path.isfile(self.filename):
                os.remove(self.filename)

    def __enter__(self):
        self.create()
        return self

    def __exit__(self, exc_type=None, exc_value=None, exc_tb=None):
        self.close()
<commit_msg>v1.0.1<commit_after>import os
import sys
import errno
import fcntl
import atexit
import signal
import logging
import tempfile

__version__ = "1.0.1"

class PidFileError(Exception):
    pass

class PidFileUnreadableError(PidFileError):
    pass

class PidFileAlreadyRunningError(PidFileError):
    pass

class PidFileAlreadyLockedError(PidFileError):
    pass

class PidFile(object):
    __slots__ = ("pid", "filename", "fh", "logger", "lock_pidfile", "chmod", "uid", "gid")

    def __init__(self, pidname=None, piddir=None, enforce_dotpid_postfix=True,
                 register_term_signal_handler=True, lock_pidfile=True,
                 chmod=0o644, uid=-1, gid=-1, force_tmpdir=False):
        self.logger = logging.getLogger("PidFile")
        if pidname is None:
            pidname = "%s.pid" % os.path.basename(sys.argv[0])
        if enforce_dotpid_postfix and not pidname.endswith(".pid"):
            pidname = "%s.pid" % pidname
        if piddir is None:
            if os.path.isdir("/var/run/") and force_tmpdir is False:
                piddir = "/var/run/"
            else:
                piddir = tempfile.gettempdir()

        if not os.path.isdir(piddir):
            os.makedirs(piddir)
        if not os.access(piddir, os.R_OK):
            raise IOError("Pid file directory '%s' cannot be read")
        if not os.access(piddir, os.W_OK):
            raise IOError("Pid file directory '%s' cannot be written to")

        self.lock_pidfile = lock_pidfile
        self.chmod = chmod
        self.uid = uid
        self.gid = gid
        self.filename = os.path.abspath(os.path.join(piddir, pidname))
        self.pid = os.getpid()

        if register_term_signal_handler:
            # Register TERM signal handler to make sure atexit runs on TERM signal
            def sigterm_noop_handler(*args, **kwargs):
                raise SystemExit(1)

            signal.signal(signal.SIGTERM, sigterm_noop_handler)

    def check(self):
        def __check(fh):
            try:
                fh.seek(0)
                pid_str = fh.read(16).split("\n", 1)[0].strip()
                if not pid_str:
                    return None
                pid = int(pid_str)
            except (IOError, ValueError) as exc:
                self.close(fh=fh)
                raise PidFileUnreadableError(exc)
            else:
                try:
                    os.kill(pid, 0)
                except OSError as exc:
                    if exc.errno == errno.ESRCH:
                        # this pid is not running
                        return None
                    self.close(fh=fh)
                    raise PidFileAlreadyRunningError(exc)
            self.close(fh=fh)
            raise PidFileAlreadyRunningError("Program already running with pid: %d" % pid)
        if self.fh is None:
            if os.path.isfile(self.filename):
                with open(self.filename, "r") as fh:
                    __check(fh)
        else:
            __check(self.fh)

    def create(self):
        self.fh = open(self.filename, 'a+')
        if self.lock_pidfile:
            try:
                fcntl.flock(self.fh.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB)
            except IOError as exc:
                self.close()
                raise PidFileAlreadyLockedError(exc)
        self.check()
        if self.chmod:
            os.fchmod(self.fh.fileno(), self.chmod)
        if self.uid >= 0 or self.gid >= 0:
            os.fchown(self.fh.fileno(), self.uid, self.gid)
        self.fh.seek(0)
        self.fh.truncate()
        # pidfile must consist of the pid and a newline character
        self.fh.write("%d\n" % self.pid)
        self.fh.flush()
        self.fh.seek(0)
        atexit.register(self.close)

    def close(self, fh=None):
        if not fh:
            fh = self.fh
        try:
            fh.close()
        except IOError as exc:
            # ignore error when file was already closed
            if exc.errno != errno.EBADF:
                raise
        finally:
            if os.path.isfile(self.filename):
                os.remove(self.filename)

    def __enter__(self):
        self.create()
        return self

    def __exit__(self, exc_type=None, exc_value=None, exc_tb=None):
        self.close()
<|endoftext|>


<commit_before>####################################################################################################
#
# Bleau Database - A database of the bouldering area of Fontainebleau
# Copyright (C) 2015 Fabrice Salvaire
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
####################################################################################################

####################################################################################################

import hashlib

# from bokeh.plotting import figure
from bokeh.embed import components
from bokeh._legacy_charts import Bar

####################################################################################################

from BleauDataBase.Statistics import CircuitStatistics

####################################################################################################

class BokehPlot:

    ##############################################

    def __init__(self, *args, **kwargs):

        script, div = components(*args, **kwargs)
        self.script = script
        self.div = div

####################################################################################################

class CircuitStatisticsData:

    ##############################################

    def __init__(self, circuits):

        self._circuit_statistics = CircuitStatistics(circuits)
        self._circuit_grade_barchart = self._make_barchart(self._circuit_statistics.circuit_grade_histogram,
                                                           'Cotation des Circuits') # Circuit Grade
        self._global_boulder_grade_barchart = self._make_barchart(self._circuit_statistics.global_boulder_grade_histogram,
                                                                  'Cotations des Blocs') # Boulder Grade
        self._boulder_grade_barchart_map = {grade:self._make_barchart(self._circuit_statistics.boulder_grade_histogram(grade),
                                                                      'Cotations des Blocs pour les circuits {}'.format(grade))
                                            for grade in self._circuit_statistics.circuit_grades}

    ##############################################

    def _make_barchart(self, histogram, title):

        grade_counters = histogram.domain()
        y_data = [grade_counter.count for grade_counter in grade_counters]
        x_data = [str(grade_counter) for grade_counter in grade_counters]
        bar = Bar(y_data, x_data, title=title, stacked=True, tools='')
        bar.toolbar_location = None

        return BokehPlot(bar)

    ##############################################

    @property
    def circuit_statistics(self):
        return self._circuit_statistics

    @property
    def circuit_grade_barchart(self):
        return self._circuit_grade_barchart

    @property
    def global_boulder_grade_barchart(self):
        return self._global_boulder_grade_barchart

    @property
    def circuit_grades(self):
        return list(self._boulder_grade_barchart_map.keys())

    def boulder_grade_barchart(self, grade):
        return self._boulder_grade_barchart_map[grade]

####################################################################################################

class CircuitStatisticsCache:

    ##############################################

    def __init__(self):

        self._cache = {}

    ##############################################

    def __getitem__(self, circuits):

        ids = [id(circuit) for circuit in circuits]
        id_string = ''.join([str(x) for x in sorted(ids)])
        key = hashlib.sha256(id_string.encode('ascii'))
        if key not in self._cache:
            self._cache[key] = CircuitStatisticsData(circuits)
        return self._cache[key]

####################################################################################################
#
# End
#
####################################################################################################
<commit_msg>make responsive bokeh plot<commit_after>####################################################################################################
#
# Bleau Database - A database of the bouldering area of Fontainebleau
# Copyright (C) 2015 Fabrice Salvaire
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
####################################################################################################

####################################################################################################

import hashlib

# from bokeh.plotting import figure
from bokeh.embed import components
from bokeh._legacy_charts import Bar

####################################################################################################

from BleauDataBase.Statistics import CircuitStatistics

####################################################################################################

class BokehPlot:

    ##############################################

    def __init__(self, *args, **kwargs):

        script, div = components(*args, **kwargs)
        self.script = script
        self.div = div

####################################################################################################

class CircuitStatisticsData:

    ##############################################

    def __init__(self, circuits):

        self._circuit_statistics = CircuitStatistics(circuits)
        self._circuit_grade_barchart = self._make_barchart(self._circuit_statistics.circuit_grade_histogram,
                                                           'Cotation des Circuits') # Circuit Grade
        self._global_boulder_grade_barchart = self._make_barchart(self._circuit_statistics.global_boulder_grade_histogram,
                                                                  'Cotations des Blocs') # Boulder Grade
        self._boulder_grade_barchart_map = {grade:self._make_barchart(self._circuit_statistics.boulder_grade_histogram(grade),
                                                                      'Cotations des Blocs pour les circuits {}'.format(grade))
                                            for grade in self._circuit_statistics.circuit_grades}

    ##############################################

    def _make_barchart(self, histogram, title):

        grade_counters = histogram.domain()
        y_data = [grade_counter.count for grade_counter in grade_counters]
        x_data = [str(grade_counter) for grade_counter in grade_counters]
        bar = Bar(y_data, x_data, title=title, stacked=True, tools='', width=300, height=200, responsive=True)
        bar.toolbar_location = None

        return BokehPlot(bar)

    ##############################################

    @property
    def circuit_statistics(self):
        return self._circuit_statistics

    @property
    def circuit_grade_barchart(self):
        return self._circuit_grade_barchart

    @property
    def global_boulder_grade_barchart(self):
        return self._global_boulder_grade_barchart

    @property
    def circuit_grades(self):
        return list(self._boulder_grade_barchart_map.keys())

    def boulder_grade_barchart(self, grade):
        return self._boulder_grade_barchart_map[grade]

####################################################################################################

class CircuitStatisticsCache:

    ##############################################

    def __init__(self):

        self._cache = {}

    ##############################################

    def __getitem__(self, circuits):

        ids = [id(circuit) for circuit in circuits]
        id_string = ''.join([str(x) for x in sorted(ids)])
        key = hashlib.sha256(id_string.encode('ascii'))
        if key not in self._cache:
            self._cache[key] = CircuitStatisticsData(circuits)
        return self._cache[key]

####################################################################################################
#
# End
#
####################################################################################################
<|endoftext|>


<commit_before>#!/usr/bin/env python

import os
import sys
import pickle
import pipes
import time
import re
import logging
import traceback

from datetime import datetime
from logging.handlers import RotatingFileHandler

from ispyb_api.dbconnection import dbconnection
from ispyb_api.mxmr import mxmr

def get_logical_arg(job, name):
    return job.args[job.args.index(name)+1]

def format_time(t):
    return time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(t))

def get_scaling_id(dir):
    '''Read the fast_dp output file, extract and return the autoProcScalingId'''
    id_file = os.path.join(dir, "ispyb_ids.xml")

    if (not os.path.isfile(id_file)) or (not os.access(id_file, os.R_OK)):
        print "Either file %s is missing or is not readable" % id_file
        return None
    xml = None
    with open(id_file, 'rb') as f:
        xml = f.read()
    if xml is None:
        return None

    m = re.match(r'.*\<autoProcScalingId\>(\d+)\<\/autoProcScalingId\>.*', xml)
    if m is None:
        return None
    else:
        return m.group(1)

def store_result(cursor, dir, scaling_id):
    '''Store results from DIMPLE pipeline'''

    pkl = os.path.join(dir, "workflow.pickle")
    if (not os.path.isfile(pkl)) or (not os.access(pkl, os.R_OK)):
        print "Either file %s is missing or is not readable" % pkl
        return None
    wf = None
    with open(pkl, "rb") as f:
        wf = pickle.load(f)
    if wf is None:
        return None

    refmac_job = [job for job in wf.jobs if job.name == 'refmac5 restr'][0]
    fb_job = [job for job in wf.jobs if job.name == 'find-blobs'][0]

    params = mxmr.get_run_params()
    params['parentid'] = scaling_id
    params['pipeline'] = 'dimple'
    params['starttime'] = format_time(wf.jobs[0].started)
    # datetime.strptime('2014-09-24 14:30:01', '%Y-%m-%d %H:%M:%S')
    params['endtime'] =  format_time(wf.jobs[-1].started + wf.jobs[-1].total_time)
    # datetime.strptime('2014-09-24 14:30:27', '%Y-%m-%d %H:%M:%S')
    params['success'] = 1

    scores = fb_job.data["scores"][:2]
    if scores:
        params['message'] = "Blob scores: " + " ".join("%.0f" % sc for sc in scores)
    else:
        params['message'] = "No blobs found"

    params['cmd_line'] = " ".join(pipes.quote(a) for a in wf.argv)
    params['input_coord_file'] = get_logical_arg(wf.jobs[0], 'XYZIN')
    params['output_coord_file'] = get_logical_arg(refmac_job, 'XYZOUT')
    params['input_MTZ_file'] = get_logical_arg(wf.jobs[0], 'HKLIN')
    params['output_MTZ_file'] = get_logical_arg(refmac_job, 'HKLOUT')
    params['run_dir'] = wf.output_dir
    params['log_file'] = wf.output_dir
    params['r_start'] = refmac_job.data["ini_overall_r"]
    params['r_end'] = refmac_job.data["overall_r"]
    params['rfree_start'] = refmac_job.data["ini_free_r"]
    params['rfree_end'] = refmac_job.data["free_r"]
    mr_id = mxmr.insert_run(cursor, params.values())

    for n,_ in enumerate(scores):
        mrblob_id = mxmr.insert_run_blob(cursor, mr_id, 'blob%dv1.png', 'blob%dv2.png', 'blob%dv3.png' % (n, n, n))

    print "done"

def store_failure(cursor, scaling_id):
    '''Store failure of DIMPLE pipeline'''
    params = mxmr.get_run_params()
    params['parentid'] = scaling_id
    params['pipeline'] = 'dimple'
    params['success'] = 0
    params['message'] = 'Unknown error'
    mr_id = mxmr.insert_run(cursor, params.values())

# Configure logging
logger = logging.getLogger()
logger.setLevel(logging.DEBUG)
formatter = logging.Formatter('* %(asctime)s [id=%(thread)d] <%(levelname)s> %(message)s')
hdlr = logging.StreamHandler(sys.stdout)
hdlr.setFormatter(formatter)
logging.getLogger().addHandler(hdlr)

log_file = None

# Add file logging
try:
    if log_file is not None:
        hdlr2 = RotatingFileHandler(filename=log_file, maxBytes=1000000, backupCount=10) # 'a', 4194304, 10)
        hdlr2.setFormatter(_formatter)
        logging.getLogger().addHandler(hdlr2)
except:
        logging.getLogger().exception("dimple2ispyb: problem setting the file logging using file %s :-(" % log_file)

if len(sys.argv) != 3:
    print("Usage: %s dimple-output-dir fast_dp-output-dir" % sys.argv[0])
    sys.exit(1)

cursor = dbconnection.connect_to_prod()

scaling_id = get_scaling_id(sys.argv[2])

if scaling_id is not None:
    try:
        store_result(cursor, sys.argv[1], scaling_id)
    except:
        print e
        logging.getLogger().exception("dimple2ispyb: Problem extracting / storing the dimple result.")
        store_failure(cursor, scaling_id)

dbconnection.disconnect()

<commit_msg>Bug fix<commit_after>#!/usr/bin/env python

import os
import sys
import pickle
import pipes
import time
import re
import logging
import traceback

from datetime import datetime
from logging.handlers import RotatingFileHandler

from ispyb_api.dbconnection import dbconnection
from ispyb_api.mxmr import mxmr

def get_logical_arg(job, name):
    return job.args[job.args.index(name)+1]

def format_time(t):
    return time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(t))

def get_scaling_id(dir):
    '''Read the fast_dp output file, extract and return the autoProcScalingId'''
    id_file = os.path.join(dir, "ispyb_ids.xml")

    if (not os.path.isfile(id_file)) or (not os.access(id_file, os.R_OK)):
        print "Either file %s is missing or is not readable" % id_file
        return None
    xml = None
    with open(id_file, 'rb') as f:
        xml = f.read()
    if xml is None:
        return None

    m = re.match(r'.*\<autoProcScalingId\>(\d+)\<\/autoProcScalingId\>.*', xml)
    if m is None:
        return None
    else:
        return m.group(1)

def store_result(cursor, dir, scaling_id):
    '''Store results from DIMPLE pipeline'''

    pkl = os.path.join(dir, "workflow.pickle")
    if (not os.path.isfile(pkl)) or (not os.access(pkl, os.R_OK)):
        print "Either file %s is missing or is not readable" % pkl
        return None
    wf = None
    with open(pkl, "rb") as f:
        wf = pickle.load(f)
    if wf is None:
        return None

    refmac_job = [job for job in wf.jobs if job.name == 'refmac5 restr'][0]
    fb_job = [job for job in wf.jobs if job.name == 'find-blobs'][0]

    params = mxmr.get_run_params()
    params['parentid'] = scaling_id
    params['pipeline'] = 'dimple'
    params['starttime'] = format_time(wf.jobs[0].started)
    # datetime.strptime('2014-09-24 14:30:01', '%Y-%m-%d %H:%M:%S')
    params['endtime'] =  format_time(wf.jobs[-1].started + wf.jobs[-1].total_time)
    # datetime.strptime('2014-09-24 14:30:27', '%Y-%m-%d %H:%M:%S')
    params['success'] = 1

    scores = fb_job.data["scores"][:2]
    if scores:
        params['message'] = "Blob scores: " + " ".join("%.0f" % sc for sc in scores)
    else:
        params['message'] = "No blobs found"

    params['cmd_line'] = " ".join(pipes.quote(a) for a in wf.argv)
    params['input_coord_file'] = get_logical_arg(wf.jobs[0], 'XYZIN')
    params['output_coord_file'] = get_logical_arg(refmac_job, 'XYZOUT')
    params['input_MTZ_file'] = get_logical_arg(wf.jobs[0], 'HKLIN')
    params['output_MTZ_file'] = get_logical_arg(refmac_job, 'HKLOUT')
    params['run_dir'] = wf.output_dir
    params['log_file'] = wf.output_dir
    params['r_start'] = refmac_job.data["ini_overall_r"]
    params['r_end'] = refmac_job.data["overall_r"]
    params['rfree_start'] = refmac_job.data["ini_free_r"]
    params['rfree_end'] = refmac_job.data["free_r"]
    mr_id = mxmr.insert_run(cursor, params.values())

    for n,_ in enumerate(scores):
        mrblob_id = mxmr.insert_run_blob(cursor, mr_id, 'blob%dv1.png', 'blob%dv2.png', 'blob%dv3.png' % (n, n, n))

    print "done"

def store_failure(cursor, scaling_id):
    '''Store failure of DIMPLE pipeline'''
    params = mxmr.get_run_params()
    params['parentid'] = scaling_id
    params['pipeline'] = 'dimple'
    params['success'] = 0
    params['message'] = 'Unknown error'
    mr_id = mxmr.insert_run(cursor, params.values())

# Configure logging
logger = logging.getLogger()
logger.setLevel(logging.DEBUG)
formatter = logging.Formatter('* %(asctime)s [id=%(thread)d] <%(levelname)s> %(message)s')
hdlr = logging.StreamHandler(sys.stdout)
hdlr.setFormatter(formatter)
logging.getLogger().addHandler(hdlr)

log_file = None

# Add file logging
try:
    if log_file is not None:
        hdlr2 = RotatingFileHandler(filename=log_file, maxBytes=1000000, backupCount=10) # 'a', 4194304, 10)
        hdlr2.setFormatter(_formatter)
        logging.getLogger().addHandler(hdlr2)
except:
        logging.getLogger().exception("dimple2ispyb: problem setting the file logging using file %s :-(" % log_file)

if len(sys.argv) != 3:
    print("Usage: %s dimple-output-dir fast_dp-output-dir" % sys.argv[0])
    sys.exit(1)

cursor = dbconnection.connect_to_prod()

scaling_id = get_scaling_id(sys.argv[2])

if scaling_id is not None:
    try:
        store_result(cursor, sys.argv[1], scaling_id)
    except:
        logging.getLogger().exception("dimple2ispyb: Problem extracting / storing the dimple result.")
        store_failure(cursor, scaling_id)

dbconnection.disconnect()

<|endoftext|>


<commit_before>#!/usr/bin/python
#
# solver_cese.py
#
# Description:
#     1D Sod Tube sover based on CESE method.
#
#     This program is implemented by OO style to be
#     a part of ipython notebook demo materials.
#
#     The derivation of the equations for the analytic solution
#     is based on the book,
#     Principles of Computational Fluid Dynamics,
#     written by Pieter Wesseling.
#     Or, people could refer to the solvcon website
#     http://www.solvcon.net/en/latest/cese.html#sod-s-shock-tube-problem
#

import numpy as np
import generator_mesh

gmesh = generator_mesh.Mesher()

# a number to claim two floating number value are equal.
delta_precision = 0.0000000000001

GAMMA = 1.4

# Sod tube initial conditions of the left and right
# of the diaphragm
rhol = 1.0
ul = 0.0
pl = 1.0
rhor = 0.125
ur = 0.0
pr = 0.1

ia = 1

a2 = 3.0 - GAMMA
a3 = a2/2.0
a4 = 1.5*(GAMMA-1.0)

class Solver():
    """
    CESE method to generate the 1D Sod tube solution
    @iteration, which is integer
    TODO:
        1. grid_size_x, mesh_x_start, and mesh_x_stop should be integer
        for gmesh.gen_mesh method, which requires integer inputs to be easier
        to generate mesh points along x. This should be smarter and allow user
        to input their own numbers. Users' inputs are 10000 times as
        the real inputs for CESE interation.

    """
    def __init__(self,
                 iteration = 100,
                 grid_size_t = 0.004,
                 mesh_t_stop = 0.2,
                 grid_size_x = 100,
                 mesh_x_start = -10050,
                 mesh_x_stop = 10050):
        self.iteration = iteration
        self.grid_size_t = grid_size_t
        self.mesh_t_stop = mesh_t_stop
        self.mesh_x_start = mesh_x_start
        self.mesh_x_stop = mesh_x_stop

        gmesh.gen_mesh(grid_size_x, mesh_x_start, mesh_x_stop)
        self.mesh_x = gmesh.get_mesh()
        self.grid_size_x = grid_size_x / 10000.0

    def get_cese_solution(self):
        """
        given the mesh size
        output the solution based on CESE method

        iteration: int, please note n iteration will has n+2 mesh points.

        """
        iteration = self.iteration
        grid_size_t = self.grid_size_t
        mesh_t_stop = self.mesh_t_stop
        grid_size_x = self.grid_size_x
        mesh_x_start = self.mesh_x_start
        mesh_x_stop = self.mesh_x_stop
        mesh_x = self.mesh_x

        #self.check_input(iteration,
        #                 grid_size_t,
        #                 mesh_t_stop,
        #                 grid_size_x,
        #                 mesh_x_start,
        #                 mesh_x_stop)

        # mesh point number along x
        mesh_pt_number_x_at_half_t = iteration + 1
        mesh_pt_number_x = iteration + 2

        # u_m of 1D Eular equation.
        # this also means the status of the gas
        # on grids.
        # prefix mtx_ means 'matrix of sth.'
        mtx_q = np.asmatrix(np.zeros(shape=(3, mesh_pt_number_x)))
        # u_m, but means 'next' u_m
        # u_m at the next time step
        mtx_qn = np.asmatrix(np.zeros(shape=(3, mesh_pt_number_x)))
        # partial matrix of q partial x
        mtx_qx = np.asmatrix(np.zeros(shape=(3, mesh_pt_number_x)))
        # partial mastrix of q partial t
        mtx_qt = np.asmatrix(np.zeros(shape=(3, mesh_pt_number_x)))
        # matrix s, it is a part of the marching mtx_qn
        mtx_s = np.asmatrix(np.zeros(shape=(3, mesh_pt_number_x)))
        vxl = np.zeros(shape=(3,1))
        vxr = np.zeros(shape=(3,1))

        mtx_f = np.asmatrix(np.zeros(shape=(3,3)))

        mtx_q[0][0] = rhol
        mtx_q[1][0] = rhol*ul
        mtx_q[2][0] = pl/(GAMMA-1.0) + 0.5*rhol*ul**2.0
        mesh_pt_number_x_at_half_t = iteration + 1
        # initialize the gas status before the diaphragm
        # was removed.
        for i in xrange(mesh_pt_number_x_at_half_t):
            mtx_q[0,i+1] = rhor
            mtx_q[1,i+1] = rhor*ur
            mtx_q[2,i+1] = pr/(GAMMA-1.0) + 0.5*rhor*ur**2.0

        # m is the number used to calculate the status before
        # the half delta t stepping is applied.
        m = 2 # move out from the diaphragm which the 0th grid.
        for i in xrange(iteration):
            self.get_cese_status_before_half_dt(m, mtx_q, mtx_f, mtx_qt, mtx_qx, mtx_s)
            # stepping into the next halt delta t
            # m mesh points along t could introduce m - 1 mesh points along t + 0.5*dt
            self.get_cese_status_after_half_dt(m, mtx_q, mtx_qn, mtx_qt, mtx_qx, mtx_s)
            #  ask the status at t + 0.5*dt to be the next status before the half delta t is applied
            m = self.push_status_along_t(m, mtx_q, mtx_qn)

        #solution__x_start_index = mesh_x.get_start_grid(mesh_pt_number_x)
        solution_x_start_index = 50
        solution = []
        for i in range(mesh_pt_number_x):
            solution_x = mesh_x[i + solution_x_start_index]
            x = solution_x
            solution_rho = mtx_q[0,i]
            solution_v = mtx_q[1,i]/mtx_q[0,i]
            solution_p = (GAMMA-1.0)*(mtx_q[2,i] - 0.5*(solution_v**2)*mtx_q[0,i])
            solution.append((solution_x, solution_rho, solution_v, solution_p))

        return solution

    def get_cese_solution_mesh_size(self,
                                    iteration = 100,
                                    grid_t = 0.004,
                                    mesh_x_start = -0.5050,
                                    mesh_x_stop = 0.5050,
                                    grid_x = 0.01):
        # iteration should be an even number to
        # make sure the grids return to the same
        # as the initial one.
        if iteration % 2 != 0:
            raise Exception(iteration, 'should be even!')
        gmesh.gen_mesh(grid_x * 10000, mesh_x_start * 10000, mesh_x_stop * 10000)
        mesh_pt_number_x_at_half_t = iteration + 1
        # grids generated by the iteration +
        # the most left grid +
        # the most right grid
        mesh_pt_number_x = iteration + 2

    def cal_cese_solution(self, initcondition, mesh, ceseparameters):
        return self.solution

    def get_cese_status_before_half_dt(self, m, mtx_q, mtx_f, mtx_qt, mtx_qx, mtx_s):
        """
        the status is
        """
        for j in xrange(m):
            w2 = mtx_q[1,j]/mtx_q[0,j]
            w3 = mtx_q[2,j]/mtx_q[0,j]
            # f[0][0] = 0.0
            mtx_f[0,1] = 1.0
            # f[0][2] = 0.0
            mtx_f[1,0] = -a3*w2**2
            mtx_f[1,1] = a2*w2
            mtx_f[1,2] = GAMMA - 1.0
            mtx_f[2,0] = (GAMMA-1.0)*w2**3 - GAMMA*w2*w3
            mtx_f[2,1] = GAMMA*w3 - (GAMMA-1.0)*w2**2
            mtx_f[2,2] = GAMMA*w2

            # (4.17) in chang95
            mtx_qt[:,j] = -1.0*mtx_f*mtx_qx[:,j]
            # (4.25) in chang95
            # the n_(fmt)_j of the last term should be substitubed
            # by the other terms.
            mtx_s[:,j] = (self.grid_size_x/4.0)*mtx_qx[:,j] + (self.grid_size_t/self.grid_size_x)*mtx_f*mtx_q[:,j] \
                        - (self.grid_size_t/self.grid_size_x)*(self.grid_size_t/4.0)*mtx_f*mtx_f*mtx_qx[:,j]
        return  m, mtx_q, mtx_f, mtx_qt, mtx_qx, mtx_s

    def get_cese_status_after_half_dt(self, m, mtx_q, mtx_qn, mtx_qt, mtx_qx, mtx_s):
        mm = m - 1
        for j in xrange(mm):
            # (4.24) in chang95
            # Please note the j+1 on the left hand side addresses
            # different t (n) from the j/j+1 on the right hand side,
            # which address the other t (n-1/2) on the space-time
            # surface.
            mtx_qn[:,j+1] = 0.5*(mtx_q[:,j] \
                                + mtx_q[:,j+1] \
                                + mtx_s[:,j] - mtx_s[:,j+1])
            # (4.27) and (4.36) in chang95
            vxl = np.asarray((mtx_qn[:,j+1] \
                              - mtx_q[:,j] - (self.grid_size_t/2.0)*mtx_qt[:,j]) \
                              /(self.grid_size_x/2.0))
            vxr = np.asarray((mtx_q[:,j+1] + (self.grid_size_t/2.0)*mtx_qt[:,j+1] \
                              - mtx_qn[:,j+1]) \
                              /(self.grid_size_x/2.0))
            # (4.39) in chang95
            mtx_qx[:,j+1] = np.asmatrix((vxl*((abs(vxr))**ia) \
                                        + vxr*((abs(vxl))**ia)) \
                                        /(((abs(vxl))**ia) \
                                            + ((abs(vxr))**ia) + 1.0E-60))

    def push_status_along_t(self, number_mesh_points_before_hdt, mtx_q, mtx_qn):
        # hdt means 0.5*grid_size_t
        for j in xrange(1,number_mesh_points_before_hdt):
            mtx_q[:,j] = mtx_qn[:,j]
        number_mesh_points_before_hdt_next_iter = number_mesh_points_before_hdt + 1
        return number_mesh_points_before_hdt_next_iter
<commit_msg>refactoring solver_cese.py: abandon variable a2, a3 and a4<commit_after>#!/usr/bin/python
#
# solver_cese.py
#
# Description:
#     1D Sod Tube sover based on CESE method.
#
#     This program is implemented by OO style to be
#     a part of ipython notebook demo materials.
#
#     The derivation of the equations for the analytic solution
#     is based on the book,
#     Principles of Computational Fluid Dynamics,
#     written by Pieter Wesseling.
#     Or, people could refer to the solvcon website
#     http://www.solvcon.net/en/latest/cese.html#sod-s-shock-tube-problem
#

import numpy as np
import generator_mesh

gmesh = generator_mesh.Mesher()

# a number to claim two floating number value are equal.
delta_precision = 0.0000000000001

GAMMA = 1.4

# Sod tube initial conditions of the left and right
# of the diaphragm
rhol = 1.0
ul = 0.0
pl = 1.0
rhor = 0.125
ur = 0.0
pr = 0.1

ia = 1

class Solver():
    """
    CESE method to generate the 1D Sod tube solution
    @iteration, which is integer
    TODO:
        1. grid_size_x, mesh_x_start, and mesh_x_stop should be integer
        for gmesh.gen_mesh method, which requires integer inputs to be easier
        to generate mesh points along x. This should be smarter and allow user
        to input their own numbers. Users' inputs are 10000 times as
        the real inputs for CESE interation.

    """
    def __init__(self,
                 iteration = 100,
                 grid_size_t = 0.004,
                 mesh_t_stop = 0.2,
                 grid_size_x = 100,
                 mesh_x_start = -10050,
                 mesh_x_stop = 10050):
        self.iteration = iteration
        self.grid_size_t = grid_size_t
        self.mesh_t_stop = mesh_t_stop
        self.mesh_x_start = mesh_x_start
        self.mesh_x_stop = mesh_x_stop

        gmesh.gen_mesh(grid_size_x, mesh_x_start, mesh_x_stop)
        self.mesh_x = gmesh.get_mesh()
        self.grid_size_x = grid_size_x / 10000.0

    def get_cese_solution(self):
        """
        given the mesh size
        output the solution based on CESE method

        iteration: int, please note n iteration will has n+2 mesh points.

        """
        iteration = self.iteration
        grid_size_t = self.grid_size_t
        mesh_t_stop = self.mesh_t_stop
        grid_size_x = self.grid_size_x
        mesh_x_start = self.mesh_x_start
        mesh_x_stop = self.mesh_x_stop
        mesh_x = self.mesh_x

        #self.check_input(iteration,
        #                 grid_size_t,
        #                 mesh_t_stop,
        #                 grid_size_x,
        #                 mesh_x_start,
        #                 mesh_x_stop)

        # mesh point number along x
        mesh_pt_number_x_at_half_t = iteration + 1
        mesh_pt_number_x = iteration + 2

        # u_m of 1D Eular equation.
        # this also means the status of the gas
        # on grids.
        # prefix mtx_ means 'matrix of sth.'
        mtx_q = np.asmatrix(np.zeros(shape=(3, mesh_pt_number_x)))
        # u_m, but means 'next' u_m
        # u_m at the next time step
        mtx_qn = np.asmatrix(np.zeros(shape=(3, mesh_pt_number_x)))
        # partial matrix of q partial x
        mtx_qx = np.asmatrix(np.zeros(shape=(3, mesh_pt_number_x)))
        # partial mastrix of q partial t
        mtx_qt = np.asmatrix(np.zeros(shape=(3, mesh_pt_number_x)))
        # matrix s, it is a part of the marching mtx_qn
        mtx_s = np.asmatrix(np.zeros(shape=(3, mesh_pt_number_x)))
        vxl = np.zeros(shape=(3,1))
        vxr = np.zeros(shape=(3,1))

        mtx_f = np.asmatrix(np.zeros(shape=(3,3)))

        mtx_q[0][0] = rhol
        mtx_q[1][0] = rhol*ul
        mtx_q[2][0] = pl/(GAMMA-1.0) + 0.5*rhol*ul**2.0
        mesh_pt_number_x_at_half_t = iteration + 1
        # initialize the gas status before the diaphragm
        # was removed.
        for i in xrange(mesh_pt_number_x_at_half_t):
            mtx_q[0,i+1] = rhor
            mtx_q[1,i+1] = rhor*ur
            mtx_q[2,i+1] = pr/(GAMMA-1.0) + 0.5*rhor*ur**2.0

        # m is the number used to calculate the status before
        # the half delta t stepping is applied.
        m = 2 # move out from the diaphragm which the 0th grid.
        for i in xrange(iteration):
            self.get_cese_status_before_half_dt(m, mtx_q, mtx_f, mtx_qt, mtx_qx, mtx_s)
            # stepping into the next halt delta t
            # m mesh points along t could introduce m - 1 mesh points along t + 0.5*dt
            self.get_cese_status_after_half_dt(m, mtx_q, mtx_qn, mtx_qt, mtx_qx, mtx_s)
            #  ask the status at t + 0.5*dt to be the next status before the half delta t is applied
            m = self.push_status_along_t(m, mtx_q, mtx_qn)

        #solution__x_start_index = mesh_x.get_start_grid(mesh_pt_number_x)
        solution_x_start_index = 50
        solution = []
        for i in range(mesh_pt_number_x):
            solution_x = mesh_x[i + solution_x_start_index]
            x = solution_x
            solution_rho = mtx_q[0,i]
            solution_v = mtx_q[1,i]/mtx_q[0,i]
            solution_p = (GAMMA-1.0)*(mtx_q[2,i] - 0.5*(solution_v**2)*mtx_q[0,i])
            solution.append((solution_x, solution_rho, solution_v, solution_p))

        return solution

    def get_cese_solution_mesh_size(self,
                                    iteration = 100,
                                    grid_t = 0.004,
                                    mesh_x_start = -0.5050,
                                    mesh_x_stop = 0.5050,
                                    grid_x = 0.01):
        # iteration should be an even number to
        # make sure the grids return to the same
        # as the initial one.
        if iteration % 2 != 0:
            raise Exception(iteration, 'should be even!')
        gmesh.gen_mesh(grid_x * 10000, mesh_x_start * 10000, mesh_x_stop * 10000)
        mesh_pt_number_x_at_half_t = iteration + 1
        # grids generated by the iteration +
        # the most left grid +
        # the most right grid
        mesh_pt_number_x = iteration + 2

    def cal_cese_solution(self, initcondition, mesh, ceseparameters):
        return self.solution

    def get_cese_status_before_half_dt(self, m, mtx_q, mtx_f, mtx_qt, mtx_qx, mtx_s):
        """
        the status is
        """
        for j in xrange(m):
            w2 = mtx_q[1,j]/mtx_q[0,j]
            w3 = mtx_q[2,j]/mtx_q[0,j]
            # f[0][0] = 0.0
            mtx_f[0,1] = 1.0
            # f[0][2] = 0.0
            mtx_f[1,0] = -((3.0-GAMMA)/2.0)*w2**2
            mtx_f[1,1] = (3.0-GAMMA)*w2
            mtx_f[1,2] = GAMMA - 1.0
            mtx_f[2,0] = (GAMMA-1.0)*w2**3 - GAMMA*w2*w3
            mtx_f[2,1] = GAMMA*w3 - (GAMMA-1.0)*w2**2
            mtx_f[2,2] = GAMMA*w2

            # (4.17) in chang95
            mtx_qt[:,j] = -1.0*mtx_f*mtx_qx[:,j]
            # (4.25) in chang95
            # the n_(fmt)_j of the last term should be substitubed
            # by the other terms.
            mtx_s[:,j] = (self.grid_size_x/4.0)*mtx_qx[:,j] + (self.grid_size_t/self.grid_size_x)*mtx_f*mtx_q[:,j] \
                        - (self.grid_size_t/self.grid_size_x)*(self.grid_size_t/4.0)*mtx_f*mtx_f*mtx_qx[:,j]
        return  m, mtx_q, mtx_f, mtx_qt, mtx_qx, mtx_s

    def get_cese_status_after_half_dt(self, m, mtx_q, mtx_qn, mtx_qt, mtx_qx, mtx_s):
        mm = m - 1
        for j in xrange(mm):
            # (4.24) in chang95
            # Please note the j+1 on the left hand side addresses
            # different t (n) from the j/j+1 on the right hand side,
            # which address the other t (n-1/2) on the space-time
            # surface.
            mtx_qn[:,j+1] = 0.5*(mtx_q[:,j] \
                                + mtx_q[:,j+1] \
                                + mtx_s[:,j] - mtx_s[:,j+1])
            # (4.27) and (4.36) in chang95
            vxl = np.asarray((mtx_qn[:,j+1] \
                              - mtx_q[:,j] - (self.grid_size_t/2.0)*mtx_qt[:,j]) \
                              /(self.grid_size_x/2.0))
            vxr = np.asarray((mtx_q[:,j+1] + (self.grid_size_t/2.0)*mtx_qt[:,j+1] \
                              - mtx_qn[:,j+1]) \
                              /(self.grid_size_x/2.0))
            # (4.39) in chang95
            mtx_qx[:,j+1] = np.asmatrix((vxl*((abs(vxr))**ia) \
                                        + vxr*((abs(vxl))**ia)) \
                                        /(((abs(vxl))**ia) \
                                            + ((abs(vxr))**ia) + 1.0E-60))

    def push_status_along_t(self, number_mesh_points_before_hdt, mtx_q, mtx_qn):
        # hdt means 0.5*grid_size_t
        for j in xrange(1,number_mesh_points_before_hdt):
            mtx_q[:,j] = mtx_qn[:,j]
        number_mesh_points_before_hdt_next_iter = number_mesh_points_before_hdt + 1
        return number_mesh_points_before_hdt_next_iter
<|endoftext|>


<commit_before>from typing import Union

from hwt.code import If
from hwt.doc_markers import internal
from hwt.hdl.assignment import Assignment
from hwt.hdl.operator import Operator
from hwt.hdl.operatorDefs import AllOps
from hwt.hdl.types.bits import Bits
from hwt.hdl.types.defs import BOOL, INT
from hwt.hdl.value import Value
from hwt.serializer.generic.context import SerializerCtx
from hwt.synthesizer.rtlLevel.mainBases import RtlSignalBase
from hwt.synthesizer.rtlLevel.rtlSignal import RtlSignal

@internal
def isResultOfTypeConversion(sig):
    if len(sig.drivers) != 1:
        return False

    if sig.hidden:
        d = sig.singleDriver()
        return d.operator != AllOps.INDEX

    return False

class VhdlSerializer_ops():
    # keep in mind that there is no such a thing in vhdl itself
    opPrecedence = {
        AllOps.DOWNTO: 8,
        AllOps.TO: 8,
        AllOps.TERNARY: 8,

        AllOps.XOR: 7,
        AllOps.AND: 7,
        AllOps.OR: 7,

        AllOps.EQ: 6,
        AllOps.NEQ: 6,
        AllOps.GT: 6,
        AllOps.LT: 6,
        AllOps.GE: 6,
        AllOps.LE: 6,

        AllOps.CONCAT: 5,
        AllOps.ADD: 5,
        AllOps.SUB: 5,

        AllOps.NEG: 4,

        AllOps.DIV: 3,
        AllOps.MUL: 3,
        AllOps.MOD: 3,

        AllOps.NOT: 2,
        AllOps.POW: 2,

        AllOps.RISING_EDGE: 1,
        AllOps.FALLING_EDGE: 1,
        AllOps.INDEX: 1,
        AllOps.CALL: 1,
        AllOps.BitsAsSigned: 1,
        AllOps.BitsAsUnsigned: 1,
        AllOps.BitsAsVec: 1,
    }
    _binOps = {
        AllOps.AND: '%s AND %s',
        AllOps.OR: '%s OR %s',
        AllOps.XOR: '%s XOR %s',
        AllOps.CONCAT: '%s & %s',
        AllOps.DIV: '%s / %s',
        AllOps.DOWNTO: '%s-1 DOWNTO %s',
        AllOps.TO: '%s-1 TO %s',
        AllOps.EQ: '%s = %s',
        AllOps.GT: '%s > %s',
        AllOps.GE: '%s >= %s',
        AllOps.LE: '%s <= %s',
        AllOps.POW: '%s ** %s',
        AllOps.LT: '%s < %s',
        AllOps.SUB: '%s - %s',
        AllOps.MUL: '%s * %s',
        AllOps.NEQ: '%s /= %s',
        AllOps.ADD: '%s + %s',
    }
    _unaryOps = {
        AllOps.NOT: "NOT %s",
        AllOps.NEG: "-(%s)",
        AllOps.RISING_EDGE: "RISING_EDGE(%s)",
        AllOps.FALLING_EDGE: "FALLING_EDGE(%s)",
        AllOps.BitsAsSigned: "SIGNED(%s)",
        AllOps.BitsAsUnsigned: "UNSIGNED(%s)",
        AllOps.BitsAsVec: "STD_LOGIC_VECTOR(%s)",
    }

    @internal
    @classmethod
    def _tmp_var_for_ternary(cls, val: RtlSignal, ctx: SerializerCtx):
        """
        Optionaly convert boolean to std_logic_vector
        """
        o = ctx.createTmpVarFn("tmpTernary", val._dtype)
        cond, ifTrue, ifFalse = val.drivers[0].operands
        if_ = If(cond)
        if_.ifTrue.append(Assignment(ifTrue, o,
                                     virtual_only=True,
                                     parentStm=if_))
        if_.ifFalse = []
        if_.ifFalse.append(Assignment(ifFalse, o,
                                      virtual_only=True,
                                      parentStm=if_))
        if_._outputs.append(o)
        for obj in (cond, ifTrue, ifFalse):
            if isinstance(obj, RtlSignalBase):
                if_._inputs.append(obj)
        o.drivers.append(if_)
        return o

    @classmethod
    def _as_Bits(cls, val: Union[RtlSignal, Value], ctx: SerializerCtx):
        if val._dtype == BOOL:
            bit1_t = Bits(1)
            o = ctx.createTmpVarFn("tmpBool2std_logic", bit1_t)
            ifTrue, ifFalse = bit1_t.from_py(1), bit1_t.from_py(0)
            if_ = If(val)
            if_.ifTrue.append(Assignment(ifTrue, o, virtual_only=True, parentStm=if_))
            if_.ifFalse = []
            if_.ifFalse.append(Assignment(ifFalse, o, virtual_only=True, parentStm=if_))
            if_._outputs.append(o)
            o.drivers.append(if_)
            return o
        else:
            assert isinstance(val._dtype, Bits), val._dtype
            return val

    @internal
    @classmethod
    def _operand(cls, operand: Union[RtlSignal, Value], i: int,
                 oper: Operator,
                 expr_requires_parenthesis: bool,
                 cancel_parenthesis: bool,
                 ctx: SerializerCtx):
        # no nested ternary in expressions like
        # ( '1'  WHEN r = f ELSE  '0' ) & "0"
        try:
            isTernaryOp = operand.hidden\
                and operand.drivers[0].operator == AllOps.TERNARY
        except (AttributeError, IndexError):
            isTernaryOp = False

        if isTernaryOp:
            # rewrite ternary operator as if
            operand = cls._tmp_var_for_ternary(operand, ctx)

        return super(VhdlSerializer_ops, cls)._operand(
                operand,
                i,
                oper,
                not isTernaryOp and expr_requires_parenthesis,
                isTernaryOp or cancel_parenthesis,
                ctx)

    @classmethod
    def Operator(cls, op: Operator, ctx: SerializerCtx):
        ops = op.operands
        o = op.operator

        op_str = cls._unaryOps.get(o, None)
        if op_str is not None:
            cancel_parenthesis = op_str[-1] == ")"
            return op_str % (cls._operand(ops[0], 0, op, False, cancel_parenthesis, ctx))

        op_str = cls._binOps.get(o, None)
        if op_str is not None:
            res_t = op.result._dtype
            if isinstance(res_t, Bits) and res_t != BOOL:
                op0 = cls._as_Bits(ops[0], ctx)
                op1 = cls._as_Bits(ops[1], ctx)
                op = op.operator._evalFn(op0, op1).drivers[0]
            return cls._bin_op(op, op_str, ctx)

        if o == AllOps.CALL:
            return "%s(%s)" % (
                cls.FunctionContainer(ops[0]),
                ", ".join(
                    # operand i does not matter as thy are all in ()
                    map(lambda op: cls._operand(op, 1, o, False, True, ctx), ops[1:])
                    )
                )
        elif o == AllOps.INDEX:
            op0, op1 = ops
            if isinstance(op0, RtlSignalBase) and isResultOfTypeConversion(op0):
                op0 = ctx.createTmpVarFn("tmpTypeConv", op0._dtype)
                op0.def_val = ops[0]

            # if the op0 is not signal or other index index operator it is extracted
            # as tmp variable
            op0_str = cls._operand(op0, 0, op, False, False, ctx)
            op1_str = cls._operand(ops[1], 1, op, False, True, ctx)

            if isinstance(op1._dtype, Bits) and op1._dtype != INT:
                sig = op1._dtype.signed
                if sig is None:
                    op1_str = "UNSIGNED(%s)" % op1_str
                op1_str = "TO_INTEGER(%s)" % op1_str

            return "%s(%s)" % (op0_str, op1_str)
        elif o == AllOps.TERNARY:
            op0 = cls.condAsHdl([ops[0]], True, ctx)
            op1 = cls._operand(ops[1], 1, op, False, False, ctx)
            op2 = cls._operand(ops[2], 2, op, False, False, ctx)
            return "%s WHEN %s ELSE %s" % (op0, op1, op2)
        else:
            raise NotImplementedError(
                "Do not know how to convert %s to vhdl" % (o))
<commit_msg>vhdl: fix missing parenthesis around different operator in sequence<commit_after>from typing import Union

from hwt.code import If
from hwt.doc_markers import internal
from hwt.hdl.assignment import Assignment
from hwt.hdl.operator import Operator
from hwt.hdl.operatorDefs import AllOps
from hwt.hdl.types.bits import Bits
from hwt.hdl.types.defs import BOOL, INT
from hwt.hdl.value import Value
from hwt.serializer.generic.context import SerializerCtx
from hwt.synthesizer.rtlLevel.mainBases import RtlSignalBase
from hwt.synthesizer.rtlLevel.rtlSignal import RtlSignal

@internal
def isResultOfTypeConversion(sig):
    if len(sig.drivers) != 1:
        return False

    if sig.hidden:
        d = sig.singleDriver()
        return d.operator != AllOps.INDEX

    return False

class VhdlSerializer_ops():
    # keep in mind that there is no such a thing in vhdl itself
    opPrecedence = {
        AllOps.DOWNTO: 8,
        AllOps.TO: 8,
        AllOps.TERNARY: 8,

        AllOps.XOR: 7,
        AllOps.AND: 7,
        AllOps.OR: 7,

        AllOps.EQ: 6,
        AllOps.NEQ: 6,
        AllOps.GT: 6,
        AllOps.LT: 6,
        AllOps.GE: 6,
        AllOps.LE: 6,

        AllOps.CONCAT: 5,
        AllOps.ADD: 5,
        AllOps.SUB: 5,

        AllOps.NEG: 4,

        AllOps.DIV: 3,
        AllOps.MUL: 3,
        AllOps.MOD: 3,

        AllOps.NOT: 2,
        AllOps.POW: 2,

        AllOps.RISING_EDGE: 1,
        AllOps.FALLING_EDGE: 1,
        AllOps.INDEX: 1,
        AllOps.CALL: 1,
        AllOps.BitsAsSigned: 1,
        AllOps.BitsAsUnsigned: 1,
        AllOps.BitsAsVec: 1,
    }
    _binOps = {
        AllOps.AND: '%s AND %s',
        AllOps.OR: '%s OR %s',
        AllOps.XOR: '%s XOR %s',
        AllOps.CONCAT: '%s & %s',
        AllOps.DIV: '%s / %s',
        AllOps.DOWNTO: '%s-1 DOWNTO %s',
        AllOps.TO: '%s-1 TO %s',
        AllOps.EQ: '%s = %s',
        AllOps.GT: '%s > %s',
        AllOps.GE: '%s >= %s',
        AllOps.LE: '%s <= %s',
        AllOps.POW: '%s ** %s',
        AllOps.LT: '%s < %s',
        AllOps.SUB: '%s - %s',
        AllOps.MUL: '%s * %s',
        AllOps.NEQ: '%s /= %s',
        AllOps.ADD: '%s + %s',
    }
    _unaryOps = {
        AllOps.NOT: "NOT %s",
        AllOps.NEG: "-(%s)",
        AllOps.RISING_EDGE: "RISING_EDGE(%s)",
        AllOps.FALLING_EDGE: "FALLING_EDGE(%s)",
        AllOps.BitsAsSigned: "SIGNED(%s)",
        AllOps.BitsAsUnsigned: "UNSIGNED(%s)",
        AllOps.BitsAsVec: "STD_LOGIC_VECTOR(%s)",
    }

    @internal
    @classmethod
    def _tmp_var_for_ternary(cls, val: RtlSignal, ctx: SerializerCtx):
        """
        Optionaly convert boolean to std_logic_vector
        """
        o = ctx.createTmpVarFn("tmpTernary", val._dtype)
        cond, ifTrue, ifFalse = val.drivers[0].operands
        if_ = If(cond)
        if_.ifTrue.append(Assignment(ifTrue, o,
                                     virtual_only=True,
                                     parentStm=if_))
        if_.ifFalse = []
        if_.ifFalse.append(Assignment(ifFalse, o,
                                      virtual_only=True,
                                      parentStm=if_))
        if_._outputs.append(o)
        for obj in (cond, ifTrue, ifFalse):
            if isinstance(obj, RtlSignalBase):
                if_._inputs.append(obj)
        o.drivers.append(if_)
        return o

    @classmethod
    def _as_Bits(cls, val: Union[RtlSignal, Value], ctx: SerializerCtx):
        if val._dtype == BOOL:
            bit1_t = Bits(1)
            o = ctx.createTmpVarFn("tmpBool2std_logic", bit1_t)
            ifTrue, ifFalse = bit1_t.from_py(1), bit1_t.from_py(0)
            if_ = If(val)
            if_.ifTrue.append(Assignment(ifTrue, o, virtual_only=True, parentStm=if_))
            if_.ifFalse = []
            if_.ifFalse.append(Assignment(ifFalse, o, virtual_only=True, parentStm=if_))
            if_._outputs.append(o)
            o.drivers.append(if_)
            return o
        else:
            assert isinstance(val._dtype, Bits), val._dtype
            return val

    @internal
    @classmethod
    def _operand(cls, operand: Union[RtlSignal, Value], i: int,
                 oper: Operator,
                 expr_requires_parenthesis: bool,
                 cancel_parenthesis: bool,
                 ctx: SerializerCtx):
        # no nested ternary in expressions like
        # ( '1'  WHEN r = f ELSE  '0' ) & "0"
        try:
            isTernaryOp = operand.hidden\
                and operand.drivers[0].operator == AllOps.TERNARY
        except (AttributeError, IndexError):
            isTernaryOp = False

        if isTernaryOp:
            # rewrite ternary operator as if
            operand = cls._tmp_var_for_ternary(operand, ctx)

        return super(VhdlSerializer_ops, cls)._operand(
                operand,
                i,
                oper,
                not isTernaryOp and expr_requires_parenthesis,
                isTernaryOp or cancel_parenthesis,
                ctx)

    @classmethod
    def Operator(cls, op: Operator, ctx: SerializerCtx):
        ops = op.operands
        o = op.operator

        op_str = cls._unaryOps.get(o, None)
        if op_str is not None:
            cancel_parenthesis = op_str[-1] == ")"
            return op_str % (cls._operand(ops[0], 0, op, False, cancel_parenthesis, ctx))

        op_str = cls._binOps.get(o, None)
        if op_str is not None:
            res_t = op.result._dtype
            if isinstance(res_t, Bits) and res_t != BOOL:
                op0 = cls._as_Bits(ops[0], ctx)
                op1 = cls._as_Bits(ops[1], ctx)
                op = op.operator._evalFn(op0, op1).drivers[0]
            return cls._bin_op(op, op_str, ctx, expr_requires_parenthesis=True)

        if o == AllOps.CALL:
            return "%s(%s)" % (
                cls.FunctionContainer(ops[0]),
                ", ".join(
                    # operand i does not matter as thy are all in ()
                    map(lambda op: cls._operand(op, 1, o, False, True, ctx), ops[1:])
                    )
                )
        elif o == AllOps.INDEX:
            op0, op1 = ops
            if isinstance(op0, RtlSignalBase) and isResultOfTypeConversion(op0):
                op0 = ctx.createTmpVarFn("tmpTypeConv", op0._dtype)
                op0.def_val = ops[0]

            # if the op0 is not signal or other index index operator it is extracted
            # as tmp variable
            op0_str = cls._operand(op0, 0, op, False, False, ctx)
            op1_str = cls._operand(ops[1], 1, op, False, True, ctx)

            if isinstance(op1._dtype, Bits) and op1._dtype != INT:
                sig = op1._dtype.signed
                if sig is None:
                    op1_str = "UNSIGNED(%s)" % op1_str
                op1_str = "TO_INTEGER(%s)" % op1_str

            return "%s(%s)" % (op0_str, op1_str)
        elif o == AllOps.TERNARY:
            op0 = cls.condAsHdl([ops[0]], True, ctx)
            op1 = cls._operand(ops[1], 1, op, False, False, ctx)
            op2 = cls._operand(ops[2], 2, op, False, False, ctx)
            return "%s WHEN %s ELSE %s" % (op0, op1, op2)
        else:
            raise NotImplementedError(
                "Do not know how to convert %s to vhdl" % (o))
<|endoftext|>