<commit_before># vim: tabstop=4 shiftwidth=4 softtabstop=4
# -*- coding: utf-8 -*-

#    Copyright (C) 2012 Yahoo! Inc. All Rights Reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

def center_text(text, fill, max_len):
    return '{0:{fill}{align}{size}}'.format(text, fill=fill, align="^", size=max_len)

def _pformat_list(lst, item_max_len):
    lines = []
    if not lst:
        lines.append("+------+")
        lines.append("'------'")
        return "\n".join(lines)
    entries = []
    max_len = 0
    for i in lst:
        entry = pformat(i, item_max_len)
        value_lines = entry.split("\n")
        if len(value_lines) > 1:
            for v in value_lines:
                max_len = max(max_len, len(v) + 2)
        else:
            max_len = max(max_len, len(entry) + 2)
        entries.append(entry)
    lines.append("+%s+" % ("-" * (max_len)))
    for i in entries:
        value_lines = i.split("\n")
        for j in range(0, len(value_lines)):
            lines.append("|%s|" % (center_text(value_lines[j], ' ', max_len)))
    lines.append("'%s'" % ("-" * (max_len)))
    return "\n".join(lines)

def _pformat_hash(hsh, item_max_len):
    lines = []
    if not hsh:
        lines.append("+-----+-----+")
        lines.append("'-----+-----'")
        return "\n".join(lines)
    # Figure out the lengths to place items in...
    max_key_len = 0
    max_value_len = 0
    entries = []
    for (k, v) in hsh.items():
        entry = ("%s" % (_pformat_escape(k, item_max_len)), "%s" % (pformat(v, item_max_len)))
        max_key_len = max(max_key_len, len(entry[0]) + 2)
        value_lines = entry[1].split("\n")
        for v in value_lines:
            max_value_len = max(max_value_len, len(v) + 2)
        entries.append(entry)
    # Now actually do the placement since we have the lengths
    lines.append("+%s+%s+" % ("-" * max_key_len, "-" * max_value_len))
    for (key, value) in entries:
        value_lines = value.split("\n")
        lines.append("|%s|%s|" % (center_text(key, ' ', max_key_len),
                                  center_text(value_lines[0], ' ', max_value_len)))
        if len(value_lines) > 1:
            for j in range(1, len(value_lines)):
                lines.append("|%s|%s|" % (center_text("-", ' ', max_key_len),
                                          center_text(value_lines[j], ' ', max_value_len)))
    lines.append("'%s+%s'" % ("-" * max_key_len, "-" * max_value_len))
    return "\n".join(lines)

def _pformat_escape(item, item_max_len):
    item = _pformat_simple(item, item_max_len)
    item = item.replace("\n", "\\n")
    item = item.replace("\t", "\\t")
    return item

def _pformat_simple(item, item_max_len):
    if item_max_len is None or item_max_len < 0:
        return  "%s" % (item)
    if item_max_len == 0:
        return ''
    item_str = "%s" % (item)
    if len(item_str) > item_max_len:
        # TODO(harlowja) use utf8 ellipse or '...'??
        item_str = item_str[0:item_max_len] + u"…"
    return item_str

def pformat(item, item_max_len):
    if isinstance(item, (list, set, tuple)):
        return _pformat_list(item, item_max_len)
    elif isinstance(item, (dict)):
        return _pformat_hash(item, item_max_len)
    else:
        return _pformat_simple(item, item_max_len)

def pprint(item, item_max_len):
    print("%s" % (pformat(item, item_max_len)))
<commit_msg>Guess that didn't work<commit_after># vim: tabstop=4 shiftwidth=4 softtabstop=4

#    Copyright (C) 2012 Yahoo! Inc. All Rights Reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

def center_text(text, fill, max_len):
    return '{0:{fill}{align}{size}}'.format(text, fill=fill, align="^", size=max_len)

def _pformat_list(lst, item_max_len):
    lines = []
    if not lst:
        lines.append("+------+")
        lines.append("'------'")
        return "\n".join(lines)
    entries = []
    max_len = 0
    for i in lst:
        entry = pformat(i, item_max_len)
        value_lines = entry.split("\n")
        if len(value_lines) > 1:
            for v in value_lines:
                max_len = max(max_len, len(v) + 2)
        else:
            max_len = max(max_len, len(entry) + 2)
        entries.append(entry)
    lines.append("+%s+" % ("-" * (max_len)))
    for i in entries:
        value_lines = i.split("\n")
        for j in range(0, len(value_lines)):
            lines.append("|%s|" % (center_text(value_lines[j], ' ', max_len)))
    lines.append("'%s'" % ("-" * (max_len)))
    return "\n".join(lines)

def _pformat_hash(hsh, item_max_len):
    lines = []
    if not hsh:
        lines.append("+-----+-----+")
        lines.append("'-----+-----'")
        return "\n".join(lines)
    # Figure out the lengths to place items in...
    max_key_len = 0
    max_value_len = 0
    entries = []
    for (k, v) in hsh.items():
        entry = ("%s" % (_pformat_escape(k, item_max_len)), "%s" % (pformat(v, item_max_len)))
        max_key_len = max(max_key_len, len(entry[0]) + 2)
        value_lines = entry[1].split("\n")
        for v in value_lines:
            max_value_len = max(max_value_len, len(v) + 2)
        entries.append(entry)
    # Now actually do the placement since we have the lengths
    lines.append("+%s+%s+" % ("-" * max_key_len, "-" * max_value_len))
    for (key, value) in entries:
        value_lines = value.split("\n")
        lines.append("|%s|%s|" % (center_text(key, ' ', max_key_len),
                                  center_text(value_lines[0], ' ', max_value_len)))
        if len(value_lines) > 1:
            for j in range(1, len(value_lines)):
                lines.append("|%s|%s|" % (center_text("-", ' ', max_key_len),
                                          center_text(value_lines[j], ' ', max_value_len)))
    lines.append("'%s+%s'" % ("-" * max_key_len, "-" * max_value_len))
    return "\n".join(lines)

def _pformat_escape(item, item_max_len):
    item = _pformat_simple(item, item_max_len)
    item = item.replace("\n", "\\n")
    item = item.replace("\t", "\\t")
    return item

def _pformat_simple(item, item_max_len):
    if item_max_len is None or item_max_len < 0:
        return  "%s" % (item)
    if item_max_len == 0:
        return ''
    item_str = "%s" % (item)
    if len(item_str) > item_max_len:
        # TODO(harlowja) use utf8 ellipse or '...'??
        item_str = item_str[0:item_max_len] + '...'
    return item_str

def pformat(item, item_max_len):
    if isinstance(item, (list, set, tuple)):
        return _pformat_list(item, item_max_len)
    elif isinstance(item, (dict)):
        return _pformat_hash(item, item_max_len)
    else:
        return _pformat_simple(item, item_max_len)

def pprint(item, item_max_len):
    print("%s" % (pformat(item, item_max_len)))
<|endoftext|>


<commit_before>import re
import time
from datetime import datetime
from queue import PriorityQueue
from .colors import uncolored

TIMESTAMP_PATTERN = "%Y-%m-%d %H:%M:%S"

def to_timestamp(t):
    return "-".center(19) if t is None else time.strftime(TIMESTAMP_PATTERN, time.localtime(t))

YEAR = time.strftime("%Y")
MONTH = time.strftime("%m")
DAY = time.strftime("%d")

TIMESTAMP_GETTERS = [

    # 01:21:27
    (re.compile(r"^(\d+:\d+:\d+)"),
     lambda ts: time.mktime(time.strptime("%s-%s-%s %s" % (YEAR, MONTH, DAY, ts), "%Y-%m-%d %H:%M:%S"))),

    # Apr 6 17:13:40
    (re.compile(r"^(\w{3} +\d+ +\d+:\d+:\d+)"),
     lambda ts: time.mktime(time.strptime("%s %s" % (YEAR, ts), "%Y %b %d %H:%M:%S"))),

    # 2018-12-15T02:11:06+0200
    (re.compile(r"^(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\+\d{4})"),
     lambda ts: datetime.strptime(ts, '%Y-%m-%dT%H:%M:%S%z').timestamp()),

    # 2018-12-15T02:11:06.123456+02:00
    (re.compile(r"^(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{6})(\+\d{2}):(\d{2})"),
     lambda *args: datetime.strptime("{}{}{}".format(*args), '%Y-%m-%dT%H:%M:%S.%f%z').timestamp()),

    # 2018-04-06 17:13:40,955
    # 2018-04-23 04:48:11,811|
    (re.compile(r"(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}),(\d{3})[| ]"),
     lambda ts, ms: time.mktime(time.strptime(ts, "%Y-%m-%d %H:%M:%S")) + float(ms) / 1000),

    # 2018-04-06 17:13:40
    # 2018-04-06 17:13:40.955356
    # [2018/04/06 17:13:40
    # [2018/04/06 17:13:40.955356
    (re.compile(r"^\[?(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2})(?:\.(\d{6}))?"),
     lambda ts, ms: time.mktime(time.strptime(ts.replace("/", "-"), "%Y-%m-%d %H:%M:%S")) + float(ms or 0) / 1000000),

    # for strace logs
    # 16255 15:08:52.554223
    (re.compile(r"\d+ (\d{2}:\d{2}:\d{2}).(\d{6})"),
     lambda ts, ms: time.mktime(time.strptime("%s-%s-%s %s" % (YEAR, MONTH, DAY, ts), "%Y-%m-%d %H:%M:%S")) + float(ms) / 1000000),
]

class TimestampedStream(object):

    def __init__(self, stream, prefix="> "):
        self.name = stream.name if hasattr(stream, "name") else repr(stream)
        self.stream = iter(stream)
        self.prefix = prefix
        self.filler = " "*len(self.prefix)
        self._untimestamp = None

    def __gt__(self, other):
        return id(self) > id(other)

    def get_next(self):
        "Get next line in the stream and the stream itself, or None if stream ended"
        try:
            line = next(self.stream)
        except StopIteration:
            return
        else:
            ts = self.get_timestamp(uncolored(line))
            return ts, (self.prefix if ts else self.filler) + line, self

    def get_timestamp(self, line):
        """
        Find the timestamp if exists, and return as a float
        """
        if not line.startswith(" "):
            if not self._untimestamp:
                for regex, converter in TIMESTAMP_GETTERS:
                    match = regex.search(line)
                    if match:
                        # cache the regex and conversion funcs for later
                        self._untimestamp = converter, regex
                        break
            else:
                converter, regex = self._untimestamp
                match = regex.search(line)
            if match:
                return converter(*match.groups())
        return 0

def iter_zipped_logs(*log_streams, prefix="DARK_GRAY@{> }@", show_intervals=None):
    """
    Line iterator that merges lines from different log streams based on their timestamp.
    Timestamp patterns are found in the TIMESTAMP_GETTERS list in this module.

    :param prefix: Prepend this prefix to each line where a timestamp was identified
    :param show_intervals: `s` or `ms` - Prepend the duration since the previous log line
    """

    # A sorted queue of (timestamp, stream) tuples (lowest-timestamp first)
    streams = PriorityQueue()
    stream_names = []
    for i, stream in enumerate(log_streams):
        if not isinstance(stream, tuple):
            tstream = TimestampedStream(stream, prefix)
        else:
            tstream = TimestampedStream(*stream)

        n = tstream.get_next()
        if n:
            stream_names.append(tstream.name)
            streams.put(n)

    last_ts = None
    if show_intervals:
        from easypy.units import Duration

        def formatted(line, current_ts, last_ts):
            fmt = "{:>7}{}"
            if (current_ts and last_ts):
                return fmt.format(Duration(current_ts - last_ts).render(show_intervals), line)
            else:
                return fmt.format("", line)
    else:
        def formatted(line, current_ts, last_ts):
            return line

    while not streams.empty():
        current_ts, line, stream = streams.get()
        yield formatted(line, current_ts, last_ts)
        last_ts = current_ts
        while True:
            n = stream.get_next()
            if not n:
                break   # stream ended
            ts, line, stream = n
            if ts and ts > current_ts:
                streams.put((ts, line, stream))
                break   # timestamp advanced
            yield formatted(line, ts, last_ts)
            if ts:
                last_ts = ts

def main():
    import sys
    import argparse
    parser = argparse.ArgumentParser(description='ZipLog - merge logs by timestamps')
    parser.add_argument(
        'logs', metavar='N', type=str, nargs='+',
        help='Log files; Use "-" for STDIN')
    parser.add_argument(
        '-i', '--interval', dest='interval', default=None,
        help="Show interval by seconds (s), or milliseconds (ms)")
    parser.add_argument(
        '-p', '--prefix', dest='prefix', default="> ",
        help="A prefix to prepend to timestamped lines")
    ns = parser.parse_args(sys.argv[1:])

    files = [sys.stdin if f == "-" else open(f) for f in ns.logs]
    try:
        for line in iter_zipped_logs(*files, show_intervals=ns.interval, prefix=ns.prefix):
            print(line, end="")
    except BrokenPipeError:
        pass

if __name__ == "__main__":
    main()
<commit_msg>ziplog: conform the default prefix for the cli and the function<commit_after>import re
import time
from datetime import datetime
from queue import PriorityQueue
from .colors import uncolored

TIMESTAMP_PATTERN = "%Y-%m-%d %H:%M:%S"

def to_timestamp(t):
    return "-".center(19) if t is None else time.strftime(TIMESTAMP_PATTERN, time.localtime(t))

YEAR = time.strftime("%Y")
MONTH = time.strftime("%m")
DAY = time.strftime("%d")

TIMESTAMP_GETTERS = [

    # 01:21:27
    (re.compile(r"^(\d+:\d+:\d+)"),
     lambda ts: time.mktime(time.strptime("%s-%s-%s %s" % (YEAR, MONTH, DAY, ts), "%Y-%m-%d %H:%M:%S"))),

    # Apr 6 17:13:40
    (re.compile(r"^(\w{3} +\d+ +\d+:\d+:\d+)"),
     lambda ts: time.mktime(time.strptime("%s %s" % (YEAR, ts), "%Y %b %d %H:%M:%S"))),

    # 2018-12-15T02:11:06+0200
    (re.compile(r"^(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\+\d{4})"),
     lambda ts: datetime.strptime(ts, '%Y-%m-%dT%H:%M:%S%z').timestamp()),

    # 2018-12-15T02:11:06.123456+02:00
    (re.compile(r"^(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{6})(\+\d{2}):(\d{2})"),
     lambda *args: datetime.strptime("{}{}{}".format(*args), '%Y-%m-%dT%H:%M:%S.%f%z').timestamp()),

    # 2018-04-06 17:13:40,955
    # 2018-04-23 04:48:11,811|
    (re.compile(r"(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}),(\d{3})[| ]"),
     lambda ts, ms: time.mktime(time.strptime(ts, "%Y-%m-%d %H:%M:%S")) + float(ms) / 1000),

    # 2018-04-06 17:13:40
    # 2018-04-06 17:13:40.955356
    # [2018/04/06 17:13:40
    # [2018/04/06 17:13:40.955356
    (re.compile(r"^\[?(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2})(?:\.(\d{6}))?"),
     lambda ts, ms: time.mktime(time.strptime(ts.replace("/", "-"), "%Y-%m-%d %H:%M:%S")) + float(ms or 0) / 1000000),

    # for strace logs
    # 16255 15:08:52.554223
    (re.compile(r"\d+ (\d{2}:\d{2}:\d{2}).(\d{6})"),
     lambda ts, ms: time.mktime(time.strptime("%s-%s-%s %s" % (YEAR, MONTH, DAY, ts), "%Y-%m-%d %H:%M:%S")) + float(ms) / 1000000),
]

class TimestampedStream(object):

    def __init__(self, stream, prefix="> "):
        self.name = stream.name if hasattr(stream, "name") else repr(stream)
        self.stream = iter(stream)
        self.prefix = prefix
        self.filler = " "*len(self.prefix)
        self._untimestamp = None

    def __gt__(self, other):
        return id(self) > id(other)

    def get_next(self):
        "Get next line in the stream and the stream itself, or None if stream ended"
        try:
            line = next(self.stream)
        except StopIteration:
            return
        else:
            ts = self.get_timestamp(uncolored(line))
            return ts, (self.prefix if ts else self.filler) + line, self

    def get_timestamp(self, line):
        """
        Find the timestamp if exists, and return as a float
        """
        if not line.startswith(" "):
            if not self._untimestamp:
                for regex, converter in TIMESTAMP_GETTERS:
                    match = regex.search(line)
                    if match:
                        # cache the regex and conversion funcs for later
                        self._untimestamp = converter, regex
                        break
            else:
                converter, regex = self._untimestamp
                match = regex.search(line)
            if match:
                return converter(*match.groups())
        return 0

def iter_zipped_logs(*log_streams, prefix="> ", show_intervals=None):
    """
    Line iterator that merges lines from different log streams based on their timestamp.
    Timestamp patterns are found in the TIMESTAMP_GETTERS list in this module.

    :param prefix: Prepend this prefix to each line where a timestamp was identified
    :param show_intervals: `s` or `ms` - Prepend the duration since the previous log line
    """

    # A sorted queue of (timestamp, stream) tuples (lowest-timestamp first)
    streams = PriorityQueue()
    stream_names = []
    for i, stream in enumerate(log_streams):
        if not isinstance(stream, tuple):
            tstream = TimestampedStream(stream, prefix)
        else:
            tstream = TimestampedStream(*stream)

        n = tstream.get_next()
        if n:
            stream_names.append(tstream.name)
            streams.put(n)

    last_ts = None
    if show_intervals:
        from easypy.units import Duration

        def formatted(line, current_ts, last_ts):
            fmt = "{:>7}{}"
            if (current_ts and last_ts):
                return fmt.format(Duration(current_ts - last_ts).render(show_intervals), line)
            else:
                return fmt.format("", line)
    else:
        def formatted(line, current_ts, last_ts):
            return line

    while not streams.empty():
        current_ts, line, stream = streams.get()
        yield formatted(line, current_ts, last_ts)
        last_ts = current_ts
        while True:
            n = stream.get_next()
            if not n:
                break   # stream ended
            ts, line, stream = n
            if ts and ts > current_ts:
                streams.put((ts, line, stream))
                break   # timestamp advanced
            yield formatted(line, ts, last_ts)
            if ts:
                last_ts = ts

def main():
    import sys
    import argparse
    parser = argparse.ArgumentParser(description='ZipLog - merge logs by timestamps')
    parser.add_argument(
        'logs', metavar='N', type=str, nargs='+',
        help='Log files; Use "-" for STDIN')
    parser.add_argument(
        '-i', '--interval', dest='interval', default=None,
        help="Show interval by seconds (s), or milliseconds (ms)")
    parser.add_argument(
        '-p', '--prefix', dest='prefix', default="> ",
        help="A prefix to prepend to timestamped lines")
    ns = parser.parse_args(sys.argv[1:])

    files = [sys.stdin if f == "-" else open(f) for f in ns.logs]
    try:
        for line in iter_zipped_logs(*files, show_intervals=ns.interval, prefix=ns.prefix):
            print(line, end="")
    except BrokenPipeError:
        pass

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before><commit_msg>Fixed a few bugs that occurred when the first periodic message was added to the transmit thread. The message was either sent faster or slower than expected.<commit_after><|endoftext|>


<commit_before># Copyright (c) 2010-2013, GEM Foundation.
#
# OpenQuake is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# OpenQuake is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with OpenQuake.  If not, see <http://www.gnu.org/licenses/>.

"""
Functions for exporting risk artifacts from the database.
"""

import os

from openquake.db import models
from openquake.export import core
from nrml.risk import writers

LOSS_CURVE_FILENAME_FMT = 'loss-curves-%(loss_curve_id)s.xml'
LOSS_MAP_FILENAME_FMT = 'loss-maps-%(loss_map_id)s-poe-%(poe)s.xml'
BCR_FILENAME_FMT = 'bcr-distribution-%(bcr_distribution_id)s.xml'

def export(output_id, target_dir):
    """
    Export the given risk calculation output from the database to the
    specified directory. See `openquake.export.hazard.export` for more
    details.
    """
    output = models.Output.objects.get(id=output_id)
    return globals().get("export_%s" % output.output_type,
                         core._export_fn_not_implemented)(
                             output, os.path.expanduser(target_dir))

def _export_common(output):
    """
    Returns a dict containing the common arguments used by nrml
    serializers to serialize the risk calculation `output`.
    """
    risk_calculation = output.oq_job.risk_calculation
    investigation_time = (
        risk_calculation.get_hazard_calculation().investigation_time)
    statistics, quantile_value = risk_calculation.hazard_statistics

    source_model_tree_path, gsim_tree_path = None, None
    if statistics is not None:
        lt_paths = risk_calculation.hazard_logic_tree_paths

        if lt_paths:
            source_model_tree_path, gsim_tree_path = [
                core.LT_PATH_JOIN_TOKEN.join(x) for x in lt_paths]

    unit = risk_calculation.exposure_model.stco_unit

    return dict(investigation_time=investigation_time,
                statistics=statistics,
                quantile_value=quantile_value,
                source_model_tree_path=source_model_tree_path,
                gsim_tree_path=gsim_tree_path,
                unit=unit)

@core.makedirs
def export_agg_loss_curve(output, target_dir):
    """
    Export `output` to `target_dir` by using a nrml loss curves
    serializer
    """
    args = _export_common(output)
    args['path'] = os.path.join(target_dir, LOSS_CURVE_FILENAME_FMT % {
        'loss_curve_id': output.loss_curve.id})
    writers.AggregateLossCurveXMLWriter(**args).serialize(
        output.loss_curve.aggregatelosscurvedata)
    return [args['path']]

@core.makedirs
def export_loss_curve(output, target_dir):
    """
    Export `output` to `target_dir` by using a nrml loss curves
    serializer
    """
    args = _export_common(output)
    args['path'] = os.path.join(target_dir, LOSS_CURVE_FILENAME_FMT % {
        'loss_curve_id': output.loss_curve.id})
    if output.loss_curve.insured:
        args['insured'] = True

    writers.LossCurveXMLWriter(**args).serialize(
        output.loss_curve.losscurvedata_set.all().order_by('asset_ref'))
    return [args['path']]

@core.makedirs
def export_loss_map(output, target_dir):
    """
    Export `output` to `target_dir` by using a nrml loss map
    serializer
    """
    risk_calculation = output.oq_job.risk_calculation
    args = _export_common(output)
    args.update(
        dict(path=os.path.join(target_dir, LOSS_MAP_FILENAME_FMT % {
            'loss_map_id': output.loss_map.id,
            'poe': output.loss_map.poe}),
            poe=output.loss_map.poe,
            loss_category=risk_calculation.exposure_model.category))
    writers.LossMapXMLWriter(**args).serialize(
        output.loss_map.lossmapdata_set.all().order_by('asset_ref'))
    return [args['path']]

@core.makedirs
def export_bcr_distribution(output, target_dir):
    """
    Export `output` to `target_dir` by using a nrml bcr distribution
    serializer
    """
    risk_calculation = output.oq_job.risk_calculation
    args = _export_common(output)

    args.update(
        dict(path=os.path.join(target_dir, BCR_FILENAME_FMT % {
            'bcr_distribution_id': output.bcr_distribution.id}),
            interest_rate=risk_calculation.interest_rate,
            asset_life_expectancy=risk_calculation.asset_life_expectancy))
    del args['investigation_time']

    writers.BCRMapXMLWriter(**args).serialize(
        output.bcr_distribution.bcrdistributiondata_set.all().order_by(
            'asset_ref'))
    return [args['path']]

@core.makedirs
def export_ins_loss_curve(*args, **kwargs):
    return export_loss_curve(*args, **kwargs)
<commit_msg>updated _export_common<commit_after># Copyright (c) 2010-2013, GEM Foundation.
#
# OpenQuake is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# OpenQuake is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with OpenQuake.  If not, see <http://www.gnu.org/licenses/>.

"""
Functions for exporting risk artifacts from the database.
"""

import os

from openquake.db import models
from openquake.export import core
from nrml.risk import writers

LOSS_CURVE_FILENAME_FMT = 'loss-curves-%(loss_curve_id)s.xml'
LOSS_MAP_FILENAME_FMT = 'loss-maps-%(loss_map_id)s-poe-%(poe)s.xml'
BCR_FILENAME_FMT = 'bcr-distribution-%(bcr_distribution_id)s.xml'

def export(output_id, target_dir):
    """
    Export the given risk calculation output from the database to the
    specified directory. See `openquake.export.hazard.export` for more
    details.
    """
    output = models.Output.objects.get(id=output_id)
    return globals().get("export_%s" % output.output_type,
                         core._export_fn_not_implemented)(
                             output, os.path.expanduser(target_dir))

def _export_common(output):
    """
    Returns a dict containing the common arguments used by nrml
    serializers to serialize the risk calculation `output`.
    """
    risk_calculation = output.oq_job.risk_calculation
    investigation_time = (
        risk_calculation.get_hazard_calculation().investigation_time)
    statistics, quantile_value = risk_calculation.hazard_statistics

    source_model_tree_path, gsim_tree_path = None, None

    if risk_calculation.calculation_mode != u'scenario':
        if not statistics:
            lt_paths = risk_calculation.hazard_logic_tree_paths

            if lt_paths:
                source_model_tree_path, gsim_tree_path = [
                    core.LT_PATH_JOIN_TOKEN.join(x) for x in lt_paths]

    unit = risk_calculation.exposure_model.stco_unit

    return dict(investigation_time=investigation_time,
                statistics=statistics,
                quantile_value=quantile_value,
                source_model_tree_path=source_model_tree_path,
                gsim_tree_path=gsim_tree_path,
                unit=unit)

@core.makedirs
def export_agg_loss_curve(output, target_dir):
    """
    Export `output` to `target_dir` by using a nrml loss curves
    serializer
    """
    args = _export_common(output)
    args['path'] = os.path.join(target_dir, LOSS_CURVE_FILENAME_FMT % {
        'loss_curve_id': output.loss_curve.id})
    writers.AggregateLossCurveXMLWriter(**args).serialize(
        output.loss_curve.aggregatelosscurvedata)
    return [args['path']]

@core.makedirs
def export_loss_curve(output, target_dir):
    """
    Export `output` to `target_dir` by using a nrml loss curves
    serializer
    """
    args = _export_common(output)
    args['path'] = os.path.join(target_dir, LOSS_CURVE_FILENAME_FMT % {
        'loss_curve_id': output.loss_curve.id})
    if output.loss_curve.insured:
        args['insured'] = True

    writers.LossCurveXMLWriter(**args).serialize(
        output.loss_curve.losscurvedata_set.all().order_by('asset_ref'))
    return [args['path']]

@core.makedirs
def export_loss_map(output, target_dir):
    """
    Export `output` to `target_dir` by using a nrml loss map
    serializer
    """
    risk_calculation = output.oq_job.risk_calculation
    args = _export_common(output)
    args.update(
        dict(path=os.path.join(target_dir, LOSS_MAP_FILENAME_FMT % {
            'loss_map_id': output.loss_map.id,
            'poe': output.loss_map.poe}),
            poe=output.loss_map.poe,
            loss_category=risk_calculation.exposure_model.category))
    writers.LossMapXMLWriter(**args).serialize(
        output.loss_map.lossmapdata_set.all().order_by('asset_ref'))
    return [args['path']]

@core.makedirs
def export_bcr_distribution(output, target_dir):
    """
    Export `output` to `target_dir` by using a nrml bcr distribution
    serializer
    """
    risk_calculation = output.oq_job.risk_calculation
    args = _export_common(output)

    args.update(
        dict(path=os.path.join(target_dir, BCR_FILENAME_FMT % {
            'bcr_distribution_id': output.bcr_distribution.id}),
            interest_rate=risk_calculation.interest_rate,
            asset_life_expectancy=risk_calculation.asset_life_expectancy))
    del args['investigation_time']

    writers.BCRMapXMLWriter(**args).serialize(
        output.bcr_distribution.bcrdistributiondata_set.all().order_by(
            'asset_ref'))
    return [args['path']]

@core.makedirs
def export_ins_loss_curve(*args, **kwargs):
    return export_loss_curve(*args, **kwargs)
<|endoftext|>


<commit_before># (C) Datadog, Inc. 2018-present
# All rights reserved
# Licensed under a 3-clause BSD style license (see LICENSE)

# To use a consistent encoding
from codecs import open
from os import path

# Always prefer setuptools over distutils
from setuptools import setup

HERE = path.abspath(path.dirname(__file__))

# Get the long description from the README file
with open(path.join(HERE, 'README.md'), encoding='utf-8') as f:
    long_description = f.read()

def read(*parts):
    with open(path.join(HERE, *parts), 'r') as fp:
        return fp.read()

# Get version info
ABOUT = {}
with open(path.join(HERE, "datadog_checks", "apache", "__about__.py")) as f:
    exec(f.read(), ABOUT)

def get_dependencies():
    dep_file = path.join(HERE, 'requirements.in')
    if not path.isfile(dep_file):
        return []

    with open(dep_file, encoding='utf-8') as f:
        return f.readlines()

CHECKS_BASE_REQ = 'datadog-checks-base>=11.2.0'

setup(
    name='datadog-apache',
    version=ABOUT["__version__"],
    description='The Apache Check',
    long_description=long_description,
    long_description_content_type='text/markdown',
    keywords='datadog agent apache check',
    # The project's main homepage.
    url='https://github.com/DataDog/integrations-core',
    # Author details
    author='Datadog',
    author_email='packages@datadoghq.com',
    # License
    license='BSD',
    # See https://pypi.python.org/pypi?%3Aaction=list_classifiers
    classifiers=[
        'Development Status :: 5 - Production/Stable',
        'Intended Audience :: Developers',
        'Intended Audience :: System Administrators',
        'Topic :: System :: Monitoring',
        'License :: OSI Approved :: BSD License',
        'Programming Language :: Python :: 2',
        'Programming Language :: Python :: 2.7',
    ],
    # The package we're going to ship
    packages=['datadog_checks.apache'],
    # Run-time dependencies
    install_requires=[CHECKS_BASE_REQ],
    extras_require={'deps': get_dependencies()},
    # Extra files to ship with the wheel package
    include_package_data=True,
)
<commit_msg>Bump base dependency (#8748)<commit_after># (C) Datadog, Inc. 2018-present
# All rights reserved
# Licensed under a 3-clause BSD style license (see LICENSE)

# To use a consistent encoding
from codecs import open
from os import path

# Always prefer setuptools over distutils
from setuptools import setup

HERE = path.abspath(path.dirname(__file__))

# Get the long description from the README file
with open(path.join(HERE, 'README.md'), encoding='utf-8') as f:
    long_description = f.read()

def read(*parts):
    with open(path.join(HERE, *parts), 'r') as fp:
        return fp.read()

# Get version info
ABOUT = {}
with open(path.join(HERE, "datadog_checks", "apache", "__about__.py")) as f:
    exec(f.read(), ABOUT)

def get_dependencies():
    dep_file = path.join(HERE, 'requirements.in')
    if not path.isfile(dep_file):
        return []

    with open(dep_file, encoding='utf-8') as f:
        return f.readlines()

CHECKS_BASE_REQ = 'datadog-checks-base>=15.0.0'

setup(
    name='datadog-apache',
    version=ABOUT["__version__"],
    description='The Apache Check',
    long_description=long_description,
    long_description_content_type='text/markdown',
    keywords='datadog agent apache check',
    # The project's main homepage.
    url='https://github.com/DataDog/integrations-core',
    # Author details
    author='Datadog',
    author_email='packages@datadoghq.com',
    # License
    license='BSD',
    # See https://pypi.python.org/pypi?%3Aaction=list_classifiers
    classifiers=[
        'Development Status :: 5 - Production/Stable',
        'Intended Audience :: Developers',
        'Intended Audience :: System Administrators',
        'Topic :: System :: Monitoring',
        'License :: OSI Approved :: BSD License',
        'Programming Language :: Python :: 2',
        'Programming Language :: Python :: 2.7',
    ],
    # The package we're going to ship
    packages=['datadog_checks.apache'],
    # Run-time dependencies
    install_requires=[CHECKS_BASE_REQ],
    extras_require={'deps': get_dependencies()},
    # Extra files to ship with the wheel package
    include_package_data=True,
)
<|endoftext|>


<commit_before><commit_msg>fixed RenameEmployeeNumberColumn migration step<commit_after><|endoftext|>


<commit_before>#!/usr/bin/env python3
import getopt
import hashlib
import os
import shutil
import subprocess
import sys
import re
from datetime import datetime
from subprocess import check_output, CalledProcessError

version = '1.2.0'

def main(argv):
    check_dependencies()

    if len(argv) != 2:
        help_info()

    inputdir = argv[0]
    outputdir = argv[1]

    if not os.path.isdir(inputdir) or not os.path.exists(inputdir):
        error('Input directory "%s" does not exist' % inputdir)
    if not os.path.exists(outputdir):
        print('Output directory "%s" does not exist, creating now' % outputdir)
        os.makedirs(outputdir)

    ignored_files = ('.DS_Store', 'Thumbs.db')

    for root, _, files in os.walk(inputdir):
        for filename in files:
            try:
                if filename in ignored_files:
                    continue
                handle_file(os.path.join(root, filename), outputdir)
            except KeyboardInterrupt:
                print(' Exiting...')
                sys.exit(0)

def check_dependencies():
    if shutil.which('exiftool') is None:
        print('Exiftool is not installed. Visit http://www.sno.phy.queensu.ca/~phil/exiftool/')
        print('On Ubuntu you can `sudo apt-get install libimage-exiftool-perl`')
        print('On Mac you can `brew install exiftool`')
        sys.exit(2)

def exif(file):
    try:
        data = check_output(['exiftool', file]).decode('UTF-8').strip().split("\\n")[0].split("\n")
        exif_data = {}
    except (CalledProcessError, UnicodeDecodeError):
        return None

    for row in data:
        opt = row.split(":")
        exif_data[opt[0].strip()] = ":".join(opt[1:]).strip()

    return exif_data

def get_date(file, exif_data):
    keys = ['Create Date', 'Date/Time Original']

    datestr = None

    for key in keys:
        if key in exif_data:
            datestr = exif_data[key]
            break

    if datestr:
        datestr = datestr.split('.')
        date = datestr[0]

        if len(datestr) > 1:
            subseconds = datestr[1]
        else:
            subseconds = ''

        search = r'(.*)([+-]\d{2}:\d{2})'
        if re.search(search, date) is not None:
            date = re.sub(search, r'\1', date)

        try:
            parsed_date_time = datetime.strptime(date, "%Y:%m:%d %H:%M:%S")
        except ValueError:
            try:
                parsed_date_time = datetime.strptime(date, "%Y-%m-%d %H:%M:%S")
            except ValueError:
                parsed_date_time = None

        return {
            'date': parsed_date_time,
            'subseconds': subseconds
        }
    else:
        # If missing datetime from exif data check if filename is in datetime format
        # E.g.: IMG_20160915_123456.jpg
        regex = re.compile('.*[_-](\d{8})[_-]?(\d{6})')
        matches = regex.findall(os.path.basename(file))

        if matches:
            try:
                datetimestr = ' '.join(list(matches[0]))
                date = datetime.strptime(datetimestr, "%Y%m%d %H%M%S")
            except ValueError:
                date = None

            if date:
                return {
                    'date': date,
                    'subseconds': ''
                }

def get_output_dir(date, outputdir):
    if outputdir.endswith(os.path.sep):
        outputdir = outputdir[:-1]

    if date:
        path = [
            outputdir,
            '%04d' % date['date'].year,
            '%02d' % date['date'].month,
            '%02d' % date['date'].day,
        ]
    else:
        path = [
            outputdir,
            'unknown',
        ]

    fullpath = os.path.sep.join(path)

    if not os.path.isdir(fullpath):
        os.makedirs(fullpath)

    return fullpath

def get_file_name(file, date):
    if date:
        filename = [
            '%04d' % date['date'].year,
            '%02d' % date['date'].month,
            '%02d' % date['date'].day,
            '-',
            '%02d' % date['date'].hour,
            '%02d' % date['date'].minute,
            '%02d' % date['date'].second,
        ]

        if date['subseconds']:
            filename.append(date['subseconds'])

        return ''.join(filename) + os.path.splitext(file)[1]

    else:
        return os.path.basename(file)

def is_image_or_video(exif_data):
    pattern = re.compile('^(image/.+|video/.+|application/vnd.adobe.photoshop)$')
    if pattern.match(exif_data['MIME Type']):
        return True
    return False

def handle_file(source_file, outputdir):
    if str.endswith(source_file, '.xmp'):
        return None

    print(source_file, end="", flush=True)

    exif_data = exif(source_file)

    if exif_data and is_image_or_video(exif_data):
        date = get_date(source_file, exif_data)
        output_dir = get_output_dir(date, outputdir)
        target_file_name = get_file_name(source_file, date).lower()
        target_file_path = os.path.sep.join([output_dir, target_file_name])
    else:
        output_dir = get_output_dir(False, outputdir)
        target_file_name = os.path.basename(source_file)
        target_file_path = os.path.sep.join([output_dir, target_file_name])

    suffix = 1
    target_file = target_file_path
    while True:
        if os.path.isfile(target_file):
            if sha256_checksum(source_file) == sha256_checksum(target_file):
                print(' => skipped, duplicated file')
                break
        else:
            shutil.copy2(source_file, target_file)
            print(' => %s' % target_file)
            handle_file_xmp(source_file, target_file_name, suffix, output_dir)
            break

        suffix += 1
        target_split = os.path.splitext(target_file_path)
        target_file = "%s-%d%s" % (target_split[0], suffix, target_split[1])

def handle_file_xmp(source_file, photo_name, suffix, exif_output_dir):
    xmp_original_with_ext = source_file + '.xmp'
    xmp_original_without_ext = os.path.splitext(source_file)[0] + '.xmp'

    suffix = '-%s' % suffix if suffix > 1 else ''

    if os.path.isfile(xmp_original_with_ext):
        xmp_original = xmp_original_with_ext
        xmp_target = '%s%s.xmp' % (photo_name, suffix)
    elif os.path.isfile(xmp_original_without_ext):
        xmp_original = xmp_original_without_ext
        xmp_target = '%s%s.xmp' % (os.path.splitext(photo_name)[0], suffix)
    else:
        xmp_original = None
        xmp_target = None

    if xmp_original:
        xmp_path = os.path.sep.join([exif_output_dir, xmp_target])
        print('%s => %s' % (xmp_original, xmp_path))
        shutil.copy2(xmp_original, xmp_path)

def sha256_checksum(filename, block_size=65536):
    sha256 = hashlib.sha256()
    with open(filename, 'rb') as f:
        for block in iter(lambda: f.read(block_size), b''):
            sha256.update(block)
    return sha256.hexdigest()

def error(message):
    print(message)
    sys.exit(2)

def help_info():
    error("""NAME
    phockup - v{version}

SYNOPSIS
    phockup INPUTDIR OUTPUTDIR

DESCRIPTION
    Phockup is a photos and videos sorting and backup tool written in Python 3.
    It organizes the media from your camera in a meaningful hierarchy and with proper file names.

ARGUMENTS
    INPUTDIR
        Specify the source directory where your photos are located

    OUTPUTDIR
        Specify the output directory where your photos should be exported
""".format(version=version))

if __name__ == '__main__':
    main(sys.argv[1:])
<commit_msg>Remove generic error message for exiftool<commit_after>#!/usr/bin/env python3
import getopt
import hashlib
import os
import shutil
import subprocess
import sys
import re
from datetime import datetime
from subprocess import check_output, CalledProcessError

version = '1.2.0'

def main(argv):
    check_dependencies()

    if len(argv) != 2:
        help_info()

    inputdir = argv[0]
    outputdir = argv[1]

    if not os.path.isdir(inputdir) or not os.path.exists(inputdir):
        error('Input directory "%s" does not exist' % inputdir)
    if not os.path.exists(outputdir):
        print('Output directory "%s" does not exist, creating now' % outputdir)
        os.makedirs(outputdir)

    ignored_files = ('.DS_Store', 'Thumbs.db')

    for root, _, files in os.walk(inputdir):
        for filename in files:
            try:
                if filename in ignored_files:
                    continue
                handle_file(os.path.join(root, filename), outputdir)
            except KeyboardInterrupt:
                print(' Exiting...')
                sys.exit(0)

def check_dependencies():
    if shutil.which('exiftool') is None:
        print('Exiftool is not installed. Visit http://www.sno.phy.queensu.ca/~phil/exiftool/')
        sys.exit(2)

def exif(file):
    try:
        data = check_output(['exiftool', file]).decode('UTF-8').strip().split("\\n")[0].split("\n")
        exif_data = {}
    except (CalledProcessError, UnicodeDecodeError):
        return None

    for row in data:
        opt = row.split(":")
        exif_data[opt[0].strip()] = ":".join(opt[1:]).strip()

    return exif_data

def get_date(file, exif_data):
    keys = ['Create Date', 'Date/Time Original']

    datestr = None

    for key in keys:
        if key in exif_data:
            datestr = exif_data[key]
            break

    if datestr:
        datestr = datestr.split('.')
        date = datestr[0]

        if len(datestr) > 1:
            subseconds = datestr[1]
        else:
            subseconds = ''

        search = r'(.*)([+-]\d{2}:\d{2})'
        if re.search(search, date) is not None:
            date = re.sub(search, r'\1', date)

        try:
            parsed_date_time = datetime.strptime(date, "%Y:%m:%d %H:%M:%S")
        except ValueError:
            try:
                parsed_date_time = datetime.strptime(date, "%Y-%m-%d %H:%M:%S")
            except ValueError:
                parsed_date_time = None

        return {
            'date': parsed_date_time,
            'subseconds': subseconds
        }
    else:
        # If missing datetime from exif data check if filename is in datetime format
        # E.g.: IMG_20160915_123456.jpg
        regex = re.compile('.*[_-](\d{8})[_-]?(\d{6})')
        matches = regex.findall(os.path.basename(file))

        if matches:
            try:
                datetimestr = ' '.join(list(matches[0]))
                date = datetime.strptime(datetimestr, "%Y%m%d %H%M%S")
            except ValueError:
                date = None

            if date:
                return {
                    'date': date,
                    'subseconds': ''
                }

def get_output_dir(date, outputdir):
    if outputdir.endswith(os.path.sep):
        outputdir = outputdir[:-1]

    if date:
        path = [
            outputdir,
            '%04d' % date['date'].year,
            '%02d' % date['date'].month,
            '%02d' % date['date'].day,
        ]
    else:
        path = [
            outputdir,
            'unknown',
        ]

    fullpath = os.path.sep.join(path)

    if not os.path.isdir(fullpath):
        os.makedirs(fullpath)

    return fullpath

def get_file_name(file, date):
    if date:
        filename = [
            '%04d' % date['date'].year,
            '%02d' % date['date'].month,
            '%02d' % date['date'].day,
            '-',
            '%02d' % date['date'].hour,
            '%02d' % date['date'].minute,
            '%02d' % date['date'].second,
        ]

        if date['subseconds']:
            filename.append(date['subseconds'])

        return ''.join(filename) + os.path.splitext(file)[1]

    else:
        return os.path.basename(file)

def is_image_or_video(exif_data):
    pattern = re.compile('^(image/.+|video/.+|application/vnd.adobe.photoshop)$')
    if pattern.match(exif_data['MIME Type']):
        return True
    return False

def handle_file(source_file, outputdir):
    if str.endswith(source_file, '.xmp'):
        return None

    print(source_file, end="", flush=True)

    exif_data = exif(source_file)

    if exif_data and is_image_or_video(exif_data):
        date = get_date(source_file, exif_data)
        output_dir = get_output_dir(date, outputdir)
        target_file_name = get_file_name(source_file, date).lower()
        target_file_path = os.path.sep.join([output_dir, target_file_name])
    else:
        output_dir = get_output_dir(False, outputdir)
        target_file_name = os.path.basename(source_file)
        target_file_path = os.path.sep.join([output_dir, target_file_name])

    suffix = 1
    target_file = target_file_path
    while True:
        if os.path.isfile(target_file):
            if sha256_checksum(source_file) == sha256_checksum(target_file):
                print(' => skipped, duplicated file')
                break
        else:
            shutil.copy2(source_file, target_file)
            print(' => %s' % target_file)
            handle_file_xmp(source_file, target_file_name, suffix, output_dir)
            break

        suffix += 1
        target_split = os.path.splitext(target_file_path)
        target_file = "%s-%d%s" % (target_split[0], suffix, target_split[1])

def handle_file_xmp(source_file, photo_name, suffix, exif_output_dir):
    xmp_original_with_ext = source_file + '.xmp'
    xmp_original_without_ext = os.path.splitext(source_file)[0] + '.xmp'

    suffix = '-%s' % suffix if suffix > 1 else ''

    if os.path.isfile(xmp_original_with_ext):
        xmp_original = xmp_original_with_ext
        xmp_target = '%s%s.xmp' % (photo_name, suffix)
    elif os.path.isfile(xmp_original_without_ext):
        xmp_original = xmp_original_without_ext
        xmp_target = '%s%s.xmp' % (os.path.splitext(photo_name)[0], suffix)
    else:
        xmp_original = None
        xmp_target = None

    if xmp_original:
        xmp_path = os.path.sep.join([exif_output_dir, xmp_target])
        print('%s => %s' % (xmp_original, xmp_path))
        shutil.copy2(xmp_original, xmp_path)

def sha256_checksum(filename, block_size=65536):
    sha256 = hashlib.sha256()
    with open(filename, 'rb') as f:
        for block in iter(lambda: f.read(block_size), b''):
            sha256.update(block)
    return sha256.hexdigest()

def error(message):
    print(message)
    sys.exit(2)

def help_info():
    error("""NAME
    phockup - v{version}

SYNOPSIS
    phockup INPUTDIR OUTPUTDIR

DESCRIPTION
    Phockup is a photos and videos sorting and backup tool written in Python 3.
    It organizes the media from your camera in a meaningful hierarchy and with proper file names.

ARGUMENTS
    INPUTDIR
        Specify the source directory where your photos are located

    OUTPUTDIR
        Specify the output directory where your photos should be exported
""".format(version=version))

if __name__ == '__main__':
    main(sys.argv[1:])
<|endoftext|>


<commit_before># Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
# For license information, please see license.txt

from __future__ import unicode_literals
import frappe
import json, os
from frappe import _
from frappe.model.document import Document
from frappe.core.doctype.role.role import get_emails_from_role
from frappe.utils import validate_email_add, nowdate
from frappe.utils.data import parse_val
from frappe.utils.jinja import validate_template
from frappe.modules.utils import export_module_json, get_doc_module
from markdown2 import markdown

class EmailAlert(Document):
	def autoname(self):
		if not self.name:
			self.name = self.subject

	def validate(self):
		validate_template(self.subject)
		validate_template(self.message)

		if self.event in ("Days Before", "Days After") and not self.date_changed:
			frappe.throw(_("Please specify which date field must be checked"))

		if self.event=="Value Change" and not self.value_changed:
			frappe.throw(_("Please specify which value field must be checked"))

		self.validate_forbidden_types()
		self.validate_condition()

	def on_update(self):
		frappe.cache().hdel('email_alerts', self.document_type)
		path = export_module_json(self, self.is_standard, self.module)
		if path:
			# js
			if not os.path.exists(path + '.md') and not os.path.exists(path + '.html'):
				with open(path + '.md', 'w') as f:
					f.write(self.message)

			# py
			if not os.path.exists(path + '.py'):
				with open(path + '.py', 'w') as f:
					f.write("""from __future__ import unicode_literals

import frappe

def get_context(context):
	# do your magic here
	pass
""")

	def validate_condition(self):
		temp_doc = frappe.new_doc(self.document_type)
		if self.condition:
			try:
				frappe.safe_eval(self.condition, None, get_context(temp_doc))
			except:
				frappe.throw(_("The Condition '{0}' is invalid").format(self.condition))

	def validate_forbidden_types(self):
		forbidden_document_types = ("Email Queue",)
		if (self.document_type in forbidden_document_types
			or frappe.get_meta(self.document_type).istable):
			# currently email alerts don't work on child tables as events are not fired for each record of child table

			frappe.throw(_("Cannot set Email Alert on Document Type {0}").format(self.document_type))

	def get_documents_for_today(self):
		'''get list of documents that will be triggered today'''
		docs = []

		diff_days = self.days_in_advance
		if self.event=="Days After":
			diff_days = -diff_days

		for name in frappe.db.sql_list("""select name from `tab{0}` where
			DATE(`{1}`) = ADDDATE(DATE(%s), INTERVAL %s DAY)""".format(self.document_type,
				self.date_changed), (nowdate(), diff_days or 0)):

			doc = frappe.get_doc(self.document_type, name)

			if self.condition and not frappe.safe_eval(self.condition, None, get_context(doc)):
				continue

			docs.append(doc)

		return docs

	def send(self, doc):
		'''Build recipients and send email alert'''

		def get_attachment(doc):
			""" check print settings are attach the pdf """
			if not self.attach_print:
				return None

			print_settings = frappe.get_doc("Print Settings", "Print Settings")
			if (doc.docstatus == 0 and not print_settings.allow_print_for_draft) or \
				(doc.docstatus == 2 and not print_settings.allow_print_for_cancelled):

				# ignoring attachment as draft and cancelled documents are not allowed to print
				status = "Draft" if doc.docstatus == 0 else "Cancelled"
				frappe.throw(_("""Not allowed to attach {0} document,
					please enable Allow Print For {0} in Print Settings""".format(status)),
					title=_("Error in Email Alert"))
			else:
				return [frappe.attach_print(doc.doctype, doc.name)]

		context = get_context(doc)
		recipients = []

		for recipient in self.recipients:
			if recipient.condition:
				if not frappe.safe_eval(recipient.condition, None, context):
					continue
			if recipient.email_by_document_field:
				if validate_email_add(doc.get(recipient.email_by_document_field)):
					recipient.email_by_document_field = doc.get(recipient.email_by_document_field).replace(",", "\n")
					recipients = recipients + recipient.email_by_document_field.split("\n")

				# else:
				# 	print "invalid email"
			if recipient.cc:
				recipient.cc = recipient.cc.replace(",", "\n")
				recipients = recipients + recipient.cc.split("\n")

			#For sending emails to specified role
			if recipient.email_by_role:
				emails = get_emails_from_role(recipient.email_by_role)

				for email in emails:
					recipients = recipients + email.split("\n")

		if not recipients:
			return

		recipients = list(set(recipients))
		subject = self.subject

		context = {"doc": doc, "alert": self, "comments": None}

		if self.is_standard:
			self.load_standard_properties(context)

		if doc.get("_comments"):
			context["comments"] = json.loads(doc.get("_comments"))

		if "{" in subject:
			subject = frappe.render_template(self.subject, context)

		attachments = get_attachment(doc)

		frappe.sendmail(recipients=recipients, subject=subject,
			message= frappe.render_template(self.message, context),
			reference_doctype = doc.doctype,
			reference_name = doc.name,
			attachments = attachments)

	def load_standard_properties(self, context):
		module = get_doc_module(self.module, self.doctype, self.name)
		if module:
			if hasattr(module, 'get_context'):
				out = module.get_context(context)
				if out: context.update(out)

			def load_template(extn):
				template_path = os.path.join(os.path.dirname(module.__file__),
					frappe.scrub(self.name) + extn)
				if os.path.exists(template_path):
					with open(template_path, 'r') as f:
						self.message = f.read()
					return True

			# get template
			if not load_template('.html'):
				if load_template('.md'):
					self.message = markdown(self.message)

@frappe.whitelist()
def get_documents_for_today(email_alert):
	email_alert = frappe.get_doc('Email Alert', email_alert)
	email_alert.check_permission('read')
	return [d.name for d in email_alert.get_documents_for_today()]

def trigger_daily_alerts():
	trigger_email_alerts(None, "daily")

def trigger_email_alerts(doc, method=None):
	if frappe.flags.in_import or frappe.flags.in_patch:
		# don't send email alerts while syncing or patching
		return

	if method == "daily":
		for alert in frappe.db.sql_list("""select name from `tabEmail Alert`
			where event in ('Days Before', 'Days After') and enabled=1"""):
			alert = frappe.get_doc("Email Alert", alert)
			for doc in alert.get_documents_for_today():
				evaluate_alert(doc, alert, alert.event)

def evaluate_alert(doc, alert, event):
	from jinja2 import TemplateError
	try:
		if isinstance(alert, basestring):
			alert = frappe.get_doc("Email Alert", alert)

		context = get_context(doc)

		if alert.condition:
			if not frappe.safe_eval(alert.condition, None, context):
				return

		if event=="Value Change" and not doc.is_new():
			try:
				db_value = frappe.db.get_value(doc.doctype, doc.name, alert.value_changed)
			except frappe.DatabaseOperationalError as e:
				if e.args[0]==1054:
					alert.db_set('enabled', 0)
					frappe.log_error('Email Alert {0} has been disabled due to missing field'.format(alert.name))
					return

			db_value = parse_val(db_value)
			if (doc.get(alert.value_changed) == db_value) or \
				(not db_value and not doc.get(alert.value_changed)):

				return # value not changed

		if event != "Value Change" and not doc.is_new():
			# reload the doc for the latest values & comments,
			# except for validate type event.
			doc = frappe.get_doc(doc.doctype, doc.name)

		alert.send(doc)
	except TemplateError:
		frappe.throw(_("Error while evaluating Email Alert {0}. Please fix your template.").format(alert))
	except Exception, e:
		frappe.log_error(message=frappe.get_traceback(), title=e)
		frappe.throw("Error in Email Alert")

def get_context(doc):
	return {"doc": doc, "nowdate": nowdate, "frappe.utils": frappe.utils}
<commit_msg>[minor] commit after sending email alert (#3361)<commit_after># Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
# For license information, please see license.txt

from __future__ import unicode_literals
import frappe
import json, os
from frappe import _
from frappe.model.document import Document
from frappe.core.doctype.role.role import get_emails_from_role
from frappe.utils import validate_email_add, nowdate
from frappe.utils.data import parse_val
from frappe.utils.jinja import validate_template
from frappe.modules.utils import export_module_json, get_doc_module
from markdown2 import markdown

class EmailAlert(Document):
	def autoname(self):
		if not self.name:
			self.name = self.subject

	def validate(self):
		validate_template(self.subject)
		validate_template(self.message)

		if self.event in ("Days Before", "Days After") and not self.date_changed:
			frappe.throw(_("Please specify which date field must be checked"))

		if self.event=="Value Change" and not self.value_changed:
			frappe.throw(_("Please specify which value field must be checked"))

		self.validate_forbidden_types()
		self.validate_condition()

	def on_update(self):
		frappe.cache().hdel('email_alerts', self.document_type)
		path = export_module_json(self, self.is_standard, self.module)
		if path:
			# js
			if not os.path.exists(path + '.md') and not os.path.exists(path + '.html'):
				with open(path + '.md', 'w') as f:
					f.write(self.message)

			# py
			if not os.path.exists(path + '.py'):
				with open(path + '.py', 'w') as f:
					f.write("""from __future__ import unicode_literals

import frappe

def get_context(context):
	# do your magic here
	pass
""")

	def validate_condition(self):
		temp_doc = frappe.new_doc(self.document_type)
		if self.condition:
			try:
				frappe.safe_eval(self.condition, None, get_context(temp_doc))
			except:
				frappe.throw(_("The Condition '{0}' is invalid").format(self.condition))

	def validate_forbidden_types(self):
		forbidden_document_types = ("Email Queue",)
		if (self.document_type in forbidden_document_types
			or frappe.get_meta(self.document_type).istable):
			# currently email alerts don't work on child tables as events are not fired for each record of child table

			frappe.throw(_("Cannot set Email Alert on Document Type {0}").format(self.document_type))

	def get_documents_for_today(self):
		'''get list of documents that will be triggered today'''
		docs = []

		diff_days = self.days_in_advance
		if self.event=="Days After":
			diff_days = -diff_days

		for name in frappe.db.sql_list("""select name from `tab{0}` where
			DATE(`{1}`) = ADDDATE(DATE(%s), INTERVAL %s DAY)""".format(self.document_type,
				self.date_changed), (nowdate(), diff_days or 0)):

			doc = frappe.get_doc(self.document_type, name)

			if self.condition and not frappe.safe_eval(self.condition, None, get_context(doc)):
				continue

			docs.append(doc)

		return docs

	def send(self, doc):
		'''Build recipients and send email alert'''

		def get_attachment(doc):
			""" check print settings are attach the pdf """
			if not self.attach_print:
				return None

			print_settings = frappe.get_doc("Print Settings", "Print Settings")
			if (doc.docstatus == 0 and not print_settings.allow_print_for_draft) or \
				(doc.docstatus == 2 and not print_settings.allow_print_for_cancelled):

				# ignoring attachment as draft and cancelled documents are not allowed to print
				status = "Draft" if doc.docstatus == 0 else "Cancelled"
				frappe.throw(_("""Not allowed to attach {0} document,
					please enable Allow Print For {0} in Print Settings""".format(status)),
					title=_("Error in Email Alert"))
			else:
				return [frappe.attach_print(doc.doctype, doc.name)]

		context = get_context(doc)
		recipients = []

		for recipient in self.recipients:
			if recipient.condition:
				if not frappe.safe_eval(recipient.condition, None, context):
					continue
			if recipient.email_by_document_field:
				if validate_email_add(doc.get(recipient.email_by_document_field)):
					recipient.email_by_document_field = doc.get(recipient.email_by_document_field).replace(",", "\n")
					recipients = recipients + recipient.email_by_document_field.split("\n")

				# else:
				# 	print "invalid email"
			if recipient.cc:
				recipient.cc = recipient.cc.replace(",", "\n")
				recipients = recipients + recipient.cc.split("\n")

			#For sending emails to specified role
			if recipient.email_by_role:
				emails = get_emails_from_role(recipient.email_by_role)

				for email in emails:
					recipients = recipients + email.split("\n")

		if not recipients:
			return

		recipients = list(set(recipients))
		subject = self.subject

		context = {"doc": doc, "alert": self, "comments": None}

		if self.is_standard:
			self.load_standard_properties(context)

		if doc.get("_comments"):
			context["comments"] = json.loads(doc.get("_comments"))

		if "{" in subject:
			subject = frappe.render_template(self.subject, context)

		attachments = get_attachment(doc)

		frappe.sendmail(recipients=recipients, subject=subject,
			message= frappe.render_template(self.message, context),
			reference_doctype = doc.doctype,
			reference_name = doc.name,
			attachments = attachments)

	def load_standard_properties(self, context):
		module = get_doc_module(self.module, self.doctype, self.name)
		if module:
			if hasattr(module, 'get_context'):
				out = module.get_context(context)
				if out: context.update(out)

			def load_template(extn):
				template_path = os.path.join(os.path.dirname(module.__file__),
					frappe.scrub(self.name) + extn)
				if os.path.exists(template_path):
					with open(template_path, 'r') as f:
						self.message = f.read()
					return True

			# get template
			if not load_template('.html'):
				if load_template('.md'):
					self.message = markdown(self.message)

@frappe.whitelist()
def get_documents_for_today(email_alert):
	email_alert = frappe.get_doc('Email Alert', email_alert)
	email_alert.check_permission('read')
	return [d.name for d in email_alert.get_documents_for_today()]

def trigger_daily_alerts():
	trigger_email_alerts(None, "daily")

def trigger_email_alerts(doc, method=None):
	if frappe.flags.in_import or frappe.flags.in_patch:
		# don't send email alerts while syncing or patching
		return

	if method == "daily":
		for alert in frappe.db.sql_list("""select name from `tabEmail Alert`
			where event in ('Days Before', 'Days After') and enabled=1"""):
			alert = frappe.get_doc("Email Alert", alert)
			for doc in alert.get_documents_for_today():
				evaluate_alert(doc, alert, alert.event)
				frappe.db.commit()

def evaluate_alert(doc, alert, event):
	from jinja2 import TemplateError
	try:
		if isinstance(alert, basestring):
			alert = frappe.get_doc("Email Alert", alert)

		context = get_context(doc)

		if alert.condition:
			if not frappe.safe_eval(alert.condition, None, context):
				return

		if event=="Value Change" and not doc.is_new():
			try:
				db_value = frappe.db.get_value(doc.doctype, doc.name, alert.value_changed)
			except frappe.DatabaseOperationalError as e:
				if e.args[0]==1054:
					alert.db_set('enabled', 0)
					frappe.log_error('Email Alert {0} has been disabled due to missing field'.format(alert.name))
					return

			db_value = parse_val(db_value)
			if (doc.get(alert.value_changed) == db_value) or \
				(not db_value and not doc.get(alert.value_changed)):

				return # value not changed

		if event != "Value Change" and not doc.is_new():
			# reload the doc for the latest values & comments,
			# except for validate type event.
			doc = frappe.get_doc(doc.doctype, doc.name)

		alert.send(doc)
	except TemplateError:
		frappe.throw(_("Error while evaluating Email Alert {0}. Please fix your template.").format(alert))
	except Exception, e:
		frappe.log_error(message=frappe.get_traceback(), title=e)
		frappe.throw("Error in Email Alert")

def get_context(doc):
	return {"doc": doc, "nowdate": nowdate, "frappe.utils": frappe.utils}
<|endoftext|>


<commit_before># Copyright (c) 2015 Clinton Knight.  All rights reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

from tempest_lib import exceptions as lib_exc  # noqa

from tempest.api.share import base
from tempest import config_share as config
from tempest import test

CONF = config.CONF

class SchedulerStatsAdminTest(base.BaseSharesAdminTest):

    @test.attr(type=["gate", "smoke", ])
    def test_pool_list(self):

        # List pools
        pool_response = self.shares_client.list_pools()
        pool_list = pool_response.get('pools')
        self.assertIsNotNone(pool_list, 'No pools returned from pools API')
        self.assertNotEmpty(pool_list)
        pool = pool_list[0]
        required_keys = {'name', 'host', 'backend', 'pool'}
        actual_keys = set(pool.keys())
        self.assertTrue(actual_keys.issuperset(required_keys))

    @test.attr(type=["gate", "smoke", ])
    def test_pool_list_with_filters(self):

        # List pools
        pool_response = self.shares_client.list_pools()
        pool_list = pool_response.get('pools')

        # Ensure we got at least one pool
        self.assertIsNotNone(pool_list, 'No pools returned from pools API')
        self.assertNotEmpty(pool_list)
        pool = pool_list[0]

        # Build search opts from data and get pools again with filter
        search_opts = {
            'host': pool.get('host'),
            'backend': pool.get('backend'),
            'pool': pool.get('pool'),
        }
        pool_response = self.shares_client.list_pools(
            search_opts=search_opts)
        filtered_pool_list = pool_response.get('pools')

        # Ensure we got exactly one pool matching the first one from above
        self.assertEqual(1, len(filtered_pool_list))
        self.assertDictEqual(pool, filtered_pool_list[0])

    @test.attr(type=["gate", "smoke", ])
    def test_pool_list_with_filters_negative(self):

        # Build search opts for a non-existent pool
        search_opts = {
            'host': 'foo',
            'backend': 'bar',
            'pool': 'shark',
        }
        pool_response = self.shares_client.list_pools(
            search_opts=search_opts)
        pool_list = pool_response.get('pools')

        # Ensure we got no pools
        self.assertEmpty(pool_list)

    @test.attr(type=["gate", "smoke", ])
    def test_pool_list_detail(self):

        # List pools
        pool_response = self.shares_client.list_pools(detail=True)
        pool_list = pool_response.get('pools')
        self.assertIsNotNone(pool_list, 'No pools returned from pools API')
        self.assertNotEmpty(pool_list)
        pool = pool_list[0]
        required_keys = {'name', 'host', 'backend', 'pool', 'capabilities'}
        actual_keys = set(pool.keys())
        self.assertTrue(actual_keys.issuperset(required_keys))

    @test.attr(type=["gate", "smoke", ])
    def test_pool_list_detail_with_filters(self):

        # List pools
        pool_response = self.shares_client.list_pools(detail=True)
        pool_list = pool_response.get('pools')

        # Ensure we got at least one pool
        self.assertIsNotNone(pool_list, 'No pools returned from pools API')
        self.assertNotEmpty(pool_list)
        pool = pool_list[0]

        # Build search opts from data and get pools again with filter
        search_opts = {
            'host': pool.get('host'),
            'backend': pool.get('backend'),
            'pool': pool.get('pool'),
        }
        pool_response = self.shares_client.list_pools(
            detail=True, search_opts=search_opts)
        filtered_pool_list = pool_response.get('pools')

        # Ensure we got exactly one pool matching the first one from above
        self.assertEqual(1, len(filtered_pool_list))
        self.assertDictEqual(pool, filtered_pool_list[0])

    @test.attr(type=["gate", "smoke", ])
    def test_pool_list_detail_with_filters_negative(self):

        # Build search opts for a non-existent pool
        search_opts = {
            'host': 'foo',
            'backend': 'bar',
            'pool': 'shark',
        }
        pool_response = self.shares_client.list_pools(
            detail=True, search_opts=search_opts)
        pool_list = pool_response.get('pools')

        # Ensure we got no pools
        self.assertEmpty(pool_list)
<commit_msg>Fix for SchedulerStatsAdminTest fails on timestamp<commit_after># Copyright (c) 2015 Clinton Knight.  All rights reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

from tempest_lib import exceptions as lib_exc  # noqa

from tempest.api.share import base
from tempest import config_share as config
from tempest import test

CONF = config.CONF

class SchedulerStatsAdminTest(base.BaseSharesAdminTest):

    @test.attr(type=["gate", "smoke", ])
    def test_pool_list(self):

        # List pools
        pool_response = self.shares_client.list_pools()
        pool_list = pool_response.get('pools')
        self.assertIsNotNone(pool_list, 'No pools returned from pools API')
        self.assertNotEmpty(pool_list)
        pool = pool_list[0]
        required_keys = {'name', 'host', 'backend', 'pool'}
        actual_keys = set(pool.keys())
        self.assertTrue(actual_keys.issuperset(required_keys))

    @test.attr(type=["gate", "smoke", ])
    def test_pool_list_with_filters(self):

        # List pools
        pool_response = self.shares_client.list_pools()
        pool_list = pool_response.get('pools')

        # Ensure we got at least one pool
        self.assertIsNotNone(pool_list, 'No pools returned from pools API')
        self.assertNotEmpty(pool_list)
        pool = pool_list[0]

        # Build search opts from data and get pools again with filter
        search_opts = {
            'host': pool.get('host'),
            'backend': pool.get('backend'),
            'pool': pool.get('pool'),
        }
        pool_response = self.shares_client.list_pools(
            search_opts=search_opts)
        filtered_pool_list = pool_response.get('pools')

        # Ensure we got exactly one pool matching the first one from above
        self.assertEqual(1, len(filtered_pool_list))

        # Match the key values, not the timestamp.
        for k, v in search_opts.items():
            self.assertEqual(v, filtered_pool_list[0][k])

    @test.attr(type=["gate", "smoke", ])
    def test_pool_list_with_filters_negative(self):

        # Build search opts for a non-existent pool
        search_opts = {
            'host': 'foo',
            'backend': 'bar',
            'pool': 'shark',
        }
        pool_response = self.shares_client.list_pools(
            search_opts=search_opts)
        pool_list = pool_response.get('pools')

        # Ensure we got no pools
        self.assertEmpty(pool_list)

    @test.attr(type=["gate", "smoke", ])
    def test_pool_list_detail(self):

        # List pools
        pool_response = self.shares_client.list_pools(detail=True)
        pool_list = pool_response.get('pools')
        self.assertIsNotNone(pool_list, 'No pools returned from pools API')
        self.assertNotEmpty(pool_list)
        pool = pool_list[0]
        required_keys = {'name', 'host', 'backend', 'pool', 'capabilities'}
        actual_keys = set(pool.keys())
        self.assertTrue(actual_keys.issuperset(required_keys))

    @test.attr(type=["gate", "smoke", ])
    def test_pool_list_detail_with_filters(self):

        # List pools
        pool_response = self.shares_client.list_pools(detail=True)
        pool_list = pool_response.get('pools')

        # Ensure we got at least one pool
        self.assertIsNotNone(pool_list, 'No pools returned from pools API')
        self.assertNotEmpty(pool_list)
        pool = pool_list[0]

        # Build search opts from data and get pools again with filter
        search_opts = {
            'host': pool.get('host'),
            'backend': pool.get('backend'),
            'pool': pool.get('pool'),
        }
        pool_response = self.shares_client.list_pools(
            detail=True, search_opts=search_opts)
        filtered_pool_list = pool_response.get('pools')

        # Ensure we got exactly one pool matching the first one from above
        self.assertEqual(1, len(filtered_pool_list))

        # Match the key values, not the timestamp.
        for k, v in search_opts.items():
            self.assertEqual(v, filtered_pool_list[0][k])

    @test.attr(type=["gate", "smoke", ])
    def test_pool_list_detail_with_filters_negative(self):

        # Build search opts for a non-existent pool
        search_opts = {
            'host': 'foo',
            'backend': 'bar',
            'pool': 'shark',
        }
        pool_response = self.shares_client.list_pools(
            detail=True, search_opts=search_opts)
        pool_list = pool_response.get('pools')

        # Ensure we got no pools
        self.assertEmpty(pool_list)
<|endoftext|>


<commit_before>#!/usr/bin/env python

"""Compute the R factor of a comparison between two scattering curves.
This will usually be one calculated and one experimental data set.

R factor is used by analogy with crystallography where:
R = sum (abs(F_expt - F_calc)) / sum (abs(F_expt))
"""

# David W. Wright - 08 October 2013

import argparse
import numpy
from sjp_util import sjp_util

def parse_arguments():
    """Parse command line arguments and ensure correct combinations present"""

    parser = argparse.ArgumentParser(
        description = 'Compute R factor from a comparison of two '
                      'scattering curves.')
    parser.add_argument(
        '-c','--calc', nargs='?', type=str, dest='calc_curve',
        help = 'Path to the input calculated curve', required=True)
    parser.add_argument(
        '-e','--expt', nargs='?', type=str, dest='expt_curve',
        help = 'Path to the input experimental curve', required=True)
    parser.add_argument(
        '-q','--qrange', nargs=2, type=float,
        help = 'Minimum and maximum Q values used in curve fitting',
        required=True)
    parser.add_argument(
        '-o','--outfile', nargs='?', type=str, dest='out_file',
        help = 'Path to the output file', required=False)
    parser.add_argument(
        '--header', action='store_true',
        help = 'Output a header (no data analysis performed)')

    args = parser.parse_args()
    return args

def load_scatter_curve(filename, q_min, q_max):
    """Load I, Q data from file and filter for values between q_min and q_max"""

    scatter_data = numpy.loadtxt(filename)
    qrange_mask = (scatter_data[:,0] >= q_min) & (scatter_data[:,0] <= q_max)

    return scatter_data[qrange_mask]

def match_scatter_curves(target_data, source_data):
    """Match I values from one data set to the Q values of another.

    Input data is two I vs Q scattering curves.
    Output is I values from the source data set matched to the Q values of
    the target set.
    """

    # Create list of calculated I values matched to the nearest experimental Q
    # Remember that the arrays in python start at 0, those in Fortran at 1
    last_source = len( source_data )
    last_target = len( target_data )

    # Initialize array to hold the calculated I values matched to
    # experimental Q values
    matched_I = numpy.zeros( last_target, dtype=float )

    # Use the old fortran routine to match the data sets by Q value
    # matched_no is the number of datapoints which contain matched data
    matched_no = sjp_util.qrange_match(target_data[:,0], source_data[:,0],
                                        source_data[:,1], last_target,
                                        last_source, matched_I)

    matched_I.resize(matched_no)

    return matched_I

def calculate_rfactor(target_data, source_data, q_min, q_max):
    """Compute R factor comparing two matched scattering curves

    Input data is two I vs Q scattering curves and the min/max Q values to use
    to compare them
    Output is the R factor and the scaling factor needed to match I from the
    target scattering curve to the source data
    """

    matched_source_I = match_scatter_curves(target_data, source_data)

    # Get average I for experimental and calculated values over matched Q range
    matched_no = len(matched_source_I)
    expt_avg = numpy.mean( target_data[0:matched_no, 1] )
    calc_avg = numpy.mean( matched_source_I )

    # Initial guess of the concentration:
    # ratio of experimental and calculated average intensities
    con = expt_avg / calc_avg

    # Call fortran code to calculate the R factor
    rfactor = sjp_util.calc_rfactor( target_data[:,0], target_data[:,1],
        matched_source_I, matched_no, q_min, q_max, con, False)

    # 1/con is the scaling factor needed to multiply experimental I values
    # to compare with calculated data
    return rfactor, 1.0/con

def main():
    # Interpret command line arguments
    args = parse_arguments()

    if (not args.header) :

        q_min = args.qrange[0]
        q_max = args.qrange[1]

        # Load the data from the two input files
        calc_data = load_scatter_curve(args.calc_curve, q_min, q_max)
        expt_data = load_scatter_curve(args.expt_curve, q_min, q_max)

        rfactor, scale = calculate_rfactor(expt_data, calc_data, q_min, q_max)

        output_data = '{0:s}\t{1:s}\t{2:0.5f}\t{3:0.5f}'
        '\t{4:0.5f}\t{5:0.5f}\n'.format(
        args.calc_curve, args.expt_curve, q_min, q_max, scale, rfactor)

        if args.out_file == None:
            print output_data
        else:
            with open(args.out_file, "a") as fle:
                fle.write(output_data)
    else:

        header = 'Calculated file\tExperimental file\tQ min'
        '\tQ max\tScaling factor\tR factor\n'

        if args.out_file == None:
            print header
        else:
            with open(args.out_file, "a") as fle:
                fle.write(header)

if __name__ == "__main__":
    main()<commit_msg>calculate_rfactor: corrected output of script<commit_after>#!/usr/bin/env python

"""Compute the R factor of a comparison between two scattering curves.
This will usually be one calculated and one experimental data set.

R factor is used by analogy with crystallography where:
R = sum (abs(F_expt - F_calc)) / sum (abs(F_expt))
"""

# David W. Wright - 08 October 2013

import argparse
import numpy
from sjp_util import sjp_util

def parse_arguments():
    """Parse command line arguments and ensure correct combinations present"""

    parser = argparse.ArgumentParser(
        description = 'Compute R factor from a comparison of two '
                      'scattering curves.')
    parser.add_argument(
        '-c','--calc', nargs='?', type=str, dest='calc_curve',
        help = 'Path to the input calculated curve', required=True)
    parser.add_argument(
        '-e','--expt', nargs='?', type=str, dest='expt_curve',
        help = 'Path to the input experimental curve', required=True)
    parser.add_argument(
        '-q','--qrange', nargs=2, type=float,
        help = 'Minimum and maximum Q values used in curve fitting',
        required=True)
    parser.add_argument(
        '-o','--outfile', nargs='?', type=str, dest='out_file',
        help = 'Path to the output file', required=False)
    parser.add_argument(
        '--header', action='store_true',
        help = 'Output a header (no data analysis performed)')

    args = parser.parse_args()
    return args

def load_scatter_curve(filename, q_min, q_max):
    """Load I, Q data from file and filter for values between q_min and q_max"""

    scatter_data = numpy.loadtxt(filename)
    qrange_mask = (scatter_data[:,0] >= q_min) & (scatter_data[:,0] <= q_max)

    return scatter_data[qrange_mask]

def match_scatter_curves(target_data, source_data):
    """Match I values from one data set to the Q values of another.

    Input data is two I vs Q scattering curves.
    Output is I values from the source data set matched to the Q values of
    the target set.
    """

    # Create list of calculated I values matched to the nearest experimental Q
    # Remember that the arrays in python start at 0, those in Fortran at 1
    last_source = len( source_data )
    last_target = len( target_data )

    # Initialize array to hold the calculated I values matched to
    # experimental Q values
    matched_I = numpy.zeros( last_target, dtype=float )

    # Use the old fortran routine to match the data sets by Q value
    # matched_no is the number of datapoints which contain matched data
    matched_no = sjp_util.qrange_match(target_data[:,0], source_data[:,0],
                                        source_data[:,1], last_target,
                                        last_source, matched_I)

    matched_I.resize(matched_no)

    return matched_I

def calculate_rfactor(target_data, source_data, q_min, q_max):
    """Compute R factor comparing two matched scattering curves

    Input data is two I vs Q scattering curves and the min/max Q values to use
    to compare them
    Output is the R factor and the scaling factor needed to match I from the
    target scattering curve to the source data
    """

    matched_source_I = match_scatter_curves(target_data, source_data)

    # Get average I for experimental and calculated values over matched Q range
    matched_no = len(matched_source_I)
    expt_avg = numpy.mean( target_data[0:matched_no, 1] )
    calc_avg = numpy.mean( matched_source_I )

    # Initial guess of the concentration:
    # ratio of experimental and calculated average intensities
    con = expt_avg / calc_avg

    # Call fortran code to calculate the R factor
    rfactor = sjp_util.calc_rfactor( target_data[:,0], target_data[:,1],
        matched_source_I, matched_no, q_min, q_max, con, False)

    # 1/con is the scaling factor needed to multiply experimental I values
    # to compare with calculated data
    return rfactor, 1.0/con

def main():
    # Interpret command line arguments
    args = parse_arguments()

    if (not args.header) :

        q_min = args.qrange[0]
        q_max = args.qrange[1]

        # Load the data from the two input files
        calc_data = load_scatter_curve(args.calc_curve, q_min, q_max)
        expt_data = load_scatter_curve(args.expt_curve, q_min, q_max)

        rfactor, scale = calculate_rfactor(expt_data, calc_data, q_min, q_max)

        output_data = '{0:s}\t{1:s}\t{2:0.5f}\t{3:0.5f}\t{4:0.5f}\t{5:0.5f}\n'.format(
                                        args.calc_curve,
                                        args.expt_curve,
                                        q_min,
                                        q_max,
                                        scale,
                                        rfactor)

        if args.out_file == None:
            print output_data
        else:
            with open(args.out_file, "a") as fle:
                fle.write(output_data)
    else:

        header = 'Calculated file\tExperimental file\tQ min'
        '\tQ max\tScaling factor\tR factor\n'

        if args.out_file == None:
            print header
        else:
            with open(args.out_file, "a") as fle:
                fle.write(header)

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before>#!/usr/bin/env python3
import getopt
import os
import re
import sys

from src.date import Date
from src.dependency import check_dependencies
from src.help import help
from src.phockup import Phockup
from src.printer import Printer

version = '1.5.0'
printer = Printer()

def main(argv):
    check_dependencies()

    move = False
    link = False
    date_regex = None
    dir_format = os.path.sep.join(['%Y', '%m', '%d'])

    try:
        opts, args = getopt.getopt(argv[2:], "d:r:mlh", ["date=", "regex=", "move", "link", "help"])
    except getopt.GetoptError:
        help(version)
        sys.exit(2)

    for opt, arg in opts:
        if opt in ("-h", "--help"):
            help(version)
            sys.exit(2)

        if opt in ("-d", "--date"):
            if not arg:
                printer.print.error("Date format cannot be empty")
            dir_format = Date().parse(arg)

        if opt in ("-m", "--move"):
            move = True
            printer.line("Using move strategy!")

        if opt in ("-l", "--link"):
            link = True
            printer.line("Using link strategy!")

        if opt in ("-r", "--regex"):
            try:
                date_regex = re.compile(arg)
            except:
                printer.error("Provided regex is invalid!")
                sys.exit(2)

    if link and move:
        printer.error("Can't use move and link strategy together!")
        sys.exit(1)

    if len(argv) < 2:
        help(version)
        sys.exit(2)

    return Phockup(
        argv[0], argv[1],
        dir_format=dir_format,
        move=move,
        link=link,
        date_regex=date_regex
    )

if __name__ == '__main__':
    try:
        main(sys.argv[1:])
    except KeyboardInterrupt:
        printer.empty().line('Exiting...')
        sys.exit(0)
<commit_msg>Bumped version to 1.5.1<commit_after>#!/usr/bin/env python3
import getopt
import os
import re
import sys

from src.date import Date
from src.dependency import check_dependencies
from src.help import help
from src.phockup import Phockup
from src.printer import Printer

version = '1.5.1'
printer = Printer()

def main(argv):
    check_dependencies()

    move = False
    link = False
    date_regex = None
    dir_format = os.path.sep.join(['%Y', '%m', '%d'])

    try:
        opts, args = getopt.getopt(argv[2:], "d:r:mlh", ["date=", "regex=", "move", "link", "help"])
    except getopt.GetoptError:
        help(version)
        sys.exit(2)

    for opt, arg in opts:
        if opt in ("-h", "--help"):
            help(version)
            sys.exit(2)

        if opt in ("-d", "--date"):
            if not arg:
                printer.print.error("Date format cannot be empty")
            dir_format = Date().parse(arg)

        if opt in ("-m", "--move"):
            move = True
            printer.line("Using move strategy!")

        if opt in ("-l", "--link"):
            link = True
            printer.line("Using link strategy!")

        if opt in ("-r", "--regex"):
            try:
                date_regex = re.compile(arg)
            except:
                printer.error("Provided regex is invalid!")
                sys.exit(2)

    if link and move:
        printer.error("Can't use move and link strategy together!")
        sys.exit(1)

    if len(argv) < 2:
        help(version)
        sys.exit(2)

    return Phockup(
        argv[0], argv[1],
        dir_format=dir_format,
        move=move,
        link=link,
        date_regex=date_regex
    )

if __name__ == '__main__':
    try:
        main(sys.argv[1:])
    except KeyboardInterrupt:
        printer.empty().line('Exiting...')
        sys.exit(0)
<|endoftext|>