<commit_before># -*- coding: utf-8 -*-
# vim: tabstop=4 shiftwidth=4 softtabstop=4
#
# Copyright (C) 2015-2019 GEM Foundation
#
# OpenQuake is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# OpenQuake is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with OpenQuake. If not, see <http://www.gnu.org/licenses/>.

import os
import re
import getpass
import collections
import h5py

from openquake.baselib import hdf5, config, performance

CALC_REGEX = r'(calc|cache)_(\d+)\.hdf5'

def get_datadir():
    """
    Extracts the path of the directory where the openquake data are stored
    from the environment ($OQ_DATADIR) or from the shared_dir in the
    configuration file.
    """
    datadir = os.environ.get('OQ_DATADIR')
    if not datadir:
        shared_dir = config.directory.shared_dir
        if shared_dir:
            datadir = os.path.join(shared_dir, getpass.getuser(), 'oqdata')
        else:  # use the home of the user
            datadir = os.path.join(os.path.expanduser('~'), 'oqdata')
    return datadir

def get_calc_ids(datadir=None):
    """
    Extract the available calculation IDs from the datadir, in order.
    """
    datadir = datadir or get_datadir()
    if not os.path.exists(datadir):
        return []
    calc_ids = set()
    for f in os.listdir(datadir):
        mo = re.match(CALC_REGEX, f)
        if mo:
            calc_ids.add(int(mo.group(2)))
    return sorted(calc_ids)

def get_last_calc_id(datadir=None):
    """
    Extract the latest calculation ID from the given directory.
    If none is found, return 0.
    """
    datadir = datadir or get_datadir()
    calcs = get_calc_ids(datadir)
    if not calcs:
        return 0
    return calcs[-1]

def hdf5new(datadir=None):
    """
    Return a new `hdf5.File by` instance with name determined by the last
    calculation in the datadir (plus one). Set the .path attribute to the
    generated filename.
    """
    datadir = datadir or get_datadir()
    if not os.path.exists(datadir):
        os.makedirs(datadir)
    calc_id = get_last_calc_id(datadir) + 1
    fname = os.path.join(datadir, 'calc_%d.hdf5' % calc_id)
    new = hdf5.File(fname, 'w')
    new.path = fname
    performance.init_performance(new)
    return new

def extract_calc_id_datadir(filename, datadir=None):
    """
    Extract the calculation ID from the given filename or integer:

    >>> extract_calc_id_datadir('/mnt/ssd/oqdata/calc_25.hdf5')
    (25, '/mnt/ssd/oqdata')
    >>> extract_calc_id_datadir('/mnt/ssd/oqdata/wrong_name.hdf5')
    Traceback (most recent call last):
       ...
    ValueError: Cannot extract calc_id from /mnt/ssd/oqdata/wrong_name.hdf5
    """
    datadir = datadir or get_datadir()
    try:
        calc_id = int(filename)
    except ValueError:
        filename = os.path.abspath(filename)
        datadir = os.path.dirname(filename)
        mo = re.match(CALC_REGEX, os.path.basename(filename))
        if mo is None:
            raise ValueError('Cannot extract calc_id from %s' % filename)
        calc_id = int(mo.group(2))
    return calc_id, datadir

def read(calc_id, mode='r', datadir=None):
    """
    :param calc_id: calculation ID or filename
    :param mode: 'r' or 'w'
    :param datadir: the directory where to look
    :returns: the corresponding DataStore instance

    Read the datastore, if it exists and it is accessible.
    """
    datadir = datadir or get_datadir()
    dstore = DataStore(calc_id, datadir, mode=mode)
    try:
        hc_id = dstore['oqparam'].hazard_calculation_id
    except KeyError:  # no oqparam
        hc_id = None
    if hc_id:
        dstore.parent = read(hc_id, datadir=os.path.dirname(dstore.filename))
    return dstore

class DataStore(collections.abc.MutableMapping):
    """
    DataStore class to store the inputs/outputs of a calculation on the
    filesystem.

    Here is a minimal example of usage:

    >>> ds = DataStore()
    >>> ds['example'] = 42
    >>> print(ds['example'][()])
    42
    >>> ds.clear()

    When reading the items, the DataStore will return a generator. The
    items will be ordered lexicographically according to their name.

    There is a serialization protocol to store objects in the datastore.
    An object is serializable if it has a method `__toh5__` returning
    an array and a dictionary, and a method `__fromh5__` taking an array
    and a dictionary and populating the object.
    For an example of use see :class:`openquake.hazardlib.site.SiteCollection`.
    """
    def __init__(self, calc_id=None, datadir=None, params=(), mode=None):
        datadir = datadir or get_datadir()
        if isinstance(calc_id, str):  # passed a real path
            self.filename = calc_id
            self.calc_id, datadir = extract_calc_id_datadir(calc_id, datadir)
        else:
            if calc_id is None:  # use a new datastore
                self.calc_id = get_last_calc_id(datadir) + 1
            elif calc_id < 0:  # use an old datastore
                calc_ids = get_calc_ids(datadir)
                try:
                    self.calc_id = calc_ids[calc_id]
                except IndexError:
                    raise IndexError(
                        'There are %d old calculations, cannot '
                        'retrieve the %s' % (len(calc_ids), calc_id))
            else:  # use the given datastore
                self.calc_id = calc_id
            self.filename = os.path.join(
                datadir, 'calc_%s.hdf5' % self.calc_id)
        if not os.path.exists(datadir):
            os.makedirs(datadir)
        self.params = params
        self.parent = ()  # can be set later
        self.datadir = datadir
        self.mode = mode or ('r+' if os.path.exists(self.filename) else 'w')
        if self.mode == 'r' and not os.path.exists(self.filename):
            raise IOError('File not found: %s' % self.filename)
        self.hdf5 = ()  # so that `key in self.hdf5` is valid
        self.open(self.mode)
        if 'w' in self.mode or '+' in self.mode:
            performance.init_performance(self.hdf5)

    def open(self, mode):
        """
        Open the underlying .hdf5 file and the parent, if any
        """
        if self.hdf5 == ():  # not already open
            kw = dict(mode=mode, libver='latest')
            if mode == 'r':
                kw['swmr'] = True
            try:
                self.hdf5 = hdf5.File(self.filename, **kw)
            except OSError as exc:
                if os.path.exists(self.filename + '~'):  # temporary file
                    self.hdf5 = hdf5.File(self.filename + '~', **kw)
                else:
                    raise OSError('%s in %s' % (exc, self.filename))

    @property
    def export_dir(self):
        """
        Return the underlying export directory
        """
        edir = getattr(self, '_export_dir', None) or self['oqparam'].export_dir
        return edir

    @export_dir.setter
    def export_dir(self, value):
        """
        Set the export directory
        """
        self._export_dir = value

    def cachepath(self):
        """
        :returns: the path to the .hdf5 cache file associated to the calc_id
        """
        return os.path.join(self.datadir, 'cache_%d.hdf5' % self.calc_id)

    def getitem(self, name):
        """
        Return a dataset by using h5py.File.__getitem__
        """
        return h5py.File.__getitem__(self.hdf5, name)

    def set_nbytes(self, key, nbytes=None):
        """
        Set the `nbytes` attribute on the HDF5 object identified by `key`.
        """
        return self.hdf5.set_nbytes(key, nbytes)

    def set_attrs(self, key, **kw):
        """
        Set the HDF5 attributes of the given key
        """
        self.hdf5.save_attrs(key, kw)

    def get_attr(self, key, name, default=None):
        """
        :param key: dataset path
        :param name: name of the attribute
        :param default: value to return if the attribute is missing
        """
        try:
            obj = h5py.File.__getitem__(self.hdf5, key)
        except KeyError:
            if self.parent != ():
                return self.parent.get_attr(key, name, default)
            else:
                raise
        try:
            return obj.attrs[name]
        except KeyError:
            if default is None:
                raise
            return default

    def get_attrs(self, key):
        """
        :param key: dataset path
        :returns: dictionary of attributes for that path
        """
        try:
            dset = h5py.File.__getitem__(self.hdf5, key)
        except KeyError:
            if self.parent != ():
                dset = h5py.File.__getitem__(self.parent.hdf5, key)
            else:
                raise
        return dict(dset.attrs)

    def create_dset(self, key, dtype, shape=(None,), compression=None,
                    fillvalue=0, attrs=None):
        """
        Create a one-dimensional HDF5 dataset.

        :param key: name of the dataset
        :param dtype: dtype of the dataset (usually composite)
        :param shape: shape of the dataset, possibly extendable
        :param compression: the kind of HDF5 compression to use
        :param attrs: dictionary of attributes of the dataset
        :returns: a HDF5 dataset
        """
        return hdf5.create(
            self.hdf5, key, dtype, shape, compression, fillvalue, attrs)

    def extend(self, key, array, **attrs):
        """
        Extend the dataset associated to the given key; create it if needed

        :param key: name of the dataset
        :param array: array to store
        :param attrs: a dictionary of attributes
        """
        try:
            dset = self.hdf5[key]
        except KeyError:
            dset = hdf5.create(self.hdf5, key, array.dtype,
                               shape=(None,) + array.shape[1:])
        hdf5.extend(dset, array)
        for k, v in attrs.items():
            dset.attrs[k] = v
        return dset

    def save(self, key, kw):
        """
        Update the object associated to `key` with the `kw` dictionary;
        works for LiteralAttrs objects and automatically flushes.
        """
        if key not in self:
            obj = hdf5.LiteralAttrs()
        else:
            obj = self[key]
        vars(obj).update(kw)
        self[key] = obj
        self.flush()

    def export_path(self, relname, export_dir=None):
        """
        Return the path of the exported file by adding the export_dir in
        front, the calculation ID at the end.

        :param relname: relative file name
        :param export_dir: export directory (if None use .export_dir)
        """
        # removing inner slashed to avoid creating intermediate directories
        name, ext = relname.replace('/', '-').rsplit('.', 1)
        newname = '%s_%s.%s' % (name, self.calc_id, ext)
        if export_dir is None:
            export_dir = self.export_dir
        return os.path.join(export_dir, newname)

    def build_fname(self, prefix, postfix, fmt, export_dir=None):
        """
        Build a file name from a realization, by using prefix and extension.

        :param prefix: the prefix to use
        :param postfix: the postfix to use (can be a realization object)
        :param fmt: the extension ('csv', 'xml', etc)
        :param export_dir: export directory (if None use .export_dir)
        :returns: relative pathname including the extension
        """
        if hasattr(postfix, 'sm_lt_path'):  # is a realization
            fname = '%s-rlz-%03d.%s' % (prefix, postfix.ordinal, fmt)
        else:
            fname = prefix + ('-%s' % postfix if postfix else '') + '.' + fmt
        return self.export_path(fname, export_dir)

    def flush(self):
        """Flush the underlying hdf5 file"""
        if self.parent != ():
            self.parent.flush()
        if self.hdf5:  # is open
            self.hdf5.flush()

    def close(self):
        """Close the underlying hdf5 file"""
        if self.parent != ():
            self.parent.flush()
            self.parent.close()
        if self.hdf5:  # is open
            self.hdf5.flush()
            self.hdf5.close()
            self.hdf5 = ()

    def clear(self):
        """Remove the datastore from the file system"""
        self.close()
        os.remove(self.filename)

    def getsize(self, key=None):
        """
        Return the size in byte of the output associated to the given key.
        If no key is given, returns the total size of all files.
        """
        if key is None:
            return os.path.getsize(self.filename)
        return hdf5.ByteCounter.get_nbytes(
            h5py.File.__getitem__(self.hdf5, key))

    def get(self, key, default):
        """
        :returns: the value associated to the datastore key, or the default
        """
        try:
            return self[key]
        except KeyError:
            return default

    @property
    def metadata(self):
        """
        :returns: datastore metadata version, date, checksum as a dictionary
        """
        a = self.hdf5.attrs
        return dict(generated_by='OpenQuake engine %s' % a['engine_version'],
                    start_date=a['date'], checksum=a['checksum32'])

    def __getitem__(self, key):
        if self.hdf5 == ():  # the datastore is closed
            raise ValueError('Cannot find %s in %s' % (key, self))
        try:
            val = self.hdf5[key]
        except KeyError:
            if self.parent != ():
                self.parent.open('r')
                try:
                    val = self.parent[key]
                except KeyError:
                    raise KeyError(
                        'No %r found in %s and ancestors' % (key, self))
            else:
                raise KeyError('No %r found in %s' % (key, self))
        return val

    def __setitem__(self, key, val):
        if key in self.hdf5:
            # there is a bug in the current version of HDF5 for composite
            # arrays: is impossible to save twice the same key; so we remove
            # the key first, then it is possible to save it again
            del self[key]
        try:
            self.hdf5[key] = val
        except RuntimeError as exc:
            raise RuntimeError('Could not save %s: %s in %s' %
                               (key, exc, self.filename))

    def __delitem__(self, key):
        del self.hdf5[key]

    def __enter__(self):
        self.was_close = self.hdf5 == ()
        if self.was_close:
            self.open(self.mode)
        return self

    def __exit__(self, etype, exc, tb):
        if self.was_close:  # and has been opened in __enter__, close it
            self.close()
        del self.was_close

    def __getstate__(self):
        # make the datastore pickleable
        return dict(mode='r',
                    parent=self.parent,
                    calc_id=self.calc_id,
                    hdf5=(),
                    filename=self.filename)

    def __iter__(self):
        if not self.hdf5:
            raise RuntimeError('%s is closed' % self)
        for path in sorted(self.hdf5):
            yield path

    def __contains__(self, key):
        return key in self.hdf5 or self.parent and key in self.parent.hdf5

    def __len__(self):
        if self.hdf5 == ():  # closed
            return 1
        return sum(1 for f in self.hdf5)

    def __hash__(self):
        return self.calc_id

    def __repr__(self):
        status = 'open' if self.hdf5 else 'closed'
        return '<%s %s %s>' % (self.__class__.__name__, self.filename, status)
<commit_msg>Removed offending reading of the ~ hdf5 file<commit_after># -*- coding: utf-8 -*-
# vim: tabstop=4 shiftwidth=4 softtabstop=4
#
# Copyright (C) 2015-2019 GEM Foundation
#
# OpenQuake is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# OpenQuake is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with OpenQuake. If not, see <http://www.gnu.org/licenses/>.

import os
import re
import getpass
import collections
import h5py

from openquake.baselib import hdf5, config, performance

CALC_REGEX = r'(calc|cache)_(\d+)\.hdf5'

def get_datadir():
    """
    Extracts the path of the directory where the openquake data are stored
    from the environment ($OQ_DATADIR) or from the shared_dir in the
    configuration file.
    """
    datadir = os.environ.get('OQ_DATADIR')
    if not datadir:
        shared_dir = config.directory.shared_dir
        if shared_dir:
            datadir = os.path.join(shared_dir, getpass.getuser(), 'oqdata')
        else:  # use the home of the user
            datadir = os.path.join(os.path.expanduser('~'), 'oqdata')
    return datadir

def get_calc_ids(datadir=None):
    """
    Extract the available calculation IDs from the datadir, in order.
    """
    datadir = datadir or get_datadir()
    if not os.path.exists(datadir):
        return []
    calc_ids = set()
    for f in os.listdir(datadir):
        mo = re.match(CALC_REGEX, f)
        if mo:
            calc_ids.add(int(mo.group(2)))
    return sorted(calc_ids)

def get_last_calc_id(datadir=None):
    """
    Extract the latest calculation ID from the given directory.
    If none is found, return 0.
    """
    datadir = datadir or get_datadir()
    calcs = get_calc_ids(datadir)
    if not calcs:
        return 0
    return calcs[-1]

def hdf5new(datadir=None):
    """
    Return a new `hdf5.File by` instance with name determined by the last
    calculation in the datadir (plus one). Set the .path attribute to the
    generated filename.
    """
    datadir = datadir or get_datadir()
    if not os.path.exists(datadir):
        os.makedirs(datadir)
    calc_id = get_last_calc_id(datadir) + 1
    fname = os.path.join(datadir, 'calc_%d.hdf5' % calc_id)
    new = hdf5.File(fname, 'w')
    new.path = fname
    performance.init_performance(new)
    return new

def extract_calc_id_datadir(filename, datadir=None):
    """
    Extract the calculation ID from the given filename or integer:

    >>> extract_calc_id_datadir('/mnt/ssd/oqdata/calc_25.hdf5')
    (25, '/mnt/ssd/oqdata')
    >>> extract_calc_id_datadir('/mnt/ssd/oqdata/wrong_name.hdf5')
    Traceback (most recent call last):
       ...
    ValueError: Cannot extract calc_id from /mnt/ssd/oqdata/wrong_name.hdf5
    """
    datadir = datadir or get_datadir()
    try:
        calc_id = int(filename)
    except ValueError:
        filename = os.path.abspath(filename)
        datadir = os.path.dirname(filename)
        mo = re.match(CALC_REGEX, os.path.basename(filename))
        if mo is None:
            raise ValueError('Cannot extract calc_id from %s' % filename)
        calc_id = int(mo.group(2))
    return calc_id, datadir

def read(calc_id, mode='r', datadir=None):
    """
    :param calc_id: calculation ID or filename
    :param mode: 'r' or 'w'
    :param datadir: the directory where to look
    :returns: the corresponding DataStore instance

    Read the datastore, if it exists and it is accessible.
    """
    datadir = datadir or get_datadir()
    dstore = DataStore(calc_id, datadir, mode=mode)
    try:
        hc_id = dstore['oqparam'].hazard_calculation_id
    except KeyError:  # no oqparam
        hc_id = None
    if hc_id:
        dstore.parent = read(hc_id, datadir=os.path.dirname(dstore.filename))
    return dstore

class DataStore(collections.abc.MutableMapping):
    """
    DataStore class to store the inputs/outputs of a calculation on the
    filesystem.

    Here is a minimal example of usage:

    >>> ds = DataStore()
    >>> ds['example'] = 42
    >>> print(ds['example'][()])
    42
    >>> ds.clear()

    When reading the items, the DataStore will return a generator. The
    items will be ordered lexicographically according to their name.

    There is a serialization protocol to store objects in the datastore.
    An object is serializable if it has a method `__toh5__` returning
    an array and a dictionary, and a method `__fromh5__` taking an array
    and a dictionary and populating the object.
    For an example of use see :class:`openquake.hazardlib.site.SiteCollection`.
    """
    def __init__(self, calc_id=None, datadir=None, params=(), mode=None):
        datadir = datadir or get_datadir()
        if isinstance(calc_id, str):  # passed a real path
            self.filename = calc_id
            self.calc_id, datadir = extract_calc_id_datadir(calc_id, datadir)
        else:
            if calc_id is None:  # use a new datastore
                self.calc_id = get_last_calc_id(datadir) + 1
            elif calc_id < 0:  # use an old datastore
                calc_ids = get_calc_ids(datadir)
                try:
                    self.calc_id = calc_ids[calc_id]
                except IndexError:
                    raise IndexError(
                        'There are %d old calculations, cannot '
                        'retrieve the %s' % (len(calc_ids), calc_id))
            else:  # use the given datastore
                self.calc_id = calc_id
            self.filename = os.path.join(
                datadir, 'calc_%s.hdf5' % self.calc_id)
        if not os.path.exists(datadir):
            os.makedirs(datadir)
        self.params = params
        self.parent = ()  # can be set later
        self.datadir = datadir
        self.mode = mode or ('r+' if os.path.exists(self.filename) else 'w')
        if self.mode == 'r' and not os.path.exists(self.filename):
            raise IOError('File not found: %s' % self.filename)
        self.hdf5 = ()  # so that `key in self.hdf5` is valid
        self.open(self.mode)
        if 'w' in self.mode or '+' in self.mode:
            performance.init_performance(self.hdf5)

    def open(self, mode):
        """
        Open the underlying .hdf5 file and the parent, if any
        """
        if self.hdf5 == ():  # not already open
            kw = dict(mode=mode, libver='latest')
            if mode == 'r':
                kw['swmr'] = True
            try:
                self.hdf5 = hdf5.File(self.filename, **kw)
            except OSError as exc:
                # there was an hdf5.File(self.filename + '~', **kw)
                raise OSError('%s in %s' % (exc, self.filename))

    @property
    def export_dir(self):
        """
        Return the underlying export directory
        """
        edir = getattr(self, '_export_dir', None) or self['oqparam'].export_dir
        return edir

    @export_dir.setter
    def export_dir(self, value):
        """
        Set the export directory
        """
        self._export_dir = value

    def cachepath(self):
        """
        :returns: the path to the .hdf5 cache file associated to the calc_id
        """
        return os.path.join(self.datadir, 'cache_%d.hdf5' % self.calc_id)

    def getitem(self, name):
        """
        Return a dataset by using h5py.File.__getitem__
        """
        return h5py.File.__getitem__(self.hdf5, name)

    def set_nbytes(self, key, nbytes=None):
        """
        Set the `nbytes` attribute on the HDF5 object identified by `key`.
        """
        return self.hdf5.set_nbytes(key, nbytes)

    def set_attrs(self, key, **kw):
        """
        Set the HDF5 attributes of the given key
        """
        self.hdf5.save_attrs(key, kw)

    def get_attr(self, key, name, default=None):
        """
        :param key: dataset path
        :param name: name of the attribute
        :param default: value to return if the attribute is missing
        """
        try:
            obj = h5py.File.__getitem__(self.hdf5, key)
        except KeyError:
            if self.parent != ():
                return self.parent.get_attr(key, name, default)
            else:
                raise
        try:
            return obj.attrs[name]
        except KeyError:
            if default is None:
                raise
            return default

    def get_attrs(self, key):
        """
        :param key: dataset path
        :returns: dictionary of attributes for that path
        """
        try:
            dset = h5py.File.__getitem__(self.hdf5, key)
        except KeyError:
            if self.parent != ():
                dset = h5py.File.__getitem__(self.parent.hdf5, key)
            else:
                raise
        return dict(dset.attrs)

    def create_dset(self, key, dtype, shape=(None,), compression=None,
                    fillvalue=0, attrs=None):
        """
        Create a one-dimensional HDF5 dataset.

        :param key: name of the dataset
        :param dtype: dtype of the dataset (usually composite)
        :param shape: shape of the dataset, possibly extendable
        :param compression: the kind of HDF5 compression to use
        :param attrs: dictionary of attributes of the dataset
        :returns: a HDF5 dataset
        """
        return hdf5.create(
            self.hdf5, key, dtype, shape, compression, fillvalue, attrs)

    def extend(self, key, array, **attrs):
        """
        Extend the dataset associated to the given key; create it if needed

        :param key: name of the dataset
        :param array: array to store
        :param attrs: a dictionary of attributes
        """
        try:
            dset = self.hdf5[key]
        except KeyError:
            dset = hdf5.create(self.hdf5, key, array.dtype,
                               shape=(None,) + array.shape[1:])
        hdf5.extend(dset, array)
        for k, v in attrs.items():
            dset.attrs[k] = v
        return dset

    def save(self, key, kw):
        """
        Update the object associated to `key` with the `kw` dictionary;
        works for LiteralAttrs objects and automatically flushes.
        """
        if key not in self:
            obj = hdf5.LiteralAttrs()
        else:
            obj = self[key]
        vars(obj).update(kw)
        self[key] = obj
        self.flush()

    def export_path(self, relname, export_dir=None):
        """
        Return the path of the exported file by adding the export_dir in
        front, the calculation ID at the end.

        :param relname: relative file name
        :param export_dir: export directory (if None use .export_dir)
        """
        # removing inner slashed to avoid creating intermediate directories
        name, ext = relname.replace('/', '-').rsplit('.', 1)
        newname = '%s_%s.%s' % (name, self.calc_id, ext)
        if export_dir is None:
            export_dir = self.export_dir
        return os.path.join(export_dir, newname)

    def build_fname(self, prefix, postfix, fmt, export_dir=None):
        """
        Build a file name from a realization, by using prefix and extension.

        :param prefix: the prefix to use
        :param postfix: the postfix to use (can be a realization object)
        :param fmt: the extension ('csv', 'xml', etc)
        :param export_dir: export directory (if None use .export_dir)
        :returns: relative pathname including the extension
        """
        if hasattr(postfix, 'sm_lt_path'):  # is a realization
            fname = '%s-rlz-%03d.%s' % (prefix, postfix.ordinal, fmt)
        else:
            fname = prefix + ('-%s' % postfix if postfix else '') + '.' + fmt
        return self.export_path(fname, export_dir)

    def flush(self):
        """Flush the underlying hdf5 file"""
        if self.parent != ():
            self.parent.flush()
        if self.hdf5:  # is open
            self.hdf5.flush()

    def close(self):
        """Close the underlying hdf5 file"""
        if self.parent != ():
            self.parent.flush()
            self.parent.close()
        if self.hdf5:  # is open
            self.hdf5.flush()
            self.hdf5.close()
            self.hdf5 = ()

    def clear(self):
        """Remove the datastore from the file system"""
        self.close()
        os.remove(self.filename)

    def getsize(self, key=None):
        """
        Return the size in byte of the output associated to the given key.
        If no key is given, returns the total size of all files.
        """
        if key is None:
            return os.path.getsize(self.filename)
        return hdf5.ByteCounter.get_nbytes(
            h5py.File.__getitem__(self.hdf5, key))

    def get(self, key, default):
        """
        :returns: the value associated to the datastore key, or the default
        """
        try:
            return self[key]
        except KeyError:
            return default

    @property
    def metadata(self):
        """
        :returns: datastore metadata version, date, checksum as a dictionary
        """
        a = self.hdf5.attrs
        return dict(generated_by='OpenQuake engine %s' % a['engine_version'],
                    start_date=a['date'], checksum=a['checksum32'])

    def __getitem__(self, key):
        if self.hdf5 == ():  # the datastore is closed
            raise ValueError('Cannot find %s in %s' % (key, self))
        try:
            val = self.hdf5[key]
        except KeyError:
            if self.parent != ():
                self.parent.open('r')
                try:
                    val = self.parent[key]
                except KeyError:
                    raise KeyError(
                        'No %r found in %s and ancestors' % (key, self))
            else:
                raise KeyError('No %r found in %s' % (key, self))
        return val

    def __setitem__(self, key, val):
        if key in self.hdf5:
            # there is a bug in the current version of HDF5 for composite
            # arrays: is impossible to save twice the same key; so we remove
            # the key first, then it is possible to save it again
            del self[key]
        try:
            self.hdf5[key] = val
        except RuntimeError as exc:
            raise RuntimeError('Could not save %s: %s in %s' %
                               (key, exc, self.filename))

    def __delitem__(self, key):
        del self.hdf5[key]

    def __enter__(self):
        self.was_close = self.hdf5 == ()
        if self.was_close:
            self.open(self.mode)
        return self

    def __exit__(self, etype, exc, tb):
        if self.was_close:  # and has been opened in __enter__, close it
            self.close()
        del self.was_close

    def __getstate__(self):
        # make the datastore pickleable
        return dict(mode='r',
                    parent=self.parent,
                    calc_id=self.calc_id,
                    hdf5=(),
                    filename=self.filename)

    def __iter__(self):
        if not self.hdf5:
            raise RuntimeError('%s is closed' % self)
        for path in sorted(self.hdf5):
            yield path

    def __contains__(self, key):
        return key in self.hdf5 or self.parent and key in self.parent.hdf5

    def __len__(self):
        if self.hdf5 == ():  # closed
            return 1
        return sum(1 for f in self.hdf5)

    def __hash__(self):
        return self.calc_id

    def __repr__(self):
        status = 'open' if self.hdf5 else 'closed'
        return '<%s %s %s>' % (self.__class__.__name__, self.filename, status)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

# Copyright (c) 2010-2011, GEM Foundation.
#
# OpenQuake is free software: you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License version 3
# only, as published by the Free Software Foundation.
#
# OpenQuake is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Lesser General Public License version 3 for more details
# (a copy is included in the LICENSE file that accompanied this code).
#
# You should have received a copy of the GNU Lesser General Public License
# version 3 along with OpenQuake.  If not, see
# <http://www.gnu.org/licenses/lgpl-3.0.txt> for a copy of the LGPLv3 License.

""" Mixins for risk jobs. """

#import probabilistic
#import classical_psha
#import scenario
<commit_msg>Removed some commented code.<commit_after># -*- coding: utf-8 -*-

# Copyright (c) 2010-2011, GEM Foundation.
#
# OpenQuake is free software: you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License version 3
# only, as published by the Free Software Foundation.
#
# OpenQuake is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Lesser General Public License version 3 for more details
# (a copy is included in the LICENSE file that accompanied this code).
#
# You should have received a copy of the GNU Lesser General Public License
# version 3 along with OpenQuake.  If not, see
# <http://www.gnu.org/licenses/lgpl-3.0.txt> for a copy of the LGPLv3 License.

""" Functionality for risk calculations. """
<|endoftext|>


<commit_before>#!/usr/bin/env python

from stat import S_IRUSR, S_IWUSR, S_IXUSR, S_IRGRP, S_IXGRP, S_IROTH, S_IXOTH
from subprocess import call
import os
import re
import sys

class DjangoSettingsManager(object):
    """
    Utility class which allows moving and copying variables inbetween several
    settings files in the project's ``settings/`` folder.
    """
    __path = ''
    __file = {}
    __data = {}
    #__match = {}
    NO_MATCH = (0, 0)

    def __init__(self, projectname, *filenames):
        """
        Constructor to add settings files (named without path and extension).
        """
        self.__path = os.path.join(projectname, 'settings')
        if not os.path.exists(self.__path):
            os.makedirs(self.__path)
        for fname in filenames:
            self.add_file(fname)

    def add_file(self, fname):
        """
        Adds a settings file (named without path and extension).
        If the related .py file doesn't exist an empty file is created.
        """
        file = open(os.path.join(self.__path, fname + '.py'), 'a+')
        self.__file[fname] = file
        self.__data[fname] = file.read()
        #self.__match[fname] = {}

    def save_files(self):
        """
        Write all changes to disk.
        """
        for fname, file in self.__file.items():
            data = self.__data[fname]
            file.write(data)

    def find_var(self, src, var):
        """
        Returns (start, stop) position of a match, or NO_MATCH i.e. (0, 0).
        A match is a variable including optional leading comment lines.
        """
        # yield cached result if available (from an earlier call)
        #if var in self.__match[src]:
        #    return self.__match[src][var]

        data = self.__data[src]

        # variable incl. leading comments, until after trailing equal sign
        # and optional line continuation mark (backslash)
        pattern = re.compile(r'([ ]*#.*\n)*[ ]*(\A|\b)' + var + r'\s*=\s*\\?\s*')
        m = pattern.search(data)
        if m == None:
            # not found
            #self.__match[src][var] = (0, 0)
            return self.NO_MATCH

        start, stop = m.span()
        stop = self._find_endofvalue(data, stop)

        #print('>>> MATCH >>>' + str((start,stop)) + '>>>' + data[start:stop] + '<<<')  # DEBUG ONLY
        #self.__match[src][var] = (start, stop)
        return (start, stop)

    def _find_endofvalue(self, data, start):
        """
        Identifies value type (str, tuple, list, dict) and returns end index.
        """
        delimiters = {
            '\"""': '\"""',
            '"': '"',
            "'": "'",
            '(': ')',
            '[': ']',
            '{': '}',
        }

        open_delim = data[start:start + 3]
        if open_delim != '\"""':
            open_delim = data[start:start + 1]

        stop = start + len(open_delim)
        try:
            close_delim = delimiters[open_delim]
            pattern = re.compile(close_delim + r'[ ]*\n?')
            m = pattern.search(data, stop)
            if m:
                ignore, stop = m.span()
            else:
                raise SyntaxError('No closing delimiter %s found' % (close_delim, ))
        except:
            # variable found instead of opening delimiter
            pattern = re.compile(r'\n')
            m = pattern.search(data, stop)
            if m:
                ignore, stop = m.span()
            else:
                stop = len(data)
        return stop

    def _append(self, dest, data):
        self.__data[dest] += data

    def append_lines(self, dest, *lines):
        if len(self.__data[dest]) > 0:
            self._append(os.linesep)
        for data in lines:
            self._append(dest, data + os.linesep)

    def delete_var(self, src, var):
        """
        Deletes a variable from a settings file.
        """
        data = self.__data[src]
        start, stop = self.find_var(src, var)
        self.__data[src] = data[:start] + data[stop:]
        #print('>>> DELETING >>>' + data[start:stop] + '<<< DELETED. ' + os.linesep + '-' * 40)  # DEBUG ONLY

    def copy_var(self, src, destinations, var):
        """
        Copies a variable from one settings file to one or more others.
        """
        start, stop = self.find_var(src, var)
        data = self.__data[src][start:stop]
        for dest in destinations:
            self._append(dest, data)

    def move_var(self, src, destinations, var):
        """
        Moves a variable from one settings file to one or more others.
        """
        self.copy_var(src, destinations, var)
        self.delete_var(src, var)

def startproject():
    """
    Starts a new django Organice project by use of django-admin.py.
    """
    usage_descr = 'django Organice setup. Start getting organiced!'

    if sys.version_info < (2, 7):
        from optparse import OptionParser  # Deprecated since version 2.7

        parser = OptionParser(description=usage_descr)
        (options, args) = parser.parse_args()
        if len(args) != 1:
            parser.error("Please specify a projectname")
        projectname = args[0]
    else:
        from argparse import ArgumentParser  # New since version 2.7

        parser = ArgumentParser(description=usage_descr)
        parser.add_argument("projectname", help="name of project to create")
        args = parser.parse_args()
        projectname = args.projectname

    mode0755 = S_IRUSR | S_IWUSR | S_IXUSR | S_IRGRP | S_IXGRP | S_IROTH | S_IXOTH
    profiles = ('develop', 'staging', 'production')
    filenames = ('__init__', 'common') + profiles

    print('Generating project %s ...' % projectname)
    call(['django-admin.py', 'startproject', projectname, '.'])

    print('Converting settings to deployment profiles (%s) ...' % ', '.join(profiles))
    os.mkdir(os.path.join(projectname, 'settings'))
    os.rename(os.path.join(projectname, 'settings.py'),
              os.path.join(projectname, 'settings', 'common.py'))
    os.chmod('manage.py', mode0755)

    settings = DjangoSettingsManager(projectname, *filenames)
    settings.append_lines('__init__',
                          '\"""',
                          'Modularized settings generated by django Organice setup. http://organice.io',
                          'This solution follows the second recommendation from',
                          'http://www.sparklewise.com/django-settings-for-production-and-development-best-practices/',
                          '\"""',
                          'from develop import *')
    for prof in profiles:
        settings.append_lines(prof,
                              '# Django project settings for %s environment' % prof.capitalize(),
                              '',
                              'from common import *')
    settings.move_var('common', profiles, 'DEBUG')
    settings.move_var('common', profiles, 'TEMPLATE_DEBUG')
    settings.move_var('common', profiles, 'ALLOWED_HOSTS')
    settings.move_var('common', profiles, 'DATABASES')
    settings.move_var('common', profiles, 'SECRET_KEY')
    settings.move_var('common', profiles, '#WSGI_APPLICATION')
    settings.save_files()
    print('Done. Enjoy your organiced day!')

if __name__ == "__main__":
    startproject()
<commit_msg>Removed __match variable (because it breaks when parts of __data are deleted)<commit_after>#!/usr/bin/env python

from stat import S_IRUSR, S_IWUSR, S_IXUSR, S_IRGRP, S_IXGRP, S_IROTH, S_IXOTH
from subprocess import call
import os
import re
import sys

class DjangoSettingsManager(object):
    """
    Utility class which allows moving and copying variables inbetween several
    settings files in the project's ``settings/`` folder.
    """
    __path = ''
    __file = {}
    __data = {}
    NO_MATCH = (0, 0)

    def __init__(self, projectname, *filenames):
        """
        Constructor to add settings files (named without path and extension).
        """
        self.__path = os.path.join(projectname, 'settings')
        if not os.path.exists(self.__path):
            os.makedirs(self.__path)
        for fname in filenames:
            self.add_file(fname)

    def add_file(self, fname):
        """
        Adds a settings file (named without path and extension).
        If the related .py file doesn't exist an empty file is created.
        """
        file = open(os.path.join(self.__path, fname + '.py'), 'a+')
        self.__file[fname] = file
        self.__data[fname] = file.read()

    def save_files(self):
        """
        Write all changes to disk.
        """
        for fname, file in self.__file.items():
            data = self.__data[fname]
            file.write(data)

    def find_var(self, src, var):
        """
        Returns (start, stop) position of a match, or NO_MATCH i.e. (0, 0).
        A match is a variable including optional leading comment lines.
        """
        data = self.__data[src]

        # variable incl. leading comments, until after trailing equal sign
        # and optional line continuation mark (backslash)
        pattern = re.compile(r'([ ]*#.*\n)*[ ]*(\A|\b)' + var + r'\s*=\s*\\?\s*')
        m = pattern.search(data)
        if m == None:
            return self.NO_MATCH

        start, stop = m.span()
        stop = self._find_endofvalue(data, stop)

        return (start, stop)

    def _find_endofvalue(self, data, start):
        """
        Identifies value type (str, tuple, list, dict) and returns end index.
        """
        delimiters = {
            '\"""': '\"""',
            '"': '"',
            "'": "'",
            '(': ')',
            '[': ']',
            '{': '}',
        }

        open_delim = data[start:start + 3]
        if open_delim != '\"""':
            open_delim = data[start:start + 1]

        stop = start + len(open_delim)
        try:
            close_delim = delimiters[open_delim]
            pattern = re.compile(close_delim + r'[ ]*\n?')
            m = pattern.search(data, stop)
            if m:
                ignore, stop = m.span()
            else:
                raise SyntaxError('No closing delimiter %s found' % (close_delim, ))
        except:
            # variable found instead of opening delimiter
            pattern = re.compile(r'\n')
            m = pattern.search(data, stop)
            if m:
                ignore, stop = m.span()
            else:
                stop = len(data)
        return stop

    def _append(self, dest, data):
        self.__data[dest] += data

    def append_lines(self, dest, *lines):
        if len(self.__data[dest]) > 0:
            self._append(os.linesep)
        for data in lines:
            self._append(dest, data + os.linesep)

    def delete_var(self, src, var):
        """
        Deletes a variable from a settings file.
        """
        data = self.__data[src]
        start, stop = self.find_var(src, var)
        self.__data[src] = data[:start] + data[stop:]

    def copy_var(self, src, destinations, var):
        """
        Copies a variable from one settings file to one or more others.
        """
        start, stop = self.find_var(src, var)
        data = self.__data[src][start:stop]
        for dest in destinations:
            self._append(dest, data)

    def move_var(self, src, destinations, var):
        """
        Moves a variable from one settings file to one or more others.
        """
        self.copy_var(src, destinations, var)
        self.delete_var(src, var)

def startproject():
    """
    Starts a new django Organice project by use of django-admin.py.
    """
    usage_descr = 'django Organice setup. Start getting organiced!'

    if sys.version_info < (2, 7):
        from optparse import OptionParser  # Deprecated since version 2.7

        parser = OptionParser(description=usage_descr)
        (options, args) = parser.parse_args()
        if len(args) != 1:
            parser.error("Please specify a projectname")
        projectname = args[0]
    else:
        from argparse import ArgumentParser  # New since version 2.7

        parser = ArgumentParser(description=usage_descr)
        parser.add_argument("projectname", help="name of project to create")
        args = parser.parse_args()
        projectname = args.projectname

    mode0755 = S_IRUSR | S_IWUSR | S_IXUSR | S_IRGRP | S_IXGRP | S_IROTH | S_IXOTH
    profiles = ('develop', 'staging', 'production')
    filenames = ('__init__', 'common') + profiles

    print('Generating project %s ...' % projectname)
    call(['django-admin.py', 'startproject', projectname, '.'])

    print('Converting settings to deployment profiles (%s) ...' % ', '.join(profiles))
    os.mkdir(os.path.join(projectname, 'settings'))
    os.rename(os.path.join(projectname, 'settings.py'),
              os.path.join(projectname, 'settings', 'common.py'))
    os.chmod('manage.py', mode0755)

    settings = DjangoSettingsManager(projectname, *filenames)
    settings.append_lines('__init__',
                          '\"""',
                          'Modularized settings generated by django Organice setup. http://organice.io',
                          'This solution follows the second recommendation from',
                          'http://www.sparklewise.com/django-settings-for-production-and-development-best-practices/',
                          '\"""',
                          'from develop import *')
    for prof in profiles:
        settings.append_lines(prof,
                              '# Django project settings for %s environment' % prof.capitalize(),
                              '',
                              'from common import *')
    settings.move_var('common', profiles, 'DEBUG')
    settings.move_var('common', profiles, 'TEMPLATE_DEBUG')
    settings.move_var('common', profiles, 'ALLOWED_HOSTS')
    settings.move_var('common', profiles, 'DATABASES')
    settings.move_var('common', profiles, 'SECRET_KEY')
    settings.move_var('common', profiles, '#WSGI_APPLICATION')
    settings.save_files()
    print('Done. Enjoy your organiced day!')

if __name__ == "__main__":
    startproject()
<|endoftext|>


<commit_before>#!/usr/bin/env python
from itertools import tee, izip, chain
import json
import git
from collections import Counter
from random import randint

filename = "/var/local/allura/tickets.json"  # Absolute path to exported tickets.json
output = "/var/local/allura/updated_tickets.json"  # Absolute path to the output
ticket_list = "/var/local/allura/ticket_ids.list"
top_usernames = "/var/local/allura/top_usernames.list"
gitrepository = "/var/local/allura"  # Path to allura repository
g = git.Git(gitrepository)

reviews = ['code-review', 'design-review']

with open(filename, 'r') as json_file:
    data = json.loads(json_file.read())

def pairwise(iterable):
    """s -> (s0,s1), (s1,s2), (s2, s3), ..."""
    a, b = tee(iterable)
    next(b, None)
    return izip(a, b)

tags = ['asf_release_1.0.0', 'asf_release_1.0.0-RC1', 'asf_release_1.0.1', 'asf_release_1.1.0', 'HEAD']

tag_log = dict()
for tag1, tag2 in pairwise(tags):
    log = g.log('%s...%s' % (tag1, tag2), '--pretty=oneline')
    tag_log[tag2] = log

ticket_tag = dict()
tickets = data.pop('tickets')
for ticket in tickets:
    for key, value in tag_log.iteritems():
        if "[#%s]" % ticket['ticket_num'] in value:
            ticket_tag[ticket['ticket_num']] = key
            continue

data.pop('milestones', None)
data.pop('saved_bins', None)

updated = []
for ticket in tickets:
    if not ticket['private'] or ticket['ticket_num'] == 6054:
        if ticket['status'] in reviews:
            ticket['status'] = 'review'

        milestone = ticket_tag.get(ticket['ticket_num'], None)
        if not milestone:
            if ticket['status'] == 'closed':
                milestone = tags[0]
        ticket['custom_fields']['_milestone'] = milestone if milestone and milestone != 'HEAD' else 'unreleased'
        if '_size' in ticket['custom_fields'].keys():
            size = ticket['custom_fields']['_size']
            if size:
                ticket['labels'].append("sf-%d" % int(size))
            ticket['custom_fields'].pop('_size', None)

        if '_qa' in ticket['custom_fields'].keys():
            ticket['custom_fields']['_reviewer'] = ticket['custom_fields']['_qa']
            ticket['custom_fields'].pop('_qa', None)
        updated.append(ticket)
tags[-1] = 'unreleased'

data['tickets'] = updated

# Remove milestones from the list
custom_fields = filter(lambda d: d.get('name') not in ['_milestone', 'name', '_size', '_qa'], data['custom_fields'])
data['custom_fields'] = custom_fields

milestones = {
    "milestones": [
        dict(name=milestone_name,
             old_name=milestone_name,
             default=False,
             complete=False,
             due_date="",
             description="")
        for milestone_name in tags
    ],
    "name": "_milestone",
    "show_in_search": False,
    "label": "Milestone",
    "type": "milestone",
    "options": ""
}
data['custom_fields'].append(milestones)
data['custom_fields'].append({
    "show_in_search": True,
    "label": "Reviewer",
    "type": "user",
    "options": "",
    "name": "_reviewer"
})
data['milestones'] = milestones
data['saved_bins'] = []

# Count top used usernames

assigned_to = [ticket.get('assigned_to', None) for ticket in updated]
reported_by = [ticket.get('reported_by', None) for ticket in updated]
reviewed_by = [ticket['custom_fields'].get('_reviewer', None) for ticket in updated]

posts = [ticket['discussion_thread']['posts'] for ticket in updated]

post_authors = [post.get('author', None) for post in list(chain(*posts))]

usernames = filter(lambda x: bool(x), chain(assigned_to, reported_by, reviewed_by, post_authors))

top_users = Counter(usernames).most_common(50)

with open(output, 'w') as outfile:
    json.dump(data, outfile, indent=2)

with open(ticket_list, 'w') as outfile:
    outfile.write('\n'.join(sorted([str(ticket['ticket_num']) for ticket in updated])))

with open(top_usernames, 'w') as outfile:
    lines = ["%s - %s" % (username, frequency) for username, frequency in top_users]
    outfile.write('\n'.join(lines))<commit_msg>[#7510] add license header and docstring to new script<commit_after>#!/usr/bin/env python
#       Licensed to the Apache Software Foundation (ASF) under one
#       or more contributor license agreements.  See the NOTICE file
#       distributed with this work for additional information
#       regarding copyright ownership.  The ASF licenses this file
#       to you under the Apache License, Version 2.0 (the
#       "License"); you may not use this file except in compliance
#       with the License.  You may obtain a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#       Unless required by applicable law or agreed to in writing,
#       software distributed under the License is distributed on an
#       "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
#       KIND, either express or implied.  See the License for the
#       specific language governing permissions and limitations
#       under the License.

from itertools import tee, izip, chain
import json
import git
from collections import Counter

'''
This script is for one-time conversion of Allura's own tickets from SourceForge
host to forge-allura.apache.org hosting

Change path variables here:
'''

filename = "/var/local/allura/tickets.json"  # Absolute path to exported tickets.json
output = "/var/local/allura/updated_tickets.json"  # Absolute path to the output
ticket_list = "/var/local/allura/ticket_ids.list"
top_usernames = "/var/local/allura/top_usernames.list"
gitrepository = "/var/local/allura"  # Path to allura repository
g = git.Git(gitrepository)

reviews = ['code-review', 'design-review']

with open(filename, 'r') as json_file:
    data = json.loads(json_file.read())

def pairwise(iterable):
    """s -> (s0,s1), (s1,s2), (s2, s3), ..."""
    a, b = tee(iterable)
    next(b, None)
    return izip(a, b)

tags = ['asf_release_1.0.0', 'asf_release_1.0.0-RC1', 'asf_release_1.0.1', 'asf_release_1.1.0', 'HEAD']

tag_log = dict()
for tag1, tag2 in pairwise(tags):
    log = g.log('%s...%s' % (tag1, tag2), '--pretty=oneline')
    tag_log[tag2] = log

ticket_tag = dict()
tickets = data.pop('tickets')
for ticket in tickets:
    for key, value in tag_log.iteritems():
        if "[#%s]" % ticket['ticket_num'] in value:
            ticket_tag[ticket['ticket_num']] = key
            continue

data.pop('milestones', None)
data.pop('saved_bins', None)

updated = []
for ticket in tickets:
    if not ticket['private'] or ticket['ticket_num'] == 6054:
        if ticket['status'] in reviews:
            ticket['status'] = 'review'

        milestone = ticket_tag.get(ticket['ticket_num'], None)
        if not milestone:
            if ticket['status'] == 'closed':
                milestone = tags[0]
        ticket['custom_fields']['_milestone'] = milestone if milestone and milestone != 'HEAD' else 'unreleased'
        if '_size' in ticket['custom_fields'].keys():
            size = ticket['custom_fields']['_size']
            if size:
                ticket['labels'].append("sf-%d" % int(size))
            ticket['custom_fields'].pop('_size', None)

        if '_qa' in ticket['custom_fields'].keys():
            ticket['custom_fields']['_reviewer'] = ticket['custom_fields']['_qa']
            ticket['custom_fields'].pop('_qa', None)
        updated.append(ticket)
tags[-1] = 'unreleased'

data['tickets'] = updated

# Remove milestones from the list
custom_fields = filter(lambda d: d.get('name') not in ['_milestone', 'name', '_size', '_qa'], data['custom_fields'])
data['custom_fields'] = custom_fields

milestones = {
    "milestones": [
        dict(name=milestone_name,
             old_name=milestone_name,
             default=False,
             complete=False,
             due_date="",
             description="")
        for milestone_name in tags
    ],
    "name": "_milestone",
    "show_in_search": False,
    "label": "Milestone",
    "type": "milestone",
    "options": ""
}
data['custom_fields'].append(milestones)
data['custom_fields'].append({
    "show_in_search": True,
    "label": "Reviewer",
    "type": "user",
    "options": "",
    "name": "_reviewer"
})
data['milestones'] = milestones
data['saved_bins'] = []

# Count top used usernames

assigned_to = [ticket.get('assigned_to', None) for ticket in updated]
reported_by = [ticket.get('reported_by', None) for ticket in updated]
reviewed_by = [ticket['custom_fields'].get('_reviewer', None) for ticket in updated]

posts = [ticket['discussion_thread']['posts'] for ticket in updated]

post_authors = [post.get('author', None) for post in list(chain(*posts))]

usernames = filter(lambda x: bool(x), chain(assigned_to, reported_by, reviewed_by, post_authors))

top_users = Counter(usernames).most_common(50)

with open(output, 'w') as outfile:
    json.dump(data, outfile, indent=2)

with open(ticket_list, 'w') as outfile:
    outfile.write('\n'.join(sorted([str(ticket['ticket_num']) for ticket in updated])))

with open(top_usernames, 'w') as outfile:
    lines = ["%s - %s" % (username, frequency) for username, frequency in top_users]
    outfile.write('\n'.join(lines))<|endoftext|>


<commit_before><commit_msg>Experiment with TransE, ComplEx and DistMult at once<commit_after><|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and contributors
# For license information, please see license.txt

from __future__ import unicode_literals
import frappe, json
from frappe import _
from frappe.model.mapper import get_mapped_doc
from frappe.utils import get_url, cint
from frappe.utils.user import get_user_fullname
from frappe.utils.print_format import download_pdf
from frappe.desk.form.load import get_attachments
from frappe.core.doctype.communication.email import make
from erpnext.accounts.party import get_party_account_currency, get_party_details
from erpnext.stock.doctype.material_request.material_request import set_missing_values
from erpnext.controllers.buying_controller import BuyingController
from erpnext.buying.utils import validate_for_items

STANDARD_USERS = ("Guest", "Administrator")

class RequestforQuotation(BuyingController):
	def validate(self):
		self.validate_duplicate_supplier()
		self.validate_supplier_list()
		validate_for_items(self)
		self.update_email_id()

	def validate_duplicate_supplier(self):
		supplier_list = [d.supplier for d in self.suppliers]
		if len(supplier_list) != len(set(supplier_list)):
			frappe.throw(_("Same supplier has been entered multiple times"))

	def validate_supplier_list(self):
		for d in self.suppliers:
			prevent_rfqs = frappe.db.get_value("Supplier", d.supplier, 'prevent_rfqs')
			if prevent_rfqs:
				standing = frappe.db.get_value("Supplier Scorecard",d.supplier, 'status')
				frappe.throw(_("RFQs are not allowed for {0} due to a scorecard standing of {1}").format(d.supplier, standing))
			warn_rfqs = frappe.db.get_value("Supplier", d.supplier, 'warn_rfqs')
			if warn_rfqs:
				standing = frappe.db.get_value("Supplier Scorecard",d.supplier, 'status')
				frappe.msgprint(_("{0} currently has a {1} Supplier Scorecard standing, and RFQs to this supplier should be issued with caution.").format(d.supplier, standing), title=_("Caution"), indicator='orange')

	def update_email_id(self):
		for rfq_supplier in self.suppliers:
			if not rfq_supplier.email_id:
				rfq_supplier.email_id = frappe.db.get_value("Contact", rfq_supplier.contact, "email_id")

	def validate_email_id(self, args):
		if not args.email_id:
			frappe.throw(_("Row {0}: For supplier {0} Email Address is required to send email").format(args.idx, args.supplier))

	def on_submit(self):
		frappe.db.set(self, 'status', 'Submitted')
		for supplier in self.suppliers:
			supplier.email_sent = 0
			supplier.quote_status = 'Pending'

	def on_cancel(self):
		frappe.db.set(self, 'status', 'Cancelled')

	def send_to_supplier(self):
		for rfq_supplier in self.suppliers:
			if rfq_supplier.send_email:
				self.validate_email_id(rfq_supplier)

				# make new user if required
				update_password_link = self.update_supplier_contact(rfq_supplier, self.get_link())

				self.update_supplier_part_no(rfq_supplier)
				self.supplier_rfq_mail(rfq_supplier, update_password_link, self.get_link())
				rfq_supplier.email_sent = 1
				rfq_supplier.save()

	def get_link(self):
		# RFQ link for supplier portal
		return get_url("/rfq/" + self.name)

	def update_supplier_part_no(self, args):
		self.vendor = args.supplier
		for item in self.items:
			item.supplier_part_no = frappe.db.get_value('Item Supplier',
				{'parent': item.item_code, 'supplier': args.supplier}, 'supplier_part_no')

	def update_supplier_contact(self, rfq_supplier, link):
		'''Create a new user for the supplier if not set in contact'''
		update_password_link = ''

		if frappe.db.exists("User", rfq_supplier.email_id):
			user = frappe.get_doc("User", rfq_supplier.email_id)
		else:
			user, update_password_link = self.create_user(rfq_supplier, link)

		self.update_contact_of_supplier(rfq_supplier, user)

		return update_password_link

	def update_contact_of_supplier(self, rfq_supplier, user):
		if rfq_supplier.contact:
			contact = frappe.get_doc("Contact", rfq_supplier.contact)
		else:
			contact = frappe.new_doc("Contact")
			contact.first_name = rfq_supplier.supplier_name or rfq_supplier.supplier
			contact.append('links', {
				'link_doctype': 'Supplier',
				'link_name': rfq_supplier.supplier
			})

		if not contact.email_id and not contact.user:
			contact.email_id = user.name
			contact.user = user.name

		contact.save(ignore_permissions=True)

	def create_user(self, rfq_supplier, link):
		user = frappe.get_doc({
			'doctype': 'User',
			'send_welcome_email': 0,
			'email': rfq_supplier.email_id,
			'first_name': rfq_supplier.supplier_name or rfq_supplier.supplier,
			'user_type': 'Website User',
			'redirect_url': link
		})
		user.save(ignore_permissions=True)
		update_password_link = user.reset_password()

		return user, update_password_link

	def supplier_rfq_mail(self, data, update_password_link, rfq_link):
		full_name = get_user_fullname(frappe.session['user'])
		if full_name == "Guest":
			full_name = "Administrator"

		args = {
			'update_password_link': update_password_link,
			'message': frappe.render_template(self.message_for_supplier, data.as_dict()),
			'rfq_link': rfq_link,
			'user_fullname': full_name
		}

		subject = _("Request for Quotation:")
		template = "templates/emails/request_for_quotation.html"
		sender = frappe.session.user not in STANDARD_USERS and frappe.session.user or None
		message = frappe.get_template(template).render(args)
		attachments = self.get_attachments()

		self.send_email(data, sender, subject, message, attachments)

	def send_email(self, data, sender, subject, message, attachments):
		make(subject = subject, content=message,recipients=data.email_id,
			sender=sender,attachments = attachments, send_email=True,
		     	doctype=self.doctype, name=self.name)["name"]

		frappe.msgprint(_("Email sent to supplier {0}").format(data.supplier))

	def get_attachments(self):
		attachments = [d.name for d in get_attachments(self.doctype, self.name)]
		attachments.append(frappe.attach_print(self.doctype, self.name, doc=self))
		return attachments

	def update_rfq_supplier_status(self, sup_name=None):
		for supplier in self.suppliers:
			if sup_name == None or supplier.supplier == sup_name:
				if supplier.quote_status != _('No Quote'):
					quote_status = _('Received')
					for item in self.items:
						sqi_count = frappe.db.sql("""
							SELECT
								COUNT(sqi.name) as count
							FROM
								`tabSupplier Quotation Item` as sqi,
								`tabSupplier Quotation` as sq
							WHERE sq.supplier = %(supplier)s
								AND sqi.docstatus = 1
								AND sqi.request_for_quotation_item = %(rqi)s
								AND sqi.parent = sq.name""",
							{"supplier": supplier.supplier, "rqi": item.name}, as_dict=1)[0]
						if (sqi_count.count) == 0:
							quote_status = _('Pending')
					supplier.quote_status = quote_status

@frappe.whitelist()
def send_supplier_emails(rfq_name):
	check_portal_enabled('Request for Quotation')
	rfq = frappe.get_doc("Request for Quotation", rfq_name)
	if rfq.docstatus==1:
		rfq.send_to_supplier()

def check_portal_enabled(reference_doctype):
	if not frappe.db.get_value('Portal Menu Item',
		{'reference_doctype': reference_doctype}, 'enabled'):
		frappe.throw(_("Request for Quotation is disabled to access from portal, for more check portal settings."))

def get_list_context(context=None):
	from erpnext.controllers.website_list_for_contact import get_list_context
	list_context = get_list_context(context)
	list_context.update({
		'show_sidebar': True,
		'show_search': True,
		'no_breadcrumbs': True,
		'title': _('Request for Quotation'),
	})
	return list_context

def get_supplier_contacts(doctype, txt, searchfield, start, page_len, filters):
	return frappe.db.sql("""select `tabContact`.name from `tabContact`, `tabDynamic Link`
		where `tabDynamic Link`.link_doctype = 'Supplier' and (`tabDynamic Link`.link_name=%(name)s
		and `tabDynamic Link`.link_name like %(txt)s) and `tabContact`.name = `tabDynamic Link`.parent
		limit %(start)s, %(page_len)s""", {"start": start, "page_len":page_len, "txt": "%%%s%%" % txt, "name": filters.get('supplier')})

# This method is used to make supplier quotation from material request form.
@frappe.whitelist()
def make_supplier_quotation(source_name, for_supplier, target_doc=None):
	def postprocess(source, target_doc):
		target_doc.supplier = for_supplier
		args = get_party_details(for_supplier, party_type="Supplier", ignore_permissions=True)
		target_doc.currency = args.currency or get_party_account_currency('Supplier', for_supplier, source.company)
		target_doc.buying_price_list = args.buying_price_list or frappe.db.get_value('Buying Settings', None, 'buying_price_list')
		set_missing_values(source, target_doc)

	doclist = get_mapped_doc("Request for Quotation", source_name, {
		"Request for Quotation": {
			"doctype": "Supplier Quotation",
			"validation": {
				"docstatus": ["=", 1]
			}
		},
		"Request for Quotation Item": {
			"doctype": "Supplier Quotation Item",
			"field_map": {
				"name": "request_for_quotation_item",
				"parent": "request_for_quotation"
			},
		}
	}, target_doc, postprocess)

	return doclist

# This method is used to make supplier quotation from supplier's portal.
@frappe.whitelist()
def create_supplier_quotation(doc):
	if isinstance(doc, basestring):
		doc = json.loads(doc)

	try:
		sq_doc = frappe.get_doc({
			"doctype": "Supplier Quotation",
			"supplier": doc.get('supplier'),
			"terms": doc.get("terms"),
			"company": doc.get("company"),
			"currency": doc.get('currency') or get_party_account_currency('Supplier', doc.get('supplier'), doc.get('company')),
			"buying_price_list": doc.get('buying_price_list') or frappe.db.get_value('Buying Settings', None, 'buying_price_list')
		})
		add_items(sq_doc, doc.get('supplier'), doc.get('items'))
		sq_doc.flags.ignore_permissions = True
		sq_doc.run_method("set_missing_values")
		sq_doc.save()
		frappe.msgprint(_("Supplier Quotation {0} created").format(sq_doc.name))
		return sq_doc.name
	except Exception:
		return None

def add_items(sq_doc, supplier, items):
	for data in items:
		if data.get("qty") > 0:
			if isinstance(data, dict):
				data = frappe._dict(data)

			create_rfq_items(sq_doc, supplier, data)

def create_rfq_items(sq_doc, supplier, data):
	sq_doc.append('items', {
		"item_code": data.item_code,
		"item_name": data.item_name,
		"description": data.description,
		"qty": data.qty,
		"rate": data.rate,
		"supplier_part_no": frappe.db.get_value("Item Supplier", {'parent': data.item_code, 'supplier': supplier}, "supplier_part_no"),
		"warehouse": data.warehouse or '',
		"request_for_quotation_item": data.name,
		"request_for_quotation": data.parent
	})

@frappe.whitelist()
def get_pdf(doctype, name, supplier_idx):
	doc = get_rfq_doc(doctype, name, supplier_idx)
	if doc:
		download_pdf(doctype, name, doc=doc)

def get_rfq_doc(doctype, name, supplier_idx):
	if cint(supplier_idx):
		doc = frappe.get_doc(doctype, name)
		args = doc.get('suppliers')[cint(supplier_idx) - 1]
		doc.update_supplier_part_no(args)
		return doc

@frappe.whitelist()
def get_item_from_material_requests_based_on_supplier(source_name, target_doc = None):
	mr_items_list = frappe.db.sql("""
		SELECT
			mr.name, mr_item.item_code
		FROM
			`tabItem` as item,
			`tabItem Supplier` as item_supp,
			`tabMaterial Request Item` as mr_item,
			`tabMaterial Request`  as mr
		WHERE item_supp.supplier = %(supplier)s
			AND item.name = item_supp.parent
			AND mr_item.parent = mr.name
			AND mr_item.item_code = item.name
			AND mr.status != "Stopped"
			AND mr.material_request_type = "Purchase"
			AND mr.docstatus = 1
			AND mr.per_ordered < 99.99""", {"supplier": source_name}, as_dict=1)

	material_requests = {}
	for d in mr_items_list:
		material_requests.setdefault(d.name, []).append(d.item_code)

	for mr, items in material_requests.items():
		target_doc = get_mapped_doc("Material Request", mr, {
			"Material Request": {
				"doctype": "Request for Quotation",
				"validation": {
					"docstatus": ["=", 1],
					"material_request_type": ["=", "Purchase"],
				}
			},
			"Material Request Item": {
				"doctype": "Request for Quotation Item",
				"condition": lambda row: row.item_code in items,
				"field_map": [
					["name", "material_request_item"],
					["parent", "material_request"],
					["uom", "uom"]
				]
			}
		}, target_doc)

	return target_doc
<commit_msg>Update request_for_quotation.py<commit_after># -*- coding: utf-8 -*-
# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and contributors
# For license information, please see license.txt

from __future__ import unicode_literals
import frappe, json
from frappe import _
from frappe.model.mapper import get_mapped_doc
from frappe.utils import get_url, cint
from frappe.utils.user import get_user_fullname
from frappe.utils.print_format import download_pdf
from frappe.desk.form.load import get_attachments
from frappe.core.doctype.communication.email import make
from erpnext.accounts.party import get_party_account_currency, get_party_details
from erpnext.stock.doctype.material_request.material_request import set_missing_values
from erpnext.controllers.buying_controller import BuyingController
from erpnext.buying.utils import validate_for_items

STANDARD_USERS = ("Guest", "Administrator")

class RequestforQuotation(BuyingController):
	def validate(self):
		self.validate_duplicate_supplier()
		self.validate_supplier_list()
		validate_for_items(self)
		self.update_email_id()

	def validate_duplicate_supplier(self):
		supplier_list = [d.supplier for d in self.suppliers]
		if len(supplier_list) != len(set(supplier_list)):
			frappe.throw(_("Same supplier has been entered multiple times"))

	def validate_supplier_list(self):
		for d in self.suppliers:
			prevent_rfqs = frappe.db.get_value("Supplier", d.supplier, 'prevent_rfqs')
			if prevent_rfqs:
				standing = frappe.db.get_value("Supplier Scorecard",d.supplier, 'status')
				frappe.throw(_("RFQs are not allowed for {0} due to a scorecard standing of {1}").format(d.supplier, standing))
			warn_rfqs = frappe.db.get_value("Supplier", d.supplier, 'warn_rfqs')
			if warn_rfqs:
				standing = frappe.db.get_value("Supplier Scorecard",d.supplier, 'status')
				frappe.msgprint(_("{0} currently has a {1} Supplier Scorecard standing, and RFQs to this supplier should be issued with caution.").format(d.supplier, standing), title=_("Caution"), indicator='orange')

	def update_email_id(self):
		for rfq_supplier in self.suppliers:
			if not rfq_supplier.email_id:
				rfq_supplier.email_id = frappe.db.get_value("Contact", rfq_supplier.contact, "email_id")

	def validate_email_id(self, args):
		if not args.email_id:
			frappe.throw(_("Row {0}: For supplier {0} Email Address is required to send email").format(args.idx, args.supplier))

	def on_submit(self):
		frappe.db.set(self, 'status', 'Submitted')
		for supplier in self.suppliers:
			supplier.email_sent = 0
			supplier.quote_status = 'Pending'

	def on_cancel(self):
		frappe.db.set(self, 'status', 'Cancelled')

	def send_to_supplier(self):
		for rfq_supplier in self.suppliers:
			if rfq_supplier.send_email:
				self.validate_email_id(rfq_supplier)

				# make new user if required
				update_password_link = self.update_supplier_contact(rfq_supplier, self.get_link())

				self.update_supplier_part_no(rfq_supplier)
				self.supplier_rfq_mail(rfq_supplier, update_password_link, self.get_link())
				rfq_supplier.email_sent = 1
				rfq_supplier.save()

	def get_link(self):
		# RFQ link for supplier portal
		return get_url("/rfq/" + self.name)

	def update_supplier_part_no(self, args):
		self.vendor = args.supplier
		for item in self.items:
			item.supplier_part_no = frappe.db.get_value('Item Supplier',
				{'parent': item.item_code, 'supplier': args.supplier}, 'supplier_part_no')

	def update_supplier_contact(self, rfq_supplier, link):
		'''Create a new user for the supplier if not set in contact'''
		update_password_link = ''

		if frappe.db.exists("User", rfq_supplier.email_id):
			user = frappe.get_doc("User", rfq_supplier.email_id)
		else:
			user, update_password_link = self.create_user(rfq_supplier, link)

		self.update_contact_of_supplier(rfq_supplier, user)

		return update_password_link

	def update_contact_of_supplier(self, rfq_supplier, user):
		if rfq_supplier.contact:
			contact = frappe.get_doc("Contact", rfq_supplier.contact)
		else:
			contact = frappe.new_doc("Contact")
			contact.first_name = rfq_supplier.supplier_name or rfq_supplier.supplier
			contact.append('links', {
				'link_doctype': 'Supplier',
				'link_name': rfq_supplier.supplier
			})

		if not contact.email_id and not contact.user:
			contact.email_id = user.name
			contact.user = user.name

		contact.save(ignore_permissions=True)

	def create_user(self, rfq_supplier, link):
		user = frappe.get_doc({
			'doctype': 'User',
			'send_welcome_email': 0,
			'email': rfq_supplier.email_id,
			'first_name': rfq_supplier.supplier_name or rfq_supplier.supplier,
			'user_type': 'Website User',
			'redirect_url': link
		})
		user.save(ignore_permissions=True)
		update_password_link = user.reset_password()

		return user, update_password_link

	def supplier_rfq_mail(self, data, update_password_link, rfq_link):
		full_name = get_user_fullname(frappe.session['user'])
		if full_name == "Guest":
			full_name = "Administrator"

		args = {
			'update_password_link': update_password_link,
			'message': frappe.render_template(self.message_for_supplier, data.as_dict()),
			'rfq_link': rfq_link,
			'user_fullname': full_name
		}

		subject = _("Request for Quotation")
		template = "templates/emails/request_for_quotation.html"
		sender = frappe.session.user not in STANDARD_USERS and frappe.session.user or None
		message = frappe.get_template(template).render(args)
		attachments = self.get_attachments()

		self.send_email(data, sender, subject, message, attachments)

	def send_email(self, data, sender, subject, message, attachments):
		make(subject = subject, content=message,recipients=data.email_id,
			sender=sender,attachments = attachments, send_email=True,
		     	doctype=self.doctype, name=self.name)["name"]

		frappe.msgprint(_("Email sent to supplier {0}").format(data.supplier))

	def get_attachments(self):
		attachments = [d.name for d in get_attachments(self.doctype, self.name)]
		attachments.append(frappe.attach_print(self.doctype, self.name, doc=self))
		return attachments

	def update_rfq_supplier_status(self, sup_name=None):
		for supplier in self.suppliers:
			if sup_name == None or supplier.supplier == sup_name:
				if supplier.quote_status != _('No Quote'):
					quote_status = _('Received')
					for item in self.items:
						sqi_count = frappe.db.sql("""
							SELECT
								COUNT(sqi.name) as count
							FROM
								`tabSupplier Quotation Item` as sqi,
								`tabSupplier Quotation` as sq
							WHERE sq.supplier = %(supplier)s
								AND sqi.docstatus = 1
								AND sqi.request_for_quotation_item = %(rqi)s
								AND sqi.parent = sq.name""",
							{"supplier": supplier.supplier, "rqi": item.name}, as_dict=1)[0]
						if (sqi_count.count) == 0:
							quote_status = _('Pending')
					supplier.quote_status = quote_status

@frappe.whitelist()
def send_supplier_emails(rfq_name):
	check_portal_enabled('Request for Quotation')
	rfq = frappe.get_doc("Request for Quotation", rfq_name)
	if rfq.docstatus==1:
		rfq.send_to_supplier()

def check_portal_enabled(reference_doctype):
	if not frappe.db.get_value('Portal Menu Item',
		{'reference_doctype': reference_doctype}, 'enabled'):
		frappe.throw(_("Request for Quotation is disabled to access from portal, for more check portal settings."))

def get_list_context(context=None):
	from erpnext.controllers.website_list_for_contact import get_list_context
	list_context = get_list_context(context)
	list_context.update({
		'show_sidebar': True,
		'show_search': True,
		'no_breadcrumbs': True,
		'title': _('Request for Quotation'),
	})
	return list_context

def get_supplier_contacts(doctype, txt, searchfield, start, page_len, filters):
	return frappe.db.sql("""select `tabContact`.name from `tabContact`, `tabDynamic Link`
		where `tabDynamic Link`.link_doctype = 'Supplier' and (`tabDynamic Link`.link_name=%(name)s
		and `tabDynamic Link`.link_name like %(txt)s) and `tabContact`.name = `tabDynamic Link`.parent
		limit %(start)s, %(page_len)s""", {"start": start, "page_len":page_len, "txt": "%%%s%%" % txt, "name": filters.get('supplier')})

# This method is used to make supplier quotation from material request form.
@frappe.whitelist()
def make_supplier_quotation(source_name, for_supplier, target_doc=None):
	def postprocess(source, target_doc):
		target_doc.supplier = for_supplier
		args = get_party_details(for_supplier, party_type="Supplier", ignore_permissions=True)
		target_doc.currency = args.currency or get_party_account_currency('Supplier', for_supplier, source.company)
		target_doc.buying_price_list = args.buying_price_list or frappe.db.get_value('Buying Settings', None, 'buying_price_list')
		set_missing_values(source, target_doc)

	doclist = get_mapped_doc("Request for Quotation", source_name, {
		"Request for Quotation": {
			"doctype": "Supplier Quotation",
			"validation": {
				"docstatus": ["=", 1]
			}
		},
		"Request for Quotation Item": {
			"doctype": "Supplier Quotation Item",
			"field_map": {
				"name": "request_for_quotation_item",
				"parent": "request_for_quotation"
			},
		}
	}, target_doc, postprocess)

	return doclist

# This method is used to make supplier quotation from supplier's portal.
@frappe.whitelist()
def create_supplier_quotation(doc):
	if isinstance(doc, basestring):
		doc = json.loads(doc)

	try:
		sq_doc = frappe.get_doc({
			"doctype": "Supplier Quotation",
			"supplier": doc.get('supplier'),
			"terms": doc.get("terms"),
			"company": doc.get("company"),
			"currency": doc.get('currency') or get_party_account_currency('Supplier', doc.get('supplier'), doc.get('company')),
			"buying_price_list": doc.get('buying_price_list') or frappe.db.get_value('Buying Settings', None, 'buying_price_list')
		})
		add_items(sq_doc, doc.get('supplier'), doc.get('items'))
		sq_doc.flags.ignore_permissions = True
		sq_doc.run_method("set_missing_values")
		sq_doc.save()
		frappe.msgprint(_("Supplier Quotation {0} created").format(sq_doc.name))
		return sq_doc.name
	except Exception:
		return None

def add_items(sq_doc, supplier, items):
	for data in items:
		if data.get("qty") > 0:
			if isinstance(data, dict):
				data = frappe._dict(data)

			create_rfq_items(sq_doc, supplier, data)

def create_rfq_items(sq_doc, supplier, data):
	sq_doc.append('items', {
		"item_code": data.item_code,
		"item_name": data.item_name,
		"description": data.description,
		"qty": data.qty,
		"rate": data.rate,
		"supplier_part_no": frappe.db.get_value("Item Supplier", {'parent': data.item_code, 'supplier': supplier}, "supplier_part_no"),
		"warehouse": data.warehouse or '',
		"request_for_quotation_item": data.name,
		"request_for_quotation": data.parent
	})

@frappe.whitelist()
def get_pdf(doctype, name, supplier_idx):
	doc = get_rfq_doc(doctype, name, supplier_idx)
	if doc:
		download_pdf(doctype, name, doc=doc)

def get_rfq_doc(doctype, name, supplier_idx):
	if cint(supplier_idx):
		doc = frappe.get_doc(doctype, name)
		args = doc.get('suppliers')[cint(supplier_idx) - 1]
		doc.update_supplier_part_no(args)
		return doc

@frappe.whitelist()
def get_item_from_material_requests_based_on_supplier(source_name, target_doc = None):
	mr_items_list = frappe.db.sql("""
		SELECT
			mr.name, mr_item.item_code
		FROM
			`tabItem` as item,
			`tabItem Supplier` as item_supp,
			`tabMaterial Request Item` as mr_item,
			`tabMaterial Request`  as mr
		WHERE item_supp.supplier = %(supplier)s
			AND item.name = item_supp.parent
			AND mr_item.parent = mr.name
			AND mr_item.item_code = item.name
			AND mr.status != "Stopped"
			AND mr.material_request_type = "Purchase"
			AND mr.docstatus = 1
			AND mr.per_ordered < 99.99""", {"supplier": source_name}, as_dict=1)

	material_requests = {}
	for d in mr_items_list:
		material_requests.setdefault(d.name, []).append(d.item_code)

	for mr, items in material_requests.items():
		target_doc = get_mapped_doc("Material Request", mr, {
			"Material Request": {
				"doctype": "Request for Quotation",
				"validation": {
					"docstatus": ["=", 1],
					"material_request_type": ["=", "Purchase"],
				}
			},
			"Material Request Item": {
				"doctype": "Request for Quotation Item",
				"condition": lambda row: row.item_code in items,
				"field_map": [
					["name", "material_request_item"],
					["parent", "material_request"],
					["uom", "uom"]
				]
			}
		}, target_doc)

	return target_doc
<|endoftext|>


<commit_before>"""
=================================================================
Test with permutations the significance of a classification score
=================================================================

This example demonstrates the use of
:func:`~sklearn.model_selection.permutation_test_score` to evaluate the
significance of a cross-validated score using permutations.

"""

# Authors:  Alexandre Gramfort <alexandre.gramfort@inria.fr>
#           Lucy Liu
# License: BSD 3 clause

# %%
# Dataset
# -------
#
# We will use the :ref:`iris_dataset`, which consists of measurements taken
# from 3 types of irises.

from sklearn.datasets import load_iris

iris = load_iris()
X = iris.data
y = iris.target

# %%
# We will also generate some random feature data (i.e., 2200 features),
# uncorrelated with the class labels in the iris dataset.

import numpy as np

n_uncorrelated_features = 2200
rng = np.random.RandomState(seed=0)
# Use same number of samples as in iris and 2200 features
X_rand = rng.normal(size=(X.shape[0], n_uncorrelated_features))

# %%
# Permutation test score
# ----------------------
#
# Next, we calculate the
# :func:`~sklearn.model_selection.permutation_test_score` using the original
# iris dataset, which strongly predict the labels and
# the randomly generated features and iris labels, which should have
# no dependency between features and labels. We use the
# :class:`~sklearn.svm.SVC` classifier and :ref:`accuracy_score` to evaluate
# the model at each round.
#
# :func:`~sklearn.model_selection.permutation_test_score` generates a null
# distribution by calculating the accuracy of the classifier
# on 1000 different permutations of the dataset, where features
# remain the same but labels undergo different permutations. This is the
# distribution for the null hypothesis which states there is no dependency
# between the features and labels. An empirical p-value is then calculated as
# the percentage of permutations for which the score obtained is greater
# that the score obtained using the original data.

from sklearn.svm import SVC
from sklearn.model_selection import StratifiedKFold
from sklearn.model_selection import permutation_test_score

clf = SVC(kernel="linear", random_state=7)
cv = StratifiedKFold(2, shuffle=True, random_state=0)

score_iris, perm_scores_iris, pvalue_iris = permutation_test_score(
    clf, X, y, scoring="accuracy", cv=cv, n_permutations=1000
)

score_rand, perm_scores_rand, pvalue_rand = permutation_test_score(
    clf, X_rand, y, scoring="accuracy", cv=cv, n_permutations=1000
)

# %%
# Original data
# ^^^^^^^^^^^^^
#
# Below we plot a histogram of the permutation scores (the null
# distribution). The red line indicates the score obtained by the classifier
# on the original data. The score is much better than those obtained by
# using permuted data and the p-value is thus very low. This indicates that
# there is a low likelihood that this good score would be obtained by chance
# alone. It provides evidence that the iris dataset contains real dependency
# between features and labels and the classifier was able to utilize this
# to obtain good results.

import matplotlib.pyplot as plt

fig, ax = plt.subplots()

ax.hist(perm_scores_iris, bins=20, density=True)
ax.axvline(score_iris, ls="--", color="r")
score_label = f"Score on original\ndata: {score_iris:.2f}\n(p-value: {pvalue_iris:.3f})"
ax.text(0.7, 10, score_label, fontsize=12)
ax.set_xlabel("Accuracy score")
_ = ax.set_ylabel("Probability")

# %%
# Random data
# ^^^^^^^^^^^
#
# Below we plot the null distribution for the randomized data. The permutation
# scores are similar to those obtained using the original iris dataset
# because the permutation always destroys any feature label dependency present.
# The score obtained on the original randomized data in this case though, is
# very poor. This results in a large p-value, confirming that there was no
# feature label dependency in the original data.

fig, ax = plt.subplots()

ax.hist(perm_scores_rand, bins=20, density=True)
ax.set_xlim(0.13)
ax.axvline(score_rand, ls="--", color="r")
score_label = f"Score on original\ndata: {score_rand:.2f}\n(p-value: {pvalue_rand:.3f})"
ax.text(0.14, 7.5, score_label, fontsize=12)
ax.set_xlabel("Accuracy score")
ax.set_ylabel("Probability")
plt.show()

# %%
# Another possible reason for obtaining a high p-value is that the classifier
# was not able to use the structure in the data. In this case, the p-value
# would only be low for classifiers that are able to utilize the dependency
# present. In our case above, where the data is random, all classifiers would
# have a high p-value as there is no structure present in the data.
#
# Finally, note that this test has been shown to produce low p-values even
# if there is only weak structure in the data [1]_.
#
# .. topic:: References:
#
#   .. [1] Ojala and Garriga. `Permutation Tests for Studying Classifier
#          Performance
#          <http://www.jmlr.org/papers/volume11/ojala10a/ojala10a.pdf>`_. The
#          Journal of Machine Learning Research (2010) vol. 11
#
<commit_msg>ENH Improve speed plot_permutation_tests_for_classification.py  (#21649)<commit_after>"""
=================================================================
Test with permutations the significance of a classification score
=================================================================

This example demonstrates the use of
:func:`~sklearn.model_selection.permutation_test_score` to evaluate the
significance of a cross-validated score using permutations.

"""

# Authors:  Alexandre Gramfort <alexandre.gramfort@inria.fr>
#           Lucy Liu
# License: BSD 3 clause

# %%
# Dataset
# -------
#
# We will use the :ref:`iris_dataset`, which consists of measurements taken
# from 3 types of irises.

from sklearn.datasets import load_iris

iris = load_iris()
X = iris.data
y = iris.target

# %%
# We will also generate some random feature data (i.e., 20 features),
# uncorrelated with the class labels in the iris dataset.

import numpy as np

n_uncorrelated_features = 20
rng = np.random.RandomState(seed=0)
# Use same number of samples as in iris and 20 features
X_rand = rng.normal(size=(X.shape[0], n_uncorrelated_features))

# %%
# Permutation test score
# ----------------------
#
# Next, we calculate the
# :func:`~sklearn.model_selection.permutation_test_score` using the original
# iris dataset, which strongly predict the labels and
# the randomly generated features and iris labels, which should have
# no dependency between features and labels. We use the
# :class:`~sklearn.svm.SVC` classifier and :ref:`accuracy_score` to evaluate
# the model at each round.
#
# :func:`~sklearn.model_selection.permutation_test_score` generates a null
# distribution by calculating the accuracy of the classifier
# on 1000 different permutations of the dataset, where features
# remain the same but labels undergo different permutations. This is the
# distribution for the null hypothesis which states there is no dependency
# between the features and labels. An empirical p-value is then calculated as
# the percentage of permutations for which the score obtained is greater
# that the score obtained using the original data.

from sklearn.svm import SVC
from sklearn.model_selection import StratifiedKFold
from sklearn.model_selection import permutation_test_score

clf = SVC(kernel="linear", random_state=7)
cv = StratifiedKFold(2, shuffle=True, random_state=0)

score_iris, perm_scores_iris, pvalue_iris = permutation_test_score(
    clf, X, y, scoring="accuracy", cv=cv, n_permutations=1000
)

score_rand, perm_scores_rand, pvalue_rand = permutation_test_score(
    clf, X_rand, y, scoring="accuracy", cv=cv, n_permutations=1000
)

# %%
# Original data
# ^^^^^^^^^^^^^
#
# Below we plot a histogram of the permutation scores (the null
# distribution). The red line indicates the score obtained by the classifier
# on the original data. The score is much better than those obtained by
# using permuted data and the p-value is thus very low. This indicates that
# there is a low likelihood that this good score would be obtained by chance
# alone. It provides evidence that the iris dataset contains real dependency
# between features and labels and the classifier was able to utilize this
# to obtain good results.

import matplotlib.pyplot as plt

fig, ax = plt.subplots()

ax.hist(perm_scores_iris, bins=20, density=True)
ax.axvline(score_iris, ls="--", color="r")
score_label = f"Score on original\ndata: {score_iris:.2f}\n(p-value: {pvalue_iris:.3f})"
ax.text(0.7, 10, score_label, fontsize=12)
ax.set_xlabel("Accuracy score")
_ = ax.set_ylabel("Probability")

# %%
# Random data
# ^^^^^^^^^^^
#
# Below we plot the null distribution for the randomized data. The permutation
# scores are similar to those obtained using the original iris dataset
# because the permutation always destroys any feature label dependency present.
# The score obtained on the original randomized data in this case though, is
# very poor. This results in a large p-value, confirming that there was no
# feature label dependency in the original data.

fig, ax = plt.subplots()

ax.hist(perm_scores_rand, bins=20, density=True)
ax.set_xlim(0.13)
ax.axvline(score_rand, ls="--", color="r")
score_label = f"Score on original\ndata: {score_rand:.2f}\n(p-value: {pvalue_rand:.3f})"
ax.text(0.14, 7.5, score_label, fontsize=12)
ax.set_xlabel("Accuracy score")
ax.set_ylabel("Probability")
plt.show()

# %%
# Another possible reason for obtaining a high p-value is that the classifier
# was not able to use the structure in the data. In this case, the p-value
# would only be low for classifiers that are able to utilize the dependency
# present. In our case above, where the data is random, all classifiers would
# have a high p-value as there is no structure present in the data.
#
# Finally, note that this test has been shown to produce low p-values even
# if there is only weak structure in the data [1]_.
#
# .. topic:: References:
#
#   .. [1] Ojala and Garriga. `Permutation Tests for Studying Classifier
#          Performance
#          <http://www.jmlr.org/papers/volume11/ojala10a/ojala10a.pdf>`_. The
#          Journal of Machine Learning Research (2010) vol. 11
#
<|endoftext|>


<commit_before>import mock

from datetime import date

from decimal import Decimal

from dateutil.relativedelta import relativedelta

from go.base.tests.utils import VumiGoDjangoTestCase

from go.billing.models import MessageCost, Account, Transaction, Statement
from go.billing import tasks

class TestMonthlyStatementTask(VumiGoDjangoTestCase):

    def setUp(self):
        super(TestMonthlyStatementTask, self).setUp()
        self.setup_user_api()

    def _mk_account(self):
        account = Account(
            user=self.django_user,
            account_number=self.user_api.user_account_key)

        account.save()
        return account

    def _mk_transaction(self, account_number, tag_pool_name='pool1',
                        tag_name="tag1",
                        message_direction=MessageCost.DIRECTION_INBOUND,
                        message_cost=100, markup_percent=10.0,
                        credit_factor=0.25, credit_amount=28,
                        status=Transaction.STATUS_COMPLETED, **kwargs):
        transaction = Transaction(
            account_number=account_number,
            tag_pool_name=tag_pool_name,
            tag_name=tag_name,
            message_direction=message_direction,
            message_cost=message_cost,
            markup_percent=Decimal(str(markup_percent)),
            credit_factor=Decimal(str(credit_factor)),
            credit_amount=credit_amount,
            status=status, **kwargs)

        transaction.save()
        return transaction

    @mock.patch('go.billing.tasks.generate_monthly_statement.s',
                new_callable=mock.MagicMock)
    def test_generate_monthly_statements(self, s):
        account = self._mk_account()
        today = date.today()
        last_month = today - relativedelta(months=1)
        self._mk_transaction(account.account_number, created=last_month,
                             last_modified=last_month)

        self._mk_transaction(
            account.account_number,
            message_direction=MessageCost.DIRECTION_OUTBOUND,
            created=last_month, last_modified=last_month)

        tasks.generate_monthly_account_statements()

        from_date = date(last_month.year, last_month.month, 1)
        to_date = date(today.year, today.month, 1) - relativedelta(days=1)
        s.assert_called_with(account.id, from_date, to_date)

    def test_generate_monthly_statement(self):
        account = self._mk_account()
        self._mk_transaction(account.account_number)
        self._mk_transaction(
            account.account_number,
            message_direction=MessageCost.DIRECTION_OUTBOUND)

        today = date.today()
        next_month = today + relativedelta(months=1)
        from_date = date(today.year, today.month, 1)
        to_date = (date(next_month.year, next_month.month, 1)
                   - relativedelta(days=1))

        result = tasks.generate_monthly_statement(account.id, from_date,
                                                  to_date)

        statement = Statement.objects.get(account=account,
                                          type=Statement.TYPE_MONTHLY)

        self.assertEqual(result, statement)
        self.assertEqual(statement.lineitem_set.count(), 2)
<commit_msg>Update new tests from develop.<commit_after>import mock

from datetime import date

from decimal import Decimal

from dateutil.relativedelta import relativedelta

from go.base.tests.helpers import GoDjangoTestCase, DjangoVumiApiHelper
from go.billing.models import MessageCost, Account, Transaction, Statement
from go.billing import tasks

class TestMonthlyStatementTask(GoDjangoTestCase):

    def setUp(self):
        self.vumi_helper = DjangoVumiApiHelper()
        self.add_cleanup(self.vumi_helper.cleanup)
        self.vumi_helper.setup_vumi_api()
        self.user_helper = self.vumi_helper.make_django_user()

    def _mk_account(self):
        account = Account(
            user=self.user_helper.get_django_user(),
            account_number=self.user_helper.account_key)

        account.save()
        return account

    def _mk_transaction(self, account_number, tag_pool_name='pool1',
                        tag_name="tag1",
                        message_direction=MessageCost.DIRECTION_INBOUND,
                        message_cost=100, markup_percent=10.0,
                        credit_factor=0.25, credit_amount=28,
                        status=Transaction.STATUS_COMPLETED, **kwargs):
        transaction = Transaction(
            account_number=account_number,
            tag_pool_name=tag_pool_name,
            tag_name=tag_name,
            message_direction=message_direction,
            message_cost=message_cost,
            markup_percent=Decimal(str(markup_percent)),
            credit_factor=Decimal(str(credit_factor)),
            credit_amount=credit_amount,
            status=status, **kwargs)

        transaction.save()
        return transaction

    @mock.patch('go.billing.tasks.generate_monthly_statement.s',
                new_callable=mock.MagicMock)
    def test_generate_monthly_statements(self, s):
        account = self._mk_account()
        today = date.today()
        last_month = today - relativedelta(months=1)
        self._mk_transaction(account.account_number, created=last_month,
                             last_modified=last_month)

        self._mk_transaction(
            account.account_number,
            message_direction=MessageCost.DIRECTION_OUTBOUND,
            created=last_month, last_modified=last_month)

        tasks.generate_monthly_account_statements()

        from_date = date(last_month.year, last_month.month, 1)
        to_date = date(today.year, today.month, 1) - relativedelta(days=1)
        s.assert_called_with(account.id, from_date, to_date)

    def test_generate_monthly_statement(self):
        account = self._mk_account()
        self._mk_transaction(account.account_number)
        self._mk_transaction(
            account.account_number,
            message_direction=MessageCost.DIRECTION_OUTBOUND)

        today = date.today()
        next_month = today + relativedelta(months=1)
        from_date = date(today.year, today.month, 1)
        to_date = (date(next_month.year, next_month.month, 1)
                   - relativedelta(days=1))

        result = tasks.generate_monthly_statement(account.id, from_date,
                                                  to_date)

        statement = Statement.objects.get(account=account,
                                          type=Statement.TYPE_MONTHLY)

        self.assertEqual(result, statement)
        self.assertEqual(statement.lineitem_set.count(), 2)
<|endoftext|>