<commit_before># -*- coding: utf-8 -*-
"""
/***************************************************************************
 Svir
                                 A QGIS plugin
 OpenQuake Social Vulnerability and Integrated Risk
                              -------------------
        begin                : 2013-10-24
        copyright            : (C) 2013 by GEM Foundation
        email                : devops@openquake.org
 ***************************************************************************/

# Copyright (c) 2010-2013, GEM Foundation.
#
# OpenQuake is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# OpenQuake is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with OpenQuake.  If not, see <http://www.gnu.org/licenses/>.
"""
from numpy import mean, std, argwhere, amax, amin, log10
from register import Register

NORMALIZATION_ALGS = Register()
RANK_VARIANTS = ['AVERAGE', 'MIN', 'MAX', 'DENSE', 'ORDINAL']
QUADRATIC_VARIANTS = ['INCREASING', 'DECREASING']

def normalize(features_dict, algorithm, variant_name="", inverse=False):
    """
    Use the chosen algorithm (and optional variant) on the input dict, and
    return a dict containing the normalized values with the original ids
    """
    ids = features_dict.keys()
    values = features_dict.values()
    normalized_list = algorithm(values, variant_name, inverse)
    return dict(zip(ids, normalized_list))

@NORMALIZATION_ALGS.add('RANK')
def rank(input_list, variant_name="AVERAGE", inverse=False):
    """Assign ranks to data, dealing with ties appropriately.

    Args:
        input_list: the list of numbers to rank
        variant_name: available variants are
            [AVERAGE, MIN, MAX, DENSE, ORDINAL] and they correspond to
            different strategies on how to cope with ties (default: AVERAGE)
        inverse: instead of giving the highest rank to the biggest input value,
            give the highest rank to the smallest input value
    Returns:
        list of ranks corresponding to the input data
    Raises:
        NotImplementedError: if variant_name is not implemented
    """
    input_copy = input_list[:]
    len_input_list = len(input_list)
    rank_list = [0] * len_input_list
    previous_ties = 0
    if not inverse:  # high values get high ranks
        # obtain a value above the maximum value contained in input_list,
        # so it will never be picked as the minimum element of the list
        above_max_input = max(input_list) + 1
        curr_idx = 1
        while curr_idx <= len_input_list:
            # get the list of indices of the min elements of input_copy.
            # note that we might have ties.
            bottom_indices = argwhere(
                input_copy == amin(input_copy)).flatten().tolist()
            bottom_amount = len(bottom_indices)
            for bottom_idx in bottom_indices:
                if variant_name == "AVERAGE":
                    # e.g., if 4 inputs are equal, and they would receive
                    # ranks from 3 to 6, all of them will obtain rank 4.5
                    # Afterwards the ranks increase from 7 on.
                    rank_list[bottom_idx] = \
                        (2 * curr_idx + bottom_amount - 1) / 2.0
                elif variant_name == "MIN":
                    # e.g., if 4 inputs are equal, and they would receive
                    # ranks from 3 to 6, all of them will obtain rank 3.
                    # Afterwards the ranks increase from 7 on.
                    rank_list[bottom_idx] = curr_idx
                elif variant_name == "MAX":
                    # e.g., if 4 inputs are equal, and they would receive
                    # ranks from 3 to 6, all of them will obtain rank 6
                    # Afterwards the ranks increase from 7 on.
                    rank_list[bottom_idx] = curr_idx + bottom_amount - 1
                elif variant_name == "DENSE":
                    # the same as for "MIN", but instead of ranking the
                    # next elements counting from 7 on, the ranks will
                    # increase from 4 on (no "jumps").
                    rank_list[bottom_idx] = curr_idx - previous_ties
                elif variant_name == "ORDINAL":
                    # the ties are ranked in a "ordinal" way, assigning
                    # the smallest rank to the leftmost tie found, and so on.
                    rank_list[bottom_idx] = curr_idx
                    curr_idx += 1
                else:
                    raise NotImplementedError(
                        "%s variant not implemented" % variant_name)
                # "discard" the tie, so it will not be found as the min in
                # the next iteration (instead of removing it, which would
                # modify the list indices, I make it bigger than the
                # biggest input)
                input_copy[bottom_idx] = above_max_input
            if variant_name != "ORDINAL":
                # necessary adjustment in case of "ORDINAL"
                curr_idx += bottom_amount
            # necessary for the "DENSE" variant, to keep a count of ties
            previous_ties += bottom_amount - 1

    else:  # inverse, i.e., small inputs get high ranks
        # same as the direct methods, but proceeding top->down
        below_min_input = min(input_list) - 1
        curr_idx = len_input_list
        while curr_idx > 0:
            top_indices = argwhere(
                input_copy == amax(input_copy)).flatten().tolist()
            top_amount = len(top_indices)
            for top_idx in top_indices:
                if variant_name == "AVERAGE":
                    rank_list[top_idx] = \
                        len_input_list - (2 * curr_idx - top_amount - 1) / 2.0
                elif variant_name == "MIN":
                    rank_list[top_idx] = len_input_list - curr_idx + 1
                elif variant_name == "MAX":
                    rank_list[top_idx] = len_input_list - curr_idx + top_amount
                elif variant_name == "DENSE":
                    rank_list[top_idx] = \
                        len_input_list - curr_idx - previous_ties + 1
                elif variant_name == "ORDINAL":
                    rank_list[top_idx] = len_input_list - curr_idx + 1
                    curr_idx -= 1
                else:
                    raise NotImplementedError(
                        "%s variant not implemented" % variant_name)
                input_copy[top_idx] = below_min_input
            if variant_name != "ORDINAL":
                curr_idx -= top_amount
            previous_ties += top_amount - 1
    return rank_list

@NORMALIZATION_ALGS.add('Z_SCORE')
def z_score(input_list, variant_name=None, inverse=False):
    """
    Direct:
        Normalized(e_i) = (e_i - mean(e)) / stddev(e)
    Inverse:
        Multiply each input by -1, before doing exactly the same
    """
    if variant_name:
        raise NotImplementedError("%s variant not implemented" % variant_name)
    mean_val = mean(input_list)
    stddev_val = std(input_list)
    input_copy = input_list[:]
    if inverse:
        # multiply each input_list element by -1
        input_copy[:] = [-x for x in input_list]
    output_list = [
        1.0 * (num - mean_val) / stddev_val for num in input_copy]
    return output_list

@NORMALIZATION_ALGS.add('MIN_MAX')
def min_max(input_list, variant_name=None, inverse=False):
    """
    Direct:
        Normalized(e_i) = (e_i - min(e)) / (max(e) - min(e))
    Inverse:
        Normalized(e_i) = 1 - [(e_i - min(e)) / (max(e) - min(e))]

    """
    if variant_name:
        raise NotImplementedError("%s variant not implemented" % variant_name)
    list_min = min(input_list)
    list_max = max(input_list)
    # Get the range of the list
    list_range = float(list_max - list_min)
    # Normalize
    if inverse:
        output_list = map(
            lambda x: 1.0 - ((x - list_min) / list_range), input_list)
    else:
        output_list = map(
            lambda x: (x - list_min) / list_range, input_list)
    return output_list

@NORMALIZATION_ALGS.add('LOG10')
def log10_(input_list, variant_name=None, inverse=False):
    """
    Accept only input_list containing positive (or zero) values
    In case of zeros, set those values to 1 (this differs from the built-in
    QGIS log10 function available in the field calculator, which returns None
    in case of zeros)
    Then use numpy.log10 function to perform the log10 transformation on the
    list of values
    """
    if variant_name:
        # TODO: Perhaps it would be better to use the variant to let the user
        #       choose if setting input zeros to one or not
        raise NotImplementedError("%s variant not implemented" % variant_name)
    if inverse:
        raise NotImplementedError(
            "Inverse transformation for log10 is not implemented")
    if any(n < 0 for n in input_list):
        raise ValueError("log10 transformation can not be performed if "
                         "the field contains negative values")
    input_copy = []
    for input_value in input_list:
        corrected_value = input_value if input_value > 0 else 1.0
        input_copy.append(corrected_value)
    output_list = list(log10(input_copy))
    return output_list

@NORMALIZATION_ALGS.add('QUADRATIC')
def simple_quadratic(input_list, variant_name="INCREASING", inverse=False):
    """
    Simple quadratic transformation (bottom = 0)
        quadratic(e_i) = (e_i - bottom)^2 / (max(e) - bottom)^2
    ==>
        simple_quadratic(e_i) = e_i^2 / max(e)^2

    Inverse:
        For each output x, the final output will be 1 - x
    """
    bottom = 0.0
    max_input = max(input_list)
    if max_input - bottom == 0:
        raise ZeroDivisionError("It is impossible to perform the "
                                "transformation if the maximum "
                                "input value is 0")
    squared_range = (max_input - bottom) ** 2
    if variant_name == "INCREASING":
        output_list = map(
            lambda x: (x - bottom) ** 2 / squared_range, input_list)
    elif variant_name == "DECREASING":
        output_list = map(
            lambda x: (max_input - (x - bottom)) ** 2 / squared_range,
            input_list)

    else:
        raise NotImplementedError("%s variant not implemented" % variant_name)
    if inverse:
        output_list[:] = [1.0 - x for x in output_list]
    return output_list

<commit_msg>Fix import<commit_after># -*- coding: utf-8 -*-
"""
/***************************************************************************
 Svir
                                 A QGIS plugin
 OpenQuake Social Vulnerability and Integrated Risk
                              -------------------
        begin                : 2013-10-24
        copyright            : (C) 2013 by GEM Foundation
        email                : devops@openquake.org
 ***************************************************************************/

# Copyright (c) 2010-2013, GEM Foundation.
#
# OpenQuake is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# OpenQuake is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with OpenQuake.  If not, see <http://www.gnu.org/licenses/>.
"""
from numpy import mean, std, argwhere, amax, amin, log10
from utils import Register

NORMALIZATION_ALGS = Register()
RANK_VARIANTS = ['AVERAGE', 'MIN', 'MAX', 'DENSE', 'ORDINAL']
QUADRATIC_VARIANTS = ['INCREASING', 'DECREASING']

def normalize(features_dict, algorithm, variant_name="", inverse=False):
    """
    Use the chosen algorithm (and optional variant) on the input dict, and
    return a dict containing the normalized values with the original ids
    """
    ids = features_dict.keys()
    values = features_dict.values()
    normalized_list = algorithm(values, variant_name, inverse)
    return dict(zip(ids, normalized_list))

@NORMALIZATION_ALGS.add('RANK')
def rank(input_list, variant_name="AVERAGE", inverse=False):
    """Assign ranks to data, dealing with ties appropriately.

    Args:
        input_list: the list of numbers to rank
        variant_name: available variants are
            [AVERAGE, MIN, MAX, DENSE, ORDINAL] and they correspond to
            different strategies on how to cope with ties (default: AVERAGE)
        inverse: instead of giving the highest rank to the biggest input value,
            give the highest rank to the smallest input value
    Returns:
        list of ranks corresponding to the input data
    Raises:
        NotImplementedError: if variant_name is not implemented
    """
    input_copy = input_list[:]
    len_input_list = len(input_list)
    rank_list = [0] * len_input_list
    previous_ties = 0
    if not inverse:  # high values get high ranks
        # obtain a value above the maximum value contained in input_list,
        # so it will never be picked as the minimum element of the list
        above_max_input = max(input_list) + 1
        curr_idx = 1
        while curr_idx <= len_input_list:
            # get the list of indices of the min elements of input_copy.
            # note that we might have ties.
            bottom_indices = argwhere(
                input_copy == amin(input_copy)).flatten().tolist()
            bottom_amount = len(bottom_indices)
            for bottom_idx in bottom_indices:
                if variant_name == "AVERAGE":
                    # e.g., if 4 inputs are equal, and they would receive
                    # ranks from 3 to 6, all of them will obtain rank 4.5
                    # Afterwards the ranks increase from 7 on.
                    rank_list[bottom_idx] = \
                        (2 * curr_idx + bottom_amount - 1) / 2.0
                elif variant_name == "MIN":
                    # e.g., if 4 inputs are equal, and they would receive
                    # ranks from 3 to 6, all of them will obtain rank 3.
                    # Afterwards the ranks increase from 7 on.
                    rank_list[bottom_idx] = curr_idx
                elif variant_name == "MAX":
                    # e.g., if 4 inputs are equal, and they would receive
                    # ranks from 3 to 6, all of them will obtain rank 6
                    # Afterwards the ranks increase from 7 on.
                    rank_list[bottom_idx] = curr_idx + bottom_amount - 1
                elif variant_name == "DENSE":
                    # the same as for "MIN", but instead of ranking the
                    # next elements counting from 7 on, the ranks will
                    # increase from 4 on (no "jumps").
                    rank_list[bottom_idx] = curr_idx - previous_ties
                elif variant_name == "ORDINAL":
                    # the ties are ranked in a "ordinal" way, assigning
                    # the smallest rank to the leftmost tie found, and so on.
                    rank_list[bottom_idx] = curr_idx
                    curr_idx += 1
                else:
                    raise NotImplementedError(
                        "%s variant not implemented" % variant_name)
                # "discard" the tie, so it will not be found as the min in
                # the next iteration (instead of removing it, which would
                # modify the list indices, I make it bigger than the
                # biggest input)
                input_copy[bottom_idx] = above_max_input
            if variant_name != "ORDINAL":
                # necessary adjustment in case of "ORDINAL"
                curr_idx += bottom_amount
            # necessary for the "DENSE" variant, to keep a count of ties
            previous_ties += bottom_amount - 1

    else:  # inverse, i.e., small inputs get high ranks
        # same as the direct methods, but proceeding top->down
        below_min_input = min(input_list) - 1
        curr_idx = len_input_list
        while curr_idx > 0:
            top_indices = argwhere(
                input_copy == amax(input_copy)).flatten().tolist()
            top_amount = len(top_indices)
            for top_idx in top_indices:
                if variant_name == "AVERAGE":
                    rank_list[top_idx] = \
                        len_input_list - (2 * curr_idx - top_amount - 1) / 2.0
                elif variant_name == "MIN":
                    rank_list[top_idx] = len_input_list - curr_idx + 1
                elif variant_name == "MAX":
                    rank_list[top_idx] = len_input_list - curr_idx + top_amount
                elif variant_name == "DENSE":
                    rank_list[top_idx] = \
                        len_input_list - curr_idx - previous_ties + 1
                elif variant_name == "ORDINAL":
                    rank_list[top_idx] = len_input_list - curr_idx + 1
                    curr_idx -= 1
                else:
                    raise NotImplementedError(
                        "%s variant not implemented" % variant_name)
                input_copy[top_idx] = below_min_input
            if variant_name != "ORDINAL":
                curr_idx -= top_amount
            previous_ties += top_amount - 1
    return rank_list

@NORMALIZATION_ALGS.add('Z_SCORE')
def z_score(input_list, variant_name=None, inverse=False):
    """
    Direct:
        Normalized(e_i) = (e_i - mean(e)) / stddev(e)
    Inverse:
        Multiply each input by -1, before doing exactly the same
    """
    if variant_name:
        raise NotImplementedError("%s variant not implemented" % variant_name)
    mean_val = mean(input_list)
    stddev_val = std(input_list)
    input_copy = input_list[:]
    if inverse:
        # multiply each input_list element by -1
        input_copy[:] = [-x for x in input_list]
    output_list = [
        1.0 * (num - mean_val) / stddev_val for num in input_copy]
    return output_list

@NORMALIZATION_ALGS.add('MIN_MAX')
def min_max(input_list, variant_name=None, inverse=False):
    """
    Direct:
        Normalized(e_i) = (e_i - min(e)) / (max(e) - min(e))
    Inverse:
        Normalized(e_i) = 1 - [(e_i - min(e)) / (max(e) - min(e))]

    """
    if variant_name:
        raise NotImplementedError("%s variant not implemented" % variant_name)
    list_min = min(input_list)
    list_max = max(input_list)
    # Get the range of the list
    list_range = float(list_max - list_min)
    # Normalize
    if inverse:
        output_list = map(
            lambda x: 1.0 - ((x - list_min) / list_range), input_list)
    else:
        output_list = map(
            lambda x: (x - list_min) / list_range, input_list)
    return output_list

@NORMALIZATION_ALGS.add('LOG10')
def log10_(input_list, variant_name=None, inverse=False):
    """
    Accept only input_list containing positive (or zero) values
    In case of zeros, set those values to 1 (this differs from the built-in
    QGIS log10 function available in the field calculator, which returns None
    in case of zeros)
    Then use numpy.log10 function to perform the log10 transformation on the
    list of values
    """
    if variant_name:
        # TODO: Perhaps it would be better to use the variant to let the user
        #       choose if setting input zeros to one or not
        raise NotImplementedError("%s variant not implemented" % variant_name)
    if inverse:
        raise NotImplementedError(
            "Inverse transformation for log10 is not implemented")
    if any(n < 0 for n in input_list):
        raise ValueError("log10 transformation can not be performed if "
                         "the field contains negative values")
    input_copy = []
    for input_value in input_list:
        corrected_value = input_value if input_value > 0 else 1.0
        input_copy.append(corrected_value)
    output_list = list(log10(input_copy))
    return output_list

@NORMALIZATION_ALGS.add('QUADRATIC')
def simple_quadratic(input_list, variant_name="INCREASING", inverse=False):
    """
    Simple quadratic transformation (bottom = 0)
        quadratic(e_i) = (e_i - bottom)^2 / (max(e) - bottom)^2
    ==>
        simple_quadratic(e_i) = e_i^2 / max(e)^2

    Inverse:
        For each output x, the final output will be 1 - x
    """
    bottom = 0.0
    max_input = max(input_list)
    if max_input - bottom == 0:
        raise ZeroDivisionError("It is impossible to perform the "
                                "transformation if the maximum "
                                "input value is 0")
    squared_range = (max_input - bottom) ** 2
    if variant_name == "INCREASING":
        output_list = map(
            lambda x: (x - bottom) ** 2 / squared_range, input_list)
    elif variant_name == "DECREASING":
        output_list = map(
            lambda x: (max_input - (x - bottom)) ** 2 / squared_range,
            input_list)

    else:
        raise NotImplementedError("%s variant not implemented" % variant_name)
    if inverse:
        output_list[:] = [1.0 - x for x in output_list]
    return output_list

<|endoftext|>


<commit_before># this file is deprecated and will soon be folded into all.py

from collections import namedtuple

from pycoin.serialize import h2b

NetworkValues = namedtuple('NetworkValues',
                           ('network_name', 'subnet_name', 'code', 'wif', 'address',
                            'pay_to_script', 'prv32', 'pub32'))

NETWORKS = (
    # VIA viacoin mainnet : xprv/xpub
    NetworkValues("Viacoin", "mainnet", "VIA", b'\xc7', b'\x47', b'\x21', h2b('0488ADE4'), h2b('0488B21E')),
    # VIA viacoin testnet : tprv/tpub
    NetworkValues("Viacoin", "testnet", "TVI", b'\xff', b'\x7f', b'\xc4', h2b('04358394'), h2b('043587CF')),

    # FTC feathercoin mainnet : xprv/xpub
    NetworkValues(
        "Feathercoin", "mainnet", "FTC", b'\x8e', b'\x0e', b'\x60', h2b('0488ADE4'), h2b('0488B21E')),
    # FTC feathercoin testnet : tprv/tpub
    NetworkValues(
        "Feathercoin", "testnet", "FTX", b'\xC1', b'\x41', b'\xc4', h2b('04358394'), h2b('043587CF')),

    # DOGE Dogecoin mainnet : dogv/dogp
    NetworkValues(
        "Dogecoin", "mainnet", "DOGE", b'\x9e', b'\x1e', b'\x16', h2b("02FD3955"), h2b("02FD3929")),
    # DOGE Dogecoin testnet : tgpv/tgub
    NetworkValues(
        "Dogecoin", "testnet", "XDT", b'\xf1', b'\x71', b'\xc4', h2b("0432a9a8"), h2b("0432a243")),

    # BC BlackCoin mainnet : bcpv/bcpb
    NetworkValues("Blackcoin", "mainnet", "BC", b'\x99', b'\x19', None, h2b("02cfbf60"), h2b("02cfbede")),

    # DRK Dash mainnet : drkv/drkp
    NetworkValues(
        "Dash", "mainnet", "DASH", b'\xcc', b'\x4c', b'\x10', h2b("02fe52f8"), h2b("02fe52cc")),

    # DRK Dash testnet : DRKV/DRKP
    NetworkValues(
        "Dash", "testnet", "tDASH", b'\xef', b'\x8c', b'\x13', h2b("3a8061a0"), h2b("3a805837")),

    # MEC Megacoin mainnet : mecv/mecp
    NetworkValues("Megacoin", "mainnet", "MEC", b'\xb2', b'\x32', None, h2b("03a04db7"), h2b("03a04d8b")),

    NetworkValues(
        "Myriadcoin", "mainnet", "MYR", b'\xb2', b'\x32', b'\x09', h2b('0488ADE4'), h2b('0488B21E')),

    NetworkValues(
        "Unobtanium", "mainnet", "UNO", b'\xe0', b'\x82', b'\x1e', h2b('0488ADE4'), h2b('0488B21E')),

    # JBS Jumbucks mainnet : jprv/jpub
    NetworkValues("Jumbucks", "mainnet", "JBS", b'\xab', b'\x2b', None, h2b('037a6460'), h2b('037a689a')),

    # MZC Mazacoin mainnet: xprv/xpub
    NetworkValues("Mazacoin", "mainnet", "MZC", b'\xe0', b'\x32', b'\9', h2b("0488ADE4"), h2b("0488B21E")),

    NetworkValues(
        "Riecoin", "mainnet", "RIC", b'\x80', b'\x3c', b'\x05', h2b('0488ADE4'), h2b('0488B21E')),

    # DFC Defcoin mainnet: dfcv/dfcp
    NetworkValues("DEFCOIN", "mainnet", "DFC", b'\x9e', b'\x1e', b'\5', h2b("02FA54D7"), h2b("02FA54AD")),

    # FAI faircoin mainnet : xprv/xpub
    NetworkValues(
        "Faircoin", "mainnet", "FAI", b'\xdf', b'\x5f', b'\x24', h2b("0488ADE4"), h2b("0488B21E")),

    # ARG argentum mainnet : xprv/xpub
    NetworkValues("Argentum", "mainnet", "ARG", b'\x97', b'\x17', b'\5', h2b("0488ADE4"), h2b("0488B21E")),

    # ZEC Zcash mainnet : xprv/xpub
    NetworkValues("Zcash", "mainnet", "ZEC", b'\x80', b'\x1C\xB8',
                  b'\x1C\xBD', h2b("0488ADE4"), h2b("0488B21E")),

    # BTCD BitcoinDark mainnet : xprv/xpub
    NetworkValues("BitcoinDark", "mainnet", "BTCD", b'\x44', b'\x3C', b'\55', h2b('0488ADE4'), h2b('0488B21E')),

    # DCR Decred mainnet : dprv/dpub
    NetworkValues("Decred", "mainnet", "DCR", b'\x22\xDE', b'\x07\x3F', b'\x07\x1A', h2b('02FDA4E8'), h2b('02FDA926')),

    # DCR Decred testnet : tprv/tpub
    NetworkValues("Decred", "testnet", "DCRT", b'\x23\x0E', b'\x0F\x21', b'\x0E\x6C', h2b('04358397'), h2b('043587D1')),

)
<commit_msg>Add PIVX network parameters.<commit_after># this file is deprecated and will soon be folded into all.py

from collections import namedtuple

from pycoin.serialize import h2b

NetworkValues = namedtuple('NetworkValues',
                           ('network_name', 'subnet_name', 'code', 'wif', 'address',
                            'pay_to_script', 'prv32', 'pub32'))

NETWORKS = (
    # VIA viacoin mainnet : xprv/xpub
    NetworkValues("Viacoin", "mainnet", "VIA", b'\xc7', b'\x47', b'\x21', h2b('0488ADE4'), h2b('0488B21E')),
    # VIA viacoin testnet : tprv/tpub
    NetworkValues("Viacoin", "testnet", "TVI", b'\xff', b'\x7f', b'\xc4', h2b('04358394'), h2b('043587CF')),

    # FTC feathercoin mainnet : xprv/xpub
    NetworkValues(
        "Feathercoin", "mainnet", "FTC", b'\x8e', b'\x0e', b'\x60', h2b('0488ADE4'), h2b('0488B21E')),
    # FTC feathercoin testnet : tprv/tpub
    NetworkValues(
        "Feathercoin", "testnet", "FTX", b'\xC1', b'\x41', b'\xc4', h2b('04358394'), h2b('043587CF')),

    # DOGE Dogecoin mainnet : dogv/dogp
    NetworkValues(
        "Dogecoin", "mainnet", "DOGE", b'\x9e', b'\x1e', b'\x16', h2b("02FD3955"), h2b("02FD3929")),
    # DOGE Dogecoin testnet : tgpv/tgub
    NetworkValues(
        "Dogecoin", "testnet", "XDT", b'\xf1', b'\x71', b'\xc4', h2b("0432a9a8"), h2b("0432a243")),

    # BC BlackCoin mainnet : bcpv/bcpb
    NetworkValues("Blackcoin", "mainnet", "BC", b'\x99', b'\x19', None, h2b("02cfbf60"), h2b("02cfbede")),

    # DRK Dash mainnet : drkv/drkp
    NetworkValues(
        "Dash", "mainnet", "DASH", b'\xcc', b'\x4c', b'\x10', h2b("02fe52f8"), h2b("02fe52cc")),

    # DRK Dash testnet : DRKV/DRKP
    NetworkValues(
        "Dash", "testnet", "tDASH", b'\xef', b'\x8c', b'\x13', h2b("3a8061a0"), h2b("3a805837")),

    # MEC Megacoin mainnet : mecv/mecp
    NetworkValues("Megacoin", "mainnet", "MEC", b'\xb2', b'\x32', None, h2b("03a04db7"), h2b("03a04d8b")),

    NetworkValues(
        "Myriadcoin", "mainnet", "MYR", b'\xb2', b'\x32', b'\x09', h2b('0488ADE4'), h2b('0488B21E')),

    NetworkValues(
        "Unobtanium", "mainnet", "UNO", b'\xe0', b'\x82', b'\x1e', h2b('0488ADE4'), h2b('0488B21E')),

    # JBS Jumbucks mainnet : jprv/jpub
    NetworkValues("Jumbucks", "mainnet", "JBS", b'\xab', b'\x2b', None, h2b('037a6460'), h2b('037a689a')),

    # MZC Mazacoin mainnet: xprv/xpub
    NetworkValues("Mazacoin", "mainnet", "MZC", b'\xe0', b'\x32', b'\9', h2b("0488ADE4"), h2b("0488B21E")),

    NetworkValues(
        "Riecoin", "mainnet", "RIC", b'\x80', b'\x3c', b'\x05', h2b('0488ADE4'), h2b('0488B21E')),

    # DFC Defcoin mainnet: dfcv/dfcp
    NetworkValues("DEFCOIN", "mainnet", "DFC", b'\x9e', b'\x1e', b'\5', h2b("02FA54D7"), h2b("02FA54AD")),

    # FAI faircoin mainnet : xprv/xpub
    NetworkValues(
        "Faircoin", "mainnet", "FAI", b'\xdf', b'\x5f', b'\x24', h2b("0488ADE4"), h2b("0488B21E")),

    # ARG argentum mainnet : xprv/xpub
    NetworkValues("Argentum", "mainnet", "ARG", b'\x97', b'\x17', b'\5', h2b("0488ADE4"), h2b("0488B21E")),

    # ZEC Zcash mainnet : xprv/xpub
    NetworkValues("Zcash", "mainnet", "ZEC", b'\x80', b'\x1C\xB8',
                  b'\x1C\xBD', h2b("0488ADE4"), h2b("0488B21E")),

    # BTCD BitcoinDark mainnet : xprv/xpub
    NetworkValues("BitcoinDark", "mainnet", "BTCD", b'\x44', b'\x3C', b'\55', h2b('0488ADE4'), h2b('0488B21E')),

    # DCR Decred mainnet : dprv/dpub
    NetworkValues("Decred", "mainnet", "DCR", b'\x22\xDE', b'\x07\x3F', b'\x07\x1A', h2b('02FDA4E8'), h2b('02FDA926')),

    # DCR Decred testnet : tprv/tpub
    NetworkValues("Decred", "testnet", "DCRT", b'\x23\x0E', b'\x0F\x21', b'\x0E\x6C', h2b('04358397'), h2b('043587D1')),

    # PIVX mainnet : dprv/dpub
    NetworkValues("PIVX", "mainnet", "PIVX", b'\xd4', b'\x1e', b'\0d', h2b('0221312b'), h2b('022d2533')),

    # PIVX testnet : tprv/tpub
    NetworkValues("PIVX", "testnet", "TPIVX", b'\xef', b'\x8b', b'\x13', h2b('3a805837'), h2b('3a8061a0')),

)
<|endoftext|>


<commit_before>#!/usr/bin/env python

from __future__ import print_function, unicode_literals

import os, sys, subprocess
from flask import Flask, send_from_directory

app = Flask(__name__)
app.config['DEBUG'] = True
here = os.getcwd()

@app.route('/')
def hello():
    return send_from_directory('statics', 'main.html')

@app.route('/f')
def forward():
    try:
        out = subprocess.check_output(os.path.join(here, 'drive.sh f'), shell=True)
    except subprocess.CalledProcessError as e:
        return 'NG: ' + unicode(e)
    return 'OK'

@app.route('/s')
def stop():
    try:
        out = subprocess.check_output(os.path.join(here, 'drive.sh s'), shell=True)
    except subprocess.CalledProcessError as e:
        return 'NG: ' + unicode(e)
    return 'OK'

@app.route('/b')
def back():
    try:
        out = subprocess.check_output(os.path.join('drive.sh b'), shell=True)
    except subprocess.CalledProcessError as e:
        return 'NG: ' + unicode(e)
    return 'OK'

@app.route('/statics/<path:path>')
def statics(path):
    return send_from_directory('statics', path)

if __name__ == '__main__':
    app.run(host='0.0.0.0', port=80, debug=True)

<commit_msg>Fix forgotten argument<commit_after>#!/usr/bin/env python

from __future__ import print_function, unicode_literals

import os, sys, subprocess
from flask import Flask, send_from_directory

app = Flask(__name__)
app.config['DEBUG'] = True
here = os.getcwd()

@app.route('/')
def hello():
    return send_from_directory('statics', 'main.html')

@app.route('/f')
def forward():
    try:
        out = subprocess.check_output(os.path.join(here, 'drive.sh f'), shell=True)
    except subprocess.CalledProcessError as e:
        return 'NG: ' + unicode(e)
    return 'OK'

@app.route('/s')
def stop():
    try:
        out = subprocess.check_output(os.path.join(here, 'drive.sh s'), shell=True)
    except subprocess.CalledProcessError as e:
        return 'NG: ' + unicode(e)
    return 'OK'

@app.route('/b')
def back():
    try:
        out = subprocess.check_output(os.path.join(here, 'drive.sh b'), shell=True)
    except subprocess.CalledProcessError as e:
        return 'NG: ' + unicode(e)
    return 'OK'

@app.route('/statics/<path:path>')
def statics(path):
    return send_from_directory('statics', path)

if __name__ == '__main__':
    app.run(host='0.0.0.0', port=80, debug=True)

<|endoftext|>


<commit_before>#!/usr/bin/env python

'''
Created on Jul 17, 2012
'''

from __future__ import division

__author__ = "Shyue Ping Ong"
__copyright__ = "Copyright 2012, The Materials Project"
__version__ = "0.1"
__maintainer__ = "Shyue Ping Ong"
__email__ = "shyue@mit.edu"
__date__ = "Jul 17, 2012"

import unittest
import numpy as np

from pymatgen.core.periodic_table import Element, Specie
from pymatgen.core.sites import Site, PeriodicSite
from pymatgen.core.lattice import Lattice

class SiteTest(unittest.TestCase):

    def setUp(self):
        self.ordered_site = Site(Element("Fe"), [0.25, 0.35, 0.45])
        self.disordered_site = Site({Element("Fe"): 0.5, Element("Mn"): 0.5},
                                    [0.25, 0.35, 0.45])
        self.propertied_site = Site(Specie("Fe", 2), [0.25, 0.35, 0.45],
                                    {'magmom': 5.1, 'charge': 4.2})

    def test_init(self):
        self.assertRaises(ValueError, Site, Specie("Fe", 2),
                          [0.25, 0.35, 0.45], {'mag': 5.1})

    def test_properties(self):
        self.assertRaises(AttributeError, getattr, self.disordered_site,
                          'specie')
        self.assertIsInstance(self.ordered_site.specie, Element)
        self.assertEqual(self.propertied_site.magmom, 5.1)
        self.assertEqual(self.propertied_site.charge, 4.2)

    def test_to_from_dict(self):
        d = self.disordered_site.to_dict
        site = Site.from_dict(d)
        self.assertEqual(site, self.disordered_site)
        self.assertNotEqual(site, self.ordered_site)
        d = self.propertied_site.to_dict
        site = Site.from_dict(d)
        self.assertEqual(site.magmom, 5.1)
        self.assertEqual(site.charge, 4.2)

    def test_hash(self):
        self.assertEqual(self.ordered_site.__hash__(), 26)
        self.assertEqual(self.disordered_site.__hash__(), 25.5)

    def test_cmp(self):
        self.assertTrue(self.ordered_site > self.disordered_site)

    def test_distance(self):
        osite = self.ordered_site
        self.assertAlmostEqual(np.linalg.norm([0.25, 0.35, 0.45]),
                               osite.distance_from_point([0, 0, 0]))
        self.assertAlmostEqual(osite.distance(self.disordered_site), 0)

class PeriodicSiteTest(unittest.TestCase):

    def setUp(self):
        self.lattice = Lattice.cubic(10.0)
        self.si = Element("Si")
        self.site = PeriodicSite("Fe", [0.25, 0.35, 0.45],
                                 self.lattice)
        self.site2 = PeriodicSite({"Si": 0.5}, [0, 0, 0], self.lattice)
        self.assertEquals(self.site2.species_and_occu, {Element('Si'): 0.5},
                          "Inconsistent site created!")
        self.propertied_site = PeriodicSite(Specie("Fe", 2),
                                            [0.25, 0.35, 0.45],
                                            self.lattice,
                                            properties={'magmom': 5.1,
                                                        'charge': 4.2})

    def test_properties(self):
        """
        Test the properties for a site
        """
        self.assertEquals(self.site.a, 0.25)
        self.assertEquals(self.site.b, 0.35)
        self.assertEquals(self.site.c, 0.45)
        self.assertEquals(self.site.x, 2.5)
        self.assertEquals(self.site.y, 3.5)
        self.assertEquals(self.site.z, 4.5)
        self.assertTrue(self.site.is_ordered)
        self.assertFalse(self.site2.is_ordered)
        self.assertEqual(self.propertied_site.magmom, 5.1)
        self.assertEqual(self.propertied_site.charge, 4.2)

    def test_distance(self):
        other_site = PeriodicSite("Fe", np.array([0, 0, 0]), self.lattice)
        self.assertAlmostEquals(self.site.distance(other_site), 6.22494979899,
                                5)

    def test_distance_from_point(self):
        self.assertNotAlmostEqual(self.site.distance_from_point([0.1, 0.1,
                                                                 0.1]),
                                  6.22494979899, 5)
        self.assertAlmostEqual(self.site.distance_from_point([0.1, 0.1, 0.1]),
                               6.0564015718906887, 5)

    def test_distance_and_image(self):
        other_site = PeriodicSite("Fe", np.array([1, 1, 1]), self.lattice)
        (distance, image) = self.site.distance_and_image(other_site)
        self.assertAlmostEquals(distance, 6.22494979899, 5)
        self.assertTrue(([-1, -1, -1] == image).all())
        (distance, image) = self.site.distance_and_image(other_site, [1, 0, 0])
        self.assertAlmostEquals(distance, 19.461500456028563, 5)
        # Test that old and new distance algo give the same ans for
        # "standard lattices"
        lattice = Lattice(np.array([[1, 0, 0], [0, 1, 0], [0, 0, 1]]))
        site1 = PeriodicSite("Fe", np.array([0.01, 0.02, 0.03]), lattice)
        site2 = PeriodicSite("Fe", np.array([0.99, 0.98, 0.97]), lattice)
        self.assertAlmostEqual(site1.distance_and_image_old(site2)[0],
                               site1.distance_and_image(site2)[0])
        lattice = Lattice.from_parameters(1, 0.01, 1, 10, 10, 10)
        site1 = PeriodicSite("Fe", np.array([0.01, 0.02, 0.03]), lattice)
        site2 = PeriodicSite("Fe", np.array([0.99, 0.98, 0.97]), lattice)
        self.assertTrue(site1.distance_and_image_old(site2)[0] >
                        site1.distance_and_image(site2)[0])
        site2 = PeriodicSite("Fe", np.random.rand(3), lattice)
        (dist_old, jimage_old) = site1.distance_and_image_old(site2)
        (dist_new, jimage_new) = site1.distance_and_image(site2)
        self.assertTrue(dist_old - dist_new > -1e-8,
                        "New distance algo should give smaller answers!")
        self.assertFalse((abs(dist_old - dist_new) < 1e-8) ^
                         (jimage_old == jimage_new).all(),
                         "If old dist == new dist, images must be the same!")

    def test_is_periodic_image(self):
        other = PeriodicSite("Fe", np.array([1.25, 2.35, 4.45]), self.lattice)
        self.assertTrue(self.site.is_periodic_image(other),
                        "This other site should be a periodic image.")
        other = PeriodicSite("Fe", np.array([1.25, 2.35, 4.46]), self.lattice)
        self.assertFalse(self.site.is_periodic_image(other),
                         "This other site should not be a periodic image.")
        other = PeriodicSite("Fe", np.array([1.25, 2.35, 4.45]),
                             Lattice.rhombohedral(2, 60))
        self.assertFalse(self.site.is_periodic_image(other),
                         "Different lattices should not be periodic images.")

    def test_equality(self):
        other_site = PeriodicSite("Fe", np.array([1, 1, 1]), self.lattice)
        self.assertTrue(self.site.__eq__(self.site))
        self.assertFalse(other_site.__eq__(self.site))
        self.assertFalse(self.site.__ne__(self.site))
        self.assertTrue(other_site.__ne__(self.site))

    def test_to_from_dict(self):
        d = self.site2.to_dict
        site = PeriodicSite.from_dict(d)
        self.assertEqual(site, self.site2)
        self.assertNotEqual(site, self.site)
        d = self.propertied_site.to_dict
        site = Site.from_dict(d)
        self.assertEqual(site.magmom, 5.1)
        self.assertEqual(site.charge, 4.2)
        site3 = PeriodicSite({"Si": 0.5, "Fe": 0.5}, [0, 0, 0], self.lattice)
        d = site3.to_dict
        site = PeriodicSite.from_dict(d)
        self.assertEqual(site.species_and_occu, site3.species_and_occu)

    def test_to_unit_cell(self):
        site = PeriodicSite("Fe", np.array([1.25, 2.35, 4.46]), self.lattice)
        site = site.to_unit_cell
        val = [0.25, 0.35, 0.46]
        self.assertTrue(np.allclose(site.frac_coords, val))

if __name__ == "__main__":
    #import sys;sys.argv = ['', 'Test.testName']
    unittest.main()
<commit_msg>More unittests for distance algo.<commit_after>#!/usr/bin/env python

'''
Created on Jul 17, 2012
'''

from __future__ import division

__author__ = "Shyue Ping Ong"
__copyright__ = "Copyright 2012, The Materials Project"
__version__ = "0.1"
__maintainer__ = "Shyue Ping Ong"
__email__ = "shyue@mit.edu"
__date__ = "Jul 17, 2012"

import unittest
import numpy as np

from pymatgen.core.periodic_table import Element, Specie
from pymatgen.core.sites import Site, PeriodicSite
from pymatgen.core.lattice import Lattice

class SiteTest(unittest.TestCase):

    def setUp(self):
        self.ordered_site = Site(Element("Fe"), [0.25, 0.35, 0.45])
        self.disordered_site = Site({Element("Fe"): 0.5, Element("Mn"): 0.5},
                                    [0.25, 0.35, 0.45])
        self.propertied_site = Site(Specie("Fe", 2), [0.25, 0.35, 0.45],
                                    {'magmom': 5.1, 'charge': 4.2})

    def test_init(self):
        self.assertRaises(ValueError, Site, Specie("Fe", 2),
                          [0.25, 0.35, 0.45], {'mag': 5.1})

    def test_properties(self):
        self.assertRaises(AttributeError, getattr, self.disordered_site,
                          'specie')
        self.assertIsInstance(self.ordered_site.specie, Element)
        self.assertEqual(self.propertied_site.magmom, 5.1)
        self.assertEqual(self.propertied_site.charge, 4.2)

    def test_to_from_dict(self):
        d = self.disordered_site.to_dict
        site = Site.from_dict(d)
        self.assertEqual(site, self.disordered_site)
        self.assertNotEqual(site, self.ordered_site)
        d = self.propertied_site.to_dict
        site = Site.from_dict(d)
        self.assertEqual(site.magmom, 5.1)
        self.assertEqual(site.charge, 4.2)

    def test_hash(self):
        self.assertEqual(self.ordered_site.__hash__(), 26)
        self.assertEqual(self.disordered_site.__hash__(), 25.5)

    def test_cmp(self):
        self.assertTrue(self.ordered_site > self.disordered_site)

    def test_distance(self):
        osite = self.ordered_site
        self.assertAlmostEqual(np.linalg.norm([0.25, 0.35, 0.45]),
                               osite.distance_from_point([0, 0, 0]))
        self.assertAlmostEqual(osite.distance(self.disordered_site), 0)

class PeriodicSiteTest(unittest.TestCase):

    def setUp(self):
        self.lattice = Lattice.cubic(10.0)
        self.si = Element("Si")
        self.site = PeriodicSite("Fe", [0.25, 0.35, 0.45],
                                 self.lattice)
        self.site2 = PeriodicSite({"Si": 0.5}, [0, 0, 0], self.lattice)
        self.assertEquals(self.site2.species_and_occu, {Element('Si'): 0.5},
                          "Inconsistent site created!")
        self.propertied_site = PeriodicSite(Specie("Fe", 2),
                                            [0.25, 0.35, 0.45],
                                            self.lattice,
                                            properties={'magmom': 5.1,
                                                        'charge': 4.2})

    def test_properties(self):
        """
        Test the properties for a site
        """
        self.assertEquals(self.site.a, 0.25)
        self.assertEquals(self.site.b, 0.35)
        self.assertEquals(self.site.c, 0.45)
        self.assertEquals(self.site.x, 2.5)
        self.assertEquals(self.site.y, 3.5)
        self.assertEquals(self.site.z, 4.5)
        self.assertTrue(self.site.is_ordered)
        self.assertFalse(self.site2.is_ordered)
        self.assertEqual(self.propertied_site.magmom, 5.1)
        self.assertEqual(self.propertied_site.charge, 4.2)

    def test_distance(self):
        other_site = PeriodicSite("Fe", np.array([0, 0, 0]), self.lattice)
        self.assertAlmostEquals(self.site.distance(other_site), 6.22494979899,
                                5)

    def test_distance_from_point(self):
        self.assertNotAlmostEqual(self.site.distance_from_point([0.1, 0.1,
                                                                 0.1]),
                                  6.22494979899, 5)
        self.assertAlmostEqual(self.site.distance_from_point([0.1, 0.1, 0.1]),
                               6.0564015718906887, 5)

    def test_distance_and_image(self):
        other_site = PeriodicSite("Fe", np.array([1, 1, 1]), self.lattice)
        (distance, image) = self.site.distance_and_image(other_site)
        self.assertAlmostEquals(distance, 6.22494979899, 5)
        self.assertTrue(([-1, -1, -1] == image).all())
        (distance, image) = self.site.distance_and_image(other_site, [1, 0, 0])
        self.assertAlmostEquals(distance, 19.461500456028563, 5)
        # Test that old and new distance algo give the same ans for
        # "standard lattices"
        lattice = Lattice(np.array([[1, 0, 0], [0, 1, 0], [0, 0, 1]]))
        site1 = PeriodicSite("Fe", np.array([0.01, 0.02, 0.03]), lattice)
        site2 = PeriodicSite("Fe", np.array([0.99, 0.98, 0.97]), lattice)
        self.assertAlmostEqual(site1.distance_and_image_old(site2)[0],
                               site1.distance_and_image(site2)[0])
        lattice = Lattice.from_parameters(1, 0.01, 1, 10, 10, 10)
        site1 = PeriodicSite("Fe", np.array([0.01, 0.02, 0.03]), lattice)
        site2 = PeriodicSite("Fe", np.array([0.99, 0.98, 0.97]), lattice)
        self.assertTrue(site1.distance_and_image_old(site2)[0] >
                        site1.distance_and_image(site2)[0])
        site2 = PeriodicSite("Fe", np.random.rand(3), lattice)
        (dist_old, jimage_old) = site1.distance_and_image_old(site2)
        (dist_new, jimage_new) = site1.distance_and_image(site2)
        self.assertTrue(dist_old - dist_new > -1e-8,
                        "New distance algo should give smaller answers!")
        self.assertFalse((abs(dist_old - dist_new) < 1e-8) ^
                         (jimage_old == jimage_new).all(),
                         "If old dist == new dist, images must be the same!")
        latt = Lattice.from_parameters(3.0, 4.0, 10.0, 3.0, 1.0, 2.0)
        site = PeriodicSite("Fe", [0.1, 0.1, 0.1], latt)
        site2 = PeriodicSite("Fe", [0.99, 0.99, 0.99], latt)
        (dist, img) = site.distance_and_image(site2)
        self.assertAlmostEqual(dist, 1.1304420998572722)
        self.assertEqual(list(img), [0, -1, -1])

    def test_is_periodic_image(self):
        other = PeriodicSite("Fe", np.array([1.25, 2.35, 4.45]), self.lattice)
        self.assertTrue(self.site.is_periodic_image(other),
                        "This other site should be a periodic image.")
        other = PeriodicSite("Fe", np.array([1.25, 2.35, 4.46]), self.lattice)
        self.assertFalse(self.site.is_periodic_image(other),
                         "This other site should not be a periodic image.")
        other = PeriodicSite("Fe", np.array([1.25, 2.35, 4.45]),
                             Lattice.rhombohedral(2, 60))
        self.assertFalse(self.site.is_periodic_image(other),
                         "Different lattices should not be periodic images.")

    def test_equality(self):
        other_site = PeriodicSite("Fe", np.array([1, 1, 1]), self.lattice)
        self.assertTrue(self.site.__eq__(self.site))
        self.assertFalse(other_site.__eq__(self.site))
        self.assertFalse(self.site.__ne__(self.site))
        self.assertTrue(other_site.__ne__(self.site))

    def test_to_from_dict(self):
        d = self.site2.to_dict
        site = PeriodicSite.from_dict(d)
        self.assertEqual(site, self.site2)
        self.assertNotEqual(site, self.site)
        d = self.propertied_site.to_dict
        site = Site.from_dict(d)
        self.assertEqual(site.magmom, 5.1)
        self.assertEqual(site.charge, 4.2)
        site3 = PeriodicSite({"Si": 0.5, "Fe": 0.5}, [0, 0, 0], self.lattice)
        d = site3.to_dict
        site = PeriodicSite.from_dict(d)
        self.assertEqual(site.species_and_occu, site3.species_and_occu)

    def test_to_unit_cell(self):
        site = PeriodicSite("Fe", np.array([1.25, 2.35, 4.46]), self.lattice)
        site = site.to_unit_cell
        val = [0.25, 0.35, 0.46]
        self.assertTrue(np.allclose(site.frac_coords, val))

if __name__ == "__main__":
    #import sys;sys.argv = ['', 'Test.testName']
    unittest.main()
<|endoftext|>


<commit_before># Testing layer to provide some of the features of PloneTestCase

from plone.app.testing import FunctionalTesting
from plone.app.testing import IntegrationTesting
from plone.app.testing import login
from plone.app.testing import logout
from plone.app.testing import PLONE_FIXTURE
from plone.app.testing import PloneSandboxLayer
from plone.app.testing import SITE_OWNER_NAME
from plone.app.testing import applyProfile
from plone.testing import z2
from Products.CMFCore.utils import getToolByName
from Products.CMFPlone.setuphandlers import setupPortalContent

import Products.ATExtensions
import Products.PloneTestCase.setup
import bika.lims
import collective.js.jqueryui
import plone.app.iterate

class BikaTestLayer(PloneSandboxLayer):

    defaultBases = (PLONE_FIXTURE,)

    def setUpZope(self, app, configurationContext):
        # Load ZCML
        self.loadZCML(package=Products.ATExtensions)
        self.loadZCML(package=plone.app.iterate)
        self.loadZCML(package=collective.js.jqueryui)
        self.loadZCML(package=bika.lims)

        # Required by Products.CMFPlone:plone-content
        z2.installProduct(app, 'Products.PythonScripts')
        z2.installProduct(app, 'bika.lims')

    def setUpPloneSite(self, portal):
        login(portal.aq_parent, SITE_OWNER_NAME)

        wf = getToolByName(portal, 'portal_workflow')
        wf.setDefaultChain('plone_workflow')
        setupPortalContent(portal)

        # make sure we have folder_listing as a template
        portal.getTypeInfo().manage_changeProperties(
            view_methods=['folder_listing'],
            default_view='folder_listing')

        applyProfile(portal, 'bika.lims:default')

        # Add some test users
        for role in ('LabManager',
                     'LabClerk',
                     'Analyst',
                     'Verifier',
                     'Sampler',
                     'Preserver',
                     'Publisher',
                     'Member',
                     'Reviewer',
                     'RegulatoryInspector'):
            username = "test_" + role.lower()
            member = portal.portal_registration.addMember(
                username,
                username,
                properties={
                    'username': username,
                    'email': username + "@example.com",
                    'fullname': username}
            )
            # Add user to all specified groups
            group_id = role + "s"
            group = portal.portal_groups.getGroupById(group_id)
            if group:
                group.addMember(username)
            # Add user to all specified roles
            member._addRole(role)
            # If user is in LabManagers, add Owner local role on clients folder
            if role == 'LabManager':
                portal.clients.manage_setLocalRoles(username, ['Owner', ])

        logout()

BIKA_TEST_FIXTURE = BikaTestLayer()

BIKA_INTEGRATION_TESTING = IntegrationTesting(
    bases=(BIKA_TEST_FIXTURE,),
    name="BikaTestingLayer:Integration")

BIKA_FUNCTIONAL_TESTING = FunctionalTesting(
    bases=(BIKA_TEST_FIXTURE,),
    name="BikaTestingLayer:Functional")

BIKA_ROBOT_TESTING = FunctionalTesting(
    bases=(BIKA_TEST_FIXTURE, z2.ZSERVER_FIXTURE),
    name="BikaTestingLayer:Robot")
<commit_msg>Add more test users in fixture<commit_after># Testing layer to provide some of the features of PloneTestCase

from plone.app.testing import FunctionalTesting
from plone.app.testing import IntegrationTesting
from plone.app.testing import login
from plone.app.testing import logout
from plone.app.testing import PLONE_FIXTURE
from plone.app.testing import PloneSandboxLayer
from plone.app.testing import SITE_OWNER_NAME
from plone.app.testing import applyProfile
from plone.testing import z2
from Products.CMFCore.utils import getToolByName
from Products.CMFPlone.setuphandlers import setupPortalContent

import Products.ATExtensions
import Products.PloneTestCase.setup
import bika.lims
import collective.js.jqueryui
import plone.app.iterate

class BikaTestLayer(PloneSandboxLayer):

    defaultBases = (PLONE_FIXTURE,)

    def setUpZope(self, app, configurationContext):
        # Load ZCML
        self.loadZCML(package=Products.ATExtensions)
        self.loadZCML(package=plone.app.iterate)
        self.loadZCML(package=collective.js.jqueryui)
        self.loadZCML(package=bika.lims)

        # Required by Products.CMFPlone:plone-content
        z2.installProduct(app, 'Products.PythonScripts')
        z2.installProduct(app, 'bika.lims')

    def setUpPloneSite(self, portal):
        login(portal.aq_parent, SITE_OWNER_NAME)

        wf = getToolByName(portal, 'portal_workflow')
        wf.setDefaultChain('plone_workflow')
        setupPortalContent(portal)

        # make sure we have folder_listing as a template
        portal.getTypeInfo().manage_changeProperties(
            view_methods=['folder_listing'],
            default_view='folder_listing')

        applyProfile(portal, 'bika.lims:default')

        # Add some test users
        for role in ('LabManager',
                     'LabClerk',
                     'Analyst',
                     'Verifier',
                     'Sampler',
                     'Preserver',
                     'Publisher',
                     'Member',
                     'Reviewer',
                     'RegulatoryInspector'):
            for user_nr in range(2):
                if user_nr == 0:
                    username = "test_" + role.lower()
                else:
                    username = "test_" + role.lower() + user_nr
                member = portal.portal_registration.addMember(
                    username,
                    username,
                    properties={
                        'username': username,
                        'email': username + "@example.com",
                        'fullname': username}
                )
                # Add user to all specified groups
                group_id = role + "s"
                group = portal.portal_groups.getGroupById(group_id)
                if group:
                    group.addMember(username)
                # Add user to all specified roles
                member._addRole(role)
                # If user is in LabManagers, add Owner local role on clients folder
                if role == 'LabManager':
                    portal.clients.manage_setLocalRoles(username, ['Owner', ])

        logout()

BIKA_TEST_FIXTURE = BikaTestLayer()

BIKA_INTEGRATION_TESTING = IntegrationTesting(
    bases=(BIKA_TEST_FIXTURE,),
    name="BikaTestingLayer:Integration")

BIKA_FUNCTIONAL_TESTING = FunctionalTesting(
    bases=(BIKA_TEST_FIXTURE,),
    name="BikaTestingLayer:Functional")

BIKA_ROBOT_TESTING = FunctionalTesting(
    bases=(BIKA_TEST_FIXTURE, z2.ZSERVER_FIXTURE),
    name="BikaTestingLayer:Robot")
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""
Created on Mon Mar 28 09:45:08 2016

@author: Suhas Somnath, Chris Smith
"""

from __future__ import division, print_function, absolute_import
import time
from warnings import warn
from multiprocessing import cpu_count
import numpy as np
from sklearn.utils import gen_batches
from sklearn.utils.extmath import randomized_svd

from ..core.processing.process import Process
from .proc_utils import get_component_slice
from ..core.io.hdf_utils import get_h5_obj_refs, check_and_link_ancillary, find_results_groups, \
    get_indices_for_region_ref, create_region_reference, calc_chunks, copy_main_attributes, copy_attributes
from pycroscopy.core.io.hdf_writer import HDFwriter
from ..core.io.io_utils import get_available_memory
from ..core.io.dtype_utils import check_dtype, stack_real_to_target_dtype
from ..core.io.virtual_data import VirtualDataset, VirtualGroup
from ..core.io.write_utils import INDICES_DTYPE

class SVD(Process):

    def __init__(self, h5_main, num_components=None):

        super(SVD, self).__init__(h5_main)
        self.process_name = 'SVD'

        '''
        Calculate the size of the main data in memory and compare to max_mem
        We use the minimum of the actual dtype's itemsize and float32 since we
        don't want to read it in yet and do the proper type conversions.
        '''
        self.data_transform_func, is_complex, is_compound, n_features, n_samples, type_mult = check_dtype(h5_main)

        if num_components is None:
            num_components = min(n_samples, n_features)
        else:
            num_components = min(n_samples, n_features, num_components)
        self.num_components = num_components
        self.parms_dict = {'num_components': num_components}
        self.duplicate_h5_groups = self._check_for_duplicates()

    def compute(self):
        """
        Computes SVD and writes results to file

        Returns
        -------
         h5_results_grp : h5py.Datagroup object
            Datagroup containing all the results
        """

        '''
        Check if a number of compnents has been set and ensure that the number is less than
        the minimum axis length of the data.  If both conditions are met, use fsvd.  If not
        use the regular svd.

        C.Smith -- We might need to put a lower limit on num_comps in the future.  I don't
                   know enough about svd to be sure.
        '''
        print('Performing SVD')

        t1 = time.time()

        U, S, V = randomized_svd(self.data_transform_func(self.h5_main), self.num_components, n_iter=3)

        print('SVD took {} seconds.  Writing results to file.'.format(round(time.time() - t1, 2)))

        self._write_results_chunk(U, S, V)
        del U, S, V

        return self.h5_results_grp

    def _write_results_chunk(self, U, S, V):
        """
        Writes the provided SVD results to file

        Parameters
        ----------
        U : array-like
            Abundance matrix
        S : array-like
            variance vector
        V : array-like
            eigenvector matrix
        """

        ds_S = VirtualDataset('S', data=np.float32(S))
        ds_S.attrs['labels'] = {'Principal Component': [slice(0, None)]}
        ds_S.attrs['units'] = ''
        ds_inds = VirtualDataset('Component_Indices', data=INDICES_DTYPE(np.arange(len(S))))
        ds_inds.attrs['labels'] = {'Principal Component': [slice(0, None)]}
        ds_inds.attrs['units'] = ''
        del S

        u_chunks = calc_chunks(U.shape, np.float32(0).itemsize)
        ds_U = VirtualDataset('U', data=np.float32(U), chunking=u_chunks)
        del U

        V = stack_real_to_target_dtype(V, self.h5_main.dtype)
        v_chunks = calc_chunks(V.shape, self.h5_main.dtype.itemsize)
        ds_V = VirtualDataset('V', data=V, chunking=v_chunks)
        del V

        '''
        Create the Group to hold the results and add the existing datasets as
        children
        '''
        grp_name = self.h5_main.name.split('/')[-1] + '-' + self.process_name + '_'
        svd_grp = VirtualGroup(grp_name, self.h5_main.parent.name[1:])
        svd_grp.add_children([ds_V, ds_S, ds_U, ds_inds])

        '''
        Write the attributes to the group
        '''
        svd_grp.attrs = self.parms_dict
        svd_grp.attrs.update({'svd_method': 'sklearn-randomized', 'last_pixel': self.h5_main.shape[0] - 1})

        '''
        Write the data and retrieve the HDF5 objects then delete the Microdatasets
        '''
        hdf = HDFwriter(self.h5_main.file)
        h5_svd_refs = hdf.write(svd_grp)

        h5_U = get_h5_obj_refs(['U'], h5_svd_refs)[0]
        h5_S = get_h5_obj_refs(['S'], h5_svd_refs)[0]
        h5_V = get_h5_obj_refs(['V'], h5_svd_refs)[0]
        h5_svd_inds = get_h5_obj_refs(['Component_Indices'], h5_svd_refs)[0]
        self.h5_results_grp = h5_S.parent

        # copy attributes
        copy_main_attributes(self.h5_main, h5_V)
        h5_V.attrs['units'] = np.array(['a. u.'], dtype='S')

        del ds_S, ds_V, ds_U, svd_grp

        # Will attempt to see if there is anything linked to this dataset.
        # Since I was meticulous about the translators that I wrote, I know I will find something here
        check_and_link_ancillary(h5_U,
                                 ['Position_Indices', 'Position_Values'],
                                 h5_main=self.h5_main)

        check_and_link_ancillary(h5_V,
                                 ['Position_Indices', 'Position_Values'],
                                 anc_refs=[h5_svd_inds, h5_S])

        check_and_link_ancillary(h5_U,
                                 ['Spectroscopic_Indices', 'Spectroscopic_Values'],
                                 anc_refs=[h5_svd_inds, h5_S])

        check_and_link_ancillary(h5_V,
                                 ['Spectroscopic_Indices', 'Spectroscopic_Values'],
                                 h5_main=self.h5_main)

        '''
        Check h5_main for plot group references.
        Copy them into V if they exist
        '''
        for key in self.h5_main.attrs.keys():
            if '_Plot_Group' not in key:
                continue

            ref_inds = get_indices_for_region_ref(self.h5_main, self.h5_main.attrs[key], return_method='corners')
            ref_inds = ref_inds.reshape([-1, 2, 2])
            ref_inds[:, 1, 0] = h5_V.shape[0] - 1

            svd_ref = create_region_reference(h5_V, ref_inds)

            h5_V.attrs[key] = svd_ref

###############################################################################

def simplified_kpca(kpca, source_data):
    """
    Performs kernel PCA on the provided dataset and returns the familiar
    eigenvector, eigenvalue, and scree matrices.

    Note that the positions in the eigenvalues may need to be transposed

    Parameters
    ----------
    kpca : KernelPCA object
        configured Kernel PCA object ready to perform analysis
    source_data : 2D numpy array
        Data arranged as [iteration, features] example - [position, time]

    Returns
    -------
    eigenvalues : 2D numpy array
        Eigenvalues in the original space arranged as [component,iteration]
    scree : 1D numpy array
        S component
    eigenvector : 2D numpy array
        Eigenvectors in the original space arranged as [component,features]

    """
    X_kpca = kpca.fit(source_data.T)
    eigenvectors = X_kpca.alphas_.T
    eigenvalues = X_kpca.fit_transform(source_data)
    # kpca_explained_variance = np.var(kpca.fit_transform(source_data), axis=0)
    # information_content = kpca_explained_variance / np.sum(kpca_explained_variance)
    scree = kpca.lambdas_
    return eigenvalues, scree, eigenvectors

def rebuild_svd(h5_main, components=None, cores=None, max_RAM_mb=1024):
    """
    Rebuild the Image from the SVD results on the windows
    Optionally, only use components less than n_comp.

    Parameters
    ----------
    h5_main : hdf5 Dataset
        dataset which SVD was performed on
    components : {int, iterable of int, slice} optional
        Defines which components to keep
        Default - None, all components kept

        Input Types
        integer : Components less than the input will be kept
        length 2 iterable of integers : Integers define start and stop of component slice to retain
        other iterable of integers or slice : Selection of component indices to retain
    cores : int, optional
        How many cores should be used to rebuild
        Default - None, all but 2 cores will be used, min 1
    max_RAM_mb : int, optional
        Maximum ammount of memory to use when rebuilding, in Mb.
        Default - 1024Mb

    Returns
    -------
    rebuilt_data : HDF5 Dataset
        the rebuilt dataset

    """

    hdf = HDFwriter(h5_main.file)
    comp_slice, num_comps = get_component_slice(components, total_components=h5_main.shape[1])
    dset_name = h5_main.name.split('/')[-1]

    # Ensuring that at least one core is available for use / 2 cores are available for other use
    max_cores = max(1, cpu_count() - 2)
    #         print('max_cores',max_cores)
    if cores is not None:
        cores = min(round(abs(cores)), max_cores)
    else:
        cores = max_cores

    max_memory = min(max_RAM_mb * 1024 ** 2, 0.75 * get_available_memory())
    if cores != 1:
        max_memory = int(max_memory / 2)

    '''
    Get the handles for the SVD results
    '''
    try:
        h5_svd = find_results_groups(h5_main, 'SVD')[-1]

        h5_S = h5_svd['S']
        h5_U = h5_svd['U']
        h5_V = h5_svd['V']

    except KeyError:
        warnstring = 'SVD Results for {dset} were not found.'.format(dset=dset_name)
        warn(warnstring)
        return
    except:
        raise

    func, is_complex, is_compound, n_features, n_samples, type_mult = check_dtype(h5_V)

    '''
    Calculate the size of a single batch that will fit in the available memory
    '''
    n_comps = h5_S[comp_slice].size
    mem_per_pix = (h5_U.dtype.itemsize + h5_V.dtype.itemsize * h5_V.shape[1]) * n_comps
    fixed_mem = h5_main.size * h5_main.dtype.itemsize

    if cores is None:
        free_mem = max_memory - fixed_mem
    else:
        free_mem = max_memory * 2 - fixed_mem

    batch_size = int(round(float(free_mem) / mem_per_pix))
    batch_slices = gen_batches(h5_U.shape[0], batch_size)

    print('Reconstructing in batches of {} positions.'.format(batch_size))
    print('Batchs should be {} Mb each.'.format(mem_per_pix * batch_size / 1024.0 ** 2))

    '''
    Loop over all batches.
    '''
    ds_V = np.dot(np.diag(h5_S[comp_slice]), func(h5_V[comp_slice, :]))
    rebuild = np.zeros((h5_main.shape[0], ds_V.shape[1]))
    for ibatch, batch in enumerate(batch_slices):
        rebuild[batch, :] += np.dot(h5_U[batch, comp_slice], ds_V)

    rebuild = stack_real_to_target_dtype(rebuild, h5_V.dtype)

    print('Completed reconstruction of data from SVD results.  Writing to file.')
    '''
    Create the Group and dataset to hold the rebuild data
    '''
    rebuilt_grp = VirtualGroup('Rebuilt_Data_', h5_svd.name[1:])

    ds_rebuilt = VirtualDataset('Rebuilt_Data', rebuild,
                                chunking=h5_main.chunks,
                                compression=h5_main.compression)
    rebuilt_grp.add_children([ds_rebuilt])

    if isinstance(comp_slice, slice):
        rebuilt_grp.attrs['components_used'] = '{}-{}'.format(comp_slice.start, comp_slice.stop)
    else:
        rebuilt_grp.attrs['components_used'] = components

    h5_refs = hdf.write(rebuilt_grp)

    h5_rebuilt = get_h5_obj_refs(['Rebuilt_Data'], h5_refs)[0]
    copy_attributes(h5_main, h5_rebuilt, skip_refs=False)

    hdf.flush()

    print('Done writing reconstructed data to file.')

    return h5_rebuilt
<commit_msg>Implemented test_on_subset(). UNTESTED<commit_after># -*- coding: utf-8 -*-
"""
Created on Mon Mar 28 09:45:08 2016

@author: Suhas Somnath, Chris Smith
"""

from __future__ import division, print_function, absolute_import
import time
from warnings import warn
from multiprocessing import cpu_count
import numpy as np
from sklearn.utils import gen_batches
from sklearn.utils.extmath import randomized_svd

from ..core.processing.process import Process
from .proc_utils import get_component_slice
from ..core.io.hdf_utils import get_h5_obj_refs, check_and_link_ancillary, find_results_groups, \
    get_indices_for_region_ref, create_region_reference, calc_chunks, copy_main_attributes, copy_attributes
from pycroscopy.core.io.hdf_writer import HDFwriter
from ..core.io.io_utils import get_available_memory
from ..core.io.dtype_utils import check_dtype, stack_real_to_target_dtype
from ..core.io.virtual_data import VirtualDataset, VirtualGroup
from ..core.io.write_utils import INDICES_DTYPE

class SVD(Process):

    def __init__(self, h5_main, num_components=None):

        super(SVD, self).__init__(h5_main)
        self.process_name = 'SVD'

        '''
        Calculate the size of the main data in memory and compare to max_mem
        We use the minimum of the actual dtype's itemsize and float32 since we
        don't want to read it in yet and do the proper type conversions.
        '''
        self.data_transform_func, is_complex, is_compound, n_features, n_samples, type_mult = check_dtype(h5_main)

        if num_components is None:
            num_components = min(n_samples, n_features)
        else:
            num_components = min(n_samples, n_features, num_components)
        self.num_components = num_components
        self.parms_dict = {'num_components': num_components}
        self.duplicate_h5_groups = self._check_for_duplicates()
        self.__u = None
        self.__v = None
        self.__s = None

    def test_on_subset(self):
        """
        Applies randomised VD to the dataset. This does NOT write results to the hdf5 file. Call compute() to write to
        the file. Handles complex, compound datasets such that the V matrix is of the same datatype as the input matrix.

        Returns
        -------
        U : numpy.ndarray
            Abundance matrix
        S : numpy.ndarray
            variance vector
        V : numpy.ndarray
            eigenvector matrix
        """
        '''
        Check if a number of compnents has been set and ensure that the number is less than
        the minimum axis length of the data.  If both conditions are met, use fsvd.  If not
        use the regular svd.

        C.Smith -- We might need to put a lower limit on num_comps in the future.  I don't
                   know enough about svd to be sure.
        '''
        print('Performing SVD')

        t1 = time.time()

        self.__u, self.__s, self.__v = randomized_svd(self.data_transform_func(self.h5_main), self.num_components,
                                                      n_iter=3)
        self.__v = stack_real_to_target_dtype(self.__v, self.h5_main.dtype)

        print('SVD took {} seconds.  Writing results to file.'.format(round(time.time() - t1, 2)))
        return self.__u, self.__s, self.__v

    def compute(self):
        """
        Computes SVD (by calling test_on_subset() if it has not already been called) and writes results to file.
        Consider calling test_on_subset() to check results before writing to file. Results are deleted from memory
        upon writing to the HDF5 file

        Returns
        -------
         h5_results_grp : h5py.Datagroup object
            Datagroup containing all the results
        """
        if self.__u is None and self.__v is None and self.__s is None:
            self.test_on_subset()

        self._write_results_chunk()
        del self.__u, self.__s, self.__v
        self.__u = None
        self.__v = None
        self.__s = None

        return self.h5_results_grp

    def _write_results_chunk(self):
        """
        Writes the provided SVD results to file

        Parameters
        ----------
        """

        ds_s = VirtualDataset('S', data=np.float32(self.__s))
        ds_s.attrs['labels'] = {'Principal Component': [slice(0, None)]}
        ds_s.attrs['units'] = ''
        ds_inds = VirtualDataset('Component_Indices', data=INDICES_DTYPE(np.arange(len(self.__s))))
        ds_inds.attrs['labels'] = {'Principal Component': [slice(0, None)]}
        ds_inds.attrs['units'] = ''

        u_chunks = calc_chunks(self.__u.shape, np.float32(0).itemsize)
        ds_u = VirtualDataset('U', data=np.float32(self.__u), chunking=u_chunks)

        v_chunks = calc_chunks(self.__v.shape, self.h5_main.dtype.itemsize)
        ds_v = VirtualDataset('V', data=self.__v, chunking=v_chunks)

        '''
        Create the Group to hold the results and add the existing datasets as
        children
        '''
        grp_name = self.h5_main.name.split('/')[-1] + '-' + self.process_name + '_'
        svd_grp = VirtualGroup(grp_name, self.h5_main.parent.name[1:])
        svd_grp.add_children([ds_v, ds_s, ds_u, ds_inds])

        '''
        Write the attributes to the group
        '''
        svd_grp.attrs = self.parms_dict
        svd_grp.attrs.update({'svd_method': 'sklearn-randomized', 'last_pixel': self.h5_main.shape[0] - 1})

        '''
        Write the data and retrieve the HDF5 objects then delete the Microdatasets
        '''
        hdf = HDFwriter(self.h5_main.file)
        h5_svd_refs = hdf.write(svd_grp)

        h5_U = get_h5_obj_refs(['U'], h5_svd_refs)[0]
        h5_S = get_h5_obj_refs(['S'], h5_svd_refs)[0]
        h5_V = get_h5_obj_refs(['V'], h5_svd_refs)[0]
        h5_svd_inds = get_h5_obj_refs(['Component_Indices'], h5_svd_refs)[0]
        self.h5_results_grp = h5_S.parent

        # copy attributes
        copy_main_attributes(self.h5_main, h5_V)
        h5_V.attrs['units'] = np.array(['a. u.'], dtype='S')

        del ds_s, ds_v, ds_u, svd_grp

        # Will attempt to see if there is anything linked to this dataset.
        # Since I was meticulous about the translators that I wrote, I know I will find something here
        check_and_link_ancillary(h5_U,
                                 ['Position_Indices', 'Position_Values'],
                                 h5_main=self.h5_main)

        check_and_link_ancillary(h5_V,
                                 ['Position_Indices', 'Position_Values'],
                                 anc_refs=[h5_svd_inds, h5_S])

        check_and_link_ancillary(h5_U,
                                 ['Spectroscopic_Indices', 'Spectroscopic_Values'],
                                 anc_refs=[h5_svd_inds, h5_S])

        check_and_link_ancillary(h5_V,
                                 ['Spectroscopic_Indices', 'Spectroscopic_Values'],
                                 h5_main=self.h5_main)

        '''
        Check h5_main for plot group references.
        Copy them into V if they exist
        '''
        for key in self.h5_main.attrs.keys():
            if '_Plot_Group' not in key:
                continue

            ref_inds = get_indices_for_region_ref(self.h5_main, self.h5_main.attrs[key], return_method='corners')
            ref_inds = ref_inds.reshape([-1, 2, 2])
            ref_inds[:, 1, 0] = h5_V.shape[0] - 1

            svd_ref = create_region_reference(h5_V, ref_inds)

            h5_V.attrs[key] = svd_ref

###############################################################################

def simplified_kpca(kpca, source_data):
    """
    Performs kernel PCA on the provided dataset and returns the familiar
    eigenvector, eigenvalue, and scree matrices.

    Note that the positions in the eigenvalues may need to be transposed

    Parameters
    ----------
    kpca : KernelPCA object
        configured Kernel PCA object ready to perform analysis
    source_data : 2D numpy array
        Data arranged as [iteration, features] example - [position, time]

    Returns
    -------
    eigenvalues : 2D numpy array
        Eigenvalues in the original space arranged as [component,iteration]
    scree : 1D numpy array
        S component
    eigenvector : 2D numpy array
        Eigenvectors in the original space arranged as [component,features]

    """
    X_kpca = kpca.fit(source_data.T)
    eigenvectors = X_kpca.alphas_.T
    eigenvalues = X_kpca.fit_transform(source_data)
    # kpca_explained_variance = np.var(kpca.fit_transform(source_data), axis=0)
    # information_content = kpca_explained_variance / np.sum(kpca_explained_variance)
    scree = kpca.lambdas_
    return eigenvalues, scree, eigenvectors

def rebuild_svd(h5_main, components=None, cores=None, max_RAM_mb=1024):
    """
    Rebuild the Image from the SVD results on the windows
    Optionally, only use components less than n_comp.

    Parameters
    ----------
    h5_main : hdf5 Dataset
        dataset which SVD was performed on
    components : {int, iterable of int, slice} optional
        Defines which components to keep
        Default - None, all components kept

        Input Types
        integer : Components less than the input will be kept
        length 2 iterable of integers : Integers define start and stop of component slice to retain
        other iterable of integers or slice : Selection of component indices to retain
    cores : int, optional
        How many cores should be used to rebuild
        Default - None, all but 2 cores will be used, min 1
    max_RAM_mb : int, optional
        Maximum ammount of memory to use when rebuilding, in Mb.
        Default - 1024Mb

    Returns
    -------
    rebuilt_data : HDF5 Dataset
        the rebuilt dataset

    """

    hdf = HDFwriter(h5_main.file)
    comp_slice, num_comps = get_component_slice(components, total_components=h5_main.shape[1])
    dset_name = h5_main.name.split('/')[-1]

    # Ensuring that at least one core is available for use / 2 cores are available for other use
    max_cores = max(1, cpu_count() - 2)
    #         print('max_cores',max_cores)
    if cores is not None:
        cores = min(round(abs(cores)), max_cores)
    else:
        cores = max_cores

    max_memory = min(max_RAM_mb * 1024 ** 2, 0.75 * get_available_memory())
    if cores != 1:
        max_memory = int(max_memory / 2)

    '''
    Get the handles for the SVD results
    '''
    try:
        h5_svd = find_results_groups(h5_main, 'SVD')[-1]

        h5_S = h5_svd['S']
        h5_U = h5_svd['U']
        h5_V = h5_svd['V']

    except KeyError:
        warnstring = 'SVD Results for {dset} were not found.'.format(dset=dset_name)
        warn(warnstring)
        return
    except:
        raise

    func, is_complex, is_compound, n_features, n_samples, type_mult = check_dtype(h5_V)

    '''
    Calculate the size of a single batch that will fit in the available memory
    '''
    n_comps = h5_S[comp_slice].size
    mem_per_pix = (h5_U.dtype.itemsize + h5_V.dtype.itemsize * h5_V.shape[1]) * n_comps
    fixed_mem = h5_main.size * h5_main.dtype.itemsize

    if cores is None:
        free_mem = max_memory - fixed_mem
    else:
        free_mem = max_memory * 2 - fixed_mem

    batch_size = int(round(float(free_mem) / mem_per_pix))
    batch_slices = gen_batches(h5_U.shape[0], batch_size)

    print('Reconstructing in batches of {} positions.'.format(batch_size))
    print('Batchs should be {} Mb each.'.format(mem_per_pix * batch_size / 1024.0 ** 2))

    '''
    Loop over all batches.
    '''
    ds_V = np.dot(np.diag(h5_S[comp_slice]), func(h5_V[comp_slice, :]))
    rebuild = np.zeros((h5_main.shape[0], ds_V.shape[1]))
    for ibatch, batch in enumerate(batch_slices):
        rebuild[batch, :] += np.dot(h5_U[batch, comp_slice], ds_V)

    rebuild = stack_real_to_target_dtype(rebuild, h5_V.dtype)

    print('Completed reconstruction of data from SVD results.  Writing to file.')
    '''
    Create the Group and dataset to hold the rebuild data
    '''
    rebuilt_grp = VirtualGroup('Rebuilt_Data_', h5_svd.name[1:])

    ds_rebuilt = VirtualDataset('Rebuilt_Data', rebuild,
                                chunking=h5_main.chunks,
                                compression=h5_main.compression)
    rebuilt_grp.add_children([ds_rebuilt])

    if isinstance(comp_slice, slice):
        rebuilt_grp.attrs['components_used'] = '{}-{}'.format(comp_slice.start, comp_slice.stop)
    else:
        rebuilt_grp.attrs['components_used'] = components

    h5_refs = hdf.write(rebuilt_grp)

    h5_rebuilt = get_h5_obj_refs(['Rebuilt_Data'], h5_refs)[0]
    copy_attributes(h5_main, h5_rebuilt, skip_refs=False)

    hdf.flush()

    print('Done writing reconstructed data to file.')

    return h5_rebuilt
<|endoftext|>


<commit_before>from CafeScraper.Scraper import Scraper
import json

scraper = Scraper()
json_dump = json.dumps(scraper.cafes_dict, default=lambda c: c.__dict__, sort_keys=True)
print(json_dump)

# save to file
with open('menus.json', 'w') as fp:
    print(json_dump, file=fp)

#
#
# for cafe in scraper.cafes:
#     #print(cafe)
#     json_dump = json.dumps(cafe, default=lambda c: c.__dict__)
#     print(json_dump)<commit_msg>Change to list of the values in the main.py.<commit_after>from CafeScraper.Scraper import Scraper
import json

scraper = Scraper()
json_dump = json.dumps(list(scraper.cafes_dict.values()), default=lambda c: c.__dict__, sort_keys=True)
print(json_dump)

# save to file
with open('menus.json', 'w') as fp:
    print(json_dump, file=fp)

#
#
# for cafe in scraper.cafes:
#     #print(cafe)
#     json_dump = json.dumps(cafe, default=lambda c: c.__dict__)
#     print(json_dump)<|endoftext|>