<commit_before>#=========================================================================
#
#  Copyright Insight Software Consortium
#
#  Licensed under the Apache License, Version 2.0 (the "License");
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0.txt
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.
#
#=========================================================================

from __future__ import print_function

import SimpleITK as sitk

import sys, time, os

if len( sys.argv ) < 3:
    print( "Usage: python " + __file__ + " <input_directory_with_DICOM_series> <output_directory>" )
    sys.exit ( 1 )

# Read the original series. First obtain the series file names using the
# image series reader. Then read each image using an image reader that is
# set to load all DICOM tags (public+private). The resulting images contain
# their DICOM meta-data dictionaries.
data_directory = sys.argv[1]
series_reader = sitk.ImageSeriesReader()
series_IDs = series_reader.GetGDCMSeriesIDs(data_directory)
if not series_IDs:
    print("ERROR: given directory \""+data_directory+"\" does not contain a DICOM series.")
    sys.exit(1)
series_file_names = series_reader.GetGDCMSeriesFileNames(data_directory, series_IDs[0])

image_reader = sitk.ImageFileReader()
image_reader.LoadPrivateTagsOn()
image_list = []
for file_name in series_file_names:
    image_reader.SetFileName(file_name)
    image_list.append(image_reader.Execute())

# Pasting all of the slices into a 3D volume requires 2D image slices and not 3D slices
# The volume's origin and direction are taken from the first slice and the spacing from
# the difference between the first two slices. Note that we are assuming we are
# dealing with a volume represented by axial slices (z spacing is difference between images).
image_list2D = [image[:,:,0] for image in image_list]
image3D = sitk.JoinSeries(image_list2D, image_list[0].GetOrigin()[2], image_list[1].GetOrigin()[2] - image_list[0].GetOrigin()[2])
image3D.SetDirection(image_list[0].GetDirection())

# Modify the image (blurring)
filtered_image = sitk.DiscreteGaussian(image3D)

# Write the 3D image as a series
# IMPORTANT: There are many DICOM tags that need to be updated when you modify an
#            original image. This is a delicate opration and requires knowlege of
#            the DICOM standard. This example only modifies some. For a more complete
#            list of tags that need to be modified see:
#                           http://gdcm.sourceforge.net/wiki/index.php/Writing_DICOM

writer = sitk.ImageFileWriter()
# Use the study/seriers/frame of reference information given in the meta-data
# dictionary and not the automatically generated information from the file IO
writer.KeepOriginalImageUIDOn()
modification_time = time.strftime("%H%M%S")
modification_date = time.strftime("%Y%m%d")
for i in range(filtered_image.GetDepth()):
    image_slice = filtered_image[:,:,i]
    original_slice = image_list[i]
    # Copy the meta-data except the rescale-intercept, rescale-slope
    for k in original_slice.GetMetaDataKeys():
        if k!="0028|1052" and k!= "0028|1053":
            image_slice.SetMetaData(k, original_slice.GetMetaData(k))
    # Set relevant keys indicating the change, modify or remove private tags as needed
    image_slice.SetMetaData("0008|0031", modification_time)
    image_slice.SetMetaData("0008|0021", modification_date)
    image_slice.SetMetaData("0008|0008", "DERIVED\SECONDARY")
    # Each of the UID components is a number (cannot start with zero) and separated by a '.'
    # We create a unique series ID using the date and time.
    image_slice.SetMetaData("0020|000e", "1.2.826.0.1.3680043.2.1125."+modification_date+".1"+modification_time)
    # Write to the output directory and add the extension dcm if not there, to force writing is in DICOM format.
    writer.SetFileName(os.path.join(sys.argv[2], os.path.basename(series_file_names[i])) + ('' if os.path.splitext(series_file_names[i])[1] == '.dcm' else '.dcm'))
    writer.Execute(image_slice)
sys.exit( 0 )
<commit_msg>Must use a ImageSeriesReader to read a DICOM series<commit_after>#=========================================================================
#
#  Copyright Insight Software Consortium
#
#  Licensed under the Apache License, Version 2.0 (the "License");
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0.txt
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.
#
#=========================================================================

from __future__ import print_function

import SimpleITK as sitk

import sys, time, os

if len( sys.argv ) < 3:
    print( "Usage: python " + __file__ + " <input_directory_with_DICOM_series> <output_directory>" )
    sys.exit ( 1 )

# Read the original series. First obtain the series file names using the
# image series reader.
data_directory = sys.argv[1]
series_IDs = sitk.ImageSeriesReader.GetGDCMSeriesIDs(data_directory)
if not series_IDs:
    print("ERROR: given directory \""+data_directory+"\" does not contain a DICOM series.")
    sys.exit(1)
series_file_names = sitk.ImageSeriesReader.GetGDCMSeriesFileNames(data_directory, series_IDs[0])

# Second, to construct a proper volume from the series we must read
# the series with the image series reader.
series_reader = sitk.ImageSeriesReader()
series_reader.SetFileNames(series_file_names)
image3D = series_reader.Execute()

# Lastly, to load the image's DICOM meta-data dictionaries, we read
# each image using an image reader that is set to load all DICOM tags
# (public+private). The resulting images contain their DICOM
# meta-data dictionaries.
image_reader = sitk.ImageFileReader()
image_reader.LoadPrivateTagsOn()
image_list = []
for file_name in series_file_names:
    image_reader.SetFileName(file_name)
    img = image_reader.Execute()
    image_list.append(img)

# Modify the image (blurring)
filtered_image = sitk.DiscreteGaussian(image3D)

# Write the 3D image as a series
# IMPORTANT: There are many DICOM tags that need to be updated when you modify an
#            original image. This is a delicate opration and requires knowlege of
#            the DICOM standard. This example only modifies some. For a more complete
#            list of tags that need to be modified see:
#                           http://gdcm.sourceforge.net/wiki/index.php/Writing_DICOM

writer = sitk.ImageFileWriter()
# Use the study/seriers/frame of reference information given in the meta-data
# dictionary and not the automatically generated information from the file IO
writer.KeepOriginalImageUIDOn()
modification_time = time.strftime("%H%M%S")
modification_date = time.strftime("%Y%m%d")
for i in range(filtered_image.GetDepth()):
    image_slice = filtered_image[:,:,i]
    original_slice = image_list[i]
    # Copy the meta-data except the rescale-intercept, rescale-slope
    for k in original_slice.GetMetaDataKeys():
        if k!="0028|1052" and k!= "0028|1053":
            image_slice.SetMetaData(k, original_slice.GetMetaData(k))
    # Set relevant keys indicating the change, modify or remove private tags as needed
    image_slice.SetMetaData("0008|0031", modification_time)
    image_slice.SetMetaData("0008|0021", modification_date)
    image_slice.SetMetaData("0008|0008", "DERIVED\SECONDARY")
    # Each of the UID components is a number (cannot start with zero) and separated by a '.'
    # We create a unique series ID using the date and time.
    image_slice.SetMetaData("0020|000e", "1.2.826.0.1.3680043.2.1125."+modification_date+".1"+modification_time)
    # Write to the output directory and add the extension dcm if not there, to force writing is in DICOM format.
    writer.SetFileName(os.path.join(sys.argv[2], os.path.basename(series_file_names[i])) + ('' if os.path.splitext(series_file_names[i])[1] == '.dcm' else '.dcm'))
    writer.Execute(image_slice)
sys.exit( 0 )
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

import os
import os.path
import sys
import click
import six
import io
import tempfile
import pathlib
import glob
import shutil
import fnmatch
import yaml
import zipfile
import stat

# WARNING! When using zip compress (file_compression=True), we got this error
# when checking:
#
# java.lang.IllegalArgumentException: Cannot open an OutputStream in 'append'
# mode on a compressed FileSystem.

def get_project_template():
    from jinja2 import Environment, PackageLoader

    loader = PackageLoader('abhealer', 'templates')

    env = Environment(loader=loader)

    return env.get_template("project.bcfg")

def folder_to_int(name):
    parts = name.split("_")
    first_part = int(parts[0])
    if len(parts) < 2:
        second_part = 0
    else:
        second_part = int(parts[1])

    return first_part * 1000 + second_part

def int_to_folder(aint):
    first_part = aint // 1000
    second_part = aint % 1000

    second_part_text = ""
    if second_part > 0:
        second_part_text = "_%s" % second_part

    return str(first_part) + second_part_text

def find_latest_dir(dest_dir):
    # Find the latest dir
    max_value = 0
    dest_dir = os.path.realpath(str(dest_dir))
    for afolder in os.listdir(dest_dir):
        if afolder.endswith("_data"):
            continue

        if afolder.lower() == "history":
            continue

        value = folder_to_int(afolder)
        if value > max_value:
            max_value = value

    if max_value <= 0:
        return

    return int_to_folder(max_value)

def get_dirs_record_path(dest_dir):
    dest_dir = pathlib.Path(dest_dir)

    folder = find_latest_dir(dest_dir)

    apath = dest_dir / folder
    apath = pathlib.Path(str(apath) + "_data")
    apath = apath / "dir_stats.yaml"

    return apath

def backup_dirs(source_dir, dest_dir):
    source_dir = os.path.realpath(str(source_dir))
    dest_dir = os.path.realpath(str(dest_dir))

    record_path = get_dirs_record_path(dest_dir)
    if record_path.exists():
        raise click.UsageError("Directories list file exists : %s" %
                               record_path)

    dir_infos = []

    print("Generating directories list : %s" % record_path)
    for root, dirs, files in os.walk(source_dir):
        for adir in dirs:
            apath = os.path.join(root, adir)
            apath = os.path.realpath(apath)

            shorten_path = apath[len(source_dir) + 1:]

            apath_object = pathlib.Path(apath)

            dir_infos.append({
                "path": shorten_path,
                'mode': os.stat(apath)[stat.ST_MODE] & 0o777,
                'user': apath_object.owner(),
                'group': apath_object.group(),
            })

    with record_path.open("w") as f:
        yaml.dump(dir_infos, f)

    print("Finished generate directories list")

def recover_dirs(source_dir, dest_dir):
    source_dir = os.path.realpath(str(source_dir))
    dest_dir = os.path.realpath(str(dest_dir))

    record_path = get_dirs_record_path(dest_dir)
    if not record_path.exists():
        raise click.UsageError(
            "Dirs file not exists : %s" % record_path)

    with record_path.open() as f:
        dir_infos = yaml.load(f)

    for dir_info in dir_infos:
        full_path = os.path.join(source_dir, dir_info["path"])
        full_path = pathlib.Path(full_path)

        if not full_path.exists():
            full_path.mkdir(mode=dir_info["mode"])
        else:
            full_path.chmod(dir_info["mode"])

    print("Recover directories permissions completed!")

def clear_dirs(dest_dir):
    # Clear empty backups that only have
    dirs = glob.glob(os.path.join(str(dest_dir), "*"))
    for adir in dirs:
        if not os.path.isdir(adir):
            continue

        adir = pathlib.Path(adir)
        data_dir = adir.parent / (adir.stem + "_data" + adir.suffix)

        if fnmatch.fnmatch(adir.stem, "*_data"):
            continue

        if not data_dir.exists():
            print("Removed damaged empty backup directory : %s" % adir)
            shutil.rmtree(str(adir), ignore_errors=True)
            continue

        # files = os.listdir(str(adir))

        # There only have .areca-empty file !

        # We must not remove the version that only have .areca-empty changes,
        # otherwise we can't restore the files.

        # Until we find out what's make this not working, MUST not enable
        # codes below:

        # if len(files) <= 1:
        #     # Removed *_data directory first, so we know if a directory
        #     # successed be removed last time.
        #     print("Removing empty backup directory : %s" % adir)
        #     shutil.rmtree(str(data_dir), ignore_errors=True)
        #     shutil.rmtree(str(adir), ignore_errors=True)

def exec_(is_backup, vars):
    temp_dir = tempfile.TemporaryDirectory(prefix="dockerred_areca")

    template = get_project_template()

    source_dir = vars["src_path"]
    source_client_dir = "/opt/source"

    # Get the parent dir and join with config file base name, don't use
    # specificed dest name
    dest_dir = pathlib.Path(vars["dst_path"])

    # If dest dir not existed, we must not do any action!
    if not dest_dir.exists():
        raise click.UsageError('Directory not existed : "%s" !' % dest_dir)

    if not is_backup:
        if os.listdir(str(source_dir)):
            raise click.UsageError(
                "Destination must be empty directory!")

    dest_client_dir = "/opt/backup"
    workspace_client_dir = "/opt/workspace"

    fixed_config_file_name = vars["project_name"] + ".bcfg"
    fixed_config_file_path = os.path.join(
        temp_dir.name, fixed_config_file_name)
    fixed_config_file_client_path = os.path.join(
        workspace_client_dir, fixed_config_file_name)

    # Change paths
    vars["src_path"] = source_client_dir
    vars["dst_path"] = dest_client_dir

    docker_image = "starofrainnight/areca-backup"

    # To fix areca won't detect delete changes only
    areca_empty_file_path = os.path.join(source_dir, ".areca-empty")
    pathlib.Path(areca_empty_file_path).touch()

    backup_script_file_name = "_backup.sh"
    backup_script_file_path = os.path.join(
        temp_dir.name, backup_script_file_name)

    config_file_client_path = os.path.join(
        workspace_client_dir, fixed_config_file_name)
    backup_script_file_client_path = os.path.join(
        workspace_client_dir, backup_script_file_name)

    with temp_dir:
        with open(backup_script_file_path, "w") as f:
            f.write("#!/bin/sh\n")
            f.write("cd /usr/local/bin\n")
            if is_backup:
                f.write("./areca_cl.sh backup -config %s -wdir %s\n" % (
                    config_file_client_path, workspace_client_dir))
            else:
                f.write("./areca_cl.sh recover "
                        "-config %s -destination %s -o -nosubdir\n" % (
                            config_file_client_path, source_client_dir))
            f.write("\n")
        os.system("chmod +x %s" % backup_script_file_path)

        print("===== script begin =====")
        with open(backup_script_file_path, "rb") as f:
            print(f.read().decode("utf-8"))
        print("===== script end =====")

        # Write fixed config file
        xml_content = template.render(vars)
        with open(fixed_config_file_path, "wb") as f:
            f.write(xml_content.encode("utf-8"))

        print("===== xml begin =====")
        print(xml_content)
        print("===== xml end =====")

        run_client_cmd = backup_script_file_client_path

        volume_options = ""
        volume_options += " -v %s:%s " % (
            os.path.abspath(temp_dir.name), workspace_client_dir)
        volume_options += " -v %s:%s " % (
            os.path.abspath(source_dir), source_client_dir)
        volume_options += " -v %s:%s " % (
            dest_dir.resolve(), dest_client_dir)

        clear_dirs(dest_dir)

        backup_cmd = "docker run -t --rm %s %s %s" % (
            volume_options, docker_image, run_client_cmd)
        print("Executing : %s" % backup_cmd)

        os.system(backup_cmd)

        if is_backup:
            backup_dirs(source_dir, dest_dir)
        else:
            recover_dirs(source_dir, dest_dir)

        # Don't remove empty dirs, they are valid either !
        clear_dirs(dest_dir)

    return 0

@click.group()
def main(args=None):
    """
    This program is a helper for dockerred Areca Backup.

    """

    """
    Areca Backup won't record the empty directories and their properties
    """

@main.command()
@click.option('-c', '--config', required=True, type=click.File())
def backup(**kwargs):
    """
    """

    config_file_name = os.path.basename(kwargs["config"].name)
    vars = yaml.load(kwargs["config"])
    vars["project_name"] = os.path.splitext(config_file_name)[0]

    return exec_(True, vars)

@main.command()
@click.option('-c', '--config', required=True, type=click.File())
@click.option('-t', '--to-path', help="Recover to path, default to src_path")
def recover(**kwargs):
    """
    """

    config_file_name = os.path.basename(kwargs["config"].name)
    vars = yaml.load(kwargs["config"])
    if kwargs["to_path"]:
        vars["src_path"] = kwargs["to_path"]

    vars["project_name"] = os.path.splitext(config_file_name)[0]

    return exec_(False, vars)

if __name__ == "__main__":
    # execute only if run as a script
    main()
<commit_msg>Map users and groups into docker container<commit_after># -*- coding: utf-8 -*-

import os
import os.path
import sys
import click
import six
import io
import tempfile
import pathlib
import glob
import shutil
import fnmatch
import yaml
import zipfile
import stat

# WARNING! When using zip compress (file_compression=True), we got this error
# when checking:
#
# java.lang.IllegalArgumentException: Cannot open an OutputStream in 'append'
# mode on a compressed FileSystem.

def get_project_template():
    from jinja2 import Environment, PackageLoader

    loader = PackageLoader('abhealer', 'templates')

    env = Environment(loader=loader)

    return env.get_template("project.bcfg")

def folder_to_int(name):
    parts = name.split("_")
    first_part = int(parts[0])
    if len(parts) < 2:
        second_part = 0
    else:
        second_part = int(parts[1])

    return first_part * 1000 + second_part

def int_to_folder(aint):
    first_part = aint // 1000
    second_part = aint % 1000

    second_part_text = ""
    if second_part > 0:
        second_part_text = "_%s" % second_part

    return str(first_part) + second_part_text

def find_latest_dir(dest_dir):
    # Find the latest dir
    max_value = 0
    dest_dir = os.path.realpath(str(dest_dir))
    for afolder in os.listdir(dest_dir):
        if afolder.endswith("_data"):
            continue

        if afolder.lower() == "history":
            continue

        value = folder_to_int(afolder)
        if value > max_value:
            max_value = value

    if max_value <= 0:
        return

    return int_to_folder(max_value)

def get_dirs_record_path(dest_dir):
    dest_dir = pathlib.Path(dest_dir)

    folder = find_latest_dir(dest_dir)

    apath = dest_dir / folder
    apath = pathlib.Path(str(apath) + "_data")
    apath = apath / "dir_stats.yaml"

    return apath

def backup_dirs(source_dir, dest_dir):
    source_dir = os.path.realpath(str(source_dir))
    dest_dir = os.path.realpath(str(dest_dir))

    record_path = get_dirs_record_path(dest_dir)
    if record_path.exists():
        raise click.UsageError("Directories list file exists : %s" %
                               record_path)

    dir_infos = []

    print("Generating directories list : %s" % record_path)
    for root, dirs, files in os.walk(source_dir):
        for adir in dirs:
            apath = os.path.join(root, adir)
            apath = os.path.realpath(apath)

            shorten_path = apath[len(source_dir) + 1:]

            apath_object = pathlib.Path(apath)

            dir_infos.append({
                "path": shorten_path,
                'mode': os.stat(apath)[stat.ST_MODE] & 0o777,
                'user': apath_object.owner(),
                'group': apath_object.group(),
            })

    with record_path.open("w") as f:
        yaml.dump(dir_infos, f)

    print("Finished generate directories list")

def recover_dirs(source_dir, dest_dir):
    source_dir = os.path.realpath(str(source_dir))
    dest_dir = os.path.realpath(str(dest_dir))

    record_path = get_dirs_record_path(dest_dir)
    if not record_path.exists():
        raise click.UsageError(
            "Dirs file not exists : %s" % record_path)

    with record_path.open() as f:
        dir_infos = yaml.load(f)

    for dir_info in dir_infos:
        full_path = os.path.join(source_dir, dir_info["path"])
        full_path = pathlib.Path(full_path)

        if not full_path.exists():
            full_path.mkdir(mode=dir_info["mode"])
        else:
            full_path.chmod(dir_info["mode"])

    print("Recover directories permissions completed!")

def clear_dirs(dest_dir):
    # Clear empty backups that only have
    dirs = glob.glob(os.path.join(str(dest_dir), "*"))
    for adir in dirs:
        if not os.path.isdir(adir):
            continue

        adir = pathlib.Path(adir)
        data_dir = adir.parent / (adir.stem + "_data" + adir.suffix)

        if fnmatch.fnmatch(adir.stem, "*_data"):
            continue

        if not data_dir.exists():
            print("Removed damaged empty backup directory : %s" % adir)
            shutil.rmtree(str(adir), ignore_errors=True)
            continue

        # files = os.listdir(str(adir))

        # There only have .areca-empty file !

        # We must not remove the version that only have .areca-empty changes,
        # otherwise we can't restore the files.

        # Until we find out what's make this not working, MUST not enable
        # codes below:

        # if len(files) <= 1:
        #     # Removed *_data directory first, so we know if a directory
        #     # successed be removed last time.
        #     print("Removing empty backup directory : %s" % adir)
        #     shutil.rmtree(str(data_dir), ignore_errors=True)
        #     shutil.rmtree(str(adir), ignore_errors=True)

def exec_(is_backup, vars):
    temp_dir = tempfile.TemporaryDirectory(prefix="dockerred_areca")

    template = get_project_template()

    source_dir = vars["src_path"]
    source_client_dir = "/opt/source"

    # Get the parent dir and join with config file base name, don't use
    # specificed dest name
    dest_dir = pathlib.Path(vars["dst_path"])

    # If dest dir not existed, we must not do any action!
    if not dest_dir.exists():
        raise click.UsageError('Directory not existed : "%s" !' % dest_dir)

    if not is_backup:
        if os.listdir(str(source_dir)):
            raise click.UsageError(
                "Destination must be empty directory!")

    dest_client_dir = "/opt/backup"
    workspace_client_dir = "/opt/workspace"

    fixed_config_file_name = vars["project_name"] + ".bcfg"
    fixed_config_file_path = os.path.join(
        temp_dir.name, fixed_config_file_name)
    fixed_config_file_client_path = os.path.join(
        workspace_client_dir, fixed_config_file_name)

    # Change paths
    vars["src_path"] = source_client_dir
    vars["dst_path"] = dest_client_dir

    docker_image = "starofrainnight/areca-backup"

    # To fix areca won't detect delete changes only
    areca_empty_file_path = os.path.join(source_dir, ".areca-empty")
    pathlib.Path(areca_empty_file_path).touch()

    backup_script_file_name = "_backup.sh"
    backup_script_file_path = os.path.join(
        temp_dir.name, backup_script_file_name)

    config_file_client_path = os.path.join(
        workspace_client_dir, fixed_config_file_name)
    backup_script_file_client_path = os.path.join(
        workspace_client_dir, backup_script_file_name)

    with temp_dir:
        with open(backup_script_file_path, "w") as f:
            f.write("#!/bin/sh\n")
            f.write("cd /usr/local/bin\n")
            if is_backup:
                f.write("./areca_cl.sh backup -config %s -wdir %s\n" % (
                    config_file_client_path, workspace_client_dir))
            else:
                f.write("./areca_cl.sh recover "
                        "-config %s -destination %s -o -nosubdir\n" % (
                            config_file_client_path, source_client_dir))
            f.write("\n")
        os.system("chmod +x %s" % backup_script_file_path)

        print("===== script begin =====")
        with open(backup_script_file_path, "rb") as f:
            print(f.read().decode("utf-8"))
        print("===== script end =====")

        # Write fixed config file
        xml_content = template.render(vars)
        with open(fixed_config_file_path, "wb") as f:
            f.write(xml_content.encode("utf-8"))

        print("===== xml begin =====")
        print(xml_content)
        print("===== xml end =====")

        run_client_cmd = backup_script_file_client_path

        volume_options = ""
        # Map users and groups
        volume_options += " -v /etc/passwd:/etc/passwd "
        volume_options += " -v /etc/group:/etc/group "
        volume_options += " -v %s:%s " % (
            os.path.abspath(temp_dir.name), workspace_client_dir)
        volume_options += " -v %s:%s " % (
            os.path.abspath(source_dir), source_client_dir)
        volume_options += " -v %s:%s " % (
            dest_dir.resolve(), dest_client_dir)

        clear_dirs(dest_dir)

        backup_cmd = "docker run -t --rm %s %s %s" % (
            volume_options, docker_image, run_client_cmd)
        print("Executing : %s" % backup_cmd)

        os.system(backup_cmd)

        if is_backup:
            backup_dirs(source_dir, dest_dir)
        else:
            recover_dirs(source_dir, dest_dir)

        # Don't remove empty dirs, they are valid either !
        clear_dirs(dest_dir)

    return 0

@click.group()
def main(args=None):
    """
    This program is a helper for dockerred Areca Backup.

    """

    """
    Areca Backup won't record the empty directories and their properties
    """

@main.command()
@click.option('-c', '--config', required=True, type=click.File())
def backup(**kwargs):
    """
    """

    config_file_name = os.path.basename(kwargs["config"].name)
    vars = yaml.load(kwargs["config"])
    vars["project_name"] = os.path.splitext(config_file_name)[0]

    return exec_(True, vars)

@main.command()
@click.option('-c', '--config', required=True, type=click.File())
@click.option('-t', '--to-path', help="Recover to path, default to src_path")
def recover(**kwargs):
    """
    """

    config_file_name = os.path.basename(kwargs["config"].name)
    vars = yaml.load(kwargs["config"])
    if kwargs["to_path"]:
        vars["src_path"] = kwargs["to_path"]

    vars["project_name"] = os.path.splitext(config_file_name)[0]

    return exec_(False, vars)

if __name__ == "__main__":
    # execute only if run as a script
    main()
<|endoftext|>


<commit_before>#!/bin/env jython
import gc
import json
import math
import sys
import traceback

import java

import org.gavrog.joss.pgraphs as pgraphs
import org.gavrog.joss.geometry as geometry

def run():
    def writeln(s=''):
        print(s)

    for source_path in sys.argv[1:]:
        index_in_source = 0

        for net in pgraphs.io.Net.iterator(source_path):
            gc.collect()

            index_in_source += 1

            writeln('{')
            writeln('  "source_path": "%s",' % source_path)
            writeln('  "index_in_source": %d,' % index_in_source)

            try:
                warnings, errors = process_net(net, writeln=writeln)
            except:
                warnings = []
                errors = ["exception thrown while processing"]
                stacktrace = traceback.format_exc()
                writeln('  "stack_trace": %s,' % json.dumps(stacktrace))

            status = "Error" if errors else "OK"

            writeln('  "warnings": %s,' % json.dumps(warnings))
            writeln('  "errors": %s,' % json.dumps(errors))
            writeln('  "status": "%s"' % status)
            writeln('}')
            writeln()

def process_net(netRaw, writeln):
    writeln('  "net_name": "%s",' % netRaw.name)

    warnings = list(netRaw.warnings)
    errors = list(netRaw.errors)

    if errors:
        return warnings, errors

    if not netRaw.isConnected():
        errors.append("disconnected net")
    elif not netRaw.isLocallyStable():
        errors.append("next-nearest neighbor collisions")
    elif not netRaw.isStable():
        warnings.append("net has collisions")

        if netRaw.isLadder():
            warnings.append("ladder net")
        elif netRaw.hasSecondOrderCollisions():
            warnings.append("possible ladder net")

    if errors:
        return warnings, errors

    if not netRaw.isMinimal():
        warnings.append("reduced from supercell")

    nodeToName, _ = node_name_mapping(netRaw.minimalImageMap())
    net = netRaw.minimalImage()

    finder = geometry.SpaceGroupFinder(net.getSpaceGroup())
    writeln('  "spacegroup_name": "%s",' % finder.groupName)
    if finder.extension:
        writeln('  "spacegroup_extension": "%s",' % finder.extension)

    orbit_reps = [list(orb)[0] for orb in net.nodeOrbits()]
    seqs = [coordination_sequence(net, v, 10) for v in orbit_reps]
    psyms = [net.pointSymbol(v) for v in orbit_reps]

    writeln('  "net_nodes_unitcell": %s,' % net.numberOfNodes())
    writeln('  "net_nodes_asym": %s,' % len(orbit_reps))

    writeln('  "net_valency": %s,' % json.dumps([s[0] for s in seqs]))
    writeln('  "net_coordination_seqs": %s,' % format_list(seqs))
    writeln('  "net_wells_point_symbols": %s,' % json.dumps(psyms))

    writeln('  "net_systre_key": "%s",' % net.systreKey)

    dof = None

    for kind in ['maximal', 'barycentric', 'relaxed']:
        try:
            e = make_embedding(net, kind)

            if dof is None:
                dof = e.degreesOfFreedom()
                writeln('  "net_degrees_of_freedom": %s,' % dof)

            if not is_positive_definite(e.gramMatrix):
                warnings.append("%s embedding has collapsed unit cell" % kind)

            pnet = pgraphs.embed.ProcessedNet(net, 'X', nodeToName, finder, e)
            write_embedding(pnet, "net_%s" % kind, writeln)
        except:
            errors.append("exception thrown in %s embedding" % kind)
            stacktrace = traceback.format_exc()
            writeln('  "stack_trace_%s": %s,' % (kind, json.dumps(stacktrace)))

    return warnings, errors

def format_list(xs):
    return '[\n      %s\n    ]' % ',\n      '.join(json.dumps(x) for x in xs)

def node_name_mapping(phi):
    imageNode2Orbit = {}
    for orbit in phi.imageGraph.nodeOrbits():
        orbit = frozenset(orbit)
        for v in orbit:
            imageNode2Orbit[v] = orbit

    orbit2name = {}
    node2name = {}
    mergedNames = []
    mergedNamesSeen = set()

    for v in phi.sourceGraph.nodes():
        name = phi.sourceGraph.getNodeName(v)
        w = phi.getImage(v)
        orbit = imageNode2Orbit[w]

        if name != orbit2name.get(orbit, name):
            pair = (name, orbit2name[orbit])
            if pair not in mergedNamesSeen:
                mergedNames.append(pair)
                mergedNamesSeen.add(pair)
        else:
            orbit2name[orbit] = name

        node2name[w] = orbit2name[orbit]

    return node2name, mergedNames

def make_embedding(graph, kind):
    embedder = pgraphs.embed.Embedder(graph, None, False)

    if kind == 'maximal':
        embedder.setMaximizeVolume(True)
        embedder.setRelaxPositions(False)
        embedder.setPasses(0)
        embedder.go(10000)
        embedder.normalize()
    else:
        embedder.setMaximizeVolume(False)
        embedder.setRelaxPositions(False)
        embedder.setPasses(0)
        embedder.go(500)
        embedder.normalize()

        embedder.setMaximizeVolume(False)
        embedder.setRelaxPositions(kind == 'relaxed')
        embedder.setPasses(3)
        embedder.go(100000 if kind == 'relaxed' else 10000)
        embedder.normalize()

    return embedder

def write_embedding(net, prefix, writeln):
    dim = net.graph.dimension
    cgd = serialized_net(net, asCGD=True)

    nodes = []
    edges = []

    for line in cgd.split('\n'):
        fields = line.strip().split()
        if len(fields) == 0:
            continue

        if fields[0] == 'CELL':
            if dim == 3:
                a, b, c, alpha, beta, gamma = map(float, fields[1:])
                writeln('  "%s_unitcell_a": %s,' % (prefix, a))
                writeln('  "%s_unitcell_b": %s,' % (prefix, b))
                writeln('  "%s_unitcell_c": %s,' % (prefix, c))
                writeln('  "%s_unitcell_alpha": %s,' % (prefix, alpha))
                writeln('  "%s_unitcell_beta": %s,' % (prefix, beta))
                writeln('  "%s_unitcell_gamma": %s,' % (prefix, gamma))
            elif dim == 2:
                a, b, gamma = map(float, fields[1:])
                writeln('  "%s_unitcell_a": %s,' % (prefix, a))
                writeln('  "%s_unitcell_b": %s,' % (prefix, b))
                writeln('  "%s_unitcell_gamma": %s,' % (prefix, gamma))
        elif fields[0] == 'CELL_VOLUME':
            volume = float(fields[1])
            writeln('  "%s_unitcell_volume": %s,' % (prefix, volume))
        elif fields[0] == 'NODE':
            nodes.append(map(float, fields[3:]))
        elif fields[0] == 'EDGE':
            edges.append(map(float, fields[1:]))
        elif fields[0] == '#' and len(fields) > 1:
            if fields[1] == 'EDGE_MIN':
                minEdge = float(fields[2])
                writeln('  "%s_shortest_edge_length": %s,' % (prefix, minEdge))
            elif fields[1] == 'EDGE_AVG':
                avgEdge = float(fields[2])
                writeln('  "%s_average_edge_length": %s,' % (prefix, avgEdge))
            elif fields[1] == 'EDGE_MAX':
                maxEdge = float(fields[2])
                writeln('  "%s_longest_edge_length": %s,' % (prefix, maxEdge))
            if fields[1] == 'ANGLE_MIN':
                minAngle = float(fields[2])
                writeln('  "%s_smallest_angle": %s,' % (prefix, minAngle))
            elif fields[1] == 'ANGLE_AVG':
                avgAngle = float(fields[2])
                writeln('  "%s_average_angle": %s,' % (prefix, avgAngle))
            elif fields[1] == 'ANGLE_MAX':
                maxAngle = float(fields[2])
                writeln('  "%s_largest_angle": %s,' % (prefix, maxAngle))
            elif fields[1] == 'SMALLEST_SEPARATION':
                sep = float(fields[2])
                writeln('  "%s_smallest_atom_separation": %s,' % (prefix, sep))

    writeln('  "%s_atoms": %s,' % (prefix, format_list(nodes)))
    writeln('  "%s_edges": %s,' % (prefix, format_list(edges)))

    cgd = serialized_net(net, asCGD=True, writeFullCell=True)

    nodes = []
    edges = []

    for line in cgd.split('\n'):
        fields = line.strip().split()
        if len(fields) == 0:
            continue

        elif fields[0] == 'NODE':
            nodes.append(map(float, fields[3:]))
        elif fields[0] == 'EDGE':
            edges.append(map(float, fields[1:]))

    writeln('  "%s_atoms_full_cell": %s,' % (prefix, format_list(nodes)))

    writeln('  "%s_half_edges_full_cell": %s,' %
        (prefix, format_list(map(halfEdge, edges)))
    )

def halfEdge(coords):
    dim = len(coords) / 2
    p = coords[:dim]
    q = coords[dim:]
    h = [ (p[i] + q[i]) / 2 for i in range(dim) ]

    return p + h

def serialized_net(net, asCGD=False, writeFullCell=False, prefix=''):
    stringWriter = java.io.StringWriter()
    writer = java.io.PrintWriter(stringWriter)
    net.writeEmbedding(writer, asCGD, writeFullCell, prefix)
    return stringWriter.toString()

def is_positive_definite(gram):
    if gram.get(0, 0).doubleValue() < 0.001:
        return False
    elif gram.getSubMatrix(0, 0, 2, 2).determinant().doubleValue() < 0.001:
        return False
    elif gram.determinant().doubleValue() < 0.001:
        return False
    else:
        return True

def coordination_sequence(net, v, n):
    cs = net.coordinationSequence(v)
    cs.next()

    out = []
    for _ in range(n):
        out.append(cs.next())

    return out

if __name__ == "__main__":
    run()
<commit_msg>jython/analyzeNetsForEpinet: do not write embedding with degenerate unit cell<commit_after>#!/bin/env jython
import gc
import json
import math
import sys
import traceback

import java

import org.gavrog.joss.pgraphs as pgraphs
import org.gavrog.joss.geometry as geometry

def run():
    def writeln(s=""):
        print(s)

    for source_path in sys.argv[1:]:
        index_in_source = 0

        for net in pgraphs.io.Net.iterator(source_path):
            gc.collect()

            index_in_source += 1

            writeln("{")
            writeln('  "source_path": "%s",' % source_path)
            writeln('  "index_in_source": %d,' % index_in_source)

            try:
                warnings, errors = process_net(net, writeln=writeln)
            except:
                warnings = []
                errors = ["exception thrown while processing"]
                stacktrace = traceback.format_exc()
                writeln('  "stack_trace": %s,' % json.dumps(stacktrace))

            status = "Error" if errors else "OK"

            writeln('  "warnings": %s,' % json.dumps(warnings))
            writeln('  "errors": %s,' % json.dumps(errors))
            writeln('  "status": "%s"' % status)
            writeln("}")
            writeln()

def process_net(netRaw, writeln):
    writeln('  "net_name": "%s",' % netRaw.name)

    warnings = list(netRaw.warnings)
    errors = list(netRaw.errors)

    if errors:
        return warnings, errors

    if not netRaw.isConnected():
        errors.append("disconnected net")
    elif not netRaw.isLocallyStable():
        errors.append("next-nearest neighbor collisions")
    elif not netRaw.isStable():
        warnings.append("net has collisions")

        if netRaw.isLadder():
            warnings.append("ladder net")
        elif netRaw.hasSecondOrderCollisions():
            warnings.append("possible ladder net")

    if errors:
        return warnings, errors

    if not netRaw.isMinimal():
        warnings.append("reduced from supercell")

    nodeToName, _ = node_name_mapping(netRaw.minimalImageMap())
    net = netRaw.minimalImage()

    finder = geometry.SpaceGroupFinder(net.getSpaceGroup())
    writeln('  "spacegroup_name": "%s",' % finder.groupName)
    if finder.extension:
        writeln('  "spacegroup_extension": "%s",' % finder.extension)

    orbit_reps = [list(orb)[0] for orb in net.nodeOrbits()]
    seqs = [coordination_sequence(net, v, 10) for v in orbit_reps]
    psyms = [net.pointSymbol(v) for v in orbit_reps]

    writeln('  "net_nodes_unitcell": %s,' % net.numberOfNodes())
    writeln('  "net_nodes_asym": %s,' % len(orbit_reps))

    writeln('  "net_valency": %s,' % json.dumps([s[0] for s in seqs]))
    writeln('  "net_coordination_seqs": %s,' % format_list(seqs))
    writeln('  "net_wells_point_symbols": %s,' % json.dumps(psyms))

    writeln('  "net_systre_key": "%s",' % net.systreKey)

    dof = None

    for kind in ["maximal", "barycentric", "relaxed"]:
        try:
            e = make_embedding(net, kind)

            if dof is None:
                dof = e.degreesOfFreedom()
                writeln('  "net_degrees_of_freedom": %s,' % dof)

            if not is_positive_definite(e.gramMatrix):
                warnings.append("%s embedding has degenerate unit cell" % kind)
            else:
                pnet = pgraphs.embed.ProcessedNet(
                    net, "X", nodeToName, finder, e
                )
                write_embedding(pnet, "net_%s" % kind, writeln)
        except:
            errors.append("exception thrown in %s embedding" % kind)
            stacktrace = traceback.format_exc()
            writeln('  "stack_trace_%s": %s,' % (kind, json.dumps(stacktrace)))

    if dof is None:
        errors.append("no successful embedding")

    return warnings, errors

def format_list(xs):
    return "[\n      %s\n    ]" % ",\n      ".join(json.dumps(x) for x in xs)

def node_name_mapping(phi):
    imageNode2Orbit = {}
    for orbit in phi.imageGraph.nodeOrbits():
        orbit = frozenset(orbit)
        for v in orbit:
            imageNode2Orbit[v] = orbit

    orbit2name = {}
    node2name = {}
    mergedNames = []
    mergedNamesSeen = set()

    for v in phi.sourceGraph.nodes():
        name = phi.sourceGraph.getNodeName(v)
        w = phi.getImage(v)
        orbit = imageNode2Orbit[w]

        if name != orbit2name.get(orbit, name):
            pair = (name, orbit2name[orbit])
            if pair not in mergedNamesSeen:
                mergedNames.append(pair)
                mergedNamesSeen.add(pair)
        else:
            orbit2name[orbit] = name

        node2name[w] = orbit2name[orbit]

    return node2name, mergedNames

def make_embedding(graph, kind):
    embedder = pgraphs.embed.Embedder(graph, None, False)

    if kind == "maximal":
        embedder.setMaximizeVolume(True)
        embedder.setRelaxPositions(False)
        embedder.setPasses(0)
        embedder.go(10000)
        embedder.normalize()
    else:
        embedder.setMaximizeVolume(False)
        embedder.setRelaxPositions(False)
        embedder.setPasses(0)
        embedder.go(500)
        embedder.normalize()

        embedder.setMaximizeVolume(False)
        embedder.setRelaxPositions(kind == "relaxed")
        embedder.setPasses(3)
        embedder.go(100000 if kind == "relaxed" else 10000)
        embedder.normalize()

    return embedder

def write_embedding(net, prefix, writeln):
    dim = net.graph.dimension
    cgd = serialized_net(net, asCGD=True)

    nodes = []
    edges = []

    for line in cgd.split("\n"):
        fields = line.strip().split()
        if len(fields) == 0:
            continue

        if fields[0] == "CELL":
            if dim == 3:
                a, b, c, alpha, beta, gamma = map(float, fields[1:])
                writeln('  "%s_unitcell_a": %s,' % (prefix, a))
                writeln('  "%s_unitcell_b": %s,' % (prefix, b))
                writeln('  "%s_unitcell_c": %s,' % (prefix, c))
                writeln('  "%s_unitcell_alpha": %s,' % (prefix, alpha))
                writeln('  "%s_unitcell_beta": %s,' % (prefix, beta))
                writeln('  "%s_unitcell_gamma": %s,' % (prefix, gamma))
            elif dim == 2:
                a, b, gamma = map(float, fields[1:])
                writeln('  "%s_unitcell_a": %s,' % (prefix, a))
                writeln('  "%s_unitcell_b": %s,' % (prefix, b))
                writeln('  "%s_unitcell_gamma": %s,' % (prefix, gamma))
        elif fields[0] == "CELL_VOLUME":
            volume = float(fields[1])
            writeln('  "%s_unitcell_volume": %s,' % (prefix, volume))
        elif fields[0] == "NODE":
            nodes.append(map(float, fields[3:]))
        elif fields[0] == "EDGE":
            edges.append(map(float, fields[1:]))
        elif fields[0] == "#" and len(fields) > 1:
            if fields[1] == "EDGE_MIN":
                minEdge = float(fields[2])
                writeln('  "%s_shortest_edge_length": %s,' % (prefix, minEdge))
            elif fields[1] == "EDGE_AVG":
                avgEdge = float(fields[2])
                writeln('  "%s_average_edge_length": %s,' % (prefix, avgEdge))
            elif fields[1] == "EDGE_MAX":
                maxEdge = float(fields[2])
                writeln('  "%s_longest_edge_length": %s,' % (prefix, maxEdge))
            if fields[1] == "ANGLE_MIN":
                minAngle = float(fields[2])
                writeln('  "%s_smallest_angle": %s,' % (prefix, minAngle))
            elif fields[1] == "ANGLE_AVG":
                avgAngle = float(fields[2])
                writeln('  "%s_average_angle": %s,' % (prefix, avgAngle))
            elif fields[1] == "ANGLE_MAX":
                maxAngle = float(fields[2])
                writeln('  "%s_largest_angle": %s,' % (prefix, maxAngle))
            elif fields[1] == "SMALLEST_SEPARATION":
                sep = float(fields[2])
                writeln('  "%s_smallest_atom_separation": %s,' % (prefix, sep))

    writeln('  "%s_atoms": %s,' % (prefix, format_list(nodes)))
    writeln('  "%s_edges": %s,' % (prefix, format_list(edges)))

    cgd = serialized_net(net, asCGD=True, writeFullCell=True)

    nodes = []
    edges = []

    for line in cgd.split("\n"):
        fields = line.strip().split()
        if len(fields) == 0:
            continue

        elif fields[0] == "NODE":
            nodes.append(map(float, fields[3:]))
        elif fields[0] == "EDGE":
            edges.append(map(float, fields[1:]))

    writeln('  "%s_atoms_full_cell": %s,' % (prefix, format_list(nodes)))

    writeln(
        '  "%s_half_edges_full_cell": %s,'
        % (prefix, format_list(map(halfEdge, edges)))
    )

def halfEdge(coords):
    dim = len(coords) / 2
    p = coords[:dim]
    q = coords[dim:]
    h = [(p[i] + q[i]) / 2 for i in range(dim)]

    return p + h

def serialized_net(net, asCGD=False, writeFullCell=False, prefix=""):
    stringWriter = java.io.StringWriter()
    writer = java.io.PrintWriter(stringWriter)
    net.writeEmbedding(writer, asCGD, writeFullCell, prefix)
    return stringWriter.toString()

def is_positive_definite(gram):
    if gram.get(0, 0).doubleValue() < 0.001:
        return False
    elif gram.getSubMatrix(0, 0, 2, 2).determinant().doubleValue() < 0.001:
        return False
    elif gram.determinant().doubleValue() < 0.001:
        return False
    else:
        return True

def coordination_sequence(net, v, n):
    cs = net.coordinationSequence(v)
    cs.next()

    out = []
    for _ in range(n):
        out.append(cs.next())

    return out

if __name__ == "__main__":
    run()
<|endoftext|>


<commit_before>"""Extracts required Google Cloud metrics using Monitoring V3 API call, parses
   the API response into a list and writes to google sheet.

   Takes VM instance name, interval start time, interval end time, alignment
   period and fio test type as command line inputs.
   Metrics extracted:
   1.Peak Cpu Utilization(%)
   2.Mean Cpu Utilization(%)
   3.Peak Network Bandwidth(By/s)
   4.Mean Network Bandwidth(By/s)
   5.Opencensus Error Count
   6.Opencensus Mean Latency(s)
   7.Read Bytes Count(By)

  Usage:
  >>python3 vm_metrics.py {instance} {start time in epoch sec} {end time in epoch sec} {period in sec} {test_type}

"""
import dataclasses
import os
import sys
import google.api_core
from google.api_core.exceptions import GoogleAPICallError
import google.cloud
from google.cloud import monitoring_v3
from gsheet import gsheet

WORKSHEET_NAME = 'vm_metrics!'

PROJECT_NAME = 'projects/gcs-fuse-test'
TEST_TYPE = 'ReadFile'

CPU_UTI_METRIC = 'compute.googleapis.com/instance/cpu/utilization'
RECEIVED_BYTES_COUNT_METRIC = 'compute.googleapis.com/instance/network/received_bytes_count'
OPS_ERROR_COUNT_METRIC = 'custom.googleapis.com/gcsfuse/fs/ops_error_count'
OPS_LATENCY_METRIC = 'custom.googleapis.com/gcsfuse/fs/ops_latency'
READ_BYTES_COUNT_METRIC = 'custom.googleapis.com/gcsfuse/gcs/read_bytes_count'

class NoValuesError(Exception):
  """API response values are missing."""

@dataclasses.dataclass
class MetricPoint:
  value: float
  start_time_sec: int
  end_time_sec: int

def _parse_metric_value_by_type(value, value_type) -> float:
  if value_type == 3:
    return value.double_value
  elif value_type == 2:
    return value.int64_value
  elif value_type == 5:
    return value.distribution_value.mean
  else:
    raise Exception('Unhandled Value type')

def _create_metric_points_from_response(metrics_response, factor):
  """Parses the given metrics API response and returns a list of MetricPoint.

    Args:
      metrics_response (object): The metrics API response
      factor (float) : Converting the API response values into appropriate unit
    Returns:
      list[MetricPoint]
  """
  metric_point_list = []
  for metric in metrics_response:
    for point in metric.points:
      value = _parse_metric_value_by_type(point.value, metric.value_type)
      metric_point = MetricPoint(value / factor,
                                 point.interval.start_time.seconds,
                                 point.interval.end_time.seconds)

      metric_point_list.append(metric_point)
  metric_point_list.reverse()
  return metric_point_list

class VmMetrics:

  def _validate_start_end_times(self, start_time_sec, end_time_sec):
    """Checks whether the start time is less than end time.

    Args:
      start_time_sec (int) : Epoch seconds
      end_time_sec (int) : Epoch seconds
    Raises:
      ValueError : When start time is after end time.
    """
    if start_time_sec < end_time_sec:
      return True
    else:
      raise ValueError('Start time should be before end time')

  def _get_api_response(self, metric_type, start_time_sec, end_time_sec,
                        instance, period, aligner, reducer, group_fields):
    """Fetches the API response for the requested metrics.

    Args:
      metric_type (str): The type of metric
      start_time_sec (int): Epoch seconds
      end_time_sec (int): Epoch seconds
      instance (str): VM instance name
      period (float): Period over which the values are aligned
      aligner(str): Operation to be applied at points of each period
      reducer(str): Operation to aggregate data points accross multiple metrics
      group_fields(list[str]): The fields we want to aggregate using reducer
    Returns:
      metrics API response (object)
    Raises:
      GoogleAPICallError

    """

    client = monitoring_v3.MetricServiceClient()
    interval = monitoring_v3.TimeInterval(
        end_time={'seconds': int(end_time_sec)},
        start_time={'seconds': int(start_time_sec)})

    aggregation = monitoring_v3.Aggregation(
        alignment_period={'seconds': period},
        per_series_aligner=getattr(monitoring_v3.Aggregation.Aligner, aligner),
        cross_series_reducer=getattr(monitoring_v3.Aggregation.Reducer,reducer),
        group_by_fields=group_fields
    )
    if(metric_type[0:7]=='compute'):
      metric_filter = (
          'metric.type = "{metric_type}" AND metric.label.instance_name '
          '={instance_name}'
          ).format(metric_type=metric_type, instance_name=instance)

    elif (metric_type[0:6] == 'custom'):
      metric_filter = (
          'metric.type = "{metric_type}" AND metric.labels.opencensus_task = '
          'ends_with("{instance_name}")'
          ).format(metric_type=metric_type, instance_name=instance)

      if (metric_type == OPS_ERROR_COUNT_METRIC):
        metric_filter = ('{} AND metric.labels.fs_op != {}'
                        ).format(metric_filter, 'GetXattr')
      elif (metric_type == OPS_LATENCY_METRIC):
        metric_filter = ('{} AND metric.labels.fs_op = {}'
                        ).format(metric_filter, TEST_TYPE)

    try:
      metrics_response = client.list_time_series({
          'name': PROJECT_NAME,
          'filter': metric_filter,
          'interval': interval,
          'view': monitoring_v3.ListTimeSeriesRequest.TimeSeriesView.FULL,
          'aggregation': aggregation,
      })
    except:
      raise GoogleAPICallError(('The request for API response of {} failed.'
                                ).format(metric_type))

    # Metrics response except OPS_ERROR_COUNT_METRIC should not be empty:
    if(metric_type != OPS_ERROR_COUNT_METRIC and metrics_response == {}):
      raise NoValuesError('No values were retrieved from the call')

    return metrics_response

  def _get_metrics(self, start_time_sec, end_time_sec, instance, period,
                   metric_type, factor, aligner, reducer='REDUCE_NONE',
                   group_fields=['metric.type']):
    """Returns the MetricPoint list for requested metric type.

    Args:
      start_time_sec (int): Epoch seconds
      end_time_sec (int): Epoch seconds
      instance (str): VM instance name
      period (float): Period over which the values are aligned
      metric_type (str): The type of metric
      factor (float) : Converting the API response values into required units.
      aligner(str): Operation to be applied at points of each period
      reducer(str): Operation to aggregate data points accross multiple metrics
      group_fields(list[str]): The fields we want to aggregate using reducer
    Returns:
      list[MetricPoint]
    """
    metrics_response = self._get_api_response(metric_type, start_time_sec,
                                              end_time_sec, instance, period,
                                              aligner, reducer, group_fields)
    metrics_data = _create_metric_points_from_response(metrics_response, factor)
    return metrics_data

  def fetch_metrics_and_write_to_google_sheet(self, start_time_sec,
                                              end_time_sec, instance,
                                              period, test_type) -> None:
    """Fetches the metrics data for all types and writes it to a google sheet.

    Args:
      start_time_sec (int): Epoch seconds
      end_time_sec (int): Epoch seconds
      instance (str): VM instance
      period (float): Period over which the values are taken
      test_type(str): The type of load test for which metrics are taken
    Returns: None
    """
    self._validate_start_end_times(start_time_sec, end_time_sec)
    global TEST_TYPE
    TEST_TYPE = test_type
    cpu_uti_peak_data = self._get_metrics(start_time_sec, end_time_sec,
                                          instance, period, CPU_UTI_METRIC,
                                          1 / 100, 'ALIGN_MAX')
    cpu_uti_mean_data = self._get_metrics(start_time_sec, end_time_sec,
                                          instance, period, CPU_UTI_METRIC,
                                          1 / 100, 'ALIGN_MEAN')
    rec_bytes_peak_data = self._get_metrics(start_time_sec, end_time_sec,
                                            instance, period,
                                            RECEIVED_BYTES_COUNT_METRIC, 60,
                                            'ALIGN_MAX')
    rec_bytes_mean_data = self._get_metrics(start_time_sec, end_time_sec,
                                            instance, period,
                                            RECEIVED_BYTES_COUNT_METRIC, 60,
                                            'ALIGN_MEAN')
    ops_latency_mean_data = self._get_metrics(start_time_sec, end_time_sec,
                                              instance, period,
                                              OPS_LATENCY_METRIC, 1,
                                              'ALIGN_DELTA')
    read_bytes_count_data = self._get_metrics(start_time_sec, end_time_sec,
                                              instance, period,
                                              READ_BYTES_COUNT_METRIC, 1,
                                              'ALIGN_DELTA')
    ops_error_count_data = self._get_metrics(start_time_sec, end_time_sec,
                                             instance, period,
                                             OPS_ERROR_COUNT_METRIC, 1,
                                             'ALIGN_DELTA', 'REDUCE_SUM',
                                             ['metric.labels'])

    # Incase OPS_ERROR_COUNT is empty, we want 0 to be dumped in the sheet:
    if(ops_error_count_data == []):
        ops_error_count_data = [MetricPoint(0, 0, 0) for point in cpu_uti_peak_data]

    metrics_data = []
    for cpu_uti_peak, cpu_uti_mean, rec_bytes_peak, rec_bytes_mean, ops_latency, read_bytes_count, ops_error_count in zip(
        cpu_uti_peak_data, cpu_uti_mean_data, rec_bytes_peak_data,
        rec_bytes_mean_data, ops_latency_mean_data, read_bytes_count_data,
        ops_error_count_data):
      metrics_data.append([
          cpu_uti_peak.start_time_sec, cpu_uti_peak.value, cpu_uti_mean.value,
          rec_bytes_peak.value, rec_bytes_mean.value,
          ops_latency.value, read_bytes_count.value, ops_error_count.value
      ])

    # Writing metrics data to google sheet
    gsheet.write_to_google_sheet(WORKSHEET_NAME, metrics_data)

def main() -> None:
  if len(sys.argv) != 6:
    raise Exception('Invalid arguments.')
  instance = sys.argv[1]
  start_time_sec = int(sys.argv[2])
  end_time_sec = int(sys.argv[3])
  period = int(sys.argv[4])
  test_type = sys.argv[5]
  vm_metrics = VmMetrics()
  vm_metrics.fetch_metrics_and_write_to_google_sheet(start_time_sec,
                                                     end_time_sec, instance,
                                                     period, test_type)

if __name__ == '__main__':
  main()
<commit_msg>Update vm_metrics.py<commit_after>"""Extracts required Google Cloud metrics using Monitoring V3 API call, parses
   the API response into a list and writes to google sheet.

   Takes VM instance name, interval start time, interval end time, alignment
   period and fio test type as command line inputs.
   Metrics extracted:
   1.Peak Cpu Utilization(%)
   2.Mean Cpu Utilization(%)
   3.Peak Network Bandwidth(By/s)
   4.Mean Network Bandwidth(By/s)
   5.Opencensus Error Count
   6.Opencensus Mean Latency(s)
   7.Read Bytes Count(By)

  Usage:
  >>python3 vm_metrics.py {instance} {start time in epoch sec} {end time in epoch sec} {period in sec} {test_type}

"""
import dataclasses
import os
import sys
import google.api_core
from google.api_core.exceptions import GoogleAPICallError
import google.cloud
from google.cloud import monitoring_v3
from gsheet import gsheet

WORKSHEET_NAME = 'vm_metrics!'

PROJECT_NAME = 'projects/gcs-fuse-test'
TEST_TYPE = 'ReadFile'

CPU_UTI_METRIC = 'compute.googleapis.com/instance/cpu/utilization'
RECEIVED_BYTES_COUNT_METRIC = 'compute.googleapis.com/instance/network/received_bytes_count'
OPS_ERROR_COUNT_METRIC = 'custom.googleapis.com/gcsfuse/fs/ops_error_count'
OPS_LATENCY_METRIC = 'custom.googleapis.com/gcsfuse/fs/ops_latency'
READ_BYTES_COUNT_METRIC = 'custom.googleapis.com/gcsfuse/gcs/read_bytes_count'

class NoValuesError(Exception):
  """API response values are missing."""

@dataclasses.dataclass
class MetricPoint:
  value: float
  start_time_sec: int
  end_time_sec: int

def _parse_metric_value_by_type(value, value_type) -> float:
  if value_type == 3:
    return value.double_value
  elif value_type == 2:
    return value.int64_value
  elif value_type == 5:
    return value.distribution_value.mean
  else:
    raise Exception('Unhandled Value type')

def _create_metric_points_from_response(metrics_response, factor):
  """Parses the given metrics API response and returns a list of MetricPoint.

    Args:
      metrics_response (object): The metrics API response
      factor (float) : Converting the API response values into appropriate unit
    Returns:
      list[MetricPoint]
  """
  metric_point_list = []
  for metric in metrics_response:
    for point in metric.points:
      value = _parse_metric_value_by_type(point.value, metric.value_type)
      metric_point = MetricPoint(value / factor,
                                 point.interval.start_time.seconds,
                                 point.interval.end_time.seconds)

      metric_point_list.append(metric_point)
  metric_point_list.reverse()
  return metric_point_list

class VmMetrics:

  def _validate_start_end_times(self, start_time_sec, end_time_sec):
    """Checks whether the start time is less than end time.

    Args:
      start_time_sec (int) : Epoch seconds
      end_time_sec (int) : Epoch seconds
    Raises:
      ValueError : When start time is after end time.
    """
    if start_time_sec < end_time_sec:
      return True
    else:
      raise ValueError('Start time should be before end time')

  def _get_api_response(self, metric_type, start_time_sec, end_time_sec,
                        instance, period, aligner, reducer, group_fields):
    """Fetches the API response for the requested metrics.

    Args:
      metric_type (str): The type of metric
      start_time_sec (int): Epoch seconds
      end_time_sec (int): Epoch seconds
      instance (str): VM instance name
      period (float): Period over which the values are aligned
      aligner(str): Operation to be applied at points of each period
      reducer(str): Operation to aggregate data points accross multiple metrics
      group_fields(list[str]): The fields we want to aggregate using reducer
    Returns:
      metrics API response (object)
    Raises:
      GoogleAPICallError

    """

    client = monitoring_v3.MetricServiceClient()
    interval = monitoring_v3.TimeInterval(
        end_time={'seconds': int(end_time_sec)},
        start_time={'seconds': int(start_time_sec)})

    aggregation = monitoring_v3.Aggregation(
        alignment_period={'seconds': period},
        per_series_aligner=getattr(monitoring_v3.Aggregation.Aligner, aligner),
        cross_series_reducer=getattr(monitoring_v3.Aggregation.Reducer,reducer),
        group_by_fields=group_fields
    )
    if(metric_type[0:7]=='compute'):
      metric_filter = (
          'metric.type = "{metric_type}" AND metric.label.instance_name '
          '={instance_name}'
          ).format(metric_type=metric_type, instance_name=instance)

    elif (metric_type[0:6] == 'custom'):
      metric_filter = (
          'metric.type = "{metric_type}" AND metric.labels.opencensus_task = '
          'ends_with("{instance_name}")'
          ).format(metric_type=metric_type, instance_name=instance)

      if (metric_type == OPS_ERROR_COUNT_METRIC):
        metric_filter = ('{} AND metric.labels.fs_op != {}'
                        ).format(metric_filter, 'GetXattr')
      elif (metric_type == OPS_LATENCY_METRIC):
        metric_filter = ('{} AND metric.labels.fs_op = {}'
                        ).format(metric_filter, TEST_TYPE)
    else:
      raise Exception('Unhandled metric type')

    try:
      metrics_response = client.list_time_series({
          'name': PROJECT_NAME,
          'filter': metric_filter,
          'interval': interval,
          'view': monitoring_v3.ListTimeSeriesRequest.TimeSeriesView.FULL,
          'aggregation': aggregation,
      })
    except:
      raise GoogleAPICallError(('The request for API response of {} failed.'
                                ).format(metric_type))

    # Metrics response except OPS_ERROR_COUNT_METRIC should not be empty:
    if(metric_type != OPS_ERROR_COUNT_METRIC and metrics_response == {}):
      raise NoValuesError('No values were retrieved from the call')

    return metrics_response

  def _get_metrics(self, start_time_sec, end_time_sec, instance, period,
                   metric_type, factor, aligner, reducer='REDUCE_NONE',
                   group_fields=['metric.type']):
    """Returns the MetricPoint list for requested metric type.

    Args:
      start_time_sec (int): Epoch seconds
      end_time_sec (int): Epoch seconds
      instance (str): VM instance name
      period (float): Period over which the values are aligned
      metric_type (str): The type of metric
      factor (float) : Converting the API response values into required units.
      aligner(str): Operation to be applied at points of each period
      reducer(str): Operation to aggregate data points accross multiple metrics
      group_fields(list[str]): The fields we want to aggregate using reducer
    Returns:
      list[MetricPoint]
    """
    metrics_response = self._get_api_response(metric_type, start_time_sec,
                                              end_time_sec, instance, period,
                                              aligner, reducer, group_fields)
    metrics_data = _create_metric_points_from_response(metrics_response, factor)
    return metrics_data

  def fetch_metrics_and_write_to_google_sheet(self, start_time_sec,
                                              end_time_sec, instance,
                                              period, test_type) -> None:
    """Fetches the metrics data for all types and writes it to a google sheet.

    Args:
      start_time_sec (int): Epoch seconds
      end_time_sec (int): Epoch seconds
      instance (str): VM instance
      period (float): Period over which the values are taken
      test_type(str): The type of load test for which metrics are taken
    Returns: None
    """
    self._validate_start_end_times(start_time_sec, end_time_sec)
    global TEST_TYPE
    TEST_TYPE = test_type
    cpu_uti_peak_data = self._get_metrics(start_time_sec, end_time_sec,
                                          instance, period, CPU_UTI_METRIC,
                                          1 / 100, 'ALIGN_MAX')
    cpu_uti_mean_data = self._get_metrics(start_time_sec, end_time_sec,
                                          instance, period, CPU_UTI_METRIC,
                                          1 / 100, 'ALIGN_MEAN')
    rec_bytes_peak_data = self._get_metrics(start_time_sec, end_time_sec,
                                            instance, period,
                                            RECEIVED_BYTES_COUNT_METRIC, 60,
                                            'ALIGN_MAX')
    rec_bytes_mean_data = self._get_metrics(start_time_sec, end_time_sec,
                                            instance, period,
                                            RECEIVED_BYTES_COUNT_METRIC, 60,
                                            'ALIGN_MEAN')
    ops_latency_mean_data = self._get_metrics(start_time_sec, end_time_sec,
                                              instance, period,
                                              OPS_LATENCY_METRIC, 1,
                                              'ALIGN_DELTA')
    read_bytes_count_data = self._get_metrics(start_time_sec, end_time_sec,
                                              instance, period,
                                              READ_BYTES_COUNT_METRIC, 1,
                                              'ALIGN_DELTA')
    ops_error_count_data = self._get_metrics(start_time_sec, end_time_sec,
                                             instance, period,
                                             OPS_ERROR_COUNT_METRIC, 1,
                                             'ALIGN_DELTA', 'REDUCE_SUM',
                                             ['metric.labels'])

    # Incase OPS_ERROR_COUNT is empty, we want 0 to be dumped in the sheet:
    if(ops_error_count_data == []):
        ops_error_count_data = [MetricPoint(0, 0, 0) for point in cpu_uti_peak_data]

    metrics_data = []
    for cpu_uti_peak, cpu_uti_mean, rec_bytes_peak, rec_bytes_mean, ops_latency, read_bytes_count, ops_error_count in zip(
        cpu_uti_peak_data, cpu_uti_mean_data, rec_bytes_peak_data,
        rec_bytes_mean_data, ops_latency_mean_data, read_bytes_count_data,
        ops_error_count_data):
      metrics_data.append([
          cpu_uti_peak.start_time_sec, cpu_uti_peak.value, cpu_uti_mean.value,
          rec_bytes_peak.value, rec_bytes_mean.value,
          ops_latency.value, read_bytes_count.value, ops_error_count.value
      ])

    # Writing metrics data to google sheet
    gsheet.write_to_google_sheet(WORKSHEET_NAME, metrics_data)

def main() -> None:
  if len(sys.argv) != 6:
    raise Exception('Invalid arguments.')
  instance = sys.argv[1]
  start_time_sec = int(sys.argv[2])
  end_time_sec = int(sys.argv[3])
  period = int(sys.argv[4])
  test_type = sys.argv[5]
  vm_metrics = VmMetrics()
  vm_metrics.fetch_metrics_and_write_to_google_sheet(start_time_sec,
                                                     end_time_sec, instance,
                                                     period, test_type)

if __name__ == '__main__':
  main()
<|endoftext|>


<commit_before># -----------------------------------------------------------------------------
# Copyright (c) 2014--, The Qiita Development Team.
#
# Distributed under the terms of the BSD 3-clause License.
#
# The full license is in the file LICENSE, distributed with this software.
# -----------------------------------------------------------------------------
from unittest import TestCase, main
from os.path import join

from qiita_core.qiita_settings import qiita_config
from qiita_pet.handlers.api_proxy.sample_template import (
    sample_template_summary_get_req, sample_template_post_req,
    sample_template_put_req, sample_template_delete_req,
    sample_template_filepaths_get_req)

class TestSampleAPI(TestCase):
    def test_sample_template_summary_get_req(self):
        obs = sample_template_summary_get_req(1, 'test@foo.bar')
        exp = {'summary': {
            'physical_specimen_location': [('ANL', 27)],
            'texture': [('63.1 sand, 17.7 silt, 19.2 clay', 9),
                        ('64.6 sand, 17.6 silt, 17.8 clay', 9),
                        ('66 sand, 16.3 silt, 17.7 clay', 9)],
            'common_name': [('rhizosphere metagenome', 9),
                            ('root metagenome', 9), ('soil metagenome', 9)],
            'water_content_soil': [('0.101', 9), ('0.164', 9), ('0.178', 9)],
            'env_feature': [('ENVO:plant-associated habitat', 27)],
            'assigned_from_geo': [('n', 27)], 'altitude': [('0.0', 27)],
            'tot_org_carb': [('3.31', 9), ('4.32', 9), ('5.0', 9)],
            'env_biome': [('ENVO:Temperate grasslands, savannas, and shrubland'
                           ' biome', 27)],
            'sample_type': [('ENVO:soil', 27)],
            'scientific_name': [('1118232', 27)],
            'host_taxid': [('3483', 27)],
            'latitude': [('0.291867635913', 1), ('3.21190859967', 1),
                         ('4.59216095574', 1), ('10.6655599093', 1),
                         ('12.6245524972', 1), ('12.7065957714', 1),
                         ('13.089194595', 1), ('23.1218032799', 1),
                         ('29.1499460692', 1), ('35.2374368957', 1),
                         ('38.2627021402', 1), ('40.8623799474', 1),
                         ('43.9614715197', 1), ('44.9725384282', 1),
                         ('53.5050692395', 1), ('57.571893782', 1),
                         ('60.1102854322', 1), ('68.0991287718', 1),
                         ('68.51099627', 1), ('74.0894932572', 1),
                         ('78.3634273709', 1), ('82.8302905615', 1),
                         ('84.0030227585', 1), ('85.4121476399', 1),
                         ('95.2060749748', 1)],
            'ph': [('6.8', 9), ('6.82', 10), ('6.94', 8)],
            'description_duplicate': [('Bucu Rhizo', 3), ('Bucu Roots', 3),
                                      ('Bucu bulk', 3), ('Burmese Rhizo', 3),
                                      ('Burmese bulk', 3), ('Burmese root', 3),
                                      ('Diesel Rhizo', 3), ('Diesel Root', 3),
                                      ('Diesel bulk', 3)],
            'elevation': [('114.0', 27)],
            'description': [('Cannabis Soil Microbiome', 27)],
            'collection_timestamp': [('2011-11-11 13:00:00', 27)],
            'physical_specimen_remaining': [('True', 27)],
            'dna_extracted': [('True', 27)],
            'taxon_id': [('410658', 9), ('939928', 9), ('1118232', 9)],
            'samp_salinity': [('7.1', 9), ('7.15', 9), ('7.44', 9)],
            'host_subject_id': [('1001:B1', 1), ('1001:B2', 1), ('1001:B3', 1),
                                ('1001:B4', 1), ('1001:B5', 1), ('1001:B6', 1),
                                ('1001:B7', 1), ('1001:B8', 1), ('1001:B9', 1),
                                ('1001:D1', 1), ('1001:D2', 1), ('1001:D3', 1),
                                ('1001:D4', 1), ('1001:D5', 1), ('1001:D6', 1),
                                ('1001:D7', 1), ('1001:D8', 1), ('1001:D9', 1),
                                ('1001:M1', 1), ('1001:M2', 1), ('1001:M3', 1),
                                ('1001:M4', 1), ('1001:M5', 1), ('1001:M6', 1),
                                ('1001:M7', 1), ('1001:M8', 1), ('1001:M9', 1)
                                ],
            'season_environment': [('winter', 27)],
            'temp': [('15.0', 27)],
            'country': [('GAZ:United States of America', 27)],
            'longitude': [('2.35063674718', 1), ('3.48274264219', 1),
                          ('6.66444220187', 1), ('15.6526750776', 1),
                          ('26.8138925876', 1), ('27.3592668624', 1),
                          ('31.2003474585', 1), ('31.6056761814', 1),
                          ('32.5563076447', 1), ('34.8360987059', 1),
                          ('42.838497795', 1), ('63.5115213108', 1),
                          ('65.3283470202', 1), ('66.1920014699', 1),
                          ('66.8954849864', 1), ('68.5041623253', 1),
                          ('68.5945325743', 1), ('70.784770579', 1),
                          ('74.423907894', 1), ('74.7123248382', 1),
                          ('82.1270418227', 1), ('82.8516734159', 1),
                          ('84.9722975792', 1), ('86.3615778099', 1),
                          ('92.5274472082', 1), ('96.0693176066', 1)],
            'tot_nitro': [('1.3', 9), ('1.41', 9), ('1.51', 9)],
            'depth': [('0.15', 27)],
            'anonymized_name': [('SKB1', 1), ('SKB2', 1), ('SKB3', 1),
                                ('SKB4', 1), ('SKB5', 1), ('SKB6', 1),
                                ('SKB7', 1), ('SKB8', 1), ('SKB9', 1),
                                ('SKD1', 1), ('SKD2', 1), ('SKD3', 1),
                                ('SKD4', 1), ('SKD5', 1), ('SKD6', 1),
                                ('SKD7', 1), ('SKD8', 1), ('SKD9', 1),
                                ('SKM1', 1), ('SKM2', 1), ('SKM3', 1),
                                ('SKM4', 1), ('SKM5', 1), ('SKM6', 1),
                                ('SKM7', 1), ('SKM8', 1), ('SKM9', 1)]},
               'num_samples': 27,
               'status': 'success',
               'message': ''}
        self.assertEqual(obs, exp)

    def test_sample_template_summary_get_req_no_access(self):
        obs = sample_template_summary_get_req(1, 'demo@microbio.me')
        exp = {'status': 'error',
               'message': 'User does not have access to study'}
        self.assertEqual(obs, exp)

    def test_sample_template_post_req(self):
        obs = sample_template_post_req(1, 'test@foo.bar', '16S',
                                       'uploaded_file.txt')
        exp = {'status': 'error',
               'message': 'Empty file passed!',
               'file': 'uploaded_file.txt'}
        self.assertEqual(obs, exp)

    def test_sample_template_post_req_no_access(self):
        obs = sample_template_post_req(1, 'demo@microbio.me', '16S',
                                       'filepath')
        exp = {'status': 'error',
               'message': 'User does not have access to study'}
        self.assertEqual(obs, exp)

    def test_sample_template_put_req(self):
        obs = sample_template_put_req(1, 'test@foo.bar',
                                      'uploaded_file.txt')
        exp = {'status': 'error',
               'message': 'Empty file passed!',
               'file': 'uploaded_file.txt'}
        self.assertEqual(obs, exp)

    def test_sample_template_put_req_no_access(self):
        obs = sample_template_put_req(1, 'demo@microbio.me', 'filepath')
        exp = {'status': 'error',
               'message': 'User does not have access to study'}
        self.assertEqual(obs, exp)

    def test_sample_template_delete_req(self):
        obs = sample_template_delete_req(1, 'test@foo.bar')
        exp = {'status': 'error',
               'message': 'Sample template can not be erased because there are'
                          ' prep templates associated.'}
        self.assertEqual(obs, exp)

    def test_sample_template_delete_req_no_access(self):
        obs = sample_template_delete_req(1, 'demo@microbio.me')
        exp = {'status': 'error',
               'message': 'User does not have access to study'}
        self.assertEqual(obs, exp)

    def test_sample_template_filepaths_get_req(self):
        obs = sample_template_filepaths_get_req(1, 'test@foo.bar')

        exp = {'status': 'success',
               'message': '',
               'filepaths': [(14, join(qiita_config.base_data_dir,
                             'templates/1_19700101-000000.txt'))]}
        self.assertEqual(obs, exp)

    def test_sample_template_filepaths_get_req_no_access(self):
        obs = sample_template_filepaths_get_req(1, 'demo@microbio.me')
        exp = {'status': 'error',
               'message': 'User does not have access to study'}
        self.assertEqual(obs, exp)

if __name__ == '__main__':
    main()
<commit_msg>fix test_sample_template_filepaths_get_req<commit_after># -----------------------------------------------------------------------------
# Copyright (c) 2014--, The Qiita Development Team.
#
# Distributed under the terms of the BSD 3-clause License.
#
# The full license is in the file LICENSE, distributed with this software.
# -----------------------------------------------------------------------------
from unittest import TestCase, main
from os.path import join

from qiita_core.util import qiita_test_checker
from qiita_db.util import get_mountpoint
from qiita_pet.handlers.api_proxy.sample_template import (
    sample_template_summary_get_req, sample_template_post_req,
    sample_template_put_req, sample_template_delete_req,
    sample_template_filepaths_get_req)

@qiita_test_checker()
class TestSampleAPI(TestCase):
    def test_sample_template_summary_get_req(self):
        obs = sample_template_summary_get_req(1, 'test@foo.bar')
        exp = {'summary': {
            'physical_specimen_location': [('ANL', 27)],
            'texture': [('63.1 sand, 17.7 silt, 19.2 clay', 9),
                        ('64.6 sand, 17.6 silt, 17.8 clay', 9),
                        ('66 sand, 16.3 silt, 17.7 clay', 9)],
            'common_name': [('rhizosphere metagenome', 9),
                            ('root metagenome', 9), ('soil metagenome', 9)],
            'water_content_soil': [('0.101', 9), ('0.164', 9), ('0.178', 9)],
            'env_feature': [('ENVO:plant-associated habitat', 27)],
            'assigned_from_geo': [('n', 27)], 'altitude': [('0.0', 27)],
            'tot_org_carb': [('3.31', 9), ('4.32', 9), ('5.0', 9)],
            'env_biome': [('ENVO:Temperate grasslands, savannas, and shrubland'
                           ' biome', 27)],
            'sample_type': [('ENVO:soil', 27)],
            'scientific_name': [('1118232', 27)],
            'host_taxid': [('3483', 27)],
            'latitude': [('0.291867635913', 1), ('3.21190859967', 1),
                         ('4.59216095574', 1), ('10.6655599093', 1),
                         ('12.6245524972', 1), ('12.7065957714', 1),
                         ('13.089194595', 1), ('23.1218032799', 1),
                         ('29.1499460692', 1), ('35.2374368957', 1),
                         ('38.2627021402', 1), ('40.8623799474', 1),
                         ('43.9614715197', 1), ('44.9725384282', 1),
                         ('53.5050692395', 1), ('57.571893782', 1),
                         ('60.1102854322', 1), ('68.0991287718', 1),
                         ('68.51099627', 1), ('74.0894932572', 1),
                         ('78.3634273709', 1), ('82.8302905615', 1),
                         ('84.0030227585', 1), ('85.4121476399', 1),
                         ('95.2060749748', 1)],
            'ph': [('6.8', 9), ('6.82', 10), ('6.94', 8)],
            'description_duplicate': [('Bucu Rhizo', 3), ('Bucu Roots', 3),
                                      ('Bucu bulk', 3), ('Burmese Rhizo', 3),
                                      ('Burmese bulk', 3), ('Burmese root', 3),
                                      ('Diesel Rhizo', 3), ('Diesel Root', 3),
                                      ('Diesel bulk', 3)],
            'elevation': [('114.0', 27)],
            'description': [('Cannabis Soil Microbiome', 27)],
            'collection_timestamp': [('2011-11-11 13:00:00', 27)],
            'physical_specimen_remaining': [('True', 27)],
            'dna_extracted': [('True', 27)],
            'taxon_id': [('410658', 9), ('939928', 9), ('1118232', 9)],
            'samp_salinity': [('7.1', 9), ('7.15', 9), ('7.44', 9)],
            'host_subject_id': [('1001:B1', 1), ('1001:B2', 1), ('1001:B3', 1),
                                ('1001:B4', 1), ('1001:B5', 1), ('1001:B6', 1),
                                ('1001:B7', 1), ('1001:B8', 1), ('1001:B9', 1),
                                ('1001:D1', 1), ('1001:D2', 1), ('1001:D3', 1),
                                ('1001:D4', 1), ('1001:D5', 1), ('1001:D6', 1),
                                ('1001:D7', 1), ('1001:D8', 1), ('1001:D9', 1),
                                ('1001:M1', 1), ('1001:M2', 1), ('1001:M3', 1),
                                ('1001:M4', 1), ('1001:M5', 1), ('1001:M6', 1),
                                ('1001:M7', 1), ('1001:M8', 1), ('1001:M9', 1)
                                ],
            'season_environment': [('winter', 27)],
            'temp': [('15.0', 27)],
            'country': [('GAZ:United States of America', 27)],
            'longitude': [('2.35063674718', 1), ('3.48274264219', 1),
                          ('6.66444220187', 1), ('15.6526750776', 1),
                          ('26.8138925876', 1), ('27.3592668624', 1),
                          ('31.2003474585', 1), ('31.6056761814', 1),
                          ('32.5563076447', 1), ('34.8360987059', 1),
                          ('42.838497795', 1), ('63.5115213108', 1),
                          ('65.3283470202', 1), ('66.1920014699', 1),
                          ('66.8954849864', 1), ('68.5041623253', 1),
                          ('68.5945325743', 1), ('70.784770579', 1),
                          ('74.423907894', 1), ('74.7123248382', 1),
                          ('82.1270418227', 1), ('82.8516734159', 1),
                          ('84.9722975792', 1), ('86.3615778099', 1),
                          ('92.5274472082', 1), ('96.0693176066', 1)],
            'tot_nitro': [('1.3', 9), ('1.41', 9), ('1.51', 9)],
            'depth': [('0.15', 27)],
            'anonymized_name': [('SKB1', 1), ('SKB2', 1), ('SKB3', 1),
                                ('SKB4', 1), ('SKB5', 1), ('SKB6', 1),
                                ('SKB7', 1), ('SKB8', 1), ('SKB9', 1),
                                ('SKD1', 1), ('SKD2', 1), ('SKD3', 1),
                                ('SKD4', 1), ('SKD5', 1), ('SKD6', 1),
                                ('SKD7', 1), ('SKD8', 1), ('SKD9', 1),
                                ('SKM1', 1), ('SKM2', 1), ('SKM3', 1),
                                ('SKM4', 1), ('SKM5', 1), ('SKM6', 1),
                                ('SKM7', 1), ('SKM8', 1), ('SKM9', 1)]},
               'num_samples': 27,
               'status': 'success',
               'message': ''}
        self.assertEqual(obs, exp)

    def test_sample_template_summary_get_req_no_access(self):
        obs = sample_template_summary_get_req(1, 'demo@microbio.me')
        exp = {'status': 'error',
               'message': 'User does not have access to study'}
        self.assertEqual(obs, exp)

    def test_sample_template_post_req(self):
        obs = sample_template_post_req(1, 'test@foo.bar', '16S',
                                       'uploaded_file.txt')
        exp = {'status': 'error',
               'message': 'Empty file passed!',
               'file': 'uploaded_file.txt'}
        self.assertEqual(obs, exp)

    def test_sample_template_post_req_no_access(self):
        obs = sample_template_post_req(1, 'demo@microbio.me', '16S',
                                       'filepath')
        exp = {'status': 'error',
               'message': 'User does not have access to study'}
        self.assertEqual(obs, exp)

    def test_sample_template_put_req(self):
        obs = sample_template_put_req(1, 'test@foo.bar',
                                      'uploaded_file.txt')
        exp = {'status': 'error',
               'message': 'Empty file passed!',
               'file': 'uploaded_file.txt'}
        self.assertEqual(obs, exp)

    def test_sample_template_put_req_no_access(self):
        obs = sample_template_put_req(1, 'demo@microbio.me', 'filepath')
        exp = {'status': 'error',
               'message': 'User does not have access to study'}
        self.assertEqual(obs, exp)

    def test_sample_template_delete_req(self):
        obs = sample_template_delete_req(1, 'test@foo.bar')
        exp = {'status': 'error',
               'message': 'Sample template can not be erased because there are'
                          ' prep templates associated.'}
        self.assertEqual(obs, exp)

    def test_sample_template_delete_req_no_access(self):
        obs = sample_template_delete_req(1, 'demo@microbio.me')
        exp = {'status': 'error',
               'message': 'User does not have access to study'}
        self.assertEqual(obs, exp)

    def test_sample_template_filepaths_get_req(self):
        templates_dir = get_mountpoint('templates')[0][1]
        obs = sample_template_filepaths_get_req(1, 'test@foo.bar')
        exp = {'status': 'success',
               'message': '',
               'filepaths': [(14, join(templates_dir,
                              '1_19700101-000000.txt'))]}
        self.assertEqual(obs, exp)

    def test_sample_template_filepaths_get_req_no_access(self):
        obs = sample_template_filepaths_get_req(1, 'demo@microbio.me')
        exp = {'status': 'error',
               'message': 'User does not have access to study'}
        self.assertEqual(obs, exp)

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>"""
The MIT License (MIT)

Copyright (c) 2015 Joshua Lynch, Sarah Nicholas

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
"""
import logging
import time

import numpy as np
import scikits.bootstrap
import sklearn.model_selection

from mrfitty.combination_fit import AllCombinationFitTask

class BestSubsetSelectionFitTask(AllCombinationFitTask):
    def __init__(
            self,
            ls,
            reference_spectrum_list,
            unknown_spectrum_list,
            energy_range_builder,
            component_count_range
    ):
        super(type(self), self).__init__(
            ls, reference_spectrum_list, unknown_spectrum_list, energy_range_builder, component_count_range
        )

    def choose_best_component_count(self, all_counts_spectrum_fit_table):
        """
        Calculate the prediction error for each subset.
        :param all_counts_spectrum_fit_table:
          dictionary with component count keys, sorted list of spectrum fit list values
        :return:
        """
        log = logging.getLogger(__name__)

        log.debug('choosing best component count from {}'.format(all_counts_spectrum_fit_table))
        component_count_to_median_cp = {
            component_count: np.Inf
            for component_count in all_counts_spectrum_fit_table.keys()
        }
        component_count_to_median_cp_ci_lo_hi = {
            component_count: (np.Inf, np.Inf)
            for component_count in all_counts_spectrum_fit_table.keys()
        }

        for component_count_i in sorted(all_counts_spectrum_fit_table.keys()):
            log.debug('calculating CI of median C_p for {} component(s)'.format(component_count_i))

            best_fit_for_component_count = all_counts_spectrum_fit_table[component_count_i][0]
            t0 = time.time()
            prediction_error_list = self.calculate_prediction_error_list(best_fit_for_component_count)
            t1 = time.time()
            log.info('%5.2fs to calculate prediction error list', t1-t0)
            component_count_to_median_cp[component_count_i] = np.median(prediction_error_list)
            component_count_to_median_cp_ci_lo_hi[component_count_i] = scikits.bootstrap.ci(
                data=prediction_error_list,
                statfunction=np.median)

        log.debug('component count to median cp: {}'.format(component_count_to_median_cp))
        log.debug('component count to median cp confidence interval: {}'.format(component_count_to_median_cp_ci_lo_hi))

        best_component_count = self.get_best_ci_component_count(component_count_to_median_cp_ci_lo_hi)
        best_fit = all_counts_spectrum_fit_table[best_component_count][0]
        log.info('best fit: {}'.format(best_fit))
        return best_fit

    @staticmethod
    def get_best_ci_component_count(component_count_to_median_cp_ci_lo_hi):
        """
        Use the 'best subset selection' criterion to choose the 'best' component count using
        confidence intervals for median C_p (prediction error). Choose the component count with
        the smallest median C_p. If two C_p confidence intervals overlap choose the lower count.

        component count  lo    hi
        1                0.3   0.4
        2                0.1   0.2

        :param component_count_to_median_cp_ci_lo_hi:
        :return: (int) best component count
        """
        log = logging.getLogger(name=__name__)
        best_component_count = max(component_count_to_median_cp_ci_lo_hi.keys())
        for n in sorted(component_count_to_median_cp_ci_lo_hi.keys())[:-1]:
            n_lo, n_hi = component_count_to_median_cp_ci_lo_hi[n]
            n_plus_1_lo, n_plus_1_hi = component_count_to_median_cp_ci_lo_hi[n+1]
            log.debug('comparing C_p ci for %d with C_p ci for %d', n, n+1)
            # must handle two cases:
            #   n_plus_1_hi >= n_lo -> choose n
            #   n_plus_1_hi <  n_lo -> try n+1
            if n_plus_1_hi >= n_lo:
                best_component_count = n
                break

        return best_component_count

    def calculate_prediction_error_list(self, fit):
        # calculate Cp and 95% confidence interval of the median
        normalized_cp_list = []
        cv = sklearn.model_selection.ShuffleSplit(n_splits=1000, test_size=0.2)
        for train_index, test_index in cv.split(fit.reference_spectra_A_df.values):
            lm = self.ls()
            lm.fit(
                fit.reference_spectra_A_df.values[train_index],
                fit.unknown_spectrum_b.values[train_index]
            )
            predicted_b = lm.predict(fit.reference_spectra_A_df.values[test_index])
            residuals = fit.unknown_spectrum_b.values[test_index] - predicted_b
            cp = np.sqrt(np.sum(np.square(residuals)))
            normalized_cp = cp / residuals.shape
            normalized_cp_list.append(normalized_cp)
        return normalized_cp_list
<commit_msg>add best_fits_plot_limit<commit_after>"""
The MIT License (MIT)

Copyright (c) 2015 Joshua Lynch, Sarah Nicholas

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
"""
import logging
import time

import numpy as np
import scikits.bootstrap
import sklearn.model_selection

from mrfitty.combination_fit import AllCombinationFitTask

class BestSubsetSelectionFitTask(AllCombinationFitTask):
    def __init__(
            self,
            ls,
            reference_spectrum_list,
            unknown_spectrum_list,
            energy_range_builder,
            best_fits_plot_limit,
            component_count_range
    ):
        super(type(self), self).__init__(
            ls, reference_spectrum_list, unknown_spectrum_list, energy_range_builder, best_fits_plot_limit, component_count_range
        )

    def choose_best_component_count(self, all_counts_spectrum_fit_table):
        """
        Calculate the prediction error for each subset.
        :param all_counts_spectrum_fit_table:
          dictionary with component count keys, sorted list of spectrum fit list values
        :return:
        """
        log = logging.getLogger(__name__)

        log.debug('choosing best component count from {}'.format(all_counts_spectrum_fit_table))
        component_count_to_median_cp = {
            component_count: np.Inf
            for component_count in all_counts_spectrum_fit_table.keys()
        }
        component_count_to_median_cp_ci_lo_hi = {
            component_count: (np.Inf, np.Inf)
            for component_count in all_counts_spectrum_fit_table.keys()
        }

        for component_count_i in sorted(all_counts_spectrum_fit_table.keys()):
            log.debug('calculating CI of median C_p for {} component(s)'.format(component_count_i))

            best_fit_for_component_count = all_counts_spectrum_fit_table[component_count_i][0]
            t0 = time.time()
            prediction_error_list = self.calculate_prediction_error_list(best_fit_for_component_count)
            t1 = time.time()
            log.info('%5.2fs to calculate prediction error list', t1-t0)
            component_count_to_median_cp[component_count_i] = np.median(prediction_error_list)
            component_count_to_median_cp_ci_lo_hi[component_count_i] = scikits.bootstrap.ci(
                data=prediction_error_list,
                statfunction=np.median)

        log.debug('component count to median cp: {}'.format(component_count_to_median_cp))
        log.debug('component count to median cp confidence interval: {}'.format(component_count_to_median_cp_ci_lo_hi))

        best_component_count = self.get_best_ci_component_count(component_count_to_median_cp_ci_lo_hi)
        best_fit = all_counts_spectrum_fit_table[best_component_count][0]
        log.info('best fit: {}'.format(best_fit))
        return best_fit

    @staticmethod
    def get_best_ci_component_count(component_count_to_median_cp_ci_lo_hi):
        """
        Use the 'best subset selection' criterion to choose the 'best' component count using
        confidence intervals for median C_p (prediction error). Choose the component count with
        the smallest median C_p. If two C_p confidence intervals overlap choose the lower count.

        component count  lo    hi
        1                0.3   0.4
        2                0.1   0.2

        :param component_count_to_median_cp_ci_lo_hi:
        :return: (int) best component count
        """
        log = logging.getLogger(name=__name__)
        best_component_count = max(component_count_to_median_cp_ci_lo_hi.keys())
        for n in sorted(component_count_to_median_cp_ci_lo_hi.keys())[:-1]:
            n_lo, n_hi = component_count_to_median_cp_ci_lo_hi[n]
            n_plus_1_lo, n_plus_1_hi = component_count_to_median_cp_ci_lo_hi[n+1]
            log.debug('comparing C_p ci for %d with C_p ci for %d', n, n+1)
            # must handle two cases:
            #   n_plus_1_hi >= n_lo -> choose n
            #   n_plus_1_hi <  n_lo -> try n+1
            if n_plus_1_hi >= n_lo:
                best_component_count = n
                break

        return best_component_count

    def calculate_prediction_error_list(self, fit):
        # calculate Cp and 95% confidence interval of the median
        normalized_cp_list = []
        cv = sklearn.model_selection.ShuffleSplit(n_splits=1000, test_size=0.2)
        for train_index, test_index in cv.split(fit.reference_spectra_A_df.values):
            lm = self.ls()
            lm.fit(
                fit.reference_spectra_A_df.values[train_index],
                fit.unknown_spectrum_b.values[train_index]
            )
            predicted_b = lm.predict(fit.reference_spectra_A_df.values[test_index])
            residuals = fit.unknown_spectrum_b.values[test_index] - predicted_b
            cp = np.sqrt(np.sum(np.square(residuals)))
            normalized_cp = cp / residuals.shape
            normalized_cp_list.append(normalized_cp)
        return normalized_cp_list
<|endoftext|>