<commit_before>import os
import sys
import shutil
import subprocess
import json

from config import rfam_config as rc

# --------------------------------------------------------------------------------------------

def map_rfam_ensembl_accession_mapping(rfam_acc_file, ensembl_acc_file, dest_dir):
    """

    :param rfam_acc_file:
    :param ensembl_acc_file:
    :param dest_dir:
    :return:
    """

    gbh_dict = {}
    ensembl_accs = {}
    rfam_accs = {}

    # initialization
    if not os.path.exists(dest_dir):
        os.mkdir(dest_dir)

    # load ensembl accessions
    fp_ensembl = open(ensembl_acc_file, 'r')

    for line in fp_ensembl:
        line = line.strip().split('\t')
        ensembl_accs[line[0].partition('.')[0]] = {"assembly_name": line[1],
                                                   "tax_id": line[2], "species_name": line[3]}

    fp_ensembl.close()

    # load rfam accessions
    fp = open(rfam_acc_file, 'r')

    for line in fp:
        line = line.strip().split('\t')
        rfam_accs[line[1].partition('.')[0]] = line[0]
    fp.close()

    for gca_acc in ensembl_accs.keys():
        if gca_acc in rfam_accs:
            gbh_dict[rfam_accs[gca_acc]] = {}
            gbh_dict[rfam_accs[gca_acc]]["assembly_name"] = ensembl_accs[gca_acc]["assembly_name"]
            gbh_dict[rfam_accs[gca_acc]]["species_name"] = ensembl_accs[gca_acc]["species_name"]

    fp_out = open(os.path.join(dest_dir, "rfam_gbh_ids.json"), 'w')
    json.dump(gbh_dict, fp_out)

# --------------------------------------------------------------------------------------------

def generate_genome_text_file_from_file(names_list, release_version, dest_dir):
    """
    Generates the genome.txt file for the genome_browser_hup given a list
    of assembly-scientific name mappings

    names_list: A tab delimited file containing mappings of the
    assembly name and the genome scientific name e.g hg38\thomo_sapiens
    version: The version of the release
    dest_dir: The destination directory where the genome.txt
    file will be generated

    return:
    """

    trackdb_url = "ftp://ftp.ebi.ac.uk/pub/databases/Rfam/%s/genome_browser_hub/%s/trackDb.txt"

    fp_in = open(names_list, 'r')

    name_mappings = {}

    # parse input file and store information in a dictionary
    for line in fp_in:
        line = line.strip().split('\t')

        if line[0] not in name_mappings:
            name_mappings[line[0]] = line[1]

    fp_in.close()

    fp_out = open(os.path.join(dest_dir, "genomes.txt"), 'w')

    for assembly_acc in name_mappings.keys():
        fp_out.write("genome %s\n" % assembly_acc)
        fp_out.write("trackDb " + trackdb_url + "\n\n" % (str(release_version),
                                             name_mappings[assembly_acc]))

    fp_out.close()

# --------------------------------------------------------------------------------------------

def generate_genome_text_file_from_dict(accession_dict, release_version, dest_dir):
    """
    Generates the genome.txt file for the genome_browser_hup given a list
    of assembly-scientific name mappings

    names_list: A tab delimited file containing mappings of the
    assembly name and the genome scientific name e.g hg38\thomo_sapiens
    version: The version of the release
    dest_dir: The destination directory where the genome.txt
    file will be generated

    return:
    """

    trackdb_url = "ftp://ftp.ebi.ac.uk/pub/databases/Rfam/%s/genome_browser_hub/%s/trackDb.txt"

    fp_out = open(os.path.join(dest_dir, "genomes.txt"), 'w')

    for genome in accession_dict.keys():
        fp_out.write("genome %s\n" % accession_dict[genome]["species_name"])
        fp_out.write("trackDb " + trackdb_url % (str(release_version),
                                             accession_dict[genome]["species_name"]) + "\n\n")

    fp_out.close()

# --------------------------------------------------------------------------------------------

def generate_hub_txt_file(release_version, dest_dir):
    """
    Generates the hub.txt file for the new Rfam release

    release_version: The Rfam release version
    dest_dir:

    return: void
    """

    # convert to string and chop off the decimals
    rel_version_int = str(release_version).partition(".")[0]
    fp_out = open(os.path.join(dest_dir, "hub.txt"), 'w')
    fp_out.write("hub rfam%s\n" % rel_version_int)
    fp_out.write("shortLabel rfam%s_ncRNA\n" % rel_version_int)
    fp_out.write("longLabel Rfam %s non-coding RNA annotation\n" % release_version)
    fp_out.write("genomesFile genomes.txt\n")
    fp_out.write("email %s\n" % rc.RFAM_EMAIL)
    fp_out.write("descriptionUrl " + rc.BROWSER_HUB_DESC_URL % str(release_version) + '\n')
    fp_out.write("hub_description.html\n")
    fp_out.close()

# --------------------------------------------------------------------------------------------

def generate_hub_description_html_file(release_version, dest_dir):
    """

    release_version:

    return: void
    """

    fp_out = open(os.path.join(dest_dir, "hub_description.html"), 'w')

    # description section
    fp_out.write("
Description
\n\n")
    fp_out.write("
\n")
    fp_out.write("These tracks show non-coding RNAs from the")
    fp_out.write("Rfam database.")
    fp_out.write("These tracks were created from the Rfam %s" % str(release_version))
    fp_out.write("covariance models (CMs), using the cmsearch program from the"
                 "Infernal package."
                 "Further information about the methods used to build Rfam families"
                 "can be found on our web page.\n"
                 "
\n\n")

    # write methods
    fp_out.write("
Methods
\n\n")
    fp_out.write("
\n")
    fp_out.write("The cmsearch program was used with the --rfam, --cut_ga and --nohmmonly"
                 "command line options to calculate the positions of non-coding RNAs on each"
                 "genome assembly. This reproduces the method used by Rfam curators when creating"
                 "each family, including the use of expertly curated thresholds for each family.\n")
    fp_out.write("
\n\n")

    # credits
    fp_out.write("
Credits
\n\n")
    fp_out.write("
\n")
    fp_out.write("This data was generated by the Rfam database."
                 " Please email"
                 " us with any questions or suggestions.\n
\n\n")

    # references
    fp_out.write("
References
\n\n
\n")
    fp_out.write("Kalvari I, Argasinska J,"
                 "Quinones-Olvera N, Nawrocki EP, Rivas E, Eddy SR, Bateman A, Finn RD, Petrov AI."
                 "Rfam 13.0: shifting to a genome-centric resource for non-coding RNA families. Nucleic"
                 "Acids Res. 2017. PMID: 29112718.\n")
    fp_out.write("
\n\n")

    fp_out.close()

# --------------------------------------------------------------------------------------------

def genome_browser_hub_id_list_parser(genome_id_list):
    """
    Parses the input file of genome browser hub and returns
    a dictionary with the ids to be used to generate all
    related sub directories and files

    genome_id_list: A tab delimited file containing the genome upids, assembly names and
    scientific names for each genome

    return: A dictionary in the form of {upid: {"assembly_name": "assembly name",
                                                "species": "scientific name"}
    """

    accession_mapings = {}

    fp_in = open(genome_id_list, 'r')

    if genome_id_list.endswith(".json"):
        accession_mapings = json.load(fp_in)

    else:
        # parse file and generate the dictionary
        for line in fp_in:
            line = line.strip().split('\t')

            if line[0] not in accession_mapings:
                accession_mapings[line[0]] = {}
                accession_mapings[line[0]]["assembly_name"] = line[1]
                accession_mapings[line[0]]["species_name"] = line[2]

    # close input file and return dictionary
    fp_in.close()
    return accession_mapings

# --------------------------------------------------------------------------------------------

def generate_trackdb_file(species, release_version, dest_dir):
    """
    Creates a new species trackDb.txt file for a given Rfam release version

    species: The name of the species directory
    release_version: The version of the Rfam release
    dest_dir: The path to the species directory where the trackDb file will
    be generated

    return: void
    """

    trackDb_fp = open(os.path.join(dest_dir, "trackDb.txt"), 'w')

    trackDb_fp.write("track Rfam\n")
    trackDb_fp.write("bigDataUrl ftp://ftp.ebi.ac.uk/pub/databases/Rfam/%s/genome_browser_hub/%s/bigBed\n" %
                     (release_version, species))
    trackDb_fp.write("shortLabel Rfam ncRNA\n"
                     "longLabel Rfam ncRNA annotations\n"
                     "type bigBed 4\n"
                     "url http://rfam.org/family/$$\n"
                     "visibility 3\n"
                     "color 102,0,0\n")
    trackDb_fp.write("html ftp://ftp.ebi.ac.uk/pub/databases/Rfam/%s/genome_browser_hub/hub_description.html\n" %
                     release_version)

    trackDb_fp.close()

# --------------------------------------------------------------------------------------------

def generate_new_genome_browser_hub_directories(genome_id_file, release_version, dest_dir, genome_project_dir):
    """
    Generates a new genome_browser_hub directory for an upcoming Rfam release

    genome_id_file: A tab delimited file containing the genome upids, assembly names and
    scientific names for each genome
    release_version: The Rfam version the genome browser hub derives from
    dest_dir: A valid path where to generate the directories. Preferably a new Rfam release
    directory

    return: void
    """

    accession_mapings = genome_browser_hub_id_list_parser(genome_id_file)

    # creating basic directory
    if not os.path.exists(dest_dir):
        os.mkdir(dest_dir)

    # create an output directory where to generate the bidbed files
    bed_files_dir = os.path.join(dest_dir, "bed_files")
    os.mkdir(bed_files_dir)

    # create a new genome_browser_hub directory
    new_gbh_dir = os.path.join(dest_dir, "genome_browser_hub")
    os.mkdir(new_gbh_dir)

    # generate genome.txt file
    generate_genome_text_file_from_dict(accession_mapings, release_version, new_gbh_dir)

    # generate hub.txt file
    generate_hub_txt_file(release_version, new_gbh_dir)

    # generate hub_description.html file
    generate_hub_description_html_file(release_version, new_gbh_dir)

    # change to bed_files dir
    os.chdir(bed_files_dir)
    # now generate genome directories, bigbed and trackDB files
    for genome in accession_mapings.keys():
        # replace spaces in species name with underscores
        # to be used as the genome directory name. If very long replace with common name
        genome_name = accession_mapings[genome]["species_name"].replace(' ', '_')
        genome_dir = os.path.join(new_gbh_dir, genome_name)
        os.mkdir(genome_dir)

        tbl_file_path = os.path.join(os.path.join(os.path.join(genome_project_dir,
                                                               genome[-3:]), genome),
                                     genome+'.tbl')

        genome_bed_dir = os.path.join(bed_files_dir, genome)
        os.mkdir(genome_bed_dir)

        tbl_copy = os.path.join(genome_bed_dir, genome+'.tbl')
        shutil.copyfile(tbl_file_path, tbl_copy)

        # call perl script to generate the bedfiles need upid.tbl file here
        cmd = "/path/to/tblout2bigBedGenomes.pl %s %s"

        subprocess.call(cmd % (tbl_copy, release_version), shell=True)
        shutil.copyfile(os.path.join(genome_bed_dir, genome+'.bigBed'), os.path.join(genome_dir, "bigBed"))

        generate_trackdb_file(genome_name, release_version, genome_dir)

    print "Done!"

# --------------------------------------------------------------------------------------------

if __name__ == '__main__':

    genome_id_list = sys.argv[1]
    release_version = sys.argv[2]
    genome_project_dir = sys.argv[3]
    dest_dir = sys.argv[4]

    generate_new_genome_browser_hub_directories(genome_id_list, release_version, dest_dir, genome_project_dir)<commit_msg>Change species_name to assembly name in genome line<commit_after>import os
import sys
import shutil
import subprocess
import json

from config import rfam_config as rc

# --------------------------------------------------------------------------------------------

def map_rfam_ensembl_accession_mapping(rfam_acc_file, ensembl_acc_file, dest_dir):
    """

    :param rfam_acc_file:
    :param ensembl_acc_file:
    :param dest_dir:
    :return:
    """

    gbh_dict = {}
    ensembl_accs = {}
    rfam_accs = {}

    # initialization
    if not os.path.exists(dest_dir):
        os.mkdir(dest_dir)

    # load ensembl accessions
    fp_ensembl = open(ensembl_acc_file, 'r')

    for line in fp_ensembl:
        line = line.strip().split('\t')
        ensembl_accs[line[0].partition('.')[0]] = {"assembly_name": line[1],
                                                   "tax_id": line[2], "species_name": line[3]}

    fp_ensembl.close()

    # load rfam accessions
    fp = open(rfam_acc_file, 'r')

    for line in fp:
        line = line.strip().split('\t')
        rfam_accs[line[1].partition('.')[0]] = line[0]
    fp.close()

    for gca_acc in ensembl_accs.keys():
        if gca_acc in rfam_accs:
            gbh_dict[rfam_accs[gca_acc]] = {}
            gbh_dict[rfam_accs[gca_acc]]["assembly_name"] = ensembl_accs[gca_acc]["assembly_name"]
            gbh_dict[rfam_accs[gca_acc]]["species_name"] = ensembl_accs[gca_acc]["species_name"]

    fp_out = open(os.path.join(dest_dir, "rfam_gbh_ids.json"), 'w')
    json.dump(gbh_dict, fp_out)

# --------------------------------------------------------------------------------------------

def generate_genome_text_file_from_file(names_list, release_version, dest_dir):
    """
    Generates the genome.txt file for the genome_browser_hup given a list
    of assembly-scientific name mappings

    names_list: A tab delimited file containing mappings of the
    assembly name and the genome scientific name e.g hg38\thomo_sapiens
    version: The version of the release
    dest_dir: The destination directory where the genome.txt
    file will be generated

    return:
    """

    trackdb_url = "ftp://ftp.ebi.ac.uk/pub/databases/Rfam/%s/genome_browser_hub/%s/trackDb.txt"

    fp_in = open(names_list, 'r')

    name_mappings = {}

    # parse input file and store information in a dictionary
    for line in fp_in:
        line = line.strip().split('\t')

        if line[0] not in name_mappings:
            name_mappings[line[0]] = line[1]

    fp_in.close()

    fp_out = open(os.path.join(dest_dir, "genomes.txt"), 'w')

    for assembly_acc in name_mappings.keys():
        fp_out.write("genome %s\n" % assembly_acc)
        fp_out.write("trackDb " + trackdb_url + "\n\n" % (str(release_version),
                                             name_mappings[assembly_acc]))

    fp_out.close()

# --------------------------------------------------------------------------------------------

def generate_genome_text_file_from_dict(accession_dict, release_version, dest_dir):
    """
    Generates the genome.txt file for the genome_browser_hup given a list
    of assembly-scientific name mappings

    names_list: A tab delimited file containing mappings of the
    assembly name and the genome scientific name e.g hg38\thomo_sapiens
    version: The version of the release
    dest_dir: The destination directory where the genome.txt
    file will be generated

    return:
    """

    trackdb_url = "ftp://ftp.ebi.ac.uk/pub/databases/Rfam/%s/genome_browser_hub/%s/trackDb.txt"

    fp_out = open(os.path.join(dest_dir, "genomes.txt"), 'w')

    for genome in accession_dict.keys():
        fp_out.write("genome %s\n" % accession_dict[genome]["assembly_name"])
        fp_out.write("trackDb " + trackdb_url % (str(release_version),
                                             accession_dict[genome]["species_name"]) + "\n\n")

    fp_out.close()

# --------------------------------------------------------------------------------------------

def generate_hub_txt_file(release_version, dest_dir):
    """
    Generates the hub.txt file for the new Rfam release

    release_version: The Rfam release version
    dest_dir:

    return: void
    """

    # convert to string and chop off the decimals
    rel_version_int = str(release_version).partition(".")[0]
    fp_out = open(os.path.join(dest_dir, "hub.txt"), 'w')
    fp_out.write("hub rfam%s\n" % rel_version_int)
    fp_out.write("shortLabel rfam%s_ncRNA\n" % rel_version_int)
    fp_out.write("longLabel Rfam %s non-coding RNA annotation\n" % release_version)
    fp_out.write("genomesFile genomes.txt\n")
    fp_out.write("email %s\n" % rc.RFAM_EMAIL)
    fp_out.write("descriptionUrl " + rc.BROWSER_HUB_DESC_URL % str(release_version) + '\n')
    fp_out.write("hub_description.html\n")
    fp_out.close()

# --------------------------------------------------------------------------------------------

def generate_hub_description_html_file(release_version, dest_dir):
    """

    release_version:

    return: void
    """

    fp_out = open(os.path.join(dest_dir, "hub_description.html"), 'w')

    # description section
    fp_out.write("
Description
\n\n")
    fp_out.write("
\n")
    fp_out.write("These tracks show non-coding RNAs from the")
    fp_out.write("Rfam database.")
    fp_out.write("These tracks were created from the Rfam %s" % str(release_version))
    fp_out.write("covariance models (CMs), using the cmsearch program from the"
                 "Infernal package."
                 "Further information about the methods used to build Rfam families"
                 "can be found on our web page.\n"
                 "
\n\n")

    # write methods
    fp_out.write("
Methods
\n\n")
    fp_out.write("
\n")
    fp_out.write("The cmsearch program was used with the --rfam, --cut_ga and --nohmmonly"
                 "command line options to calculate the positions of non-coding RNAs on each"
                 "genome assembly. This reproduces the method used by Rfam curators when creating"
                 "each family, including the use of expertly curated thresholds for each family.\n")
    fp_out.write("
\n\n")

    # credits
    fp_out.write("
Credits
\n\n")
    fp_out.write("
\n")
    fp_out.write("This data was generated by the Rfam database."
                 " Please email"
                 " us with any questions or suggestions.\n
\n\n")

    # references
    fp_out.write("
References
\n\n
\n")
    fp_out.write("Kalvari I, Argasinska J,"
                 "Quinones-Olvera N, Nawrocki EP, Rivas E, Eddy SR, Bateman A, Finn RD, Petrov AI."
                 "Rfam 13.0: shifting to a genome-centric resource for non-coding RNA families. Nucleic"
                 "Acids Res. 2017. PMID: 29112718.\n")
    fp_out.write("
\n\n")

    fp_out.close()

# --------------------------------------------------------------------------------------------

def genome_browser_hub_id_list_parser(genome_id_list):
    """
    Parses the input file of genome browser hub and returns
    a dictionary with the ids to be used to generate all
    related sub directories and files

    genome_id_list: A tab delimited file containing the genome upids, assembly names and
    scientific names for each genome

    return: A dictionary in the form of {upid: {"assembly_name": "assembly name",
                                                "species": "scientific name"}
    """

    accession_mapings = {}

    fp_in = open(genome_id_list, 'r')

    if genome_id_list.endswith(".json"):
        accession_mapings = json.load(fp_in)

    else:
        # parse file and generate the dictionary
        for line in fp_in:
            line = line.strip().split('\t')

            if line[0] not in accession_mapings:
                accession_mapings[line[0]] = {}
                accession_mapings[line[0]]["assembly_name"] = line[1]
                accession_mapings[line[0]]["species_name"] = line[2]

    # close input file and return dictionary
    fp_in.close()
    return accession_mapings

# --------------------------------------------------------------------------------------------

def generate_trackdb_file(species, release_version, dest_dir):
    """
    Creates a new species trackDb.txt file for a given Rfam release version

    species: The name of the species directory
    release_version: The version of the Rfam release
    dest_dir: The path to the species directory where the trackDb file will
    be generated

    return: void
    """

    trackDb_fp = open(os.path.join(dest_dir, "trackDb.txt"), 'w')

    trackDb_fp.write("track Rfam\n")
    trackDb_fp.write("bigDataUrl ftp://ftp.ebi.ac.uk/pub/databases/Rfam/%s/genome_browser_hub/%s/bigBed\n" %
                     (release_version, species))
    trackDb_fp.write("shortLabel Rfam ncRNA\n"
                     "longLabel Rfam ncRNA annotations\n"
                     "type bigBed 4\n"
                     "url http://rfam.org/family/$$\n"
                     "visibility 3\n"
                     "color 102,0,0\n")
    trackDb_fp.write("html ftp://ftp.ebi.ac.uk/pub/databases/Rfam/%s/genome_browser_hub/hub_description.html\n" %
                     release_version)

    trackDb_fp.close()

# --------------------------------------------------------------------------------------------

def generate_new_genome_browser_hub_directories(genome_id_file, release_version, dest_dir, genome_project_dir):
    """
    Generates a new genome_browser_hub directory for an upcoming Rfam release

    genome_id_file: A tab delimited file containing the genome upids, assembly names and
    scientific names for each genome
    release_version: The Rfam version the genome browser hub derives from
    dest_dir: A valid path where to generate the directories. Preferably a new Rfam release
    directory

    return: void
    """

    accession_mapings = genome_browser_hub_id_list_parser(genome_id_file)

    # creating basic directory
    if not os.path.exists(dest_dir):
        os.mkdir(dest_dir)

    # create an output directory where to generate the bidbed files
    bed_files_dir = os.path.join(dest_dir, "bed_files")
    os.mkdir(bed_files_dir)

    # create a new genome_browser_hub directory
    new_gbh_dir = os.path.join(dest_dir, "genome_browser_hub")
    os.mkdir(new_gbh_dir)

    # generate genome.txt file
    generate_genome_text_file_from_dict(accession_mapings, release_version, new_gbh_dir)

    # generate hub.txt file
    generate_hub_txt_file(release_version, new_gbh_dir)

    # generate hub_description.html file
    generate_hub_description_html_file(release_version, new_gbh_dir)

    # change to bed_files dir
    os.chdir(bed_files_dir)
    # now generate genome directories, bigbed and trackDB files
    for genome in accession_mapings.keys():
        # replace spaces in species name with underscores
        # to be used as the genome directory name. If very long replace with common name
        genome_name = accession_mapings[genome]["species_name"].replace(' ', '_')
        genome_dir = os.path.join(new_gbh_dir, genome_name)
        os.mkdir(genome_dir)

        tbl_file_path = os.path.join(os.path.join(os.path.join(genome_project_dir,
                                                               genome[-3:]), genome),
                                     genome+'.tbl')

        genome_bed_dir = os.path.join(bed_files_dir, genome)
        os.mkdir(genome_bed_dir)

        tbl_copy = os.path.join(genome_bed_dir, genome+'.tbl')
        shutil.copyfile(tbl_file_path, tbl_copy)

        # call perl script to generate the bedfiles need upid.tbl file here
        cmd = "/path/to/tblout2bigBedGenomes.pl %s %s"

        subprocess.call(cmd % (tbl_copy, release_version), shell=True)
        shutil.copyfile(os.path.join(genome_bed_dir, genome+'.bigBed'), os.path.join(genome_dir, "bigBed"))

        generate_trackdb_file(genome_name, release_version, genome_dir)

    print "Done!"

# --------------------------------------------------------------------------------------------

if __name__ == '__main__':

    genome_id_list = sys.argv[1]
    release_version = sys.argv[2]
    genome_project_dir = sys.argv[3]
    dest_dir = sys.argv[4]

    generate_new_genome_browser_hub_directories(genome_id_list, release_version, dest_dir, genome_project_dir)<|endoftext|>


<commit_before>from ctk_cli import CLIArgumentParser
import histomicstk as htk
import numpy as np
import json
import scipy as sp
import skimage.io
import skimage.measure

import logging
logging.basicConfig()

stainColorMap = {
    'hematoxylin': [0.65, 0.70, 0.29],
    'eosin':       [0.07, 0.99, 0.11],
    'dab':         [0.27, 0.57, 0.78],
    'null':        [0.0, 0.0, 0.0]
}

def main(args):

    #
    # Read Input Image
    #
    print('>> Reading input image')

    imInput = skimage.io.imread(args.inputImageFile)[:, :, :3]

    #
    # Perform color normalization
    #
    print('>> Performing color normalization')

    # transform input image to LAB color space
    imInputLAB = htk.RudermanLABFwd(imInput)

    # compute mean and stddev of input in LAB color space
    Mu = np.zeros(3)
    Sigma = np.zeros(3)

    for i in range(3):
        Mu[i] = imInputLAB[:, :, i].mean()
        Sigma[i] = (imInputLAB[:, :, i] - Mu[i]).std()

    # perform reinhard normalization
    imNmzd = htk.ReinhardNorm(imInput, Mu, Sigma)

    #
    # Perform color deconvolution
    #
    print('>> Performing color deconvolution')

    stainColor_1 = stainColorMap[args.stain_1]
    stainColor_2 = stainColorMap[args.stain_2]
    stainColor_3 = stainColorMap[args.stain_3]

    W = np.array([stainColor_1, stainColor_2, stainColor_3]).T

    imDeconvolved = htk.ColorDeconvolution(imNmzd, W)

    imNucleiStain = imDeconvolved.Stains[:, :, 0].astype(np.float)

    #
    # Perform nuclei segmentation
    #
    print('>> Performing nuclei segmentation')

    # segment foreground
    imFgndMask = sp.ndimage.morphology.binary_fill_holes(
        imNucleiStain < args.foreground_threshold)

    # run adaptive multi-scale LoG filter
    imLog = htk.cLoG(imNucleiStain, imFgndMask,
                     SigmaMin=args.min_radius * np.sqrt(2),
                     SigmaMax=args.max_radius * np.sqrt(2))

    imNucleiSegMask, Seeds, Max = htk.MaxClustering(
        imLog, imFgndMask, args.local_max_search_radius)

    # filter out small objects
    imNucleiSegMask = htk.FilterLabel(
        imNucleiSegMask, Lower=args.min_nucleus_area).astype(np.int)

    #
    # Generate annotations
    #
    objProps = skimage.measure.regionprops(imNucleiSegMask)

    print 'Number of nuclei = ', len(objProps)

    # create basic schema
    annotation = {
        "name":          "Nuclei",
        "description":   "Nuclei bounding boxes from a segmentation algorithm",
        "attributes": {
            "algorithm": {
                "color_normalization": "ReinhardNorm",
                "color_deconvolution": "ColorDeconvolution",
                "nuclei_segmentation": ["cLOG",
                                        "MaxClustering",
                                        "FilterLabel"]
            }
        },
        "elements": []
    }

    # add each nucleus as an element into the annotation schema
    for i in range(len(objProps)):

        c = [objProps[i].centroid[1], objProps[i].centroid[0], 0]
        width = objProps[i].bbox[3] - objProps[i].bbox[1] + 1
        height = objProps[i].bbox[2] - objProps[i].bbox[0] + 1

        cur_bbox = {
            "type":        "rectangle",
            "center":      c,
            "width":       width,
            "height":      height,
            "rotation":    0
        }

        annotation["elements"].append(cur_bbox)

    #
    # Save output segmentation mask
    #
    print('>> Outputting nuclei segmentation mask')

    skimage.io.imsave(args.outputNucleiMaskFile, imNucleiSegMask)

    #
    # Save output annotation
    #
    print('>> Outputting nuclei annotation')

    with open(args.outputNucleiAnnotationFile, 'w') as annotationFile:
        json.dump(annotation, annotationFile, indent=2, sort_keys=False)

if __name__ == "__main__":
    main(CLIArgumentParser().parse_args())
<commit_msg>Allow the segmentation script to take a girder item as input<commit_after>from ctk_cli import CLIArgumentParser
import histomicstk as htk
import numpy as np
import json
import os
import scipy as sp
import skimage.io
import skimage.measure

import logging
logging.basicConfig()

stainColorMap = {
    'hematoxylin': [0.65, 0.70, 0.29],
    'eosin':       [0.07, 0.99, 0.11],
    'dab':         [0.27, 0.57, 0.78],
    'null':        [0.0, 0.0, 0.0]
}

def read_image(file):
    return skimage.io.imread(file)[:, :, :3]

def main(args):

    #
    # Read Input Image
    #
    print('>> Reading input image')

    imInput = None
    imageFile = args.inputImageFile
    if os.path.isdir(imageFile):
        dir = imageFile
        for file in os.listdir(dir):
            imageFile = os.path.join(dir, file)
            try:
                imInput = read_image(imageFile)
                break
            except Exception:
                print('>> Could not open %s' % imageFile)
    else:
        imInput = read_image(imageFile)

    if imInput is None:
        raise Exception('No image could be opened in %s' % dir)
    #
    # Perform color normalization
    #
    print('>> Performing color normalization')

    # transform input image to LAB color space
    imInputLAB = htk.RudermanLABFwd(imInput)

    # compute mean and stddev of input in LAB color space
    Mu = np.zeros(3)
    Sigma = np.zeros(3)

    for i in range(3):
        Mu[i] = imInputLAB[:, :, i].mean()
        Sigma[i] = (imInputLAB[:, :, i] - Mu[i]).std()

    # perform reinhard normalization
    imNmzd = htk.ReinhardNorm(imInput, Mu, Sigma)

    #
    # Perform color deconvolution
    #
    print('>> Performing color deconvolution')

    stainColor_1 = stainColorMap[args.stain_1]
    stainColor_2 = stainColorMap[args.stain_2]
    stainColor_3 = stainColorMap[args.stain_3]

    W = np.array([stainColor_1, stainColor_2, stainColor_3]).T

    imDeconvolved = htk.ColorDeconvolution(imNmzd, W)

    imNucleiStain = imDeconvolved.Stains[:, :, 0].astype(np.float)

    #
    # Perform nuclei segmentation
    #
    print('>> Performing nuclei segmentation')

    # segment foreground
    imFgndMask = sp.ndimage.morphology.binary_fill_holes(
        imNucleiStain < args.foreground_threshold)

    # run adaptive multi-scale LoG filter
    imLog = htk.cLoG(imNucleiStain, imFgndMask,
                     SigmaMin=args.min_radius * np.sqrt(2),
                     SigmaMax=args.max_radius * np.sqrt(2))

    imNucleiSegMask, Seeds, Max = htk.MaxClustering(
        imLog, imFgndMask, args.local_max_search_radius)

    # filter out small objects
    imNucleiSegMask = htk.FilterLabel(
        imNucleiSegMask, Lower=args.min_nucleus_area).astype(np.int)

    #
    # Generate annotations
    #
    objProps = skimage.measure.regionprops(imNucleiSegMask)

    print 'Number of nuclei = ', len(objProps)

    # create basic schema
    annotation = {
        "name":          "Nuclei",
        "description":   "Nuclei bounding boxes from a segmentation algorithm",
        "attributes": {
            "algorithm": {
                "color_normalization": "ReinhardNorm",
                "color_deconvolution": "ColorDeconvolution",
                "nuclei_segmentation": ["cLOG",
                                        "MaxClustering",
                                        "FilterLabel"]
            }
        },
        "elements": []
    }

    # add each nucleus as an element into the annotation schema
    for i in range(len(objProps)):

        c = [objProps[i].centroid[1], objProps[i].centroid[0], 0]
        width = objProps[i].bbox[3] - objProps[i].bbox[1] + 1
        height = objProps[i].bbox[2] - objProps[i].bbox[0] + 1

        cur_bbox = {
            "type":        "rectangle",
            "center":      c,
            "width":       width,
            "height":      height,
            "rotation":    0
        }

        annotation["elements"].append(cur_bbox)

    #
    # Save output segmentation mask
    #
    print('>> Outputting nuclei segmentation mask')

    skimage.io.imsave(args.outputNucleiMaskFile, imNucleiSegMask)

    #
    # Save output annotation
    #
    print('>> Outputting nuclei annotation')

    with open(args.outputNucleiAnnotationFile, 'w') as annotationFile:
        json.dump(annotation, annotationFile, indent=2, sort_keys=False)

if __name__ == "__main__":
    main(CLIArgumentParser().parse_args())
<|endoftext|>


<commit_before>from collections import defaultdict
from row import Row, StreamInfo, PointStream
from device import Device
from sensor import Sensor

def make_row_generator(rows):
    """"Utility function for converting a list to a generator.

    :param list d: the list to convert
    :rtype: generator"""

    for r in rows:
        yield Row(r)

def make_device_generator(devices):
    for d in devices:
        sensors = []
        for s in d['sensors']:
            sensors.append(Sensor(s['key'], s.get('name', ''),
                                  s['attributes']))
        yield Device(d['key'], d.get('name', ''), d['attributes'], sensors)

def check_response(resp):
    """Utility function for checking the status of a cursor increment.  Raises
    an exception if the call to the paginated link returns anything other than
    a 200.

    :param resp: the response to check
    :type resp: :class:`tempodb.response.Response` object
    :raises ValueError: if the response is not a 200
    :rtype: None"""

    s = resp.status
    if s != 200:
        raise ValueError(
            'TempoIQ API returned %d as status when 200 was expected' % s)

class Cursor(object):
    """An iterable cursor over data retrieved from the TempoDB API.  The
    cursor will make network requests to fetch more data as needed, until
    the API returns no more data.  It can be used with the standard
    iterable interface::

        >>> data = [d for d in response.data]"""

    def __init__(self, data,  response):
        self.response = response
        self.data = make_row_generator(data)

    def __iter__(self):
        while True:
            try:
                x = self.data.next()
                yield x
            except StopIteration:
                self._fetch_next()

    def _fetch_next(self):
        raise StopIteration

class DeviceCursor(Cursor):
    """The data attribute holds the actual data from the request.

    Additionally, the raw response object is available as the response
    attribute of the cursor.

    :param response: the raw response object
    :type response: :class:`tempodb.response.Response"""

    def __init__(self, response, data, fetcher):
        self.response = response
        self.fetcher = fetcher
        self._raw_data = data
        self.data = make_device_generator(data['data'])

    def _fetch_next(self):
        try:
            cursor_obj = self._raw_data['next_page']['next_query']
            new_data = self.fetcher(cursor_obj)
            self._raw_data = new_data
            self.response.data = new_data
            self.data = make_device_generator(new_data['data'])
        except KeyError:
            raise StopIteration

class DataPointsCursor(Cursor):
    """The data attribute holds the actual data from the request.

    Additionally, the raw response object is available as the response
    attribute of the cursor.

    :param response: the raw response object
    :type response: :class:`tempodb.response.Response"""

    def __init__(self, response, data, fetcher):
        self.response = response
        self.fetcher = fetcher
        self._raw_data = data
        self.data = make_row_generator(data['data'])

    def _fetch_next(self):
        try:
            cursor_obj = self._raw_data['next_page']['next_query']
            new_data = self.fetcher(cursor_obj)
            self._raw_data = new_data
            self.response.data = new_data
            self.data = make_row_generator(new_data['data'])
        except KeyError:
            raise StopIteration
        except Exception:
            raise

class Page(object):
    def __init__(self, data, cursor_obj):
        self.data = data
        self.cursor_obj = cursor_obj

    def garbage_collect(self):
        self.data = None

    def is_active(self):
        if self.data is not None:
            return True
        return False

    def reconstruct(self, cursor):
        try:
            self.data = cursor.fetcher(self.cursor_obj)['data']
        except Exception:
            raise

class StreamManager(object):
    MAX_PAGES = 100

    def __init__(self, cursor, starting_data, page_size):
        cursor_obj = starting_data['next_page']['next_query']
        self.pages = {0: Page(starting_data['data'], cursor_obj)}
        self.active_pages = 1
        self.receiver_pointers = {}
        self.active_pointers = defaultdict(set)
        self.cursor = cursor
        self.page_size = page_size

    def _garbage_collect(self):
        to_collect = self.active_pages - StreamManager.MAX_PAGES
        max_iters = 2
        current_iters = 0
        while to_collect > 0 and current_iters < max_iters:
            for k in self.pages:
                if len(self.active_pointers[k]) == 0:
                    page = self.pages[k]
                    page.garbage_collect()
                    self.active_pages -= 1
                    #dont want to be too aggressive here, we are guaranteeing
                    #max memory used and trying to keep network traffic
                    #to a minimum
                    to_collect -= 1
                    if to_collect == 0:
                        break
            current_iters += 1

    def _update_active_pointers(self, key, page_num):
        if page_num == 0:
            self.active_pointers[page_num].add(key)
        elif key not in self.active_pointers[page_num]:
            old_page = page_num - 1
            self.active_pointers[old_page].remove(key)
            self.active_pointers[page_num].add(key)

    def fetch_next(self):
        next_idx = self.pages.keys()[-1] + 1
        new_data, cursor_obj = self.cursor._fetch_next()
        page = Page(new_data['data'], cursor_obj)
        self.pages[next_idx] = page
        self.active_pages += 1
        return page

    def next(self, receiver):
        if not self.receiver_pointers.get(receiver.key):
            self.receiver_pointers[receiver.key] = 0
            self.active_pointers[0].add(receiver.key)

        current_idx = self.receiver_pointers[receiver.key]
        page_num = current_idx / self.page_size
        item = current_idx % self.page_size
        self.receiver_pointers[receiver.key] = current_idx + 1
        self._update_active_pointers(receiver.key, page_num)

        page = self.pages.get(page_num)
        if page is None:
            page = self.fetch_next()
            self._garbage_collect()
        if not page.is_active():
            self.reconstruct_page(page)
        return page.data[item]

    def reconstruct_page(self, page):
        page.reconstruct(self.cursor)
        self.active_pages += 1

class StreamResponseCursor(Cursor):
    def __init__(self, response, data, fetcher):
        self.response = response
        self.fetcher = fetcher
        self._raw_data = data
        self.page_size = len(data['data'])
        self.stream_info = StreamInfo(data['streams'])
        self.manager = StreamManager(self, data, self.page_size)

    def _fetch_next(self):
        try:
            cursor_obj = self._raw_data['next_page']['next_query']
            new_data = self.fetcher(cursor_obj)
            self._raw_data = new_data
            return new_data, cursor_obj
        except KeyError:
            raise StopIteration
        except Exception:
            raise

    def bind_stream(self, **kwargs):
        stream_info = self.stream_info.get_one(**kwargs)
        return PointStream(stream_info, self.manager)

    @property
    def streams(self):
        return [PointStream(si, self.manager) for
                si in self.stream_info.headers]
<commit_msg>maintain row style iterator interface<commit_after>from collections import defaultdict
from row import Row, StreamInfo, PointStream
from device import Device
from sensor import Sensor

def make_row_generator(rows):
    """"Utility function for converting a list to a generator.

    :param list d: the list to convert
    :rtype: generator"""

    for r in rows:
        yield Row(r)

def make_device_generator(devices):
    for d in devices:
        sensors = []
        for s in d['sensors']:
            sensors.append(Sensor(s['key'], s.get('name', ''),
                                  s['attributes']))
        yield Device(d['key'], d.get('name', ''), d['attributes'], sensors)

def check_response(resp):
    """Utility function for checking the status of a cursor increment.  Raises
    an exception if the call to the paginated link returns anything other than
    a 200.

    :param resp: the response to check
    :type resp: :class:`tempodb.response.Response` object
    :raises ValueError: if the response is not a 200
    :rtype: None"""

    s = resp.status
    if s != 200:
        raise ValueError(
            'TempoIQ API returned %d as status when 200 was expected' % s)

class Cursor(object):
    """An iterable cursor over data retrieved from the TempoDB API.  The
    cursor will make network requests to fetch more data as needed, until
    the API returns no more data.  It can be used with the standard
    iterable interface::

        >>> data = [d for d in response.data]"""

    def __init__(self, data,  response):
        self.response = response
        self.data = make_row_generator(data)

    def __iter__(self):
        while True:
            try:
                x = self.data.next()
                yield x
            except StopIteration:
                self._fetch_next()

    def _fetch_next(self):
        raise StopIteration

class DeviceCursor(Cursor):
    """The data attribute holds the actual data from the request.

    Additionally, the raw response object is available as the response
    attribute of the cursor.

    :param response: the raw response object
    :type response: :class:`tempodb.response.Response"""

    def __init__(self, response, data, fetcher):
        self.response = response
        self.fetcher = fetcher
        self._raw_data = data
        self.data = make_device_generator(data['data'])

    def _fetch_next(self):
        try:
            cursor_obj = self._raw_data['next_page']['next_query']
            new_data = self.fetcher(cursor_obj)
            self._raw_data = new_data
            self.response.data = new_data
            self.data = make_device_generator(new_data['data'])
        except KeyError:
            raise StopIteration

class DataPointsCursor(Cursor):
    """The data attribute holds the actual data from the request.

    Additionally, the raw response object is available as the response
    attribute of the cursor.

    :param response: the raw response object
    :type response: :class:`tempodb.response.Response"""

    def __init__(self, response, data, fetcher):
        self.response = response
        self.fetcher = fetcher
        self._raw_data = data
        self.data = make_row_generator(data['data'])

    def _fetch_next(self):
        try:
            cursor_obj = self._raw_data['next_page']['next_query']
            new_data = self.fetcher(cursor_obj)
            self._raw_data = new_data
            self.response.data = new_data
            self.data = make_row_generator(new_data['data'])
        except KeyError:
            raise StopIteration
        except Exception:
            raise

class Page(object):
    def __init__(self, data, cursor_obj):
        self.data = data
        self.cursor_obj = cursor_obj

    def garbage_collect(self):
        self.data = None

    def is_active(self):
        if self.data is not None:
            return True
        return False

    def reconstruct(self, cursor):
        try:
            self.data = cursor.fetcher(self.cursor_obj)['data']
        except Exception:
            raise

class StreamManager(object):
    MAX_PAGES = 100

    def __init__(self, cursor, starting_data, page_size):
        cursor_obj = starting_data['next_page']['next_query']
        self.pages = {0: Page(starting_data['data'], cursor_obj)}
        self.active_pages = 1
        self.receiver_pointers = {}
        self.active_pointers = defaultdict(set)
        self.cursor = cursor
        self.page_size = page_size

    def _garbage_collect(self):
        to_collect = self.active_pages - StreamManager.MAX_PAGES
        max_iters = 2
        current_iters = 0
        while to_collect > 0 and current_iters < max_iters:
            for k in self.pages:
                if len(self.active_pointers[k]) == 0:
                    page = self.pages[k]
                    page.garbage_collect()
                    self.active_pages -= 1
                    #dont want to be too aggressive here, we are guaranteeing
                    #max memory used and trying to keep network traffic
                    #to a minimum
                    to_collect -= 1
                    if to_collect == 0:
                        break
            current_iters += 1

    def _update_active_pointers(self, key, page_num):
        if page_num == 0:
            self.active_pointers[page_num].add(key)
        elif key not in self.active_pointers[page_num]:
            old_page = page_num - 1
            self.active_pointers[old_page].remove(key)
            self.active_pointers[page_num].add(key)

    def fetch_next(self):
        next_idx = self.pages.keys()[-1] + 1
        new_data, cursor_obj = self.cursor._fetch_next()
        page = Page(new_data['data'], cursor_obj)
        self.pages[next_idx] = page
        self.active_pages += 1
        return page

    def next(self, receiver):
        if not self.receiver_pointers.get(receiver.key):
            self.receiver_pointers[receiver.key] = 0
            self.active_pointers[0].add(receiver.key)

        current_idx = self.receiver_pointers[receiver.key]
        page_num = current_idx / self.page_size
        item = current_idx % self.page_size
        self.receiver_pointers[receiver.key] = current_idx + 1
        self._update_active_pointers(receiver.key, page_num)

        page = self.pages.get(page_num)
        if page is None:
            page = self.fetch_next()
            self._garbage_collect()
        if not page.is_active():
            self.reconstruct_page(page)
        return page.data[item]

    def reconstruct_page(self, page):
        page.reconstruct(self.cursor)
        self.active_pages += 1

class StreamResponseCursor(Cursor):
    def __init__(self, response, data, fetcher):
        self.response = response
        self.fetcher = fetcher
        self._raw_data = data
        self.page_size = len(data['data'])
        self.stream_info = StreamInfo(data['streams'])
        self.manager = StreamManager(self, data, self.page_size)

    def __iter__(self):
        streams = self.streams
        iterators = [(s, iter(s)) for s in streams]
        for stream, it in iterators:
            yield ((stream.device, stream.sensor), it.next())

    def _fetch_next(self):
        try:
            cursor_obj = self._raw_data['next_page']['next_query']
            new_data = self.fetcher(cursor_obj)
            self._raw_data = new_data
            return new_data, cursor_obj
        except KeyError:
            raise StopIteration
        except Exception:
            raise

    def bind_stream(self, **kwargs):
        stream_info = self.stream_info.get_one(**kwargs)
        return PointStream(stream_info, self.manager)

    @property
    def streams(self):
        return [PointStream(si, self.manager) for
                si in self.stream_info.headers]
<|endoftext|>


<commit_before>__version__ = '1.1'

import namespaces as ns

from tendrl.commons import objects
from tendrl.commons import flows
from tendrl.commons.objects import atoms
from tendrl.commons import etcdobj
from tendrl.commons import log

class CommonNS(object):
    def __init__(self):
        super(CommonNS, self).__init__()
        import __builtin__
        setattr(__builtin__, "tendrl_ns", self)

        ns_str = self.to_str.split(".")[-1]
        # Create the component namespace
        setattr(self, ns_str,
                ns.Namespace(objects=ns.Namespace(), flows=ns.Namespace()))
        self.register_subclasses_to_ns()

        ns_obj = getattr(self, ns_str)
        # Definitions
        self.definitions = ns_obj.objects.Definition()

        # Config
        self.config = ns_obj.objects.Config()

        # etcd_orm
        etcd_kwargs = {'port': self.config.data['etcd_port'],
                       'host': self.config.data["etcd_connection"]}
        self.etcd_orm = etcdobj.Server(etcd_kwargs=etcd_kwargs)

        # NodeContext
        self.node_context = ns_obj.objects.NodeContext()

        self.tendrl_context = ns_obj.objects.TendrlContext()

        log.setup_logging(
            self.config.data['log_cfg_path'],
        )

    def get_ns(self):
        # eg: input : "tendrl.node_agent", return: "node_agent"
        return getattr(self, self.to_str.split(".")[-1])

        # Create the "tendrl_ns.node_agent.objects.$obj.{atoms, flows} NS
    def add_object(self, name, obj_class):
        # obj is the actual instance of that Tendrl object
        # name of object as defined in Tendrl definitions
        self.get_ns().objects[name] = obj_class

        # This is to link atoms and flows (insdie obj) to the obj ns
        private_name = "_" + name
        self.get_ns().objects[private_name] = ns.Namespace()

        if 'atoms' not in self.get_ns().objects[private_name]:
            self.get_ns().objects[private_name]['atoms'] = ns.Namespace()

        if "flows" not in self.get_ns().objects[private_name]:
            self.get_ns().objects[private_name]['flows'] = ns.Namespace()

    def get_object(self, name):
        return self.get_ns().objects[name]

    def get_atom(self, obj_name, atom_name):
        private_name = "_" + obj_name
        return self.get_ns().objects[private_name]['atoms'][atom_name]

    def get_obj_flow(self, obj_name, flow_name):
        private_name = "_" + obj_name
        return self.get_ns().objects[private_name]['flows'][flow_name]

    def add_atom(self, obj_name, atom_name, atom_class):
        private_name = "_" + obj_name
        self.get_ns().objects[private_name]['atoms'][atom_name] = atom_class
        atom_class.obj = self.get_object(obj_name)

    def add_obj_flow(self, obj_name, flow_name, flow_class):
        private_name = "_" + obj_name
        self.get_ns().objects[private_name]['flows'][flow_name] = flow_class
        flow_class.obj = self.get_object(obj_name)

    def add_flow(self, name, flow_class):
        # flow is the actual instance of that Tendrl flow
        # name of object as defined in Tendrl definitions
        self.get_ns().flows[name] = flow_class

    def get_flow(self, name):
        return self.get_ns().flows[name]

    def register_subclasses_to_ns(self):
        # registers all subclasses of BaseObject, BaseFlow, BaseAtom to
        # tendrl_ns
        def _discover_subclasses(cls, _seen=None):
            if not isinstance(cls, type):
                raise TypeError('New-style class required' % cls)
            if _seen is None: _seen = set()
            try:
                subs = cls.__subclasses__()
            except TypeError:  # fails only when cls is type
                subs = cls.__subclasses__(cls)
            for sub in subs:
                if sub not in _seen:
                    _seen.add(sub)
                    yield sub
                    for sub in _discover_subclasses(sub, _seen):
                        yield sub
        for base_class in [objects.BaseObject, flows.BaseFlow, atoms.BaseAtom]:
            for sub_class in _discover_subclasses(base_class):
                base_name = base_class.__name__
                sub_name = sub_class.__name__
                if "Object" in base_name:
                    self.add_object(sub_name, sub_class)
                if "Atom" in base_name:
                    self.add_atom(sub_class.obj.__name__, sub_name, sub_class)
                if "Flow" in base_name:
                    if hasattr(sub_class, "obj"):
                        self.add_obj_flow(sub_class.obj.__name__, sub_name,
                                          sub_class)
                    else:
                        self.add_flow(sub_name, sub_class)<commit_msg>Update __init__.py<commit_after>__version__ = '1.2'

import namespaces as ns

from tendrl.commons import objects
from tendrl.commons import flows
from tendrl.commons.objects import atoms
from tendrl.commons import etcdobj
from tendrl.commons import log

class CommonNS(object):
    def __init__(self):
        super(CommonNS, self).__init__()
        import __builtin__
        setattr(__builtin__, "tendrl_ns", self)

        ns_str = self.to_str.split(".")[-1]
        # Create the component namespace
        setattr(self, ns_str,
                ns.Namespace(objects=ns.Namespace(), flows=ns.Namespace()))
        self.register_subclasses_to_ns()

        ns_obj = getattr(self, ns_str)
        # Definitions
        self.definitions = ns_obj.objects.Definition()

        # Config
        self.config = ns_obj.objects.Config()

        # etcd_orm
        etcd_kwargs = {'port': self.config.data['etcd_port'],
                       'host': self.config.data["etcd_connection"]}
        self.etcd_orm = etcdobj.Server(etcd_kwargs=etcd_kwargs)

        # NodeContext
        self.node_context = ns_obj.objects.NodeContext()

        self.tendrl_context = ns_obj.objects.TendrlContext()

        log.setup_logging(
            self.config.data['log_cfg_path'],
        )

    def get_ns(self):
        # eg: input : "tendrl.node_agent", return: "node_agent"
        return getattr(self, self.to_str.split(".")[-1])

        # Create the "tendrl_ns.node_agent.objects.$obj.{atoms, flows} NS
    def add_object(self, name, obj_class):
        # obj is the actual instance of that Tendrl object
        # name of object as defined in Tendrl definitions
        self.get_ns().objects[name] = obj_class

        # This is to link atoms and flows (insdie obj) to the obj ns
        private_name = "_" + name
        self.get_ns().objects[private_name] = ns.Namespace()

        if 'atoms' not in self.get_ns().objects[private_name]:
            self.get_ns().objects[private_name]['atoms'] = ns.Namespace()

        if "flows" not in self.get_ns().objects[private_name]:
            self.get_ns().objects[private_name]['flows'] = ns.Namespace()

    def get_object(self, name):
        return self.get_ns().objects[name]

    def get_atom(self, obj_name, atom_name):
        private_name = "_" + obj_name
        return self.get_ns().objects[private_name]['atoms'][atom_name]

    def get_obj_flow(self, obj_name, flow_name):
        private_name = "_" + obj_name
        return self.get_ns().objects[private_name]['flows'][flow_name]

    def add_atom(self, obj_name, atom_name, atom_class):
        private_name = "_" + obj_name
        self.get_ns().objects[private_name]['atoms'][atom_name] = atom_class
        atom_class.obj = self.get_object(obj_name)

    def add_obj_flow(self, obj_name, flow_name, flow_class):
        private_name = "_" + obj_name
        self.get_ns().objects[private_name]['flows'][flow_name] = flow_class
        flow_class.obj = self.get_object(obj_name)

    def add_flow(self, name, flow_class):
        # flow is the actual instance of that Tendrl flow
        # name of object as defined in Tendrl definitions
        self.get_ns().flows[name] = flow_class

    def get_flow(self, name):
        return self.get_ns().flows[name]

    def register_subclasses_to_ns(self):
        # registers all subclasses of BaseObject, BaseFlow, BaseAtom to
        # tendrl_ns
        def _discover_subclasses(cls, _seen=None):
            if not isinstance(cls, type):
                raise TypeError('New-style class required' % cls)
            if _seen is None: _seen = set()
            try:
                subs = cls.__subclasses__()
            except TypeError:  # fails only when cls is type
                subs = cls.__subclasses__(cls)
            for sub in subs:
                if sub not in _seen:
                    _seen.add(sub)
                    yield sub
                    for sub in _discover_subclasses(sub, _seen):
                        yield sub
        for base_class in [objects.BaseObject, flows.BaseFlow, atoms.BaseAtom]:
            for sub_class in _discover_subclasses(base_class):
                base_name = base_class.__name__
                sub_name = sub_class.__name__
                if "Object" in base_name:
                    self.add_object(sub_name, sub_class)
                if "Atom" in base_name:
                    self.add_atom(sub_class.obj.__name__, sub_name, sub_class)
                if "Flow" in base_name:
                    if hasattr(sub_class, "obj"):
                        self.add_obj_flow(sub_class.obj.__name__, sub_name,
                                          sub_class)
                    else:
                        self.add_flow(sub_name, sub_class)
<|endoftext|>


<commit_before># Copyright 2018 The TensorFlow Hub Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""TensorFlow Hub Library."""
# pylint: disable=g-import-not-at-top
# pylint: disable=g-statement-before-imports

# Ensure running under a supported version of Python.
def _ensure_python_version():
  """Raises ImportError if sys.version_info is too old."""
  import sys
  #
  # Update this whenever we need to depend on a newer Python version.
  #
  required_python_version = (3, 5)
  if sys.version_info[0:2] < required_python_version:
    raise ImportError(
        "This version of tensorflow_hub requires Python {required} or newer; "
        "instead detected version {present}".format(
            required=".".join(str(x) for x in required_python_version),
            present=sys.version))

_ensure_python_version()

# Ensure TensorFlow is importable and its version is sufficiently recent. This
# needs to happen before anything else, since the imports below will try to
# import tensorflow, too.
def _ensure_tf_install():
  """Attempt to import tensorflow, and ensure its version is sufficient.

  Raises:
    ImportError: if either tensorflow is not importable or its version is
    inadequate.
  """
  try:
    import tensorflow as tf
  except ImportError:
    # Print more informative error message, then reraise.
    print(
        "\n\nFailed to import tensorflow. Please note that tensorflow is not "
        "installed by default when you install tensorflow_hub. This is so that "
        "users can decide which tensorflow package to use. "
        "To use tensorflow_hub, please install a current version of tensorflow "
        "by following the instructions at https://tensorflow.org/install and "
        "https://tensorflow.org/hub/installation.\n\n")
    raise

  import distutils.version

  #
  # Update this whenever we need to depend on a newer TensorFlow release.
  #
  # NOTE: Put only numeric release versions here, like "1.2.3", and be aware
  # that they will also allow release candidates and even any nightly build
  # starting just after the previous release was cut. That's because
  # distutils.version.LooseVersion does not understand 'dev' and 'rc' tags;
  # it just does a lexicgraphic comparison after splitting on dots
  # and character class transitions.
  #
  required_tensorflow_version = "1.15.0"
  if (distutils.version.LooseVersion(tf.__version__) <
      distutils.version.LooseVersion(required_tensorflow_version)):
    raise ImportError(
        "\n\nThis version of tensorflow_hub requires tensorflow "
        "version >= {required}; Detected an installation of version {present}. "
        "To proceed, please upgrade tensorflow by following the instructions "
        "at https://tensorflow.org/install and "
        "https://tensorflow.org/hub/installation.\n\n".format(
            required=required_tensorflow_version,
            present=tf.__version__))

_ensure_tf_install()

from tensorflow_hub.estimator import LatestModuleExporter
from tensorflow_hub.estimator import register_module_for_export
from tensorflow_hub.feature_column import image_embedding_column
from tensorflow_hub.feature_column import sparse_text_embedding_column
from tensorflow_hub.feature_column import text_embedding_column
from tensorflow_hub.feature_column_v2 import text_embedding_column_v2
from tensorflow_hub.image_util import attach_image_module_info
from tensorflow_hub.image_util import get_expected_image_size
from tensorflow_hub.image_util import get_num_image_channels
from tensorflow_hub.image_util import ImageModuleInfo
from tensorflow_hub.keras_layer import KerasLayer
from tensorflow_hub.module import eval_function_for_module
from tensorflow_hub.module import load_module_spec
from tensorflow_hub.module import Module
from tensorflow_hub.module_spec import ModuleSpec
from tensorflow_hub.module_v2 import load
from tensorflow_hub.module_v2 import resolve
from tensorflow_hub.native_module import add_signature
from tensorflow_hub.native_module import attach_message
from tensorflow_hub.native_module import create_module_spec
from tensorflow_hub.saved_model_module import create_module_spec_from_saved_model
from tensorflow_hub.version import __version__

# pylint: disable=g-import-not-at-top
# pylint: disable=g-bad-import-order
from tensorflow_hub.config import _run
_run()

# The package `tensorflow_hub.tools` is available separately for import, but
# it is not meant to be available as attribute of the tensorflow_hub module.
from tensorflow_hub import tools
del tools
# pylint: enable=g-bad-import-order
# pylint: enable=g-import-not-at-top

# If __all__ is defined the doc generator script only documents the listed
# objects (__all__ defines which symbols you get with
# `from tensorflow_hub import *`).
__all__ = [
    "LatestModuleExporter",
    "register_module_for_export",
    "image_embedding_column",
    "sparse_text_embedding_column",
    "text_embedding_column",
    "text_embedding_column_v2",
    "attach_image_module_info",
    "get_expected_image_size",
    "get_num_image_channels",
    "ImageModuleInfo",
    "KerasLayer",
    "Module",
    "ModuleSpec",
    "add_signature",
    "attach_message",
    "create_module_spec",
    "create_module_spec_from_saved_model",
    "load",
    "load_module_spec",
    "resolve",
]
<commit_msg>Update __init__.py<commit_after># Copyright 2018 The TensorFlow Hub Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""TensorFlow Hub Library."""
# pylint: disable=g-import-not-at-top
# pylint: disable=g-statement-before-imports

# Ensure running under a supported version of Python.
def _ensure_python_version():
  """Raises ImportError if sys.version_info is too old."""
  import sys
  #
  # Update this whenever we need to depend on a newer Python version.
  #
  required_python_version = (3, 5)
  if sys.version_info[0:2] < required_python_version:
    raise ImportError(
        "This version of tensorflow_hub requires Python {required} or newer; "
        "instead detected version {present}".format(
            required=".".join(str(x) for x in required_python_version),
            present=sys.version))

_ensure_python_version()

# Ensure TensorFlow is importable and its version is sufficiently recent. This
# needs to happen before anything else, since the imports below will try to
# import tensorflow, too.
def _ensure_tf_install():
  """Attempt to import tensorflow, and ensure its version is sufficient.

  Raises:
    ImportError: if either tensorflow is not importable or its version is
    inadequate.
  """
  try:
    import tensorflow as tf
  except ImportError:
    # Print more informative error message, then reraise.
    print(
        "\n\nFailed to import tensorflow. Please note that tensorflow is not "
        "installed by default when you install tensorflow_hub. This is so that "
        "users can decide which tensorflow package to use. "
        "To use tensorflow_hub, please install a current version of tensorflow "
        "by following the instructions at https://tensorflow.org/install and "
        "https://tensorflow.org/hub/installation.\n\n")
    raise

  import distutils.version

  #
  # Update this whenever we need to depend on a newer TensorFlow release.
  #
  # NOTE: Put only numeric release versions here, like "1.2.3", and be aware
  # that they will also allow release candidates and even any nightly build
  # starting just after the previous release was cut. That's because
  # distutils.version.LooseVersion does not understand 'dev' and 'rc' tags;
  # it just does a lexicgraphic comparison after splitting on dots
  # and character class transitions.
  #
  required_tensorflow_version = "1.15.0"
  if (distutils.version.LooseVersion(tf.__version__) <
      distutils.version.LooseVersion(required_tensorflow_version)):
    raise ImportError(
        "\n\nThis version of tensorflow_hub requires tensorflow "
        "version >= {required}; Detected an installation of version {present}. "
        "To proceed, please upgrade tensorflow by following the instructions "
        "at https://tensorflow.org/install and "
        "https://tensorflow.org/hub/installation.\n\n".format(
            required=required_tensorflow_version,
            present=tf.__version__))

_ensure_tf_install()

from tensorflow_hub.estimator import LatestModuleExporter
from tensorflow_hub.estimator import register_module_for_export
from tensorflow_hub.feature_column import image_embedding_column
from tensorflow_hub.feature_column import sparse_text_embedding_column
from tensorflow_hub.feature_column import text_embedding_column
from tensorflow_hub.feature_column_v2 import text_embedding_column_v2
from tensorflow_hub.image_util import attach_image_module_info
from tensorflow_hub.image_util import get_expected_image_size
from tensorflow_hub.image_util import get_num_image_channels
from tensorflow_hub.image_util import ImageModuleInfo
from tensorflow_hub.keras_layer import KerasLayer
from tensorflow_hub.module import eval_function_for_module
from tensorflow_hub.module import load_module_spec
from tensorflow_hub.module import Module
from tensorflow_hub.module_spec import ModuleSpec
from tensorflow_hub.module_v2 import load
from tensorflow_hub.module_v2 import resolve
from tensorflow_hub.module_v2 import DISABLE_CERT_VALIDATION
from tensorflow_hub.native_module import add_signature
from tensorflow_hub.native_module import attach_message
from tensorflow_hub.native_module import create_module_spec
from tensorflow_hub.saved_model_module import create_module_spec_from_saved_model
from tensorflow_hub.version import __version__

# pylint: disable=g-import-not-at-top
# pylint: disable=g-bad-import-order
from tensorflow_hub.config import _run
_run()

# The package `tensorflow_hub.tools` is available separately for import, but
# it is not meant to be available as attribute of the tensorflow_hub module.
from tensorflow_hub import tools
del tools
# pylint: enable=g-bad-import-order
# pylint: enable=g-import-not-at-top

# If __all__ is defined the doc generator script only documents the listed
# objects (__all__ defines which symbols you get with
# `from tensorflow_hub import *`).
__all__ = [
    "LatestModuleExporter",
    "register_module_for_export",
    "image_embedding_column",
    "sparse_text_embedding_column",
    "text_embedding_column",
    "text_embedding_column_v2",
    "attach_image_module_info",
    "get_expected_image_size",
    "get_num_image_channels",
    "ImageModuleInfo",
    "KerasLayer",
    "Module",
    "ModuleSpec",
    "add_signature",
    "attach_message",
    "create_module_spec",
    "create_module_spec_from_saved_model",
    "DISABLE_CERT_VALIDATION",
    "load",
    "load_module_spec",
    "resolve",
]
<|endoftext|>


<commit_before>from zeit.cms.checkout.interfaces import ICheckinManager
from zeit.cms.i18n import MessageFactory as _
from zeit.cms.workflow.interfaces import IPublishInfo
import gocept.form.grouped
import zeit.cms.browser.form
import zeit.cms.interfaces
import zeit.cms.settings.interfaces
import zeit.content.article.article
import zeit.edit.interfaces
import zeit.push.interfaces
import zope.formlib.form
import zope.i18n
import zope.interface
import zope.schema

class IPushServices(zope.interface.Interface):

    mobile = zope.schema.Bool(
        title=_('breaking-news-mobile'), required=False, default=True)
    homepage = zope.schema.Bool(
        title=_('breaking-news-homepage'), required=False, default=True)
    social = zope.schema.Bool(
        title=_('breaking-news-social'), required=False, default=True)

class Add(zeit.cms.browser.form.AddForm,
          zeit.cms.browser.form.CharlimitMixin):

    factory = zeit.content.article.article.Article
    next_view = 'do-publish'

    form_fields = (
        zope.formlib.form.FormFields(
            zeit.content.article.interfaces.IArticle).select(
                'ressort', 'sub_ressort', '__name__') +
        zope.formlib.form.FormFields(
            zeit.content.article.interfaces.IBreakingNews).select('title') +
        zope.formlib.form.FormFields(
            zeit.content.article.edit.interfaces.IBreakingNewsBody) +
        zope.formlib.form.FormFields(
            zeit.content.article.interfaces.IArticle).select(
                'commentsAllowed', 'commentsPremoderate') +
        zope.formlib.form.FormFields(
            IPushServices)
    )

    field_groups = (
        gocept.form.grouped.Fields('', (
            'ressort', 'sub_ressort', 'title', '__name__', 'text',
            'commentsAllowed', 'commentsPremoderate',
            'homepage', 'mobile', 'social')),
    )

    def setUpWidgets(self, *args, **kw):
        super(Add, self).setUpWidgets(*args, **kw)
        self.set_charlimit('title')
        self.widgets['title'].cssClass = 'breakingnews-title'
        self.widgets['__name__'].cssClass = 'breakingnews-filename'
        if not self.widgets['text'].hasInput():
            self.widgets['text'].setRenderedValue(
                zope.i18n.translate(
                    self.form_fields['text'].field.default,
                    context=self.request))

    @zope.formlib.form.action(
        _('Publish and push'), condition=zope.formlib.form.haveInputWidgets)
    def handle_add(self, action, data):
        self.createAndAdd(data)

    def create(self, data):
        message_config = []
        if data.pop('mobile', False):
            source = zeit.push.interfaces.PAYLOAD_TEMPLATE_SOURCE.factory
            # XXX hard-coded value
            template = source.find('eilmeldung.json')
            message_config.append({
                'type': 'mobile', 'enabled': True,
                'title': source.getDefaultTitle(template),
                'payload_template': template.__name__,
            })
        if data.pop('homepage', False):
            message_config.append(
                {'type': 'homepage', 'enabled': True})
        if data.pop('social', False):
            message_config.append(
                {'type': 'facebook', 'enabled': True,
                 'override_text': data['title'],
                 'account': zeit.push.facebook.facebookAccountSource(
                     self.context).MAIN_ACCOUNT})
            message_config.append(
                {'type': 'twitter', 'enabled': True,
                 'account': zeit.push.twitter.twitterAccountSource(
                     self.context).MAIN_ACCOUNT})

        article = super(Add, self).create(data)
        # XXX Duplicated from .form.AddAndCheckout
        settings = zeit.cms.settings.interfaces.IGlobalSettings(
            self.context)
        article.year = settings.default_year
        article.volume = settings.default_volume

        article.is_amp = True

        push = zeit.push.interfaces.IPushMessages(article)
        push.short_text = data['title']
        push.message_config = message_config

        body = article.body
        body.create_item('image')
        if len(body) == 2:
            # The business rule is that each article should start with an image
            # block. Now IBreakingNewsBody caused a paragraph to be created
            # (which already happened in super), but since factories only
            # append at the end, we need to swap the blocks.
            body.updateOrder(tuple(reversed(body.keys())))

        zope.event.notify(zope.lifecycleevent.ObjectCreatedEvent(article))

        return article

    def add(self, object, container=None):
        super(Add, self).add(object, container)
        zeit.content.article.interfaces.IBreakingNews(
            self._created_object).is_breaking = True
        # We need to check out the new article so that AfterCheckout events are
        # run (which e.g. set default values of ICommonMetadata fields), but
        # the user won't want to edit anything right now, so we check in
        # immediately (and redirect to a view that triggers publishing).
        self._created_object = ICheckinManager(
            self._created_object).checkin()
        self._checked_out = False

        IPublishInfo(self._created_object).urgent = True

class Retract(object):

    @property
    def breakingnews(self):
        return zeit.content.article.interfaces.IBreakingNews(self.context)

    @property
    def is_breaking(self):
        return self.breakingnews.is_breaking

    @property
    def banner_published(self):
        return IPublishInfo(self.banner).published

    @property
    def banner_matches(self):
        return self.breakingnews.banner_matches(self.banner)

    @property
    def banner(self):
        notifier = zope.component.getUtility(
            zeit.push.interfaces.IPushNotifier, name='homepage')
        return zeit.cms.interfaces.ICMSContent(notifier.uniqueId)
<commit_msg>Simplify spelling<commit_after>from zeit.cms.checkout.interfaces import ICheckinManager
from zeit.cms.i18n import MessageFactory as _
from zeit.cms.workflow.interfaces import IPublishInfo
import gocept.form.grouped
import zeit.cms.browser.form
import zeit.cms.interfaces
import zeit.cms.settings.interfaces
import zeit.content.article.article
import zeit.edit.interfaces
import zeit.push.interfaces
import zope.formlib.form
import zope.i18n
import zope.interface
import zope.schema

class IPushServices(zope.interface.Interface):

    mobile = zope.schema.Bool(
        title=_('breaking-news-mobile'), required=False, default=True)
    homepage = zope.schema.Bool(
        title=_('breaking-news-homepage'), required=False, default=True)
    social = zope.schema.Bool(
        title=_('breaking-news-social'), required=False, default=True)

class Add(zeit.cms.browser.form.AddForm,
          zeit.cms.browser.form.CharlimitMixin):

    factory = zeit.content.article.article.Article
    next_view = 'do-publish'

    form_fields = (
        zope.formlib.form.FormFields(
            zeit.content.article.interfaces.IArticle).select(
                '__name__', 'ressort', 'sub_ressort',
                'commentsAllowed', 'commentsPremoderate') +
        zope.formlib.form.FormFields(
            zeit.content.article.interfaces.IBreakingNews).select('title') +
        zope.formlib.form.FormFields(
            zeit.content.article.edit.interfaces.IBreakingNewsBody) +
        zope.formlib.form.FormFields(
            IPushServices)
    )

    field_groups = (
        gocept.form.grouped.Fields('', (
            'ressort', 'sub_ressort', 'title', '__name__', 'text',
            'commentsAllowed', 'commentsPremoderate',
            'homepage', 'mobile', 'social')),
    )

    def setUpWidgets(self, *args, **kw):
        super(Add, self).setUpWidgets(*args, **kw)
        self.set_charlimit('title')
        self.widgets['title'].cssClass = 'breakingnews-title'
        self.widgets['__name__'].cssClass = 'breakingnews-filename'
        if not self.widgets['text'].hasInput():
            self.widgets['text'].setRenderedValue(
                zope.i18n.translate(
                    self.form_fields['text'].field.default,
                    context=self.request))

    @zope.formlib.form.action(
        _('Publish and push'), condition=zope.formlib.form.haveInputWidgets)
    def handle_add(self, action, data):
        self.createAndAdd(data)

    def create(self, data):
        message_config = []
        if data.pop('mobile', False):
            source = zeit.push.interfaces.PAYLOAD_TEMPLATE_SOURCE.factory
            # XXX hard-coded value
            template = source.find('eilmeldung.json')
            message_config.append({
                'type': 'mobile', 'enabled': True,
                'title': source.getDefaultTitle(template),
                'payload_template': template.__name__,
            })
        if data.pop('homepage', False):
            message_config.append(
                {'type': 'homepage', 'enabled': True})
        if data.pop('social', False):
            message_config.append(
                {'type': 'facebook', 'enabled': True,
                 'override_text': data['title'],
                 'account': zeit.push.facebook.facebookAccountSource(
                     self.context).MAIN_ACCOUNT})
            message_config.append(
                {'type': 'twitter', 'enabled': True,
                 'account': zeit.push.twitter.twitterAccountSource(
                     self.context).MAIN_ACCOUNT})

        article = super(Add, self).create(data)
        # XXX Duplicated from .form.AddAndCheckout
        settings = zeit.cms.settings.interfaces.IGlobalSettings(
            self.context)
        article.year = settings.default_year
        article.volume = settings.default_volume

        article.is_amp = True

        push = zeit.push.interfaces.IPushMessages(article)
        push.short_text = data['title']
        push.message_config = message_config

        body = article.body
        body.create_item('image')
        if len(body) == 2:
            # The business rule is that each article should start with an image
            # block. Now IBreakingNewsBody caused a paragraph to be created
            # (which already happened in super), but since factories only
            # append at the end, we need to swap the blocks.
            body.updateOrder(tuple(reversed(body.keys())))

        zope.event.notify(zope.lifecycleevent.ObjectCreatedEvent(article))

        return article

    def add(self, object, container=None):
        super(Add, self).add(object, container)
        zeit.content.article.interfaces.IBreakingNews(
            self._created_object).is_breaking = True
        # We need to check out the new article so that AfterCheckout events are
        # run (which e.g. set default values of ICommonMetadata fields), but
        # the user won't want to edit anything right now, so we check in
        # immediately (and redirect to a view that triggers publishing).
        self._created_object = ICheckinManager(
            self._created_object).checkin()
        self._checked_out = False

        IPublishInfo(self._created_object).urgent = True

class Retract(object):

    @property
    def breakingnews(self):
        return zeit.content.article.interfaces.IBreakingNews(self.context)

    @property
    def is_breaking(self):
        return self.breakingnews.is_breaking

    @property
    def banner_published(self):
        return IPublishInfo(self.banner).published

    @property
    def banner_matches(self):
        return self.breakingnews.banner_matches(self.banner)

    @property
    def banner(self):
        notifier = zope.component.getUtility(
            zeit.push.interfaces.IPushNotifier, name='homepage')
        return zeit.cms.interfaces.ICMSContent(notifier.uniqueId)
<|endoftext|>


<commit_before>import mock
import unittest2

from oslo_config import cfg

from st2reactor.container.sensor_wrapper import SensorService

# This trigger has schema that uses all property types
TEST_SCHEMA = {
    'type': 'object',
    'additionalProperties': False,
    'properties': {
        'age': {'type': 'integer'},
        'name': {'type': 'string', 'required': True},
        'address': {'type': 'string', 'default': '-'},
        'career': {'type': 'array'},
        'married': {'type': 'boolean'},
        'awards': {'type': 'object'},
        'income': {'anyOf': [{'type': 'integer'}, {'type': 'string'}]},
    },
}

class TriggerTypeMock(object):
    def __init__(self, schema={}):
        self.payload_schema = schema

class SensorServiceTestCase(unittest2.TestCase):
    def setUp(self):
        def side_effect(trigger, payload, trace_context):
            self._dispatched_count += 1

        self.sensor_service = SensorService(mock.MagicMock())
        self.sensor_service._dispatcher = mock.Mock()
        self.sensor_service._dispatcher.dispatch = mock.MagicMock(side_effect=side_effect)
        self._dispatched_count = 0

        # reset default configuration value
        cfg.CONF.sensorcontainer.permit_invalid_payload = True

    @mock.patch('st2common.services.triggers.get_trigger_type_db',
                mock.MagicMock(return_value=TriggerTypeMock(TEST_SCHEMA)))
    def test_dispatch_success(self):
        # define a valid payload
        payload = {
            'name': 'John Doe',
            'age': 25,
            'career': ['foo, Inc.', 'bar, Inc.'],
            'married': True,
            'awards': {'2016': ['hoge prize', 'fuga prize']},
            'income': 50000,
            'country': 'US',
        }

        # dispatching a trigger
        self.sensor_service.dispatch('trigger-name', payload)

        # This assumed that the target tirgger dispatched
        self.assertEqual(self._dispatched_count, 1)

    @mock.patch('st2common.services.triggers.get_trigger_type_db',
                mock.MagicMock(return_value=TriggerTypeMock(TEST_SCHEMA)))
    def test_dispatch_failure_caused_by_incorrect_type(self):
        # define a invalid payload (the type of 'age' is incorrect)
        payload = {
            'name': 'John Doe',
            'age': '25',
        }

        # set config to stop dispatching when the payload comply with target trigger_type
        cfg.CONF.sensorcontainer.permit_invalid_payload = False

        self.sensor_service.dispatch('trigger-name', payload)

        # This assumed that the target tirgger doesn't dispatched
        self.assertEqual(self._dispatched_count, 0)

    @mock.patch('st2common.services.triggers.get_trigger_type_db',
                mock.MagicMock(return_value=TriggerTypeMock(TEST_SCHEMA)))
    def test_dispatch_failure_caused_by_lack_of_parameter(self):
        # define a invalid payload (lack of required property)
        payload = {
            'age': 25,
        }
        cfg.CONF.sensorcontainer.permit_invalid_payload = False

        self.sensor_service.dispatch('trigger-name', payload)
        self.assertEqual(self._dispatched_count, 0)

        # reset config to permit force dispatching
        cfg.CONF.sensorcontainer.permit_invalid_payload = True

        self.sensor_service.dispatch('trigger-name', payload)
        self.assertEqual(self._dispatched_count, 1)

    @mock.patch('st2common.services.triggers.get_trigger_type_db',
                mock.MagicMock(return_value=TriggerTypeMock(TEST_SCHEMA)))
    def test_dispatch_failure_caused_by_too_much_parameter(self):
        # define a invalid payload ('hobby' is extra)
        payload = {
            'name': 'John Doe',
            'hobby': 'programming',
        }
        cfg.CONF.sensorcontainer.permit_invalid_payload = False

        self.sensor_service.dispatch('trigger-name', payload)
        self.assertEqual(self._dispatched_count, 0)

    @mock.patch('st2common.services.triggers.get_trigger_type_db',
                mock.MagicMock(return_value=TriggerTypeMock(TEST_SCHEMA)))
    def test_dispatch_success_with_multiple_type_value(self):
        payload = {
            'name': 'John Doe',
            'income': 1234,
        }

        cfg.CONF.sensorcontainer.permit_invalid_payload = False

        self.sensor_service.dispatch('trigger-name', payload)

        # reset payload which can have different type
        payload['income'] = 'secret'

        self.sensor_service.dispatch('trigger-name', payload)
        self.assertEqual(self._dispatched_count, 2)

    @mock.patch('st2common.services.triggers.get_trigger_type_db',
                mock.MagicMock(return_value=TriggerTypeMock(TEST_SCHEMA)))
    def test_dispatch_success_with_null(self):
        payload = {
            'name': 'John Doe',
            'age': None,
        }

        cfg.CONF.sensorcontainer.permit_invalid_payload = False

        self.sensor_service.dispatch('trigger-name', payload)
        self.assertEqual(self._dispatched_count, 1)

    @mock.patch('st2common.services.triggers.get_trigger_type_db',
                mock.MagicMock(return_value=TriggerTypeMock()))
    def test_dispatch_success_without_payload_schema(self):
        # the case trigger has no property
        self.sensor_service.dispatch('trigger-name', {})
        self.assertEqual(self._dispatched_count, 1)

    @mock.patch('st2common.services.triggers.get_trigger_type_db',
                mock.MagicMock(return_value=None))
    def test_dispatch_success_without_trigger_type(self):
        self.sensor_service.dispatch('trigger-name', {})
        self.assertEqual(self._dispatched_count, 1)

    @mock.patch('st2common.services.triggers.get_trigger_type_db',
                mock.MagicMock(return_value=None))
    def test_dispatch_failure_without_trigger_ref(self):
        cfg.CONF.sensorcontainer.permit_invalid_payload = False

        self.sensor_service.dispatch('', {})
        self.assertEqual(self._dispatched_count, 0)
<commit_msg>Update affected tests.<commit_after>import mock
import unittest2

from oslo_config import cfg

from st2reactor.container.sensor_wrapper import SensorService

# This trigger has schema that uses all property types
TEST_SCHEMA = {
    'type': 'object',
    'additionalProperties': False,
    'properties': {
        'age': {'type': 'integer'},
        'name': {'type': 'string', 'required': True},
        'address': {'type': 'string', 'default': '-'},
        'career': {'type': 'array'},
        'married': {'type': 'boolean'},
        'awards': {'type': 'object'},
        'income': {'anyOf': [{'type': 'integer'}, {'type': 'string'}]},
    },
}

class TriggerTypeMock(object):
    def __init__(self, schema={}):
        self.payload_schema = schema

class SensorServiceTestCase(unittest2.TestCase):
    def setUp(self):
        def side_effect(trigger, payload, trace_context):
            self._dispatched_count += 1

        self.sensor_service = SensorService(mock.MagicMock())
        self.sensor_service._dispatcher = mock.Mock()
        self.sensor_service._dispatcher.dispatch = mock.MagicMock(side_effect=side_effect)
        self._dispatched_count = 0

        # reset default configuration value
        cfg.CONF.system.validate_trigger_parameters = False

    @mock.patch('st2common.services.triggers.get_trigger_type_db',
                mock.MagicMock(return_value=TriggerTypeMock(TEST_SCHEMA)))
    def test_dispatch_success(self):
        # define a valid payload
        payload = {
            'name': 'John Doe',
            'age': 25,
            'career': ['foo, Inc.', 'bar, Inc.'],
            'married': True,
            'awards': {'2016': ['hoge prize', 'fuga prize']},
            'income': 50000,
            'country': 'US',
        }

        # dispatching a trigger
        self.sensor_service.dispatch('trigger-name', payload)

        # This assumed that the target tirgger dispatched
        self.assertEqual(self._dispatched_count, 1)

    @mock.patch('st2common.services.triggers.get_trigger_type_db',
                mock.MagicMock(return_value=TriggerTypeMock(TEST_SCHEMA)))
    def test_dispatch_failure_caused_by_incorrect_type(self):
        # define a invalid payload (the type of 'age' is incorrect)
        payload = {
            'name': 'John Doe',
            'age': '25',
        }

        # set config to stop dispatching when the payload comply with target trigger_type
        cfg.CONF.system.validate_trigger_parameters = True

        self.sensor_service.dispatch('trigger-name', payload)

        # This assumed that the target trigger isn't dispatched
        self.assertEqual(self._dispatched_count, 0)

        # reset config to permit force dispatching
        cfg.CONF.system.validate_trigger_parameters = False

        self.sensor_service.dispatch('trigger-name', payload)
        self.assertEqual(self._dispatched_count, 1)

    @mock.patch('st2common.services.triggers.get_trigger_type_db',
                mock.MagicMock(return_value=TriggerTypeMock(TEST_SCHEMA)))
    def test_dispatch_failure_caused_by_lack_of_required_parameter(self):
        # define a invalid payload (lack of required property)
        payload = {
            'age': 25,
        }
        cfg.CONF.system.validate_trigger_parameters = True

        self.sensor_service.dispatch('trigger-name', payload)
        self.assertEqual(self._dispatched_count, 0)

        # reset config to permit force dispatching
        cfg.CONF.system.validate_trigger_parameters = False

        self.sensor_service.dispatch('trigger-name', payload)
        self.assertEqual(self._dispatched_count, 1)

    @mock.patch('st2common.services.triggers.get_trigger_type_db',
                mock.MagicMock(return_value=TriggerTypeMock(TEST_SCHEMA)))
    def test_dispatch_failure_caused_by_extra_parameter(self):
        # define a invalid payload ('hobby' is extra)
        payload = {
            'name': 'John Doe',
            'hobby': 'programming',
        }
        cfg.CONF.system.validate_trigger_parameters = True

        self.sensor_service.dispatch('trigger-name', payload)
        self.assertEqual(self._dispatched_count, 0)

    @mock.patch('st2common.services.triggers.get_trigger_type_db',
                mock.MagicMock(return_value=TriggerTypeMock(TEST_SCHEMA)))
    def test_dispatch_success_with_multiple_type_value(self):
        payload = {
            'name': 'John Doe',
            'income': 1234,
        }

        cfg.CONF.system.validate_trigger_parameters = True

        self.sensor_service.dispatch('trigger-name', payload)

        # reset payload which can have different type
        payload['income'] = 'secret'

        self.sensor_service.dispatch('trigger-name', payload)
        self.assertEqual(self._dispatched_count, 2)

    @mock.patch('st2common.services.triggers.get_trigger_type_db',
                mock.MagicMock(return_value=TriggerTypeMock(TEST_SCHEMA)))
    def test_dispatch_success_with_null(self):
        payload = {
            'name': 'John Doe',
            'age': None,
        }

        cfg.CONF.system.validate_trigger_parameters = True

        self.sensor_service.dispatch('trigger-name', payload)
        self.assertEqual(self._dispatched_count, 1)

    @mock.patch('st2common.services.triggers.get_trigger_type_db',
                mock.MagicMock(return_value=TriggerTypeMock()))
    def test_dispatch_success_without_payload_schema(self):
        # the case trigger has no property
        self.sensor_service.dispatch('trigger-name', {})
        self.assertEqual(self._dispatched_count, 1)

    @mock.patch('st2common.services.triggers.get_trigger_type_db',
                mock.MagicMock(return_value=None))
    def test_dispatch_success_without_trigger_type(self):
        self.sensor_service.dispatch('trigger-name', {})
        self.assertEqual(self._dispatched_count, 1)

    @mock.patch('st2common.services.triggers.get_trigger_type_db',
                mock.MagicMock(return_value=None))
    def test_dispatch_trigger__type_not_in_db_should_still_dispatch(self):
        cfg.CONF.system.validate_trigger_parameters = True

        self.sensor_service.dispatch('not-in-database-ref', {})
        self.assertEqual(self._dispatched_count, 1)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

from __future__ import absolute_import, division, print_function
from __future__ import unicode_literals

import nose
from nose.tools import *
from sknano.generators import BilayerGrapheneGenerator

def test1():
    blg = BilayerGrapheneGenerator(length=1, width=1, layer_rotation_angle=45)

if __name__ == '__main__':
    nose.runmodule()
<commit_msg>Updating unittest for BilayerGrapheneGenerator.<commit_after># -*- coding: utf-8 -*-

from __future__ import absolute_import, division, print_function
from __future__ import unicode_literals

import nose
from nose.tools import *
from sknano.generators import BilayerGrapheneGenerator
from sknano.testing import GeneratorTestFixtures

class TestCase(GeneratorTestFixtures):

    def test1(self):
        generator = BilayerGrapheneGenerator(length=1, width=1,
                                             layer_rotation_angle=45)
        generator.save_data()
        self.tmpdata.append(generator.fname)

if __name__ == '__main__':
    nose.runmodule()
<|endoftext|>