<commit_before>#author: Tobias Andermann, tobias.andermann@bioenv.gu.se

'''
Clean and trim raw Illumina read files
'''

import os
import sys
import glob
import csv
import shutil
import subprocess
import numpy as np
import pandas as pd
from secapr.helpers import FullPaths

def add_arguments(parser):
    parser.add_argument(
        '--input',
        required=True,
        action=FullPaths,
        default=None,
        help='The directory containing the .fastq or .fq files (raw read files). Files can be zipped or unzipped.'
    )
    parser.add_argument(
        '--sample_annotation_file',
        required=True,
        action=FullPaths,
        default=None,
        help='A simple comma-delimited text file containing the sample names in the first column and the name-stem of the corresponding fastq files in the second column (file should not have any column headers).'
    )
    parser.add_argument(
        '--output',
        required=True,
        action=FullPaths,
        default=None,
        help='The output directory where the cleaned reads will be stored.'
    )
    parser.add_argument(
        '--read_min',
        type=int,
        default=400000,
        help='Set the minimum read count threshold. Any sample with fewer reads than this minimum threshold will not be processed further. Default: %(default)s'
    )
    parser.add_argument(
        '--qualified_quality_phred',
        type=int,
        default=20,
        help='Specifies how accurate the match between any adapter etc. sequence must be against a read. For more information see trimmoatic tutorial. Default: %(default)s'
    )
    parser.add_argument(
        '--unqualified_percent_limit',
        type=int,
        default=40,
        help='Set the maximum percent of low-quality nucleotides allowed. Any read with a higher percentage of unqualified (low quality) nucleotides will be discarded. Default: %(default)s'
    )
    parser.add_argument(
        '--cut_window_size',
        type=int,
        default=5,
        help='Set the size of the moving window (in nucleotides) for quality trimming. The window will start moving from the end toward the beginning of the read, applying the quality threshold set with the --cut_mean_quality flag. Default: %(default)s'
    )
    parser.add_argument(
        '--cut_mean_quality',
        type=int,
        default=20,
        help='Set quality threshold for moving window. If the mean quality across the window drops below this threshold, the nucleotides within the window are removed (cut), as well as all trailing nucleotides. Default: %(default)s'
    )
    parser.add_argument(
        '--trim_front',
        type=int,
        default=0,
        help='Remove this number of nucleotides from the beginning of each read. Default: %(default)s'
    )
    parser.add_argument(
        '--trim_tail',
        type=int,
        default=0,
        help='Remove this number of nucleotides from the end of each read. Default: %(default)s'
    )
    parser.add_argument(
        '--required_read_length',
        type=int,
        default=0,
        help='Set this value to only allow reads to pass which are equal to or longer than this threshold. Default: %(default)s'
    )
    parser.add_argument(
        '--disable_complexity_filter',
        action='store_true',
        default=False,
        help='Use this flag if you want to disable the removal of low-complexity reads, e.g. AAAAAAACCCCCCCCAAAAAAAAAAAAAAAAAAGGGGG (activated by default).'
    )
    parser.add_argument(
        '--complexity_threshold',
        type=int,
        default=10,
        help='Reads below this complexity threshold will be removed (consult fastp manual for more explanation). Default: %(default)s'
    )
    parser.add_argument(
        '--disable_poly_g_trimming',
        action='store_true',
        default=False,
        help='Use this flag if you want to disable trimming of G repeats at the end of the read. Poly-G read ends are common when working with very short fragments. (activated by default).'
    )
    parser.add_argument(
        '--poly_g_min_len',
        type=int,
        default=7,
        help='Specifies the length of the nucleotide repeat region at end of read to be trimmed. Default: %(default)s'
    )
    parser.add_argument(
        '--disable_poly_x_trimming',
        action='store_true',
        default=False,
        help='Use this flag if you want to disable trimming of nucleotide repeats of any kind at the end of the read, e.g. poly-A tails (activated by default).'
    )
    parser.add_argument(
        '--poly_x_min_len',
        type=int,
        default=7,
        help='Specifies the length of the nucleotide repeat region at end of read to be trimmed. Default: %(default)s'
    )

def longest_common_substring(s1, s2):
    m = [[0] * (1 + len(s2)) for i in range(1 + len(s1))]
    longest, x_longest = 0, 0
    for x in range(1, 1 + len(s1)):
        for y in range(1, 1 + len(s2)):
            if s1[x - 1] == s2[y - 1]:
                m[x][y] = m[x - 1][y - 1] + 1
                if m[x][y] > longest:
                    longest = m[x][y]
                    x_longest = x
            else:
                m[x][y] = 0
    return s1[x_longest - longest: x_longest]

def read_count(input):
    # Use bash command to count the read number
    number_reads = subprocess.getstatusoutput("gzip -cd %s | wc -l | awk '{ print $1/4 }' " %input)[1]
    num = int(number_reads)
    return num

def quality_trim(read1_path,read2_path,sample_id,out_dir,args,pair_index=0):
    # Get all user settings
    qualified_quality_phred = args.qualified_quality_phred
    unqualified_percent_limit = args.unqualified_percent_limit
    cut_window_size = args.cut_window_size
    cut_mean_quality = args.cut_mean_quality
    trim_front = args.trim_front
    trim_tail = args.trim_tail
    required_read_length = args.required_read_length
    disable_complexity_filter = args.disable_complexity_filter
    complexity_threshold = args.complexity_threshold
    disable_poly_g_trimming = args.disable_poly_g_trimming
    poly_g_min_len = args.poly_g_min_len
    disable_poly_x_trimming = args.disable_poly_x_trimming
    poly_x_min_len = args.poly_x_min_len
    #merge_overlapping_reads = args.merge_overlapping_reads
    #! allow for manual addition of adapters
    # Start processing
    print(('#' * 50))
    print(("Processing %s...\n" %sample_id))
    # Forward and backward read file paths
    # Names of output files
    output_sample_dir = os.path.join(out_dir,sample_id)
    if not os.path.exists(output_sample_dir):
        os.makedirs(output_sample_dir)
    outpath_r1 = os.path.join(output_sample_dir, '%s_%i_clean-READ1.fastq.gz'%(sample_id,pair_index))
    outpath_r2 = os.path.join(output_sample_dir, '%s_%i_clean-READ2.fastq.gz'%(sample_id,pair_index))
    outpath_unpaired = os.path.join(output_sample_dir, '%s_%i_clean-unpaired.fastq.gz' % (sample_id, pair_index))
    # Command for fastp trimming and cleaning
    command1 = [
        "fastp",
        "--in1",
        read1_path,
        "--in2",
        read2_path,
        "--out1",
        outpath_r1,
        "--out2",
        outpath_r2,
        "--unpaired1",
        outpath_unpaired,
        "--unpaired2",
        outpath_unpaired,
        "-h",
        os.path.join(output_sample_dir,'%s_%i_fastp.html'%(sample_id,pair_index)),
        "-j",
        os.path.join(output_sample_dir,'%s_%i_fastp.json'%(sample_id,pair_index)),
        "--qualified_quality_phred",
        qualified_quality_phred,
        "--unqualified_percent_limit",
        unqualified_percent_limit,
        "--cut_tail",
        "--cut_window_size",
        cut_window_size,
        "--cut_mean_quality",
        cut_mean_quality,
        "--trim_front1",
        trim_front,
        "--trim_tail1",
        trim_tail
    ]
    if required_read_length == 0:
        command1.append("--disable_length_filtering")
    else:
        command1 += ["--length_required", required_read_length]
    if not disable_complexity_filter:
        command1.append("--low_complexity_filter")
        command1 += ["--complexity_threshold", complexity_threshold]
    if not disable_poly_g_trimming:
        command1.append("--trim_poly_g")
        command1 += ["--poly_g_min_len", poly_g_min_len]
    if not disable_poly_x_trimming:
        command1.append("--trim_poly_x")
        command1 += ["--poly_x_min_len",poly_x_min_len]
    command1 = list(np.array(command1).astype(str))
    stats_file = os.path.join(output_sample_dir,'fastp_out_%i.txt'%pair_index)
    with open(stats_file, 'w') as log_err_file:
        p1 = subprocess.Popen(command1, stderr=log_err_file)
        p1.communicate()
        print("%s successfully cleaned and trimmed. Stats are printed into %s" %(sample_id, stats_file))
        print("#" * 50)
    return(output_sample_dir)

def main(args):
    # Set working directory
    work_dir = args.input
    out_dir = args.output
    # Print the user-set read-threshold
    read_threshold = args.read_min
    print(("\n\n[Info:] Samples with a total read-count of less than %d (forward + reverse reads) are not being processed. If required you can set a different threshold, using the --read_min flag.\n" %read_threshold))

    # Read txt-file with new sample names
    name_info_file = args.sample_annotation_file
    name_info = pd.read_csv(name_info_file,header=None)
    name_info_dict = dict(name_info.values)
    # flip keys and values in dict for easier lookup of file names
    name_info_dict = {name_info_dict[k]:k for k in name_info_dict}

    # Get all fastq files belonging to each name (can be multiple runs)
    included_extenstions = ['fastq','fq','fastq.gz','fq.gz']
    file_list = [fn for fn in sorted(os.listdir(work_dir)) if any([fn.endswith(ext) for ext in included_extenstions])]
    if len(file_list) == 0:
        sys.exit('SECAPR-ERROR: No FASTQ files were found. Check if correct path is provided for --input flag')
    sample_list = [[name_info_dict[name_stem],i] for name_stem in name_info_dict.keys() for i in file_list if i.startswith(name_stem)]
    sample_filename_dict = {}
    for sample_id,filename in sample_list:
        sample_filename_dict.setdefault(sample_id,[])
        sample_filename_dict[sample_id].append(filename)
    final_sample_filename_dict = {}
    for sample_id,filenamelist in sample_filename_dict.items():
        if len(filenamelist)>2: # if more than two files per sample, make sure to identify the pairs correctly
            shared_stems = []
            for i in filenamelist:
                longest_common_substrings = [longest_common_substring(i,j) for j in filenamelist]
                selected_longest_common_substrings = [i for i in longest_common_substrings if not any([i.endswith(ext) for ext in included_extenstions])]
                longest_common_substrings_length = np.array([len(i) for i in selected_longest_common_substrings])
                best_match = np.sort(longest_common_substrings_length)[-1] # pick the longest, because the match with itself is already filtered out in previous step
                shared_file_stem = selected_longest_common_substrings[np.where(longest_common_substrings_length==best_match)[0][0]]
                shared_stems.append(shared_file_stem)
            paired_filename_list = []
            for namestem in np.unique(shared_stems):
                paired_filename_list.append([os.path.join(work_dir,i) for i in filenamelist if i.startswith(namestem)])
        else:
            paired_filename_list = [os.path.join(work_dir,i) for i in filenamelist]
        final_sample_filename_dict.setdefault(sample_id,paired_filename_list)

    # Count total number of reads of a sample
    final_sample_list = []
    raw_read_counts = []
    for sample_id, fastq_files in final_sample_filename_dict.items():
        print("%s: Counting all reads (forward + reverse) belonging to this sample..."%sample_id)
        fastq_files = list(np.array(fastq_files).flatten())
        raw_total_read_count = sum([read_count(i) for i in fastq_files])
        print(raw_total_read_count)
        if raw_total_read_count >= read_threshold:
            final_sample_list.append(sample_id)
            raw_read_counts.append(raw_total_read_count)
        else:
            print('***Not enough reads found for sample %s. Excluding this sample from further processing. Adjust the --read_min flag to set lower minimum read count threshold.***' %sample_id )
            pass

    # Clean and trim with fastp
    #! Allow manual input of adapter sequences
    #! Allow merging of reads
    clean_reads_outdirs = {}
    for sample_id in final_sample_list:
        read_files = final_sample_filename_dict[sample_id]
        if len(np.array(read_files).flatten()) == 2:
            sample_outdir = quality_trim(read_files[0],read_files[1],sample_id,out_dir,args,pair_index=0)
        else:
            for i,read_pair in enumerate(read_files):
                sample_outdir = quality_trim(read_pair[0], read_pair[1], sample_id, out_dir, args, pair_index=i)
        clean_reads_outdirs.setdefault(sample_id,sample_outdir)

    # Count all remaining clean reads per sample
    clean_read_counts = []
    for sample_id in final_sample_list:
        sample_outdir = clean_reads_outdirs[sample_id]
        print("%s: Counting remaining clean reads (forward + reverse) belonging to this sample..."%sample_id)
        fastq_files = glob.glob(os.path.join(sample_outdir,"*.fastq.gz"))
        cleaned_total_read_count = sum([read_count(i) for i in fastq_files])
        print(cleaned_total_read_count)
        clean_read_counts.append(cleaned_total_read_count)

    stats_df = pd.DataFrame(np.vstack([final_sample_list,raw_read_counts,clean_read_counts]).T,columns = ['sample_id','raw_reads_total','cleaned_reads_total'])
    stats_df.to_csv(os.path.join(out_dir,'sample_stats.txt'),sep = '\t',index=False)

<commit_msg>fixed longest common substring function<commit_after>#author: Tobias Andermann, tobias.andermann@bioenv.gu.se

'''
Clean and trim raw Illumina read files
'''

import os
import sys
import glob
import csv
import shutil
import subprocess
import numpy as np
import pandas as pd
from secapr.helpers import FullPaths

def add_arguments(parser):
    parser.add_argument(
        '--input',
        required=True,
        action=FullPaths,
        default=None,
        help='The directory containing the .fastq or .fq files (raw read files). Files can be zipped or unzipped.'
    )
    parser.add_argument(
        '--sample_annotation_file',
        required=True,
        action=FullPaths,
        default=None,
        help='A simple comma-delimited text file containing the sample names in the first column and the name-stem of the corresponding fastq files in the second column (file should not have any column headers).'
    )
    parser.add_argument(
        '--output',
        required=True,
        action=FullPaths,
        default=None,
        help='The output directory where the cleaned reads will be stored.'
    )
    parser.add_argument(
        '--read_min',
        type=int,
        default=400000,
        help='Set the minimum read count threshold. Any sample with fewer reads than this minimum threshold will not be processed further. Default: %(default)s'
    )
    parser.add_argument(
        '--qualified_quality_phred',
        type=int,
        default=20,
        help='Specifies how accurate the match between any adapter etc. sequence must be against a read. For more information see trimmoatic tutorial. Default: %(default)s'
    )
    parser.add_argument(
        '--unqualified_percent_limit',
        type=int,
        default=40,
        help='Set the maximum percent of low-quality nucleotides allowed. Any read with a higher percentage of unqualified (low quality) nucleotides will be discarded. Default: %(default)s'
    )
    parser.add_argument(
        '--cut_window_size',
        type=int,
        default=5,
        help='Set the size of the moving window (in nucleotides) for quality trimming. The window will start moving from the end toward the beginning of the read, applying the quality threshold set with the --cut_mean_quality flag. Default: %(default)s'
    )
    parser.add_argument(
        '--cut_mean_quality',
        type=int,
        default=20,
        help='Set quality threshold for moving window. If the mean quality across the window drops below this threshold, the nucleotides within the window are removed (cut), as well as all trailing nucleotides. Default: %(default)s'
    )
    parser.add_argument(
        '--trim_front',
        type=int,
        default=0,
        help='Remove this number of nucleotides from the beginning of each read. Default: %(default)s'
    )
    parser.add_argument(
        '--trim_tail',
        type=int,
        default=0,
        help='Remove this number of nucleotides from the end of each read. Default: %(default)s'
    )
    parser.add_argument(
        '--required_read_length',
        type=int,
        default=0,
        help='Set this value to only allow reads to pass which are equal to or longer than this threshold. Default: %(default)s'
    )
    parser.add_argument(
        '--disable_complexity_filter',
        action='store_true',
        default=False,
        help='Use this flag if you want to disable the removal of low-complexity reads, e.g. AAAAAAACCCCCCCCAAAAAAAAAAAAAAAAAAGGGGG (activated by default).'
    )
    parser.add_argument(
        '--complexity_threshold',
        type=int,
        default=10,
        help='Reads below this complexity threshold will be removed (consult fastp manual for more explanation). Default: %(default)s'
    )
    parser.add_argument(
        '--disable_poly_g_trimming',
        action='store_true',
        default=False,
        help='Use this flag if you want to disable trimming of G repeats at the end of the read. Poly-G read ends are common when working with very short fragments. (activated by default).'
    )
    parser.add_argument(
        '--poly_g_min_len',
        type=int,
        default=7,
        help='Specifies the length of the nucleotide repeat region at end of read to be trimmed. Default: %(default)s'
    )
    parser.add_argument(
        '--disable_poly_x_trimming',
        action='store_true',
        default=False,
        help='Use this flag if you want to disable trimming of nucleotide repeats of any kind at the end of the read, e.g. poly-A tails (activated by default).'
    )
    parser.add_argument(
        '--poly_x_min_len',
        type=int,
        default=7,
        help='Specifies the length of the nucleotide repeat region at end of read to be trimmed. Default: %(default)s'
    )

def longest_common_substring(s1, s2):
    m = []
    for i, letter in enumerate(list(s1)):
        if letter == list(s2)[i]:
            m.append(letter)
        else:
            break
    return ''.join(m)

# def longest_common_substring(s1, s2):
#     m = [[0] * (1 + len(s2)) for i in range(1 + len(s1))]
#     longest, x_longest = 0, 0
#     for x in range(1, 1 + len(s1)):
#         for y in range(1, 1 + len(s2)):
#             if s1[x - 1] == s2[y - 1]:
#                 m[x][y] = m[x - 1][y - 1] + 1
#                 if m[x][y] > longest:
#                     longest = m[x][y]
#                     x_longest = x
#             else:
#                 m[x][y] = 0
#     return s1[x_longest - longest: x_longest]

def read_count(input):
    # Use bash command to count the read number
    number_reads = subprocess.getstatusoutput("gzip -cd %s | wc -l | awk '{ print $1/4 }' " %input)[1]
    num = int(number_reads)
    return num

def quality_trim(read1_path,read2_path,sample_id,out_dir,args,pair_index=0):
    # Get all user settings
    qualified_quality_phred = args.qualified_quality_phred
    unqualified_percent_limit = args.unqualified_percent_limit
    cut_window_size = args.cut_window_size
    cut_mean_quality = args.cut_mean_quality
    trim_front = args.trim_front
    trim_tail = args.trim_tail
    required_read_length = args.required_read_length
    disable_complexity_filter = args.disable_complexity_filter
    complexity_threshold = args.complexity_threshold
    disable_poly_g_trimming = args.disable_poly_g_trimming
    poly_g_min_len = args.poly_g_min_len
    disable_poly_x_trimming = args.disable_poly_x_trimming
    poly_x_min_len = args.poly_x_min_len
    #merge_overlapping_reads = args.merge_overlapping_reads
    #! allow for manual addition of adapters
    # Start processing
    print(('#' * 50))
    print(("Processing %s...\n" %sample_id))
    # Forward and backward read file paths
    # Names of output files
    output_sample_dir = os.path.join(out_dir,sample_id)
    if not os.path.exists(output_sample_dir):
        os.makedirs(output_sample_dir)
    outpath_r1 = os.path.join(output_sample_dir, '%s_%i_clean-READ1.fastq.gz'%(sample_id,pair_index))
    outpath_r2 = os.path.join(output_sample_dir, '%s_%i_clean-READ2.fastq.gz'%(sample_id,pair_index))
    outpath_unpaired = os.path.join(output_sample_dir, '%s_%i_clean-unpaired.fastq.gz' % (sample_id, pair_index))
    # Command for fastp trimming and cleaning
    command1 = [
        "fastp",
        "--in1",
        read1_path,
        "--in2",
        read2_path,
        "--out1",
        outpath_r1,
        "--out2",
        outpath_r2,
        "--unpaired1",
        outpath_unpaired,
        "--unpaired2",
        outpath_unpaired,
        "-h",
        os.path.join(output_sample_dir,'%s_%i_fastp.html'%(sample_id,pair_index)),
        "-j",
        os.path.join(output_sample_dir,'%s_%i_fastp.json'%(sample_id,pair_index)),
        "--qualified_quality_phred",
        qualified_quality_phred,
        "--unqualified_percent_limit",
        unqualified_percent_limit,
        "--cut_tail",
        "--cut_window_size",
        cut_window_size,
        "--cut_mean_quality",
        cut_mean_quality,
        "--trim_front1",
        trim_front,
        "--trim_tail1",
        trim_tail
    ]
    if required_read_length == 0:
        command1.append("--disable_length_filtering")
    else:
        command1 += ["--length_required", required_read_length]
    if not disable_complexity_filter:
        command1.append("--low_complexity_filter")
        command1 += ["--complexity_threshold", complexity_threshold]
    if not disable_poly_g_trimming:
        command1.append("--trim_poly_g")
        command1 += ["--poly_g_min_len", poly_g_min_len]
    if not disable_poly_x_trimming:
        command1.append("--trim_poly_x")
        command1 += ["--poly_x_min_len",poly_x_min_len]
    command1 = list(np.array(command1).astype(str))
    stats_file = os.path.join(output_sample_dir,'fastp_out_%i.txt'%pair_index)
    with open(stats_file, 'w') as log_err_file:
        p1 = subprocess.Popen(command1, stderr=log_err_file)
        p1.communicate()
        print("%s successfully cleaned and trimmed. Stats are printed into %s" %(sample_id, stats_file))
        print("#" * 50)
    return(output_sample_dir)

def main(args):
    # Set working directory
    work_dir = args.input
    out_dir = args.output
    # Print the user-set read-threshold
    read_threshold = args.read_min
    print(("\n\n[Info:] Samples with a total read-count of less than %d (forward + reverse reads) are not being processed. If required you can set a different threshold, using the --read_min flag.\n" %read_threshold))

    # Read txt-file with new sample names
    name_info_file = args.sample_annotation_file
    name_info = pd.read_csv(name_info_file,header=None)
    name_info_dict = dict(name_info.values)
    # flip keys and values in dict for easier lookup of file names
    name_info_dict = {name_info_dict[k]:k for k in name_info_dict}

    # Get all fastq files belonging to each name (can be multiple runs)
    included_extenstions = ['fastq','fq','fastq.gz','fq.gz']
    file_list = [fn for fn in sorted(os.listdir(work_dir)) if any([fn.endswith(ext) for ext in included_extenstions])]
    if len(file_list) == 0:
        sys.exit('SECAPR-ERROR: No FASTQ files were found. Check if correct path is provided for --input flag')
    sample_list = [[name_info_dict[name_stem],i] for name_stem in name_info_dict.keys() for i in file_list if i.startswith(name_stem)]
    sample_filename_dict = {}
    for sample_id,filename in sample_list:
        sample_filename_dict.setdefault(sample_id,[])
        sample_filename_dict[sample_id].append(filename)
    final_sample_filename_dict = {}
    for sample_id,filenamelist in sample_filename_dict.items():
        if len(filenamelist)>2: # if more than two files per sample, make sure to identify the pairs correctly
            shared_stems = []
            for i in filenamelist:
                longest_common_substrings = [longest_common_substring(i,j) for j in filenamelist]
                selected_longest_common_substrings = [i for i in longest_common_substrings if not any([i.endswith(ext) for ext in included_extenstions])]
                longest_common_substrings_length = np.array([len(i) for i in selected_longest_common_substrings])
                best_match = np.sort(longest_common_substrings_length)[-1] # pick the longest, because the match with itself is already filtered out in previous step
                shared_file_stem = selected_longest_common_substrings[np.where(longest_common_substrings_length==best_match)[0][0]]
                shared_stems.append(shared_file_stem)
            paired_filename_list = []
            for namestem in np.unique(shared_stems):
                paired_filename_list.append([os.path.join(work_dir,i) for i in filenamelist if i.startswith(namestem)])
        else:
            paired_filename_list = [os.path.join(work_dir,i) for i in filenamelist]
        final_sample_filename_dict.setdefault(sample_id,paired_filename_list)

    # Count total number of reads of a sample
    final_sample_list = []
    raw_read_counts = []
    for sample_id, fastq_files in final_sample_filename_dict.items():
        print("%s: Counting all reads (forward + reverse) belonging to this sample..."%sample_id)
        fastq_files = list(np.array(fastq_files).flatten())
        raw_total_read_count = sum([read_count(i) for i in fastq_files])
        print(raw_total_read_count)
        if raw_total_read_count >= read_threshold:
            final_sample_list.append(sample_id)
            raw_read_counts.append(raw_total_read_count)
        else:
            print('***Not enough reads found for sample %s. Excluding this sample from further processing. Adjust the --read_min flag to set lower minimum read count threshold.***' %sample_id )
            pass

    # Clean and trim with fastp
    #! Allow manual input of adapter sequences
    #! Allow merging of reads
    clean_reads_outdirs = {}
    for sample_id in final_sample_list:
        read_files = final_sample_filename_dict[sample_id]
        if len(np.array(read_files).flatten()) == 2:
            sample_outdir = quality_trim(read_files[0],read_files[1],sample_id,out_dir,args,pair_index=0)
        else:
            for i,read_pair in enumerate(read_files):
                sample_outdir = quality_trim(read_pair[0], read_pair[1], sample_id, out_dir, args, pair_index=i)
        clean_reads_outdirs.setdefault(sample_id,sample_outdir)

    # Count all remaining clean reads per sample
    clean_read_counts = []
    for sample_id in final_sample_list:
        sample_outdir = clean_reads_outdirs[sample_id]
        print("%s: Counting remaining clean reads (forward + reverse) belonging to this sample..."%sample_id)
        fastq_files = glob.glob(os.path.join(sample_outdir,"*.fastq.gz"))
        cleaned_total_read_count = sum([read_count(i) for i in fastq_files])
        print(cleaned_total_read_count)
        clean_read_counts.append(cleaned_total_read_count)

    stats_df = pd.DataFrame(np.vstack([final_sample_list,raw_read_counts,clean_read_counts]).T,columns = ['sample_id','raw_reads_total','cleaned_reads_total'])
    stats_df.to_csv(os.path.join(out_dir,'sample_stats.txt'),sep = '\t',index=False)

<|endoftext|>


<commit_before># -*- coding: utf-8-*-
import json
import requests

from datetime import date

WORDS = ["LIVETICKER"]

def getAktuelleSaison():
    """ Merke dir die aktuelle Saison, bis zum Juni läuft die Saison aus dem Vorjahr noch """
    if date.today().month < 7:
        return date.today().year - 1
    return date.today().year

def getAktuellerSpieltag(liga):
    """ Merke dir den aktuellen Spieltag aus der übergebenen Liga """
    return json.loads(requests.get("http://www.openligadb.de/api/getcurrentgroup/" + liga).text)["GroupOrderID"]

def getLetzteAenderung(liga, saison, spieltag):
    """ Hole dir den Timestamp der letzten Änderung vom aktuellen Spieltag """
    return json.loads(requests.get(("http://www.openligadb.de/api/getlastchangedate/" + liga + "/" + str(saison) + "/" + str(spieltag))).text)

def getErgebnis(team, liga, saison, spieltag):
    """ Hole dir das Ergebnis zu den übergebenen Parametern
            Returns: Ein dict mit dem aktuellen Ergebnis zu dem Spiel mit den Parametern
    """
    ergebnis = json.loads(requests.get("http://www.openligadb.de/api/getmatchdata/" + liga + "/" + str(saison) + "/" + str(spieltag)).text)
    for spiel in ergebnis["spiele"]:
        if getTeam(spiel, 1) == team or getTeam(spiel, 2) == team:
            return spiel
    return {}

def getTeam(ergebnis, teamNr):
    """ Hole dir den Namen des Teams aus dem Ergebnis dict """
    return ergebnis["Team" + str(teamNr)]["TeamName"]

def getTore(ergebnis, teamNr):
    """ Hole dir die Anzahl der Tore vom Team aus dem Ergebnis dict """
    return ergebnis["ScoreTeam" + str(teamNr)]

def getLetztesTor(ergebnis):
    """ Hole dir die maximal GoalID aus dem Goals Array vom Ergebnis """
    letztesTor = ["GoalID" : "0"]
    for goal in ergebnis["Goals"]:
        if goal["GoalID"] > letztesTor["GoalID"]:
            letztesTor = goal
    return letztesTor

def formatiereTor(tor):
    """ Formatiere ein dict mit der Form:
            Team1
                TeamName
            Team2
                TeamName
            ScoreTeam1
            ScoreTeam2
            MatchMinute
            GoalGetterName

        Returns: den formatierten String
    """
    heimTeam = ergebnis["Team1"]["TeamName"]
    auswaertsTeam = ergebnis["Team2"]["TeamName"]
    heimTore = tor["ScoreTeam1"]
    auswaertsTore = tor["ScoreTeam2"]
    spielminute = tor["MatchMinute"]
    torschuetze = tor["GoalGetterName"]
    return ("Tor durch " + torschuetze + " in der " + str(spielminute) + ". Spielminute! Neuer Zwischenstand: "
            + heimTeam + " " + str(heimTore) + " : " + str(auswaertsTore) + " " + auswaertsTeam)

def handle(text, mic, profile):
    """ Gib den aktuellen Zwischenstand aus dem Spiel des unterstützten Teams zurück.
    """
    if "Liveticker" in profile:
        if "Team" in profile["Liveticker"] and "Liga" in profile["Liveticker"]:
            #merke dir das Team
            team = self.profile["Liveticker"]["Team"]
            #merke dir die Liga
            liga = self.profile["Liveticker"]["Liga"]

            ergebnis = getErgebnis(team, liga, getAktuelleSaison(), getAktuellerSpieltag(liga))
            if ergebnis:
                heimTeam = getTeam(ergebnis, 1)
                auswaertsTeam = getTeam(ergebnis, 2)
                heimTore = getTore(ergebnis, 1)
                auswaertsTore = getTore(ergebnis, 2)
                if ergebnis["MatchIsFinished"]:
                    output = "Das Endergebnis aus dem Spiel %s gegen %s lautet... %s zu %s." % (heimTeam, auswaertsTeam, heimTore, auswaertsTore)
                else:
                    output = "Der aktuelle Zwischenstand aus dem Spiel %s gegen %s lautet... %s zu %s." % (heimTeam, auswaertsTeam, heimTore, auswaertsTore)
                mic.say(output)
            else:
                mic.say("Fehler beim Auslesen vom Ergebnis. Bitte überprüfen Sie ihre Profildatei und geben Sie richtige Werte für ihr Team und die Liga ein!")
        else:
            mic.say("Sie müssen Angaben zu dem Team und der Liga des Teams in der Profildatei machen! Zum Beispiel 'Werder Bremen' und 'bl1' für die erste Bundesliga.")
    else:
        mic.say("Sie müssen einen Eintrag in der Profildatei mit dem Namen Liveticker machen!")

def isValid(text):
    """ Gib True zurück wenn der Input was mit Liveticker zu tun hat """
    return bool(re.search(r'\bliveticker\b', text, re.IGNORECASE))<commit_msg>Liveticker Modul auf OOP umgebaut<commit_after># -*- coding: utf-8-*-
import json
import requests

from classes.liveticker import Liveticker
from datetime import date

# expected words to start the module
WORDS = ["LIVETICKER"]

def handle(text, mic, profile):
    """ Gib den aktuellen Zwischenstand aus dem Spiel des unterstützten Teams zurück.
    """
    if "Liveticker" in profile:
        if "Team" in profile["Liveticker"] and "Liga" in profile["Liveticker"]:
            #merke dir das Team
            team = self.profile["Liveticker"]["Team"]
            #merke dir die Liga
            liga = self.profile["Liveticker"]["Liga"]

            lt = Liveticker(team, liga)

            ergebnis = lt.getErgebnis()

            if ergebnis:
                heimTeam = lt.getTeam(ergebnis, 1)
                auswaertsTeam = lt.getTeam(ergebnis, 2)
                heimTore = lt.getTore(ergebnis, 1)
                auswaertsTore = lt.getTore(ergebnis, 2)
                if ergebnis["MatchIsFinished"]:
                    output = "Das Endergebnis aus dem Spiel %s gegen %s lautet... %s zu %s." % (heimTeam, auswaertsTeam, heimTore, auswaertsTore)
                else:
                    output = "Der aktuelle Zwischenstand aus dem Spiel %s gegen %s lautet... %s zu %s." % (heimTeam, auswaertsTeam, heimTore, auswaertsTore)
                mic.say(output)
            else:
                mic.say("Fehler beim Auslesen vom Ergebnis. Bitte überprüfen Sie ihre Profildatei und geben Sie richtige Werte für ihr Team und die Liga ein!")
        else:
            mic.say("Sie müssen Angaben zu dem Team und der Liga des Teams in der Profildatei machen! Zum Beispiel 'Werder Bremen' und 'bl1' für die erste Bundesliga.")
    else:
        mic.say("Sie müssen einen Eintrag in der Profildatei mit dem Namen Liveticker haben!")

def isValid(text):
    """ Gib True zurück wenn der Input was mit Liveticker zu tun hat """
    return bool(re.search(r'\bliveticker\b', text, re.IGNORECASE))<|endoftext|>


<commit_before>from django.template import loader, RequestContext
from django.core.exceptions import ImproperlyConfigured
from django.http import (HttpResponse, HttpResponseRedirect, Http404,
                         HttpResponseForbidden)
from django.db.models import Model, get_model
from django.contrib.contenttypes.models import ContentType
from secretballot.models import Vote

def vote(request, content_type, object_id, vote, can_vote_test=None,
              redirect_url=None, template_name=None, template_loader=loader,
              extra_context=None, context_processors=None, mimetype=None):

    # get the token from a SecretBallotMiddleware
    if not hasattr(request, 'secretballot_token'):
        raise ImproperlyConfigured('To use secretballot a SecretBallotMiddleware must be installed. (see secretballot/middleware.py)')
    token = request.secretballot_token

    if isinstance(content_type, ContentType):
        pass
    elif isinstance(content_type, Model):
        content_type = ContentType.objects.get_for_model(content_type)
    elif isinstance(content_type, basestring) and '.' in content_type:
        content_type = ContentType.objects.get(app_label=app, model__iexact=modelname)
    else:
        raise ValueError('content_type must be an instance of ContentType, a model, or "app.modelname" string')

    # do the action
    if vote:

        # 404 if object to be voted upon doesn't exist
        if content_type.model_class().objects.filter(pk=object_id).count() == 0:
            raise Http404

        # if there is a can_vote_test func specified, test then 403 if needed
        if can_vote_test:
            if not can_vote_test(request, content_type, object_id, vote):
                return HttpResponseForbidden("vote was forbidden")

        vobj,new = Vote.objects.get_or_create(content_type=content_type,
                                              object_id=object_id, token=token,
                                              defaults={'vote':vote})
        if not new:
            vobj.vote = vote
            vobj.save()
    else:
        Vote.objects.filter(content_type=content_type,
                            object_id=object_id, token=token).delete()

    # build the response
    if redirect_url:
        return HttpResponseRedirect(redirect_url)
    elif template_name:
        content_obj = content_type.get_object_for_this_type(pk=object_id)
        c = RequestContext(request, {'content_obj':content_obj},
                           context_processors)

        # copy extra_context into context, calling any callables
        for k,v in extra_context.items():
            if callable(v):
                c[k] = v()
            else:
                c[k] = v

        t = template_loader.get_template(template_name)
        body = t.render(c)
    else:
        votes = Vote.objects.filter(content_type=content_type,
                                    object_id=object_id).count()
        body = "{'num_votes':%d}" % votes

    return HttpResponse(body, mimetype=mimetype)

<commit_msg>fix parent checking in generic view<commit_after>from django.template import loader, RequestContext
from django.core.exceptions import ImproperlyConfigured
from django.http import (HttpResponse, HttpResponseRedirect, Http404,
                         HttpResponseForbidden)
from django.db.models import get_model
from django.db.models.base import ModelBase
from django.contrib.contenttypes.models import ContentType
from secretballot.models import Vote

def vote(request, content_type, object_id, vote, can_vote_test=None,
              redirect_url=None, template_name=None, template_loader=loader,
              extra_context=None, context_processors=None, mimetype=None):

    # get the token from a SecretBallotMiddleware
    if not hasattr(request, 'secretballot_token'):
        raise ImproperlyConfigured('To use secretballot a SecretBallotMiddleware must be installed. (see secretballot/middleware.py)')
    token = request.secretballot_token

    if isinstance(content_type, ContentType):
        pass
    elif isinstance(content_type, ModelBase):
        content_type = ContentType.objects.get_for_model(content_type)
    elif isinstance(content_type, basestring) and '.' in content_type:
        content_type = ContentType.objects.get(app_label=app, model__iexact=modelname)
    else:
        raise ValueError('content_type must be an instance of ContentType, a model, or "app.modelname" string')

    # do the action
    if vote:

        # 404 if object to be voted upon doesn't exist
        if content_type.model_class().objects.filter(pk=object_id).count() == 0:
            raise Http404

        # if there is a can_vote_test func specified, test then 403 if needed
        if can_vote_test:
            if not can_vote_test(request, content_type, object_id, vote):
                return HttpResponseForbidden("vote was forbidden")

        vobj,new = Vote.objects.get_or_create(content_type=content_type,
                                              object_id=object_id, token=token,
                                              defaults={'vote':vote})
        if not new:
            vobj.vote = vote
            vobj.save()
    else:
        Vote.objects.filter(content_type=content_type,
                            object_id=object_id, token=token).delete()

    # build the response
    if redirect_url:
        return HttpResponseRedirect(redirect_url)
    elif template_name:
        content_obj = content_type.get_object_for_this_type(pk=object_id)
        c = RequestContext(request, {'content_obj':content_obj},
                           context_processors)

        # copy extra_context into context, calling any callables
        for k,v in extra_context.items():
            if callable(v):
                c[k] = v()
            else:
                c[k] = v

        t = template_loader.get_template(template_name)
        body = t.render(c)
    else:
        votes = Vote.objects.filter(content_type=content_type,
                                    object_id=object_id).count()
        body = "{'num_votes':%d}" % votes

    return HttpResponse(body, mimetype=mimetype)

<|endoftext|>


<commit_before>from datetime import datetime

from django.core.urlresolvers import reverse
from django.http import HttpResponseRedirect, HttpResponse
from django.shortcuts import render_to_response
from django.template import RequestContext
from django.utils import simplejson
from django.views.i18n import javascript_catalog

from sendinel.backend.authhelper import check_and_delete_authentication_call, \
                                    delete_timed_out_authentication_calls, \
                                    format_phonenumber
from sendinel.backend.models import Patient, Sendable, \
                                    InfoService, Subscription
from sendinel.web.forms import HospitalAppointmentForm
from sendinel.settings import   AUTH_NUMBER, \
                                BLUETOOTH_SERVER_ADDRESS, \
                                AUTHENTICATION_CALL_TIMEOUT, \
                                COUNTRY_CODE_PHONE, START_MOBILE_PHONE, \
                                ADMIN_MEDIA_PREFIX
from sendinel.backend import bluetooth
from sendinel.logger import logger, log_request

@log_request
def index(request):
    informationservices = InfoService.objects.all()
    return render_to_response('web/index.html',
                              locals(),
                              context_instance = RequestContext(request))

@log_request
def choose_language(request):
    return render_to_response('web/language_choose.html',
                              locals(),
                              context_instance = RequestContext(request))

def jsi18n(request):
    js_info_web = {
        'packages': ('sendinel')
    }
    return javascript_catalog(request, packages = js_info_web)

@log_request
def create_appointment(request):
    admin_media_prefix = ADMIN_MEDIA_PREFIX
    nexturl = ""
    backurl = reverse('web_index')
    if request.method == "POST":
        form = HospitalAppointmentForm(request.POST)
        if form.is_valid():
            appointment = form.save(commit=False)
            patient = Patient()
            request.session['appointment'] = appointment
            request.session['patient'] = patient

            logger.info("Create appointment via %s" %
                            appointment.way_of_communication)
            if appointment.way_of_communication == 'bluetooth':
                return HttpResponseRedirect(reverse("web_list_devices") + \
                                "?next=" + reverse("web_appointment_send"))
            elif appointment.way_of_communication in ('sms', 'voice' ):
                return HttpResponseRedirect( \
                    reverse("web_authenticate_phonenumber") + "?next=" + \
                    reverse("web_appointment_save"))
            else:
                logger.error("Unknown way of communication selected.")
                raise Exception ("Unknown way of communication %s " \
                                   %appointment.way_of_communication) +\
                                "(this is neither bluetooth nor sms or voice)"
        else:
            logger.info("create_appointment: Invalid form.")
            return render_to_response('web/appointment_create.html',
                                locals(),
                                context_instance=RequestContext(request))
    else:
        #TODO: initiale Dateneintraege
        form = HospitalAppointmentForm()
        return render_to_response('web/appointment_create.html',
                                locals(),
                                context_instance=RequestContext(request))

@log_request
def save_appointment(request):
    nexturl = reverse("web_index")
    appointment = request.session.get('appointment', None)
    patient = request.session.get('patient', None)
    if not appointment or not patient:
        logger.warning("save_appointment: no appointment/patient in session")
        return HttpResponseRedirect(reverse(create_appointment))

    # TODO Rueckgabe testen, Fehlerbehandlung
    patient.phone_number = request.session['authenticate_phonenumber']['number']

    logger.info("Saving appointment: %s with patient: %s"
                    % (appointment, patient.phone_number))

    appointment.save_with_patient(patient)

    return render_to_response('web/appointment_saved.html',
                            locals(),
                            context_instance=RequestContext(request))

@log_request
def send_appointment(request):
    if (request.method == "POST"):
        appointment = request.session.get('appointment', None)
        mac_address = request.POST['device_mac'].strip()

        logger.info("sending appointment to mac_address: " + mac_address)
        logger.info("appointment data: " + str(appointment))

        appointment.bluetooth_mac_address = mac_address
        output_data = appointment.get_data_for_sending()
        result = output_data.send()
        if(result):
            return HttpResponse(status = 200)
        else:
            return HttpResponse(status = 500)

    backurl = reverse("web_list_devices")
    url = reverse("web_appointment_send")
    next = reverse("web_index")
    mac_address = request.GET['device_mac'].strip()

    return render_to_response('web/send_bluetooth_appointment.html',
                                locals(),
                                context_instance=RequestContext(request))

@log_request
def authenticate_phonenumber(request):
    nexturl = ''
    next = ''
    ajax_url= reverse('web_check_call_received')
    backurl = reverse('web_index')
    if request.method == "POST":
        backurl = reverse('web_authenticate_phonenumber')
        try:
            number = fill_authentication_session_variable(request)
            logger.info("Starting authentication with %s" % AUTH_NUMBER)
            auth_number = AUTH_NUMBER
            next = reverse('web_index')
            return render_to_response('web/authenticate_phonenumber_call.html',
                              locals(),
                              context_instance = RequestContext(request))
        except ValueError as e:
            error = e

    logger.info("Deleting timed out authentication calls.")
    delete_timed_out_authentication_calls()

    return render_to_response('web/authenticate_phonenumber.html',
                              locals(),
                              context_instance = RequestContext(request))

@log_request
def check_call_received(request):
    response_dict = {}

    try:
        response_dict["status"] = "failed"

        number = request.session['authenticate_phonenumber']['number']
        start_time = request.session['authenticate_phonenumber']['start_time']

        if (start_time + AUTHENTICATION_CALL_TIMEOUT) >= datetime.now():
            if check_and_delete_authentication_call(number):
                response_dict["status"] = "received"
                logger.info("check_call_received: call received.")
            else:
                response_dict["status"] = "waiting"
    except KeyError:
        pass

    return HttpResponse(content = simplejson.dumps(response_dict),
                        content_type = "application/json")

@log_request
def list_bluetooth_devices(request):
    next = request.GET.get('next', '')
    backurl = reverse("web_appointment_create")
    return render_to_response('web/list_devices.html',
                                locals(),
                                context_instance=RequestContext(request))

@log_request
def get_bluetooth_devices(request):
    response_dict = {}
    devices_list = []

    try:
        devices = bluetooth.get_discovered_devices(BLUETOOTH_SERVER_ADDRESS)
        for device in devices.items():
            device_dict = {}
            device_dict["name"] = device[1]
            device_dict["mac"] = device[0]
            devices_list.append(device_dict)
        response_dict["devices"] = devices_list

        logger.debug("Got Bluetooth Devices: %s"% str(devices_list))

        return HttpResponse(content = simplejson.dumps(response_dict),
                            content_type = "application/json")
    except Exception as e:
        logger.error("get_bluetooth_devices from %s failed: %s" %
                        (BLUETOOTH_SERVER_ADDRESS, str(e)))
        # TODO write bluetooth error to log file
        return HttpResponse(status = 500)

@log_request
def register_infoservice(request, id):
    ajax_url= reverse('web_check_call_received')
    if request.method == "POST":
        request.session['way_of_communication'] = \
                                        request.POST['way_of_communication']
        try:
            number = fill_authentication_session_variable(request)
            auth_number = AUTH_NUMBER
            backurl = reverse('web_infoservice_register',  kwargs = {'id': id})
            next = reverse('web_infoservice_register_save', kwargs = {'id': id})
            url = reverse('web_check_call_received')
            return render_to_response('web/authenticate_phonenumber_call.html',
                locals(),
                context_instance = RequestContext(request))
        except ValueError as e:
            error = e
    infoservice = InfoService.objects.filter(pk = id)[0].name
    backurl = reverse("web_index")
    return render_to_response('web/infoservice_register.html',
                              locals(),
                              context_instance = RequestContext(request))

@log_request
def save_registration_infoservice(request, id):
    patient = Patient(phone_number = \
                      request.session['authenticate_phonenumber']['number'])
    patient.save()
    way_of_communication = request.session['way_of_communication']
    infoservice = InfoService.objects.filter(pk = id)[0]
    subscription = Subscription(patient = patient,
                                way_of_communication = way_of_communication,
                                infoservice = infoservice)
    subscription.save()
    logger.info("Saved subscription %s.", str(subscription))

    return HttpResponseRedirect(reverse('web_index'))

@log_request
def fill_authentication_session_variable(request):
    number = request.POST["number"].strip()
    number = format_phonenumber(number, COUNTRY_CODE_PHONE, START_MOBILE_PHONE)
    request.session['authenticate_phonenumber'] = \
                            { 'number': number,
                              'start_time': datetime.now() }
    return number
<commit_msg>reset next url in authenticate_call to authenticate_save<commit_after>from datetime import datetime

from django.core.urlresolvers import reverse
from django.http import HttpResponseRedirect, HttpResponse
from django.shortcuts import render_to_response
from django.template import RequestContext
from django.utils import simplejson
from django.views.i18n import javascript_catalog

from sendinel.backend.authhelper import check_and_delete_authentication_call, \
                                    delete_timed_out_authentication_calls, \
                                    format_phonenumber
from sendinel.backend.models import Patient, Sendable, \
                                    InfoService, Subscription
from sendinel.web.forms import HospitalAppointmentForm
from sendinel.settings import   AUTH_NUMBER, \
                                BLUETOOTH_SERVER_ADDRESS, \
                                AUTHENTICATION_CALL_TIMEOUT, \
                                COUNTRY_CODE_PHONE, START_MOBILE_PHONE, \
                                ADMIN_MEDIA_PREFIX
from sendinel.backend import bluetooth
from sendinel.logger import logger, log_request

@log_request
def index(request):
    informationservices = InfoService.objects.all()
    return render_to_response('web/index.html',
                              locals(),
                              context_instance = RequestContext(request))

@log_request
def choose_language(request):
    return render_to_response('web/language_choose.html',
                              locals(),
                              context_instance = RequestContext(request))

def jsi18n(request):
    js_info_web = {
        'packages': ('sendinel')
    }
    return javascript_catalog(request, packages = js_info_web)

@log_request
def create_appointment(request):
    admin_media_prefix = ADMIN_MEDIA_PREFIX
    nexturl = ""
    backurl = reverse('web_index')
    if request.method == "POST":
        form = HospitalAppointmentForm(request.POST)
        if form.is_valid():
            appointment = form.save(commit=False)
            patient = Patient()
            request.session['appointment'] = appointment
            request.session['patient'] = patient

            logger.info("Create appointment via %s" %
                            appointment.way_of_communication)
            if appointment.way_of_communication == 'bluetooth':
                return HttpResponseRedirect(reverse("web_list_devices") + \
                                "?next=" + reverse("web_appointment_send"))
            elif appointment.way_of_communication in ('sms', 'voice' ):
                return HttpResponseRedirect( \
                    reverse("web_authenticate_phonenumber") + "?next=" + \
                    reverse("web_appointment_save"))
            else:
                logger.error("Unknown way of communication selected.")
                raise Exception ("Unknown way of communication %s " \
                                   %appointment.way_of_communication) +\
                                "(this is neither bluetooth nor sms or voice)"
        else:
            logger.info("create_appointment: Invalid form.")
            return render_to_response('web/appointment_create.html',
                                locals(),
                                context_instance=RequestContext(request))
    else:
        #TODO: initiale Dateneintraege
        form = HospitalAppointmentForm()
        return render_to_response('web/appointment_create.html',
                                locals(),
                                context_instance=RequestContext(request))

@log_request
def save_appointment(request):
    nexturl = reverse("web_index")
    appointment = request.session.get('appointment', None)
    patient = request.session.get('patient', None)
    if not appointment or not patient:
        logger.warning("save_appointment: no appointment/patient in session")
        return HttpResponseRedirect(reverse(create_appointment))

    # TODO Rueckgabe testen, Fehlerbehandlung
    patient.phone_number = request.session['authenticate_phonenumber']['number']

    logger.info("Saving appointment: %s with patient: %s"
                    % (appointment, patient.phone_number))

    appointment.save_with_patient(patient)

    return render_to_response('web/appointment_saved.html',
                            locals(),
                            context_instance=RequestContext(request))

@log_request
def send_appointment(request):
    if (request.method == "POST"):
        appointment = request.session.get('appointment', None)
        mac_address = request.POST['device_mac'].strip()

        logger.info("sending appointment to mac_address: " + mac_address)
        logger.info("appointment data: " + str(appointment))

        appointment.bluetooth_mac_address = mac_address
        output_data = appointment.get_data_for_sending()
        result = output_data.send()
        if(result):
            return HttpResponse(status = 200)
        else:
            return HttpResponse(status = 500)

    backurl = reverse("web_list_devices")
    url = reverse("web_appointment_send")
    next = reverse("web_index")
    mac_address = request.GET['device_mac'].strip()

    return render_to_response('web/send_bluetooth_appointment.html',
                                locals(),
                                context_instance=RequestContext(request))

@log_request
def authenticate_phonenumber(request):
    nexturl = ''
    next = ''
    ajax_url= reverse('web_check_call_received')
    backurl = reverse('web_index')
    if request.method == "POST":
        backurl = reverse('web_authenticate_phonenumber')
        try:
            number = fill_authentication_session_variable(request)
            logger.info("Starting authentication with %s" % AUTH_NUMBER)
            auth_number = AUTH_NUMBER
            next = request.GET.get('next', reverse('web_appointment_save'))
            return render_to_response('web/authenticate_phonenumber_call.html',
                              locals(),
                              context_instance = RequestContext(request))
        except ValueError as e:
            error = e

    logger.info("Deleting timed out authentication calls.")
    delete_timed_out_authentication_calls()

    return render_to_response('web/authenticate_phonenumber.html',
                              locals(),
                              context_instance = RequestContext(request))

@log_request
def check_call_received(request):
    response_dict = {}

    try:
        response_dict["status"] = "failed"

        number = request.session['authenticate_phonenumber']['number']
        start_time = request.session['authenticate_phonenumber']['start_time']

        if (start_time + AUTHENTICATION_CALL_TIMEOUT) >= datetime.now():
            if check_and_delete_authentication_call(number):
                response_dict["status"] = "received"
                logger.info("check_call_received: call received.")
            else:
                response_dict["status"] = "waiting"
    except KeyError:
        pass

    return HttpResponse(content = simplejson.dumps(response_dict),
                        content_type = "application/json")

@log_request
def list_bluetooth_devices(request):
    next = request.GET.get('next', '')
    backurl = reverse("web_appointment_create")
    return render_to_response('web/list_devices.html',
                                locals(),
                                context_instance=RequestContext(request))

@log_request
def get_bluetooth_devices(request):
    response_dict = {}
    devices_list = []

    try:
        devices = bluetooth.get_discovered_devices(BLUETOOTH_SERVER_ADDRESS)
        for device in devices.items():
            device_dict = {}
            device_dict["name"] = device[1]
            device_dict["mac"] = device[0]
            devices_list.append(device_dict)
        response_dict["devices"] = devices_list

        logger.debug("Got Bluetooth Devices: %s"% str(devices_list))

        return HttpResponse(content = simplejson.dumps(response_dict),
                            content_type = "application/json")
    except Exception as e:
        logger.error("get_bluetooth_devices from %s failed: %s" %
                        (BLUETOOTH_SERVER_ADDRESS, str(e)))
        # TODO write bluetooth error to log file
        return HttpResponse(status = 500)

@log_request
def register_infoservice(request, id):
    ajax_url= reverse('web_check_call_received')
    if request.method == "POST":
        request.session['way_of_communication'] = \
                                        request.POST['way_of_communication']
        try:
            number = fill_authentication_session_variable(request)
            auth_number = AUTH_NUMBER
            backurl = reverse('web_infoservice_register',  kwargs = {'id': id})
            next = reverse('web_infoservice_register_save', kwargs = {'id': id})
            url = reverse('web_check_call_received')
            return render_to_response('web/authenticate_phonenumber_call.html',
                locals(),
                context_instance = RequestContext(request))
        except ValueError as e:
            error = e
    infoservice = InfoService.objects.filter(pk = id)[0].name
    backurl = reverse("web_index")
    return render_to_response('web/infoservice_register.html',
                              locals(),
                              context_instance = RequestContext(request))

@log_request
def save_registration_infoservice(request, id):
    patient = Patient(phone_number = \
                      request.session['authenticate_phonenumber']['number'])
    patient.save()
    way_of_communication = request.session['way_of_communication']
    infoservice = InfoService.objects.filter(pk = id)[0]
    subscription = Subscription(patient = patient,
                                way_of_communication = way_of_communication,
                                infoservice = infoservice)
    subscription.save()
    logger.info("Saved subscription %s.", str(subscription))

    return HttpResponseRedirect(reverse('web_index'))

@log_request
def fill_authentication_session_variable(request):
    number = request.POST["number"].strip()
    number = format_phonenumber(number, COUNTRY_CODE_PHONE, START_MOBILE_PHONE)
    request.session['authenticate_phonenumber'] = \
                            { 'number': number,
                              'start_time': datetime.now() }
    return number
<|endoftext|>


<commit_before># Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.

import datetime

from senlin.common import exception
from senlin.common.i18n import _
from senlin.common.i18n import _LE
from senlin.common.i18n import _LW
from senlin.db import api as db_api
from senlin.engine import event as events
from senlin.openstack.common import log as logging
from senlin.profiles import base as profile_base

LOG = logging.getLogger(__name__)

class Node(object):
    '''A node is an object that can belong to at most one single cluster.

    All operations are performed without further checking because the
    checkings are supposed to be de done before/after/during an action is
    excuted.
    '''

    statuses = (
        INIT, ACTIVE, ERROR, DELETED,
        CREATING, UPDATING, DELETING,
    ) = (
        'INIT', 'ACTIVE', 'ERROR', 'DELETED',
        'CREATING', 'UPDATING', 'DELETING',
    )

    def __init__(self, name, profile_id, cluster_id, **kwargs):
        self.id = kwargs.get('id', None)
        if name:
            self.name = name
        else:
            # TODO(Qiming): Use self.physical_resource_name() to
            #               generate a unique name
            self.name = 'node-name-tmp'

        self.physical_id = kwargs.get('physical_id', '')
        self.profile_id = profile_id
        self.cluster_id = cluster_id
        self.index = kwargs.get('index', -1)
        self.role = kwargs.get('role', '')

        self.init_time = kwargs.get('init_time', None)
        self.created_time = kwargs.get('created_time', None)
        self.updated_time = kwargs.get('updated_time', None)
        self.deleted_time = kwargs.get('deleted_time', None)

        self.status = kwargs.get('status', self.INIT)
        self.status_reason = kwargs.get('status_reason', 'Initializing')
        self.data = kwargs.get('data', {})
        self.tags = kwargs.get('tags', {})
        self.rt = {}

        ctx = kwargs.get('context', None)
        if ctx:
            self.rt = {
                'profile': profile_base.Profile.load(ctx, self.profile_id),
            }

    def store(self, context):
        '''Store the node record into database table.

        The invocation of DB API could be a node_create or a node_update,
        depending on whether node has an ID assigned.
        '''

        values = {
            'name': self.name,
            'physical_id': self.physical_id,
            'cluster_id': self.cluster_id,
            'profile_id': self.profile_id,
            'index': self.index,
            'role': self.role,
            'init_time': self.init_time,
            'created_time': self.created_time,
            'updated_time': self.updated_time,
            'deleted_time': self.deleted_time,
            'status': self.status,
            'status_reason': self.status_reason,
            'data': self.data,
            'tags': self.tags,
        }

        if self.id:
            values['updated_time'] = datetime.datetime.utcnow()
            db_api.node_update(context, self.id, values)
            # TODO(Qiming): create event/log
        else:
            values['init_time'] = datetime.datetime.utcnow()
            node = db_api.node_create(context, values)
            # TODO(Qiming): create event/log
            self.id = node.id

        return self.id

    @classmethod
    def from_db_record(cls, context, record):
        '''Construct a node object from database record.

        :param context: the context used for DB operations;
        :param record: a DB node object that contains all fields;
        '''
        kwargs = {
            'id': record.id,
            'physical_id': record.physical_id,
            'index': record.index,
            'role': record.role,
            'init_time': record.init_time,
            'created_time': record.created_time,
            'updated_time': record.updated_time,
            'deleted_time': record.deleted_time,
            'status': record.status,
            'status_reason': record.status_reason,
            'data': record.data,
            'tags': record.tags,
            'context': context,
        }
        return cls(record.name, record.profile_id, record.cluster_id,
                   **kwargs)

    @classmethod
    def load(cls, context, node_id):
        '''Retrieve a node from database.'''

        record = db_api.node_get(context, node_id)

        if record is None:
            msg = _('No node with id "%s" exists') % node_id
            raise exception.NotFound(msg)

        return cls.from_db_record(context, record)

    @classmethod
    def load_all(cls, context, cluster_id=None, show_deleted=False,
                 limit=None, marker=None, sort_keys=None, sort_dir=None,
                 filters=None, tenant_safe=True):
        '''Retrieve all nodes of from database.'''

        records = db_api.node_get_all(context, cluster_id=cluster_id,
                                      show_deleted=show_deleted,
                                      limit=limit, marker=marker,
                                      sort_keys=sort_keys, sort_dir=sort_dir,
                                      filters=filters,
                                      tenant_safe=tenant_safe)

        for record in records:
            yield cls.from_db_record(context, record)

    @classmethod
    def delete(cls, context, node_id, force=False):
        db_api.node_delete(context, node_id, force)

    def to_dict(self):
        node_dict = {
            'id': self.id,
            'name': self.name,
            'cluster_id': self.cluster_id,
            'physical_id': self.physical_id,
            'profile_id': self.profile_id,
            'index': self.index,
            'role': self.role,
            'init_time': self.init_time,
            'created_time': self.created_time,
            'updated_time': self.updated_time,
            'deleted_time': self.deleted_time,
            'status': self.status,
            'status_reason': self.status_reason,
            'data': self.data,
            'tags': self.tags,
            'profile_name': self.rt['profile'].name,
        }
        return node_dict

    @classmethod
    def from_dict(cls, **kwargs):
        return cls(**kwargs)

    def set_status(self, context, status, reason=None):
        values = {}
        now = datetime.datetime.utcnow()
        if status == self.ACTIVE and self.status == self.CREATING:
            values['created_time'] = now
        elif status == self.DELETED:
            values['deleted_time'] = now
        else:
            values['updated_time'] = now

        self.status = status
        values['status'] = status
        if reason:
            values['status_reason'] = reason
        db_api.node_update(context, self.id, values)
        # TODO(anyone): generate event record

    def do_create(self, context):
        # TODO(Qiming): log events?
        if self.status != self.INIT:
            LOG.error(_LE('Node in wrong status'))
            return False

        self.set_status(context, self.CREATING, reason='Creation in progress')
        physical_id = profile_base.Profile.create_object(context, self)
        if not physical_id:
            return False

        self.physical_id = physical_id
        db_api.node_update(context, self.id, {'physical_id': physical_id})

        self.set_status(context, self.ACTIVE, reason='Creation succeeded')
        return True

    def do_delete(self, context):
        if not self.physical_id:
            return True

        # TODO(Qiming): check if actions are working on it and can be canceled
        # TODO(Qiming): log events
        res = profile_base.delete_object(self)
        if res:
            db_api.delete_node(self.id)
            return True
        else:
            return False

    def do_update(self, context, new_profile_id):
        if not new_profile_id:
            raise exception.ProfileNotSpecified()

        if new_profile_id == self.profile_id:
            return True

        if not self.physical_id:
            return False

        # Check if profile types match
        old_profile = db_api.get_profile(context, self.profile_id)
        new_profile = db_api.get_profile(context, new_profile_id)
        if old_profile.type != new_profile.type:
            events.warning(_LW('Node cannot be updated to a different '
                               'profile type (%(oldt)s->%(newt)s)'),
                           {'oldt': old_profile.type,
                            'newt': new_profile.type})
            return False

        res = profile_base.update_object(self, new_profile_id)
        if res:
            self.rt['profile'] = profile_base.load(context,
                                                   new_profile_id)
            self.profile_id = new_profile_id
            self.updated_time = datetime.datetime.utcnow()
            db_api.node_update(context, self.id,
                               {'profile_id': self.profile_id,
                                'updated_time': self.updated_time})

        return res

    def do_join(self, context, cluster_id):
        if self.cluster_id == cluster_id:
            return True

        db_api.node_migrate(context, self.id, self.cluster_id,
                            cluster_id)

        self.updated_time = datetime.datetime.utcnow()
        db_api.node_update(context, self.id,
                           {'updated_time': self.updated_time})
        return True

    def do_leave(self, context):
        if self.cluster_id is None:
            return True

        db_api.node_migrate(context, self.id, self.cluster_id, None)
        self.updated_time = datetime.datetime.utcnow()
        db_api.node_update(context, self.id,
                           {'updated_time': self.updated_time})

        return True
<commit_msg>Fixed node-delete and node-update path<commit_after># Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.

import datetime

from senlin.common import exception
from senlin.common.i18n import _
from senlin.common.i18n import _LE
from senlin.common.i18n import _LW
from senlin.db import api as db_api
from senlin.engine import event as events
from senlin.openstack.common import log as logging
from senlin.profiles import base as profile_base

LOG = logging.getLogger(__name__)

class Node(object):
    '''A node is an object that can belong to at most one single cluster.

    All operations are performed without further checking because the
    checkings are supposed to be de done before/after/during an action is
    excuted.
    '''

    statuses = (
        INIT, ACTIVE, ERROR, DELETED,
        CREATING, UPDATING, DELETING,
    ) = (
        'INIT', 'ACTIVE', 'ERROR', 'DELETED',
        'CREATING', 'UPDATING', 'DELETING',
    )

    def __init__(self, name, profile_id, cluster_id, **kwargs):
        self.id = kwargs.get('id', None)
        if name:
            self.name = name
        else:
            # TODO(Qiming): Use self.physical_resource_name() to
            #               generate a unique name
            self.name = 'node-name-tmp'

        self.physical_id = kwargs.get('physical_id', '')
        self.profile_id = profile_id
        self.cluster_id = cluster_id
        self.index = kwargs.get('index', -1)
        self.role = kwargs.get('role', '')

        self.init_time = kwargs.get('init_time', None)
        self.created_time = kwargs.get('created_time', None)
        self.updated_time = kwargs.get('updated_time', None)
        self.deleted_time = kwargs.get('deleted_time', None)

        self.status = kwargs.get('status', self.INIT)
        self.status_reason = kwargs.get('status_reason', 'Initializing')
        self.data = kwargs.get('data', {})
        self.tags = kwargs.get('tags', {})
        self.rt = {}

        ctx = kwargs.get('context', None)
        if ctx:
            self.rt = {
                'profile': profile_base.Profile.load(ctx, self.profile_id),
            }

    def store(self, context):
        '''Store the node record into database table.

        The invocation of DB API could be a node_create or a node_update,
        depending on whether node has an ID assigned.
        '''

        values = {
            'name': self.name,
            'physical_id': self.physical_id,
            'cluster_id': self.cluster_id,
            'profile_id': self.profile_id,
            'index': self.index,
            'role': self.role,
            'init_time': self.init_time,
            'created_time': self.created_time,
            'updated_time': self.updated_time,
            'deleted_time': self.deleted_time,
            'status': self.status,
            'status_reason': self.status_reason,
            'data': self.data,
            'tags': self.tags,
        }

        if self.id:
            values['updated_time'] = datetime.datetime.utcnow()
            db_api.node_update(context, self.id, values)
            # TODO(Qiming): create event/log
        else:
            values['init_time'] = datetime.datetime.utcnow()
            node = db_api.node_create(context, values)
            # TODO(Qiming): create event/log
            self.id = node.id

        return self.id

    @classmethod
    def from_db_record(cls, context, record):
        '''Construct a node object from database record.

        :param context: the context used for DB operations;
        :param record: a DB node object that contains all fields;
        '''
        kwargs = {
            'id': record.id,
            'physical_id': record.physical_id,
            'index': record.index,
            'role': record.role,
            'init_time': record.init_time,
            'created_time': record.created_time,
            'updated_time': record.updated_time,
            'deleted_time': record.deleted_time,
            'status': record.status,
            'status_reason': record.status_reason,
            'data': record.data,
            'tags': record.tags,
            'context': context,
        }
        return cls(record.name, record.profile_id, record.cluster_id,
                   **kwargs)

    @classmethod
    def load(cls, context, node_id):
        '''Retrieve a node from database.'''

        record = db_api.node_get(context, node_id)

        if record is None:
            msg = _('No node with id "%s" exists') % node_id
            raise exception.NotFound(msg)

        return cls.from_db_record(context, record)

    @classmethod
    def load_all(cls, context, cluster_id=None, show_deleted=False,
                 limit=None, marker=None, sort_keys=None, sort_dir=None,
                 filters=None, tenant_safe=True):
        '''Retrieve all nodes of from database.'''

        records = db_api.node_get_all(context, cluster_id=cluster_id,
                                      show_deleted=show_deleted,
                                      limit=limit, marker=marker,
                                      sort_keys=sort_keys, sort_dir=sort_dir,
                                      filters=filters,
                                      tenant_safe=tenant_safe)

        for record in records:
            yield cls.from_db_record(context, record)

    @classmethod
    def delete(cls, context, node_id, force=False):
        db_api.node_delete(context, node_id, force)

    def to_dict(self):
        node_dict = {
            'id': self.id,
            'name': self.name,
            'cluster_id': self.cluster_id,
            'physical_id': self.physical_id,
            'profile_id': self.profile_id,
            'index': self.index,
            'role': self.role,
            'init_time': self.init_time,
            'created_time': self.created_time,
            'updated_time': self.updated_time,
            'deleted_time': self.deleted_time,
            'status': self.status,
            'status_reason': self.status_reason,
            'data': self.data,
            'tags': self.tags,
            'profile_name': self.rt['profile'].name,
        }
        return node_dict

    @classmethod
    def from_dict(cls, **kwargs):
        return cls(**kwargs)

    def set_status(self, context, status, reason=None):
        values = {}
        now = datetime.datetime.utcnow()
        if status == self.ACTIVE and self.status == self.CREATING:
            values['created_time'] = now
        elif status == self.DELETED:
            values['physical_id'] = ''
            values['deleted_time'] = now
        elif status == self.ACTIVE and self.status == self.UPDATING:
            values['updated_time'] = now

        self.status = status
        values['status'] = status
        if reason:
            values['status_reason'] = reason
        db_api.node_update(context, self.id, values)
        # TODO(anyone): generate event record

    def do_create(self, context):
        # TODO(Qiming): log events?
        if self.status != self.INIT:
            LOG.error(_LE('Node in wrong status'))
            return False

        self.set_status(context, self.CREATING, reason='Creation in progress')
        physical_id = profile_base.Profile.create_object(context, self)
        if not physical_id:
            return False

        self.physical_id = physical_id
        db_api.node_update(context, self.id, {'physical_id': physical_id})

        self.set_status(context, self.ACTIVE, reason='Creation succeeded')
        return True

    def do_delete(self, context):
        def _delete_db_record(node_id):
            self.set_status(context, self.DELETED, reason='Deletion succeeded')
            db_api.node_delete(context, self.id)
            return

        if not self.physical_id:
            _delete_db_record(self.id)
            return True

        # TODO(Qiming): check if actions are working on it and can be canceled
        # TODO(Qiming): log events
        self.set_status(context, self.DELETING, reason='Deletion in progress')
        res = profile_base.Profile.delete_object(context, self)
        if res:
            _delete_db_record(self.id)
            return True
        else:
            self.set_status(context, self.ERROR, reason='Deletion failed')
            return False

    def do_update(self, context, new_profile_id):
        if not new_profile_id:
            raise exception.ProfileNotSpecified()

        if new_profile_id == self.profile_id:
            return True

        if not self.physical_id:
            return False

        # Check if profile types match
        old_profile = db_api.get_profile(context, self.profile_id)
        new_profile = db_api.get_profile(context, new_profile_id)
        if old_profile.type != new_profile.type:
            events.warning(_LW('Node cannot be updated to a different '
                               'profile type (%(oldt)s->%(newt)s)'),
                           {'oldt': old_profile.type,
                            'newt': new_profile.type})
            return False

        res = profile_base.update_object(self, new_profile_id)
        if res:
            self.rt['profile'] = profile_base.load(context,
                                                   new_profile_id)
            self.profile_id = new_profile_id
            self.updated_time = datetime.datetime.utcnow()
            db_api.node_update(context, self.id,
                               {'profile_id': self.profile_id,
                                'updated_time': self.updated_time})

        return res

    def do_join(self, context, cluster_id):
        if self.cluster_id == cluster_id:
            return True

        db_api.node_migrate(context, self.id, self.cluster_id,
                            cluster_id)

        self.updated_time = datetime.datetime.utcnow()
        db_api.node_update(context, self.id,
                           {'updated_time': self.updated_time})
        return True

    def do_leave(self, context):
        if self.cluster_id is None:
            return True

        db_api.node_migrate(context, self.id, self.cluster_id, None)
        self.updated_time = datetime.datetime.utcnow()
        db_api.node_update(context, self.id,
                           {'updated_time': self.updated_time})

        return True
<|endoftext|>


<commit_before>import random

### DATA ###

data = {
	"APPLE" : {
			"sentences" :

				[

				{"string": "It is MOD DESC", "type": 0}, {"string": "This light is MOD DESC", "type": 0}, {"string": "Oh, it's MOD DESC", "type": 0}, {"string": "Oh, that's MOD DESC", "type": 0}, {"string": "MOD DESC now", "type": 0}, {"string": "Well, that's MOD DESC", "type": 0}, {"string": "I am MOD DESC", "type": 1}, {"string": "Am I DESC?", "type": 2}, {"string": "Is it DESC?", "type": 3}, {"string": "Well now I'm MOD DESC", "type": 4}, {"string": "My skin is MOD DESC", "type": 4}, {"string": "I feel DESC me", "type": 5}

				]

			,

			"descriptors" :

				[

				["light", 0, lambda m: m > 0.5 and m < 1.0], ["bright", 0, lambda m: m > 0.6 and m < 1.0], ["beaming", 0, lambda m: m > 0.8 and m < 1.0], ["dark", 0, lambda m: m > 0.0 and m < 0.5], ["murky", 0, lambda m: m > 0.0 and m < 0.5], ["shadowy", 0, lambda m: m > 0.0 and m < 0.5], ["basking", 1, lambda m: m > 0.5 and m < 1.0], ["shaded", 1, lambda m: m > 0.0 and m < 0.5], ["ripening", 1, lambda m: m > 0.5 and m < 1.0], ["ripe", 2, lambda m: m > 0.0 and m < 1.0], ["night", 3, lambda m: m > 0.0 and m < 0.5], ["daytime", 3, lambda m: m > 0.5 and m < 1.0], ["bathed in light", 4, lambda m: m > 0.5 and m < 1.0], ["illuminated", 4, lambda m: m > 0.6 and m < 1.0], ["red", 4, lambda m: m > 0.0 and m < 1.0] , ["light shining on", 5, lambda m: m > 0.5 and m < 1.0], ["shadow falling on", 5, lambda m: m > 0.5 and m < 1.0]

				]
			,
			"dreams" :

				[
				"I used to be quite small, but I am larger now", "When I was green, there was much more firmness inside", "The time I fell to the ground, there had been less light that day", "I haven't noticed any bees in a while", "There were big green leaves atop my crown"
				]

	},
	"MUG" : {
			"sentences" :

				[

				{"string": "It is MOD DESC", "type": 0}, {"string": "It is MOD DESC now, would you like some DRINK?", "type": 0}, {"string": "Oh, it's getting MOD DESC. Might be time for DRINK.", "type": 0}, {"string": "Aren't you MOD DESC now? Some DRINK might be a good idea.", "type": 0}, {"string": "Sure is MOD DESC. Why don't you pour youself a cup of DRINK?", "type": 0}

				]

			,

			"descriptors" :

				[

				["warm", 0, lambda m: m > 0.5 and m < 1.0], ["temperate", 0, lambda m: m > 0.4 and m < 0.6], ["cozy", 0, lambda m: m > 0.5 and m < 0.7], ["hot", 0, lambda m: m > 0.7 and m < 1.0], ["sweltering", 0, lambda m: m > 0.8 and m < 1.0], ["cool", 0, lambda m: m > 0.2 and m < 0.5], ["cold", 0, lambda m: m > 0.0 and m < 0.3], ["chilly", 0, lambda m: m > 0.0 and m < 0.3], ["freezing", 0, lambda m: m > 0.0 and m < 0.3], ["brisk", 0, lambda m: m > 0.2 and m < 0.5], ["lukewarm", 0, lambda m: m > 0.4 and m < 0.6]

				]
			,
			"drinks" :

				[

				["tea", lambda m: m > 0.0 and m < 0.5], ["coffee", lambda m: m > 0.0 and m < 0.5], ["cocoa", lambda m: m > 0.0 and m < 0.5], ["hot toddy", lambda m: m > 0.0 and m < 0.5], ["warm milk", lambda m: m > 0.0 and m < 0.5], ["hot cider", lambda m: m > 0.0 and m < 0.5], ["lemonade", lambda m: m > 0.5 and m < 1.0], ["water", lambda m: m > 0.5 and m < 1.0], ["iced tea", lambda m: m > 0.5 and m < 1.0], ["iced coffee", lambda m: m > 0.5 and m < 1.0], ["soda", lambda m: m > 0.5 and m < 1.0], ["seltzer", lambda m: m > 0.5 and m < 1.0], ["cold drink", lambda m: m > 0.5 and m < 1.0], ["hot or cold beverage", lambda m: m > 0.4 and m < 0.6]

				]
			,
			"dreams" :

				[
				"Sometimes I spin around and around and get very hot", "I once sat unused in darkness for a very long time", "There was a day when I was used 15 times. There were many cups of tea.", "When the spoon stirs, it clinks and clanks against my sides", "Ice dropped in makes liquid slosh around me"
				]

	},
	"BLOCKS" : {
			"sentences" :

				[

				{"string": "It is DESC", "type": 0}, {"string": "DESC here", "type": 1}, {"string": "DESC with us now", "type": 1}, {"string": "DESC present", "type": 1}, {"string": "DESC near", "type": 1}, {"string": "We see DESC", "type": 2}, {"string": "We see DESC has arrived", "type": 3}, {"string": "We see DESC is present", "type": 3}, {"string": "We see DESC has come", "type": 3}, {"string": "Oh, DESC now", "type": 4}, {"string": "Hello", "type": 5}, {"string": "Have you been here long?", "type": 5}, {"string": "Are you here to play with us?", "type": 5}, {"string": "Will we be one on top of the other again?", "type": 5}, {"string": "Will we be separated again?", "type": 5}

				]

			,

			"descriptors" :

				[

				["busy", 0, lambda m: m > 0.5 and m < 1.0], ["crowded", 0, lambda m: m > 0.8 and m < 1.0], ["packed", 0, lambda m: m > 0.8 and m < 1.0], ["teeming", 0, lambda m: m > 0.8 and m < 1.0], ["populated", 0, lambda m: m > 0.5 and m < 1.0], ["not empty", 0, lambda m: m > 0.2 and m < 1.0], ["empty", 0, lambda m: m > 0.0 and m < 0.3], ["idle", 0, lambda m: m > 0.0 and m < 0.2], ["clear of people", 0, lambda m: m > 0.0 and m < 0.2], ["devoid of people", 0, lambda m: m > 0.0 and m < 0.2], ["someone is", 1, lambda m: m > 0.3 and m < 1.0], ["people are", 1, lambda m: m > 0.3 and m < 1.0], ["a person is", 1, lambda m: m > 0.3 and m < 1.0], ["nobody is", 1, lambda m: m > 0.0 and m < 0.3], ["some people are", 0, lambda m: m > 0.3 and m < 1.0], ["few people are", 1, lambda m: m > 0.0 and m < 0.3], ["no one", 2, lambda m: m > 0.0 and m < 0.2], ["someone", 2, lambda m: m > 0.2 and m < 1.0], ["people", 2, lambda m: m > 0.2 and m < 1.0], ["you", 2, lambda m: m > 0.2 and m < 1.0], ["a group of people", 2, lambda m: m > 0.4 and m < 1.0], ["an assembly of people", 2, lambda m: m > 0.4 and m < 1.0], ["someone", 3, lambda m: m > 0.3 and m < 1.0], ["no one", 3, lambda m: m > 0.0 and m < 0.3], ["you are here", 4, lambda m: m > 0.3 and m < 1.0], ["we have company", 4, lambda m: m > 0.3 and m < 1.0], ["you have gone", 4, lambda m: m > 0.0 and m < 0.3], ["nobody is here", 4, lambda m: m > 0.0 and m < 0.3], ["", 5, lambda m: m > 0.0 and m < 1.0]

				]
			,
			"dreams" :

				[
				"We have seen many more people today than usual", "A long time ago, we sat next to one another in a container", "62 days ago, we were stacked unusually high", "Once, we moved around for a long time before we rested", "There used to be 26 of us", "Sometimes, nobody comes for a long time"
				]

	},
	"modifiers" :

				[

					["a tiny bit", lambda m: m > 0.0 and m < 0.2], ["a bit", lambda m: m > 0.0 and m < 0.5], ["a little", lambda m: m > 0.0 and m < 0.5], ["quite", lambda m: m > 0.5 and m < 1.0], ["very", lambda m: m > 0.5 and m < 1.0], ["extremely", lambda m: m > 0.8 and m < 1.0], ["fiercely", lambda m: m > 0.8 and m < 1.0], ["somewhat", lambda m: m > 0.3 and m < 0.6], ["fairly", lambda m: m > 0.3 and m < 0.6], ["rather", lambda m: m > 0.5 and m < 0.7]

				]
}

###  FUNCTION ###
def generate(obj, reading, diff, dream=False):
	print 'received %s %s %s %s' % (obj, reading, diff, dream)

	if dream==True:
		output = random.choice(data[obj]["dreams"])
		return output

	else:
		sentence = random.choice(data[obj]["sentences"])
		type = sentence["type"]
		descriptor = None
		modifier = ""

		while descriptor is None:
			d = random.choice(data[obj]["descriptors"])
			if d[1] == type:
				if d[2](reading):
					descriptor = d[0]
			else:
				print d[1]

		if obj == "APPLE" or obj == "MUG":
			if random.random() > 0.5:
				while modifier is "":
					m = random.choice(data["modifiers"])
					if m[1](reading):
						modifier = m[0]

		if obj == "MUG":
			drink = None
			while drink is None:
				d = random.choice(data[obj]["drinks"])
				if d[1](reading):
					drink = d[0]
			output = sentence["string"].replace("DESC", descriptor).replace("MOD", modifier).replace("DRINK", drink).lstrip()

		elif obj == "APPLE":
			output = sentence["string"].replace("DESC", descriptor).replace("MOD", modifier).lstrip()

		elif obj == "BLOCKS":
			if "DESC" in sentence["string"]:
				output = sentence["string"].replace("DESC", descriptor).lstrip()
			else:
				output = sentence["string"].lstrip()

		print 'returning %s'%(output[0].upper() + output[1:])
		return output[0].upper() + output[1:]

### EXAMPLE ###
# sentence = generate("BLOCKS", random.random(), random.random())
# sentence = generate("APPLE", random.random(), random.random(), True)
# print sentence

<commit_msg>sentence generator hangs indefinitely on the random.randint lines, altering these to random.choice<commit_after>import random

### DATA ###

data = {
	"APPLE" : {
			"sentences" :

				[

				{"string": "It is MOD DESC", "type": 0}, {"string": "This light is MOD DESC", "type": 0}, {"string": "Oh, it's MOD DESC", "type": 0}, {"string": "Oh, that's MOD DESC", "type": 0}, {"string": "MOD DESC now", "type": 0}, {"string": "Well, that's MOD DESC", "type": 0}, {"string": "I am MOD DESC", "type": 1}, {"string": "Am I DESC?", "type": 2}, {"string": "Is it DESC?", "type": 3}, {"string": "Well now I'm MOD DESC", "type": 4}, {"string": "My skin is MOD DESC", "type": 4}, {"string": "I feel DESC me", "type": 5}

				]

			,

			"descriptors" :

				[

				["light", 0, lambda m: m > 0.5 and m < 1.0], ["bright", 0, lambda m: m > 0.6 and m < 1.0], ["beaming", 0, lambda m: m > 0.8 and m < 1.0], ["dark", 0, lambda m: m > 0.0 and m < 0.5], ["murky", 0, lambda m: m > 0.0 and m < 0.5], ["shadowy", 0, lambda m: m > 0.0 and m < 0.5], ["basking", 1, lambda m: m > 0.5 and m < 1.0], ["shaded", 1, lambda m: m > 0.0 and m < 0.5], ["ripening", 1, lambda m: m > 0.5 and m < 1.0], ["ripe", 2, lambda m: m > 0.0 and m < 1.0], ["night", 3, lambda m: m > 0.0 and m < 0.5], ["daytime", 3, lambda m: m > 0.5 and m < 1.0], ["bathed in light", 4, lambda m: m > 0.5 and m < 1.0], ["illuminated", 4, lambda m: m > 0.6 and m < 1.0], ["red", 4, lambda m: m > 0.0 and m < 1.0] , ["light shining on", 5, lambda m: m > 0.5 and m < 1.0], ["shadow falling on", 5, lambda m: m > 0.5 and m < 1.0]

				]
			,
			"dreams" :

				[
				"I used to be quite small, but I am larger now", "When I was green, there was much more firmness inside", "The time I fell to the ground, there had been less light that day", "I haven't noticed any bees in a while", "There were big green leaves atop my crown"
				]

	},
	"MUG" : {
			"sentences" :

				[

				{"string": "It is MOD DESC", "type": 0}, {"string": "It is MOD DESC now, would you like some DRINK?", "type": 0}, {"string": "Oh, it's getting MOD DESC. Might be time for DRINK.", "type": 0}, {"string": "Aren't you MOD DESC now? Some DRINK might be a good idea.", "type": 0}, {"string": "Sure is MOD DESC. Why don't you pour youself a cup of DRINK?", "type": 0}

				]

			,

			"descriptors" :

				[

				["warm", 0, lambda m: m > 0.5 and m < 1.0], ["temperate", 0, lambda m: m > 0.4 and m < 0.6], ["cozy", 0, lambda m: m > 0.5 and m < 0.7], ["hot", 0, lambda m: m > 0.7 and m < 1.0], ["sweltering", 0, lambda m: m > 0.8 and m < 1.0], ["cool", 0, lambda m: m > 0.2 and m < 0.5], ["cold", 0, lambda m: m > 0.0 and m < 0.3], ["chilly", 0, lambda m: m > 0.0 and m < 0.3], ["freezing", 0, lambda m: m > 0.0 and m < 0.3], ["brisk", 0, lambda m: m > 0.2 and m < 0.5], ["lukewarm", 0, lambda m: m > 0.4 and m < 0.6]

				]
			,
			"drinks" :

				[

				["tea", lambda m: m > 0.0 and m < 0.5], ["coffee", lambda m: m > 0.0 and m < 0.5], ["cocoa", lambda m: m > 0.0 and m < 0.5], ["hot toddy", lambda m: m > 0.0 and m < 0.5], ["warm milk", lambda m: m > 0.0 and m < 0.5], ["hot cider", lambda m: m > 0.0 and m < 0.5], ["lemonade", lambda m: m > 0.5 and m < 1.0], ["water", lambda m: m > 0.5 and m < 1.0], ["iced tea", lambda m: m > 0.5 and m < 1.0], ["iced coffee", lambda m: m > 0.5 and m < 1.0], ["soda", lambda m: m > 0.5 and m < 1.0], ["seltzer", lambda m: m > 0.5 and m < 1.0], ["cold drink", lambda m: m > 0.5 and m < 1.0], ["hot or cold beverage", lambda m: m > 0.4 and m < 0.6]

				]
			,
			"dreams" :

				[
				"Sometimes I spin around and around and get very hot", "I once sat unused in darkness for a very long time", "There was a day when I was used 15 times. There were many cups of tea.", "When the spoon stirs, it clinks and clanks against my sides", "Ice dropped in makes liquid slosh around me"
				]

	},
	"BLOCKS" : {
			"sentences" :

				[

				{"string": "It is DESC", "type": 0}, {"string": "DESC here", "type": 1}, {"string": "DESC with us now", "type": 1}, {"string": "DESC present", "type": 1}, {"string": "DESC near", "type": 1}, {"string": "We see DESC", "type": 2}, {"string": "We see DESC has arrived", "type": 3}, {"string": "We see DESC is present", "type": 3}, {"string": "We see DESC has come", "type": 3}, {"string": "Oh, DESC now", "type": 4}, {"string": "Hello", "type": 5}, {"string": "Have you been here long?", "type": 5}, {"string": "Are you here to play with us?", "type": 5}, {"string": "Will we be one on top of the other again?", "type": 5}, {"string": "Will we be separated again?", "type": 5}

				]

			,

			"descriptors" :

				[

				["busy", 0, lambda m: m > 0.5 and m < 1.0], ["crowded", 0, lambda m: m > 0.8 and m < 1.0], ["packed", 0, lambda m: m > 0.8 and m < 1.0], ["teeming", 0, lambda m: m > 0.8 and m < 1.0], ["populated", 0, lambda m: m > 0.5 and m < 1.0], ["not empty", 0, lambda m: m > 0.2 and m < 1.0], ["empty", 0, lambda m: m > 0.0 and m < 0.3], ["idle", 0, lambda m: m > 0.0 and m < 0.2], ["clear of people", 0, lambda m: m > 0.0 and m < 0.2], ["devoid of people", 0, lambda m: m > 0.0 and m < 0.2], ["someone is", 1, lambda m: m > 0.3 and m < 1.0], ["people are", 1, lambda m: m > 0.3 and m < 1.0], ["a person is", 1, lambda m: m > 0.3 and m < 1.0], ["nobody is", 1, lambda m: m > 0.0 and m < 0.3], ["some people are", 0, lambda m: m > 0.3 and m < 1.0], ["few people are", 1, lambda m: m > 0.0 and m < 0.3], ["no one", 2, lambda m: m > 0.0 and m < 0.2], ["someone", 2, lambda m: m > 0.2 and m < 1.0], ["people", 2, lambda m: m > 0.2 and m < 1.0], ["you", 2, lambda m: m > 0.2 and m < 1.0], ["a group of people", 2, lambda m: m > 0.4 and m < 1.0], ["an assembly of people", 2, lambda m: m > 0.4 and m < 1.0], ["someone", 3, lambda m: m > 0.3 and m < 1.0], ["no one", 3, lambda m: m > 0.0 and m < 0.3], ["you are here", 4, lambda m: m > 0.3 and m < 1.0], ["we have company", 4, lambda m: m > 0.3 and m < 1.0], ["you have gone", 4, lambda m: m > 0.0 and m < 0.3], ["nobody is here", 4, lambda m: m > 0.0 and m < 0.3], ["", 5, lambda m: m > 0.0 and m < 1.0]

				]
			,
			"dreams" :

				[
				"We have seen many more people today than usual", "A long time ago, we sat next to one another in a container", "62 days ago, we were stacked unusually high", "Once, we moved around for a long time before we rested", "There used to be 26 of us", "Sometimes, nobody comes for a long time"
				]

	},
	"modifiers" :

				[

					["a tiny bit", lambda m: m > 0.0 and m < 0.2], ["a bit", lambda m: m > 0.0 and m < 0.5], ["a little", lambda m: m > 0.0 and m < 0.5], ["quite", lambda m: m > 0.5 and m < 1.0], ["very", lambda m: m > 0.5 and m < 1.0], ["extremely", lambda m: m > 0.8 and m < 1.0], ["fiercely", lambda m: m > 0.8 and m < 1.0], ["somewhat", lambda m: m > 0.3 and m < 0.6], ["fairly", lambda m: m > 0.3 and m < 0.6], ["rather", lambda m: m > 0.5 and m < 0.7]

				]
}

###  FUNCTION ###
def generate(obj, reading, diff, dream=False):
	print 'received %s %s %s %s' % (obj, reading, diff, dream)

	if dream==True:
		output = random.choice(data[obj]["dreams"])
		return output

	else:
		sentence = random.choice(data[obj]["sentences"])
		print sentence
		type = sentence["type"]
		descriptor = None
		modifier = ""

		while descriptor is None:
			d = random.choice(data[obj]["descriptors"])
			if d[1] == type:
				if d[2](reading):
					descriptor = d[0]

		if obj == "APPLE" or obj == "MUG":
			if random.random() > 0.5:
				while modifier is "":
					m = random.choice(data["modifiers"])
					if m[1](reading):
						modifier = m[0]

		if obj == "MUG":
			drink = None
			while drink is None:
				d = random.choice(data[obj]["drinks"])
				if d[1](reading):
					drink = d[0]
			output = sentence["string"].replace("DESC", descriptor).replace("MOD", modifier).replace("DRINK", drink).lstrip()

		elif obj == "APPLE":
			output = sentence["string"].replace("DESC", descriptor).replace("MOD", modifier).lstrip()

		elif obj == "BLOCKS":
			if "DESC" in sentence["string"]:
				output = sentence["string"].replace("DESC", descriptor).lstrip()
			else:
				output = sentence["string"].lstrip()

		print 'returning %s'%(output[0].upper() + output[1:])
		return output[0].upper() + output[1:]

### EXAMPLE ###
# sentence = generate("BLOCKS", random.random(), random.random())
# sentence = generate("APPLE", random.random(), random.random(), True)
# print sentence

<|endoftext|>