<commit_before>import re
from pyramid.view import view_config
from ..contentbase import (
    Root
)
from ..indexing import ELASTIC_SEARCH

sanitize_search_string_re = re.compile(r'[\\\+\-\&\|\!\(\)\{\}\[\]\^\~\:\/\\\*\?]')

def get_filtered_query(term, fields, search_fields):
    return {
        'explain': True,
        'query': {
            'filtered': {
                'query': {
                    'queryString': {
                        'query': term,
                        'analyze_wildcard': True,
                        'analyzer': 'encoded_search_analyzer',
                        'default_operator': 'AND',
                        'fields': search_fields
                    }
                },
                'filter': {
                    'and': {
                        'filters': []
                    }
                }
            }
        },
        'highlight': {
            'fields': {
                '_all': {}
            }
        },
        'facets': {},
        'fields': fields
    }

def sanitize_search_string(text):
    return sanitize_search_string_re.sub(r'\\\g<0>', text)

@view_config(name='search', context=Root, request_method='GET', permission='view')
def search(context, request):
    ''' Search view connects to ElasticSearch and returns the results'''

    result = context.__json__(request)
    params = request.params
    root = request.root
    result.update({
        '@id': '/search/',
        '@type': ['search'],
        'title': 'Search',
        'facets': [],
        '@graph': [],
        'columns': {},
        'count': 0,
        'filters': [],
        'notification': ''
    })

    qs = request.environ.get('QUERY_STRING')
    if qs:
        result['@id'] = '/search/?%s' % qs

    es = request.registry[ELASTIC_SEARCH]
    if 'limit' in params:
        if params['limit'] == 'all':
            size = 99999
        else:
            if params['limit'].isdigit():
                size = params['limit']
            else:
                size = 100
    else:
        size = 100

    try:
        search_term = params['searchTerm'].strip()
        search_term = sanitize_search_string(search_term)
        # Handling whitespaces in the search term
        if not search_term:
            result['notification'] = 'Please enter search term'
            return result
    except:
        if 'type' in params:
            if params['type'] == '*':
                result['notification'] = 'Please enter search term'
                return result
            else:
                search_term = "*"
        else:
            result['notification'] = 'Please enter search term'
            return result

    try:
        search_type = root.collections[params['type']].item_type
        collections = root.by_item_type.keys()
        # handling invalid item types
        if search_type not in collections:
            result['notification'] = '\'' + search_type + '\' is not a valid \'item type\''
            return result
    except:
        # Handling search type
        search_type = '*'
        if not search_term:
            result['notification'] = 'Please enter search term'
            return result
    if search_term == '*' and search_type == '*':
        result['notification'] = 'Please enter search term'
        return result

    # Building query for filters
    fields = ['object.@id', 'object.@type']
    search_fields = []
    if search_type == '*':
        doc_types = ['antibody_approval', 'biosample', 'experiment', 'target', 'dataset']
    else:
        doc_types = [search_type]

    collections = root.by_item_type
    for doc_type in doc_types:
        collection = root[doc_type]
        schema = collection.schema
        for column in collection.columns:
            fields.append('object.' + column)
            result['columns'].update({column: collection.columns[column]})
        # Adding search fields and boost values
        for value in schema.get('boost_values', ()):
            search_fields = search_fields + ['object.' + value, 'object.' + value + '.standard^2', 'object.' + value + '.untouched^3']

    # Builds filtered query which supports multiple facet selection
    query = get_filtered_query(search_term, list(set(fields)), search_fields)

    # Setting filters
    for key, value in params.iteritems():
        if key not in ['type', 'searchTerm', 'limit', 'format']:
            if value == 'other':
                query['query']['filtered']['filter']['and']['filters'] \
                    .append({'missing': {'field': 'object.' + key}})
            else:
                query['query']['filtered']['filter']['and']['filters'] \
                    .append({'bool': {'must': {'term': {'object.' + key + '.untouched': value}}}})
            result['filters'].append({key: value})

    # Adding facets to the query
    facets = []
    if len(doc_types) > 1:
        facets = [{'Data Type': 'object.@type.untouched'}]
        for facet in facets:
            face = {'terms': {'field': '', 'size': 99999}}
            face['terms']['field'] = facet[facet.keys()[0]]
            query['facets'][facet.keys()[0]] = face
    else:
        facets = root[doc_types[0]].schema['facets']
        for facet in facets:
            face = {'terms': {'field': '', 'size': 99999}}
            face['terms']['field'] = 'object.' + facet[facet.keys()[0]] + '.untouched'
            query['facets'][facet.keys()[0]] = face
            for f in result['filters']:
                if facet[facet.keys()[0]] == f.keys()[0]:
                    del(query['facets'][facet.keys()[0]])

    # Execute the query
    results = es.search(query, index='encoded', doc_type=doc_types, size=size)

    # Loading facets in to the results
    if 'facets' in results:
        facet_results = results['facets']
        if len(doc_types) > 1:
            for facet in facets:
                if facet.keys()[0] in facet_results:
                    face = {}
                    face['field'] = 'type'
                    face[facet.keys()[0]] = []
                    for term in facet_results[facet.keys()[0]]['terms']:
                        if term['term'] in doc_types:
                            face[facet.keys()[0]].append({root.by_item_type[term['term']].__name__: term['count']})
                    result['facets'].append(face)
        else:
            for facet in facets:
                if facet.keys()[0] in facet_results:
                    face = {}
                    face['field'] = facet[facet.keys()[0]]
                    face[facet.keys()[0]] = []
                    for term in facet_results[facet.keys()[0]]['terms']:
                        face[facet.keys()[0]].append({term['term']: term['count']})
                    if len(face[facet.keys()[0]]) > 1:
                        result['facets'].append(face)

    # Loading result rows
    for hit in results['hits']['hits']:
        result_hit = hit['fields']
        result_hit_new = {}
        for c in result_hit:
            result_hit_new[c[7:]] = result_hit[c]
        result['@graph'].append(result_hit_new)

    # Adding count
    result['count'] = results['hits']['total']
    if len(result['@graph']):
        result['notification'] = 'Success'
    else:
        if len(search_term) < 3:
            result['notification'] = 'No results found. Search term should be at least 3 characters long.'
        else:
            result['notification'] = 'No results found'
    return result
<commit_msg>Added "principals_allowed_view" to the filters<commit_after>import re
from pyramid.view import view_config
from ..contentbase import (
    Root
)
from ..indexing import ELASTIC_SEARCH
from pyramid.security import effective_principals

sanitize_search_string_re = re.compile(r'[\\\+\-\&\|\!\(\)\{\}\[\]\^\~\:\/\\\*\?]')

def get_filtered_query(term, fields, search_fields, principals):
    return {
        'explain': True,
        'query': {
            'filtered': {
                'query': {
                    'queryString': {
                        'query': term,
                        'analyze_wildcard': True,
                        'analyzer': 'encoded_search_analyzer',
                        'default_operator': 'AND',
                        'fields': search_fields
                    }
                },
                'filter': {
                    'and': {
                        'filters': [
                            {
                                'bool': {
                                    'must': {
                                        'terms': {
                                            'principals_allowed_view': principals
                                        }
                                    }
                                }
                            }
                        ]
                    }
                }
            }
        },
        'highlight': {
            'fields': {
                '_all': {}
            }
        },
        'facets': {},
        'fields': fields
    }

def sanitize_search_string(text):
    return sanitize_search_string_re.sub(r'\\\g<0>', text)

@view_config(name='search', context=Root, request_method='GET', permission='view')
def search(context, request):
    ''' Search view connects to ElasticSearch and returns the results'''

    result = context.__json__(request)
    params = request.params
    root = request.root
    result.update({
        '@id': '/search/',
        '@type': ['search'],
        'title': 'Search',
        'facets': [],
        '@graph': [],
        'columns': {},
        'count': 0,
        'filters': [],
        'notification': ''
    })

    qs = request.environ.get('QUERY_STRING')
    principals = effective_principals(request)

    if qs:
        result['@id'] = '/search/?%s' % qs

    es = request.registry[ELASTIC_SEARCH]
    if 'limit' in params:
        if params['limit'] == 'all':
            size = 99999
        else:
            if params['limit'].isdigit():
                size = params['limit']
            else:
                size = 100
    else:
        size = 100

    try:
        search_term = params['searchTerm'].strip()
        search_term = sanitize_search_string(search_term)
        # Handling whitespaces in the search term
        if not search_term:
            result['notification'] = 'Please enter search term'
            return result
    except:
        if 'type' in params:
            if params['type'] == '*':
                result['notification'] = 'Please enter search term'
                return result
            else:
                search_term = "*"
        else:
            result['notification'] = 'Please enter search term'
            return result

    try:
        search_type = root.collections[params['type']].item_type
        collections = root.by_item_type.keys()
        # handling invalid item types
        if search_type not in collections:
            result['notification'] = '\'' + search_type + '\' is not a valid \'item type\''
            return result
    except:
        # Handling search type
        search_type = '*'
        if not search_term:
            result['notification'] = 'Please enter search term'
            return result
    if search_term == '*' and search_type == '*':
        result['notification'] = 'Please enter search term'
        return result

    # Building query for filters
    fields = ['object.@id', 'object.@type']
    search_fields = []
    if search_type == '*':
        doc_types = ['antibody_approval', 'biosample', 'experiment', 'target', 'dataset']
    else:
        doc_types = [search_type]

    collections = root.by_item_type
    for doc_type in doc_types:
        collection = root[doc_type]
        schema = collection.schema
        for column in collection.columns:
            fields.append('object.' + column)
            result['columns'].update({column: collection.columns[column]})
        # Adding search fields and boost values
        for value in schema.get('boost_values', ()):
            search_fields = search_fields + ['object.' + value, 'object.' + value + '.standard^2', 'object.' + value + '.untouched^3']

    # Builds filtered query which supports multiple facet selection
    query = get_filtered_query(search_term, list(set(fields)), search_fields, principals)

    # Setting filters
    for key, value in params.iteritems():
        if key not in ['type', 'searchTerm', 'limit', 'format']:
            if value == 'other':
                query['query']['filtered']['filter']['and']['filters'] \
                    .append({'missing': {'field': 'object.' + key}})
            else:
                query['query']['filtered']['filter']['and']['filters'] \
                    .append({'bool': {'must': {'term': {'object.' + key + '.untouched': value}}}})
            result['filters'].append({key: value})

    # Adding facets to the query
    facets = []
    if len(doc_types) > 1:
        facets = [{'Data Type': 'object.@type.untouched'}]
        for facet in facets:
            face = {'terms': {'field': '', 'size': 99999}}
            face['terms']['field'] = facet[facet.keys()[0]]
            query['facets'][facet.keys()[0]] = face
    else:
        facets = root[doc_types[0]].schema['facets']
        for facet in facets:
            face = {'terms': {'field': '', 'size': 99999}}
            face['terms']['field'] = 'object.' + facet[facet.keys()[0]] + '.untouched'
            query['facets'][facet.keys()[0]] = face
            for f in result['filters']:
                if facet[facet.keys()[0]] == f.keys()[0]:
                    del(query['facets'][facet.keys()[0]])

    import pdb; pdb.set_trace();
    # Execute the query
    results = es.search(query, index='encoded', doc_type=doc_types, size=size)

    # Loading facets in to the results
    if 'facets' in results:
        facet_results = results['facets']
        if len(doc_types) > 1:
            for facet in facets:
                if facet.keys()[0] in facet_results:
                    face = {}
                    face['field'] = 'type'
                    face[facet.keys()[0]] = []
                    for term in facet_results[facet.keys()[0]]['terms']:
                        if term['term'] in doc_types:
                            face[facet.keys()[0]].append({root.by_item_type[term['term']].__name__: term['count']})
                    result['facets'].append(face)
        else:
            for facet in facets:
                if facet.keys()[0] in facet_results:
                    face = {}
                    face['field'] = facet[facet.keys()[0]]
                    face[facet.keys()[0]] = []
                    for term in facet_results[facet.keys()[0]]['terms']:
                        face[facet.keys()[0]].append({term['term']: term['count']})
                    if len(face[facet.keys()[0]]) > 1:
                        result['facets'].append(face)

    # Loading result rows
    for hit in results['hits']['hits']:
        result_hit = hit['fields']
        result_hit_new = {}
        for c in result_hit:
            result_hit_new[c[7:]] = result_hit[c]
        result['@graph'].append(result_hit_new)

    # Adding count
    result['count'] = results['hits']['total']
    if len(result['@graph']):
        result['notification'] = 'Success'
    else:
        if len(search_term) < 3:
            result['notification'] = 'No results found. Search term should be at least 3 characters long.'
        else:
            result['notification'] = 'No results found'
    return result
<|endoftext|>


<commit_before>#!/usr/bin/env python

import json
import logging
import os
from collections import defaultdict
import pandas as pd

LOG = logging.getLogger(__name__)

"""
    Serotype prediction for E. coli
"""

def predict_serotype(blast_output_file, ectyper_dict_file, predictions_file, detailed=False):
    """
    Predict the serotype of all genomes, given the blast output of the markers against the genomes

    :param blast_output_file: Results of allele file against the genomes of interest
    :param ectyper_dict_file: JSON file of known alleles and their O and H mappings
    :param predictions_file: The CSV output of the program
    :param detailed: BOOL: generate detailed output or not
    :return: The CSV formatted predictions file
    """

    basename, extension = os.path.splitext(predictions_file)
    parsed_output_file = ''.join([basename, '_raw', extension])

    LOG.info("Predicting serotype from blast output")
    output_df = blast_output_to_df(blast_output_file)
    ectyper_df = ectyper_dict_to_df(ectyper_dict_file)

    # Merge output_df and ectyper_df
    output_df = output_df.merge(ectyper_df, left_on='qseqid', right_on='name', how='left')
    predictions_dict = {}

    # Select individual genomes
    output_df['genome_name'] = output_df['sseqid'].str.split('|').str[1]
    # Initialize constants
    gene_pairs = {'wzx':'wzy', 'wzy':'wzx', 'wzm':'wzt', 'wzt':'wzm'}
    predictions_columns = ['O_prediction', 'O_info', 'H_prediction', 'H_info']
    gene_list = ['wzx', 'wzy', 'wzm', 'wzt', 'fliC', 'fllA', 'flkA', 'flmA', 'flnA']
    if detailed:
        # Add gene lists for detailed output report
        for gene in gene_list:
            predictions_columns.append(gene)

    # Make prediction for each genome based on blast output
    for genome_name, per_genome_df in output_df.groupby('genome_name'):
        predictions_dict[genome_name] = get_prediction(
            per_genome_df, predictions_columns, gene_pairs, detailed)
    predictions_df = pd.DataFrame(predictions_dict).transpose()

    if predictions_df.empty:
        predictions_df = pd.DataFrame(columns=predictions_columns)
    predictions_df = predictions_df[predictions_columns]
    store_df(output_df, parsed_output_file)
    store_df(predictions_df, predictions_file)

    LOG.info("Serotype prediction completed")
    return predictions_file

def get_prediction(per_genome_df, predictions_columns, gene_pairs, detailed):
    """
     Make serotype prediction for a single genome based on the blast output

    :param per_genome_df: The blastn results for the given genome
    :param predictions_columns: Columns to be filled with data from this function
    :param gene_pairs: Required gene pairs for the O-prediction logic
    :param detailed: BOOL: Whether to report detailed output or not
    :return:
    """

    # Extract the needed information from the blast results
    useful_columns = [
        'gene', 'serotype', 'score', 'name', 'desc', 'pident', 'qcovhsp', 'qseqid', 'sseqid', 'sseq'
    ]
    per_genome_df = per_genome_df.sort_values(['gene', 'serotype', 'score'], ascending=False)
    per_genome_df = per_genome_df[~per_genome_df.duplicated(['gene', 'serotype'])]
    predictors_df = per_genome_df[useful_columns]
    predictors_df = predictors_df.sort_values('score', ascending=False)

    predictions = {}
    for column in predictions_columns:
        predictions[column] = None

    for predicting_antigen in ['O', 'H']:
        genes_pool = defaultdict(list)

        for index, row in predictors_df.iterrows():
            gene = row['gene']
            if detailed:
                predictions[gene] = True

            if not predictions[predicting_antigen + '_prediction']:
                serotype = row['serotype']
                if serotype[0] is not predicting_antigen:
                    continue
                genes_pool[gene].append(serotype)
                prediction = None

                if len(serotype) < 1:
                    continue

                antigen = serotype[0].upper()
                if antigen != predicting_antigen:
                    continue

                if gene in gene_pairs.keys():
                    predictions[antigen + '_info'] = 'Only unpaired alignments found'
                    # Pair gene logic
                    potential_pairs = genes_pool.get(gene_pairs.get(gene))
                    if row['score'] != 1:
                        LOG.debug("NEW ALLELE for {}: {}".format(gene, row['sseq']))

                    if potential_pairs is None:
                        continue
                    if serotype in potential_pairs:
                        prediction = serotype
                else:
                    # Normal logic
                    prediction = serotype

                if prediction is None:
                    continue

                predictions[antigen+'_info'] = 'Alignment found'
                predictions[predicting_antigen + '_prediction'] = prediction

        # No alignment found
        # Make prediction based on non-paired gene
        # if only one non-paired gene is avaliable, and no paired-genes are available
        if predictions.get(predicting_antigen + '_prediction') is None:
            if len(genes_pool) == 1:
                serotypes = list(genes_pool.values())[0]
                if len(serotypes) == 1:
                    predictions[antigen + '_info'] = 'Lone unpaired alignment found'
                    predictions[predicting_antigen + '_prediction'] = serotypes[0]
    return predictions

def blast_output_to_df(blast_output_file):
    """
    Convert the raw Blast output to a DataFrame

    :param blast_output_file: Blast results to convert
    :return: DataFrame of the blast results
    """

    output_data = []
    with open(blast_output_file, 'r') as fh:
        for line in fh:
            fields = line.strip().split()
            entry = {
                'qseqid': fields[0],
                'qlen': fields[1],
                'sseqid': fields[2],
                'length': fields[3],
                'pident': fields[4],
                'sstart': fields[5],
                'send': fields[6],
                'sframe': fields[7],
                'qcovhsp': fields[8],
                'sseq': fields[9]
            }
            output_data.append(entry)
    df = pd.DataFrame(output_data)

    if not output_data:
        LOG.warning("No hits found for blast output file {}".format(blast_output_file))

        # Return empty dataframe with correct columns
        return pd.DataFrame(
            columns=[
                'length', 'pident', 'qcovhsp',
                'qlen', 'qseqid', 'send',
                'sframe', 'sseqid', 'sstart', 'sseq'
            ])
    else:
        df['score'] = df['pident'].astype(float)*df['qcovhsp'].astype(float)/10000
        return df

def ectyper_dict_to_df(ectyper_dict_file):
    """
    Load all the known alleles for the O and H genes, and store them as a DataFrame.
    :param ectyper_dict_file: JSON file of all known O- and H- alleles
    :return: DataFrame of the JSON file
    """

    with open(ectyper_dict_file) as fh:
        ectyper_dict = json.load(fh)
        temp_list = []
        for antigen, alleles in ectyper_dict.items():
            for name, allele in alleles.items():
                new_entry = {
                    'serotype': allele.get('allele'),
                    'name': name,
                    'gene': allele.get('gene'),
                    'desc': allele.get('desc')
                }
                temp_list.append(new_entry)
        df = pd.DataFrame(temp_list)
        return df

def store_df(src_df, dst_file):
    """
    Store a DataFrame as a file. Append to an existing file, or create a new one
    if one does not exist.

    :param src_df: DataFrame to be stored
    :param dst_file: file to be appened to or created
    :return: None
    """

    if os.path.isfile(dst_file):
        with open(dst_file, 'a') as fh:
            src_df.to_csv(fh, header=False)
    else:
        with open(dst_file, 'w') as fh:
            src_df.to_csv(fh, header=True, index_label='genome')

def report_result(csv_file):
    """
    Outputs the results of the given file to the log.

    :param csv_file: File whose contents will be added to the log
    :return: None
    """

    df = pd.read_csv(csv_file)
    if df.empty:
        LOG.info('No prediction was made because no alignment was found')
        return
    LOG.info('\n{0}'.format(df.to_string(index=False)))

def add_non_predicted(all_genomes_list, predictions_file):
    """
    Add genomes that do not show up in the blast results to the prediction file

    :param all_genomes_list: the list of genomes given by the user
    :param predictions_file: the file containing the ectyper predictions
    :return: modified prediction file
    """

    # genome names are given without the filename extension
    genome_names = []
    for g in all_genomes_list:
        gname = os.path.splitext(os.path.split(g)[1])[0]
        genome_names.append(gname)

    df = pd.read_csv(predictions_file)
    df = df.merge(pd.DataFrame(genome_names, columns=['genome']), on='genome', how='outer')
    df.fillna({'O_info':'No alignment found', 'H_info':'No alignment found'}, inplace=True)
    df.fillna('-', inplace=True)
    df.to_csv(predictions_file, index=False)
    return predictions_file
<commit_msg>Increased information for new alleles in debug<commit_after>#!/usr/bin/env python

import json
import logging
import os
from collections import defaultdict
import pandas as pd

LOG = logging.getLogger(__name__)

"""
    Serotype prediction for E. coli
"""

def predict_serotype(blast_output_file, ectyper_dict_file, predictions_file, detailed=False):
    """
    Predict the serotype of all genomes, given the blast output of the markers against the genomes

    :param blast_output_file: Results of allele file against the genomes of interest
    :param ectyper_dict_file: JSON file of known alleles and their O and H mappings
    :param predictions_file: The CSV output of the program
    :param detailed: BOOL: generate detailed output or not
    :return: The CSV formatted predictions file
    """

    basename, extension = os.path.splitext(predictions_file)
    parsed_output_file = ''.join([basename, '_raw', extension])

    LOG.info("Predicting serotype from blast output")
    output_df = blast_output_to_df(blast_output_file)
    ectyper_df = ectyper_dict_to_df(ectyper_dict_file)

    # Merge output_df and ectyper_df
    output_df = output_df.merge(ectyper_df, left_on='qseqid', right_on='name', how='left')
    predictions_dict = {}

    # Select individual genomes
    output_df['genome_name'] = output_df['sseqid'].str.split('|').str[1]
    # Initialize constants
    gene_pairs = {'wzx':'wzy', 'wzy':'wzx', 'wzm':'wzt', 'wzt':'wzm'}
    predictions_columns = ['O_prediction', 'O_info', 'H_prediction', 'H_info']
    gene_list = ['wzx', 'wzy', 'wzm', 'wzt', 'fliC', 'fllA', 'flkA', 'flmA', 'flnA']
    if detailed:
        # Add gene lists for detailed output report
        for gene in gene_list:
            predictions_columns.append(gene)

    # Make prediction for each genome based on blast output
    for genome_name, per_genome_df in output_df.groupby('genome_name'):
        predictions_dict[genome_name] = get_prediction(
            per_genome_df, predictions_columns, gene_pairs, detailed)
    predictions_df = pd.DataFrame(predictions_dict).transpose()

    if predictions_df.empty:
        predictions_df = pd.DataFrame(columns=predictions_columns)
    predictions_df = predictions_df[predictions_columns]
    store_df(output_df, parsed_output_file)
    store_df(predictions_df, predictions_file)

    LOG.info("Serotype prediction completed")
    return predictions_file

def get_prediction(per_genome_df, predictions_columns, gene_pairs, detailed):
    """
     Make serotype prediction for a single genome based on the blast output

    :param per_genome_df: The blastn results for the given genome
    :param predictions_columns: Columns to be filled with data from this function
    :param gene_pairs: Required gene pairs for the O-prediction logic
    :param detailed: BOOL: Whether to report detailed output or not
    :return:
    """

    # Extract the needed information from the blast results
    useful_columns = [
        'gene', 'serotype', 'score', 'name', 'desc', 'pident', 'qcovhsp', 'qseqid', 'sseqid', 'sseq'
    ]
    per_genome_df = per_genome_df.sort_values(['gene', 'serotype', 'score'], ascending=False)
    per_genome_df = per_genome_df[~per_genome_df.duplicated(['gene', 'serotype'])]
    predictors_df = per_genome_df[useful_columns]
    predictors_df = predictors_df.sort_values('score', ascending=False)

    predictions = {}
    for column in predictions_columns:
        predictions[column] = None

    for predicting_antigen in ['O', 'H']:
        genes_pool = defaultdict(list)

        for index, row in predictors_df.iterrows():
            gene = row['gene']
            if detailed:
                predictions[gene] = True

            if not predictions[predicting_antigen + '_prediction']:
                serotype = row['serotype']
                if serotype[0] is not predicting_antigen:
                    continue
                genes_pool[gene].append(serotype)
                prediction = None

                if len(serotype) < 1:
                    continue

                antigen = serotype[0].upper()
                if antigen != predicting_antigen:
                    continue

                if gene in gene_pairs.keys():
                    predictions[antigen + '_info'] = 'Only unpaired alignments found'
                    # Pair gene logic
                    potential_pairs = genes_pool.get(gene_pairs.get(gene))
                    if row['score'] != 1:
                        LOG.debug("NEW ALLELE closest to query {}, name: {}, score: {}, sequence: {}".format(row['qseqid'], row['sseqid'], row['score'], row['sseq']))

                    if potential_pairs is None:
                        continue
                    if serotype in potential_pairs:
                        prediction = serotype
                else:
                    # Normal logic
                    prediction = serotype

                if prediction is None:
                    continue

                predictions[antigen+'_info'] = 'Alignment found'
                predictions[predicting_antigen + '_prediction'] = prediction

        # No alignment found
        # Make prediction based on non-paired gene
        # if only one non-paired gene is avaliable, and no paired-genes are available
        if predictions.get(predicting_antigen + '_prediction') is None:
            if len(genes_pool) == 1:
                serotypes = list(genes_pool.values())[0]
                if len(serotypes) == 1:
                    predictions[antigen + '_info'] = 'Lone unpaired alignment found'
                    predictions[predicting_antigen + '_prediction'] = serotypes[0]
    return predictions

def blast_output_to_df(blast_output_file):
    """
    Convert the raw Blast output to a DataFrame

    :param blast_output_file: Blast results to convert
    :return: DataFrame of the blast results
    """

    output_data = []
    with open(blast_output_file, 'r') as fh:
        for line in fh:
            fields = line.strip().split()
            entry = {
                'qseqid': fields[0],
                'qlen': fields[1],
                'sseqid': fields[2],
                'length': fields[3],
                'pident': fields[4],
                'sstart': fields[5],
                'send': fields[6],
                'sframe': fields[7],
                'qcovhsp': fields[8],
                'sseq': fields[9]
            }
            output_data.append(entry)
    df = pd.DataFrame(output_data)

    if not output_data:
        LOG.warning("No hits found for blast output file {}".format(blast_output_file))

        # Return empty dataframe with correct columns
        return pd.DataFrame(
            columns=[
                'length', 'pident', 'qcovhsp',
                'qlen', 'qseqid', 'send',
                'sframe', 'sseqid', 'sstart', 'sseq'
            ])
    else:
        df['score'] = df['pident'].astype(float)*df['qcovhsp'].astype(float)/10000
        return df

def ectyper_dict_to_df(ectyper_dict_file):
    """
    Load all the known alleles for the O and H genes, and store them as a DataFrame.
    :param ectyper_dict_file: JSON file of all known O- and H- alleles
    :return: DataFrame of the JSON file
    """

    with open(ectyper_dict_file) as fh:
        ectyper_dict = json.load(fh)
        temp_list = []
        for antigen, alleles in ectyper_dict.items():
            for name, allele in alleles.items():
                new_entry = {
                    'serotype': allele.get('allele'),
                    'name': name,
                    'gene': allele.get('gene'),
                    'desc': allele.get('desc')
                }
                temp_list.append(new_entry)
        df = pd.DataFrame(temp_list)
        return df

def store_df(src_df, dst_file):
    """
    Store a DataFrame as a file. Append to an existing file, or create a new one
    if one does not exist.

    :param src_df: DataFrame to be stored
    :param dst_file: file to be appened to or created
    :return: None
    """

    if os.path.isfile(dst_file):
        with open(dst_file, 'a') as fh:
            src_df.to_csv(fh, header=False)
    else:
        with open(dst_file, 'w') as fh:
            src_df.to_csv(fh, header=True, index_label='genome')

def report_result(csv_file):
    """
    Outputs the results of the given file to the log.

    :param csv_file: File whose contents will be added to the log
    :return: None
    """

    df = pd.read_csv(csv_file)
    if df.empty:
        LOG.info('No prediction was made because no alignment was found')
        return
    LOG.info('\n{0}'.format(df.to_string(index=False)))

def add_non_predicted(all_genomes_list, predictions_file):
    """
    Add genomes that do not show up in the blast results to the prediction file

    :param all_genomes_list: the list of genomes given by the user
    :param predictions_file: the file containing the ectyper predictions
    :return: modified prediction file
    """

    # genome names are given without the filename extension
    genome_names = []
    for g in all_genomes_list:
        gname = os.path.splitext(os.path.split(g)[1])[0]
        genome_names.append(gname)

    df = pd.read_csv(predictions_file)
    df = df.merge(pd.DataFrame(genome_names, columns=['genome']), on='genome', how='outer')
    df.fillna({'O_info':'No alignment found', 'H_info':'No alignment found'}, inplace=True)
    df.fillna('-', inplace=True)
    df.to_csv(predictions_file, index=False)
    return predictions_file
<|endoftext|>


<commit_before>#!/usr/bin/env python
'''
CREATED:2014-03-18 by Justin Salamon <justin.salamon@nyu.edu>

Compute melody extraction evaluation measures

Usage:

./melody_eval.py TRUTH.TXT PREDICTION.TXT
(CSV files also accepted)

For a detailed explanation of the measures please refer to:

J. Salamon, E. Gomez, D. P. W. Ellis and G. Richard, "Melody Extraction
from Polyphonic Music Signals: Approaches, Applications and Challenges",
IEEE Signal Processing Magazine, 31(2):118-134, Mar. 2014.
'''

import numpy as np
import argparse
import sys
import os
from collections import OrderedDict
import mir_eval

def evaluate(reference_file, estimated_file, hop=None):
    '''
    Evaluate two melody (predominant f0) transcriptions, where the first is
    treated as the reference (ground truth) and the second as the estimate to
    be evaluated (prediction).

    Input:
        truth_file - path to reference file. File should contain 2 columns:
              column 1 with timestamps and column 2 the corresponding reference
              frequency values in Hz (see *).
        prediction_file - path to estimate file. File should contain with 2 columns:
              column 1 with timestamps and column 2 the corresponding estimate
              frequency values in Hz (see **).
        hop - the desired hop size (in seconds) to compare the reference and
              estimate sequences (see ***)

    Output:
        M - ordered dictionary containing 5 evaluation measures:
        voicing recall rate -       Fraction of voiced frames in ref estimated as voiced in est
        voicing false alarm rate -  Fraction of unvoiced frames in ref estimated as voiced in est
        raw pitch -                 Fraction of voiced frames in ref for which
                                    est gives a correct pitch estimate (within 50 cents)
        raw chroma -                Same as raw pitch, but ignores octave errors
        overall accuracy -          Overall performance measure combining pitch and voicing

    *    Unvoiced frames should be indicated by 0 Hz.
    **   Unvoiced frames can be indicated either by 0 Hz or by a negative Hz
         value - negative values represent the algorithm's pitch estimate for
         frames it has determined as unvoiced, in case they are in fact voiced.
    ***  For a frame-by-frame comparison, both sequences are resampled using
         the provided hop size (in seconds), the default being 10 ms. The
         frequency values of the resampled sequences are obtained via linear
         interpolation of the original frequency values converted to a cent
         scale.
    '''
    # load the data
    ref_time, ref_freq = mir_eval.io.load_time_series(reference_file)
    est_time, est_freq = mir_eval.io.load_time_series(estimated_file)
    # Convert to reference/estimated voicing/frequency (cent) arrays
    ref_voicing, est_voicing, ref_cent, est_cent = mir_eval.melody.to_cent_voicing(ref_time,
                                                                                ref_freq,
                                                                                est_time,
                                                                                est_freq,
                                                                                hop=hop)

    # Compute metrics
    M = OrderedDict()

    for name, metric in mir_eval.melody.METRICS.items():
        if metric == mir_eval.melody.voicing_measures:
            M[name] = metric(ref_voicing, est_voicing)
        else:
            M[name] = metric(ref_voicing, est_voicing, ref_cent, est_cent)

    return M

def print_evaluation(prediction_file, M):
    '''
    Pretty print the melody extraction evaluation measures
    '''
    keys = M.keys()
    keys.append(os.path.basename(prediction_file))
    print '%s\t%s\t%s\t%s\t%s\t(%s)' % tuple(keys)
    print '%.3f\t\t%.3f\t\t%.3f\t\t%.3f\t\t%.3f' % tuple(M.values())

def process_arguments():
    '''Argparse function to get the program parameters'''

    parser = argparse.ArgumentParser(description='mir_eval melody extraction evaluation')

    parser.add_argument('reference_file',
                        action = 'store',
                        help = 'path to the ground truth annotation')

    parser.add_argument('estimated_file',
                        action = 'store',
                        help = 'path to the estimation file')

    parser.add_argument("--hop", type=float,
                    help="hop size (in seconds) to use for the evaluation")

    return vars(parser.parse_args(sys.argv[1:]))

if __name__ == '__main__':
    # Get the parameters
    parameters = process_arguments()

    # Compute all the scores
    scores = evaluate(**parameters)

    # Print the scores
    print_evaluation(parameters['reference_file'], scores)
<commit_msg>PEP8/Pylint/Pyflakes cleanup, and removing use of METRICS dictionary<commit_after>#!/usr/bin/env python
'''
CREATED:2014-03-18 by Justin Salamon <justin.salamon@nyu.edu>

Compute melody extraction evaluation measures

Usage:

./melody_eval.py TRUTH.TXT PREDICTION.TXT
(CSV files also accepted)

For a detailed explanation of the measures please refer to:

J. Salamon, E. Gomez, D. P. W. Ellis and G. Richard, "Melody Extraction
from Polyphonic Music Signals: Approaches, Applications and Challenges",
IEEE Signal Processing Magazine, 31(2):118-134, Mar. 2014.
'''

import argparse
import sys
import os
from collections import OrderedDict
import mir_eval

def evaluate(reference_file, estimated_file, hop=None):
    '''
    Evaluate two melody (predominant f0) transcriptions, where the first is
    treated as the reference (ground truth) and the second as the estimate to
    be evaluated (prediction).

    :parameters:
        - truth_file : str
            path to reference file. File should contain 2 columns:
            column 1 with timestamps and column 2 the corresponding reference
            frequency values in Hz (see *).
        - prediction_file : str
            path to estimate file. File should contain with 2
            columns: column 1 with timestamps and column 2 the corresponding
            estimate frequency values in Hz (see **).
        - hop : float
            the desired hop size (in seconds) to compare the reference and
            estimate sequences (see ***)

    :returns:
        - M : collections.OrderedDict
            ordered dictionary containing 5 evaluation measures:
            voicing recall rate - Fraction of voiced frames in ref estimated as
                                  voiced in est
            voicing false alarm rate - Fraction of unvoiced frames in ref
                                       estimated as voiced in est
            raw pitch - Fraction of voiced frames in ref for which
                        est gives a correct pitch estimate (within 50 cents)
            raw chroma - Same as raw pitch, but ignores octave errors
            overall accuracy - Overall performance measure combining pitch and
                               voicing

    *    Unvoiced frames should be indicated by 0 Hz.
    **   Unvoiced frames can be indicated either by 0 Hz or by a negative Hz
         value - negative values represent the algorithm's pitch estimate for
         frames it has determined as unvoiced, in case they are in fact voiced.
    ***  For a frame-by-frame comparison, both sequences are resampled using
         the provided hop size (in seconds), the default being 10 ms. The
         frequency values of the resampled sequences are obtained via linear
         interpolation of the original frequency values converted to a cent
         scale.
    '''
    # load the data
    ref_time, ref_freq = mir_eval.io.load_time_series(reference_file)
    est_time, est_freq = mir_eval.io.load_time_series(estimated_file)
    # Convert to reference/estimated voicing/frequency (cent) arrays
    ref_voicing, est_voicing, ref_cent, est_cent = \
        mir_eval.melody.to_cent_voicing(ref_time, ref_freq, est_time, est_freq,
                                        hop=hop)

    # Compute metrics
    M = OrderedDict()

    M['Voicing Measures'] = mir_eval.melody.voicing_measures(ref_voicing,
                                                             est_voicing)
    M['Raw Pitch Accuracy'] = mir_eval.melody.raw_pitch_accuracy(ref_voicing,
                                                                 est_voicing,
                                                                 ref_cent,
                                                                 est_cent)
    M['Raw Chroma Accuracy'] = mir_eval.melody.raw_chroma_accuracy(ref_voicing,
                                                                   est_voicing,
                                                                   ref_cent,
                                                                   est_cent)
    M['Overall Accuracy'] = mir_eval.melody.raw_pitch_accuracy(ref_voicing,
                                                               est_voicing,
                                                               ref_cent,
                                                               est_cent)
    return M

def print_evaluation(prediction_file, M):
    '''
    Pretty print the melody extraction evaluation measures
    '''
    keys = M.keys()
    keys.append(os.path.basename(prediction_file))
    print '%s\t%s\t%s\t%s\t%s\t(%s)' % tuple(keys)
    print '%.3f\t\t%.3f\t\t%.3f\t\t%.3f\t\t%.3f' % tuple(M.values())

def process_arguments():
    '''Argparse function to get the program parameters'''

    parser = argparse.ArgumentParser(description='mir_eval melody extraction '
                                                 'evaluation')

    parser.add_argument('reference_file',
                        action='store',
                        help='path to the ground truth annotation')

    parser.add_argument('estimated_file',
                        action='store',
                        help='path to the estimation file')

    parser.add_argument("--hop", type=float,
                    help="hop size (in seconds) to use for the evaluation")

    return vars(parser.parse_args(sys.argv[1:]))

if __name__ == '__main__':
    # Get the parameters
    parameters = process_arguments()

    # Compute all the scores
    scores = evaluate(**parameters)

    # Print the scores
    print_evaluation(parameters['reference_file'], scores)
<|endoftext|>


<commit_before># PyQuantFi - randomBase.py
# (c) 2012 Nick Collins

from random import random
from normals import inverseCumulativeNormal

class RandomBase(object):
    """
    Abstract random number class.
    Uses the standard library's random() for uniform random variables and
    normals.inverseCumulativeNormal() to transform them.
    """

    def __init__(self, dim):
        self.dim = dim

    def getUniforms(self, N):
        return ([random() for x in range(self.dim)] for x in range(N))

    def getGaussians(self, N):
        return ([inverseCumulativeNormal(x) for x in v] for v in self.getUniforms(N))

class ParkMiller(object):
    """
    Park-Miller random number generator.
    stream() method returns a generator producing pseudo-random
    numbers in the interval [1, 2147483646].
    """

    def __init__(self,seed=1):
        self._const_a = 16807
        self._const_m = 2147483647
        self._const_q = 127773
        self._const_r = 2836
        self._seed = max(int(seed),1)
        self.maximum = self._const_m - 1
        self.minimum = 1

    def stream(self,N):
        a, m, q, r = self._const_a, self._const_m, self._const_q, self._const_r
        count = 0
        while count < N:
            k = self._seed // q # // ensures integer division for Python 3.
            self._seed = (a * (self._seed - k * q) - k * r)
            if self._seed < 0:
                self._seed += m
            yield self._seed
            count += 1

class RandomParkMiller(RandomBase):
    """
    A RandomBase class using the ParkMiller class
    to generate the uniform random variables.
    """

    def __init__(self,dim,seed=1):
        self.dim = dim
        self._seed = seed
        self._pm = ParkMiller(seed)
        self._r = 1/(1. + self._pm.maximum)

    def getUniforms(self,N):
        count = 0
        while count < N:
            yield [x * self._r for x in self._pm.stream(self.dim)]
            count += 1

    def skip(self, nPaths):
        for i in self.getUniforms(nPaths):
            pass

    def reset(self):
        self._pm = ParkMiller(self.seed)

    def _getSeed(self):
        return self._seed

    def _setSeed(self,seed):
        self._seed = seed
        self.reset(seed)

    seed = property(_getSeed, _setSeed)

class AntiThetic(RandomBase):
    """
    Anti-thetic sampling class: wraps another RandomBase class.
    Currently this only works properly for streams of an even length.
    """

    def __init__(self, base):
        self._base = base
        self._oddEven = True

    def getUniforms(self, N):
        for v in self._base.getUniforms(N/2):
            yield v
            yield [1-x for x in v]

    def _setSeed(self, seed):
        self._base.seed = seed
        self._oddEven = True

    def skip(self, nPaths):
        self._base.skip(nPaths / 2)

    def reset(self):
        self._base.reset()
        self._oddEven = True

def loop(N, s):
    """
    Returns a generator of length N, returning
    a loop of values modulo s.
    """
    i = 0
    while i < N:
        yield i % s
        i += 1

class SimpleStratifiedPM(RandomBase):
    """
    Stratified random sampling based on the RandomParkMiller class.
    """

    def __init__(self,seed=1,segments=2**8):
        self.dim = 1
        self._rpm = RandomParkMiller(1,seed)
        self._seed = seed
        self._segments = segments

    def getUniforms(self, N):
        s = self._segments
        return ([(l + x[0])/s] for l, x in zip(loop(N, s), self._rpm.getUniforms(N)))

# Testing

if __name__ == "__main__":
    rv = AntiThetic(SimpleStratifiedPM(1,16))
    N = 2**8
    r = []
    for v in rv.getUniforms(N):
        r.append(v)
    r = [x[0] for x in r]
    mean = sum(r)/N
    var = sum((x-mean)**2 for x in r)/N
    print("mean = %f" % mean)
    print("variance = %f" % var)
<commit_msg>added getter and setter to link AntiThetic's dimension to it's base class<commit_after># PyQuantFi - randomBase.py
# (c) 2012 Nick Collins

from random import random
from normals import inverseCumulativeNormal

class RandomBase(object):
    """
    Abstract random number class.
    Uses the standard library's random() for uniform random variables and
    normals.inverseCumulativeNormal() to transform them.
    """

    def __init__(self, dim):
        self.dim = dim

    def getUniforms(self, N):
        return ([random() for x in range(self.dim)] for x in range(N))

    def getGaussians(self, N):
        return ([inverseCumulativeNormal(x) for x in v] for v in self.getUniforms(N))

class ParkMiller(object):
    """
    Park-Miller random number generator.
    stream() method returns a generator producing pseudo-random
    numbers in the interval [1, 2147483646].
    """

    def __init__(self,seed=1):
        self._const_a = 16807
        self._const_m = 2147483647
        self._const_q = 127773
        self._const_r = 2836
        self._seed = max(int(seed),1)
        self.maximum = self._const_m - 1
        self.minimum = 1

    def stream(self,N):
        a, m, q, r = self._const_a, self._const_m, self._const_q, self._const_r
        count = 0
        while count < N:
            k = self._seed // q # // ensures integer division for Python 3.
            self._seed = (a * (self._seed - k * q) - k * r)
            if self._seed < 0:
                self._seed += m
            yield self._seed
            count += 1

class RandomParkMiller(RandomBase):
    """
    A RandomBase class using the ParkMiller class
    to generate the uniform random variables.
    """

    def __init__(self,dim,seed=1):
        self.dim = dim
        self._seed = seed
        self._pm = ParkMiller(seed)
        self._r = 1/(1. + self._pm.maximum)

    def getUniforms(self,N):
        count = 0
        while count < N:
            yield [x * self._r for x in self._pm.stream(self.dim)]
            count += 1

    def skip(self, nPaths):
        for i in self.getUniforms(nPaths):
            pass

    def reset(self):
        self._pm = ParkMiller(self.seed)

    def _getSeed(self):
        return self._seed

    def _setSeed(self,seed):
        self._seed = seed
        self.reset(seed)

    seed = property(_getSeed, _setSeed)

class AntiThetic(RandomBase):
    """
    Anti-thetic sampling class: wraps another RandomBase class.
    Currently this only works properly for streams of an even length.
    """

    def __init__(self, base):
        self._base = base
        self._oddEven = True

    def getUniforms(self, N):
        for v in self._base.getUniforms(N/2):
            yield v
            yield [1-x for x in v]

    def _setSeed(self, seed):
        self._base.seed = seed
        self._oddEven = True

    def skip(self, nPaths):
        self._base.skip(nPaths / 2)

    def reset(self):
        self._base.reset()
        self._oddEven = True

    def _getDim(self):
        return self._base.dim

    def _setDim(self, dim):
        self._base.dim = dim

    dim = property(_getDim, _setDim)

def loop(N, s):
    """
    Returns a generator of length N, returning
    a loop of values modulo s.
    """
    i = 0
    while i < N:
        yield i % s
        i += 1

class SimpleStratifiedPM(RandomBase):
    """
    Stratified random sampling based on the RandomParkMiller class.
    """

    def __init__(self,seed=1,segments=2**8):
        self.dim = 1
        self._rpm = RandomParkMiller(1,seed)
        self._seed = seed
        self._segments = segments

    def getUniforms(self, N):
        s = self._segments
        return ([(l + x[0])/s] for l, x in zip(loop(N, s), self._rpm.getUniforms(N)))

# Testing

if __name__ == "__main__":
    rv = AntiThetic(SimpleStratifiedPM(1,16))
    N = 2**8
    r = []
    for v in rv.getUniforms(N):
        r.append(v)
    r = [x[0] for x in r]
    mean = sum(r)/N
    var = sum((x-mean)**2 for x in r)/N
    print("mean = %f" % mean)
    print("variance = %f" % var)
<|endoftext|>


<commit_before>"""
Django settings for bootstrap_lti project.

For more information on this file, see
https://docs.djangoproject.com/en/1.6/topics/settings/

For the full list of settings and their values, see
https://docs.djangoproject.com/en/1.6/ref/settings/
"""

# Build paths inside the project like this: os.path.join(BASE_DIR, ...)
import os
from os.path import abspath, basename, dirname, join, normpath
from django.core.urlresolvers import reverse_lazy
from sys import path
from .secure import SECURE_SETTINGS

BASE_DIR = os.path.dirname(os.path.dirname(__file__))

# Absolute filesystem path to the Django project config directory:
# (this is the parent of the directory where this file resides,
# since this file is now inside a 'settings' pacakge directory)
DJANGO_PROJECT_CONFIG = dirname(dirname(abspath(__file__)))

# Absolute filesystem path to the top-level project folder:
# (this is one directory up from the project config directory)
SITE_ROOT = dirname(DJANGO_PROJECT_CONFIG)

# Site name:
SITE_NAME = basename(SITE_ROOT)

# Add our project to our pythonpath, this way we don't need to type our project
# name in our dotted import paths:
path.append(SITE_ROOT)

# Quick-start development settings - unsuitable for production
# See https://docs.djangoproject.com/en/1.6/howto/deployment/checklist/

# SECURITY WARNING: keep the secret key used in production secret!
SECRET_KEY = SECURE_SETTINGS.get('django_secret_key')

DEBUG = SECURE_SETTINGS.get('enable_debug', False)

TEMPLATES = [
    {
        'BACKEND': 'django.template.backends.django.DjangoTemplates',
        'DIRS': [],
        'APP_DIRS': True,
        'OPTIONS': {
            'context_processors': [
                'django.template.context_processors.debug',
                'django.template.context_processors.request',
                'django.contrib.auth.context_processors.auth',
                'django.contrib.messages.context_processors.messages',
            ],
            'debug': DEBUG,
            'loaders': [
                'django.template.loaders.filesystem.Loader',
                'django.template.loaders.app_directories.Loader',
            ]
        },
    },
]

ALLOWED_HOSTS = []

# Application definition

INSTALLED_APPS = (
    'django.contrib.admin',
    'django.contrib.auth',
    'django.contrib.contenttypes',
    'django.contrib.sessions',
    'django.contrib.messages',
    'django.contrib.staticfiles',
    'crispy_forms',
    'edx2canvas',
)

MIDDLEWARE_CLASSES = (
    'django.contrib.sessions.middleware.SessionMiddleware',
    'django.middleware.common.CommonMiddleware',
    'cached_auth.Middleware',
    'django_auth_lti.middleware.LTIAuthMiddleware',
    'django.contrib.messages.middleware.MessageMiddleware',
)
AUTHENTICATION_BACKENDS = (
    'django_auth_lti.backends.LTIAuthBackend',
)

CRISPY_TEMPLATE_PACK = 'bootstrap3'

LOGIN_URL = reverse_lazy('lti_auth_error')

ROOT_URLCONF = 'edx_in_canvas.urls'

WSGI_APPLICATION = 'edx_in_canvas.wsgi.application'

# Database
# https://docs.djangoproject.com/en/1.6/ref/settings/#databases

DATABASES = {
    'default': {
        'ENGINE': 'django.db.backends.sqlite3',
        'NAME': os.path.join(BASE_DIR, 'db.sqlite3'),
    }
}

# Internationalization
# https://docs.djangoproject.com/en/1.6/topics/i18n/

LANGUAGE_CODE = 'en-us'

TIME_ZONE = 'UTC'

USE_I18N = True

USE_L10N = True

USE_TZ = True

# Static files (CSS, JavaScript, Images)
# https://docs.djangoproject.com/en/1.6/howto/static-files/

STATIC_URL = '/lti_tools/static/'

# Additional locations of static files
STATICFILES_DIRS = (
    # Put strings here, like "/home/html/static" or "C:/www/django/static".
    # Always use forward slashes, even on Windows.
    # Don't forget to use absolute paths, not relative paths.
    normpath(join(SITE_ROOT, 'static')),
)
STATIC_ROOT = normpath(join(SITE_ROOT, 'http_static'))

# Logging
_DEFAULT_LOG_LEVEL = SECURE_SETTINGS.get('log_level', 'DEBUG')
_LOG_ROOT = SECURE_SETTINGS.get('log_root', '')
# Turn off default Django logging
# https://docs.djangoproject.com/en/1.8/topics/logging/#disabling-logging-configuration
LOGGING_CONFIG = None
LOGGING = {
    'version': 1,
    'disable_existing_loggers': False,
    'formatters': {
        'verbose': {
            'format': '%(levelname)s\t%(asctime)s.%(msecs)03dZ\t%(name)s:%(lineno)s\t%(message)s',
            'datefmt': '%Y-%m-%dT%H:%M:%S'
        },
        'simple': {
            'format': '%(levelname)s\t%(name)s:%(lineno)s\t%(message)s',
        },
    },
    'handlers': {
        'default': {
            'class': 'logging.handlers.WatchedFileHandler',
            'level': _DEFAULT_LOG_LEVEL,
            'formatter': 'verbose',
            'filename': os.path.join(_LOG_ROOT, 'django-canvas_integration.log'),
        },
    },
    # This is the default logger for any apps or libraries that use the logger
    # package, but are not represented in the `loggers` dict below.  A level
    # must be set and handlers defined.  Setting this logger is equivalent to
    # setting and empty string logger in the loggers dict below, but the separation
    # here is a bit more explicit.  See link for more details:
    # https://docs.python.org/2.7/library/logging.config.html#dictionary-schema-details
    'root': {
        'level': WARNING,
        'handlers': ['default'],
    },
    'loggers': {
        'edx2canvas': {
            'handlers': ['default'],
            'level': _DEFAULT_LOG_LEVEL,
            'propagate': False,
        },
    },
}

LTI_OAUTH_CREDENTIALS = SECURE_SETTINGS.get('lti_oauth_credentials', None)

CANVAS_DOMAIN = SECURE_SETTINGS.get('canvas_domain', None)
CANVAS_SDK_SETTINGS = {
    'auth_token': SECURE_SETTINGS.get('canvas_token', None),
    'base_api_url': '{}/api'.format(CANVAS_DOMAIN),
    'max_retries': 3,
    'per_page': 40,
}

EDX_URL_BASE = SECURE_SETTINGS.get('edx_url_base', None)
EXTERNAL_TOOL_DOMAIN = SECURE_SETTINGS.get('external_tool_domain', None)

EDX_LTI_PREVIEW_SETTINGS = {
    'url_base': '{}/lti_provider/courses'.format(EDX_URL_BASE),
    'key': SECURE_SETTINGS.get('edx_lti_preview_key', None),
    'secret': SECURE_SETTINGS.get('edx_lti_preview_secret', None),
    'tool_consumer_instance_guid': SECURE_SETTINGS.get('preview_tool_consumer_guid', None),
    'user_id': SECURE_SETTINGS.get('preview_user_id', None),
    'roles': u'[student]',
    'context_id': u'CanvasInstallationContext',
    'lti_version': u'LTI-1p0'
}

EDX_LTI_KEY = SECURE_SETTINGS.get('edx_lti_provider_key', None)
EDX_LTI_SECRET = SECURE_SETTINGS.get('edx_lti_provider_secret', None)

CANVAS_OAUTH_CLIENT_ID = SECURE_SETTINGS.get('canvas_oauth_client_id', None)
CANVAS_OAUTH_CLIENT_KEY = SECURE_SETTINGS.get('canvas_oauth_client_key', None)
<commit_msg>Fixing root logging level to _DEFAULT_LOG_LEVEL.<commit_after>"""
Django settings for bootstrap_lti project.

For more information on this file, see
https://docs.djangoproject.com/en/1.6/topics/settings/

For the full list of settings and their values, see
https://docs.djangoproject.com/en/1.6/ref/settings/
"""

# Build paths inside the project like this: os.path.join(BASE_DIR, ...)
import os
from os.path import abspath, basename, dirname, join, normpath
from django.core.urlresolvers import reverse_lazy
from sys import path
from .secure import SECURE_SETTINGS

BASE_DIR = os.path.dirname(os.path.dirname(__file__))

# Absolute filesystem path to the Django project config directory:
# (this is the parent of the directory where this file resides,
# since this file is now inside a 'settings' pacakge directory)
DJANGO_PROJECT_CONFIG = dirname(dirname(abspath(__file__)))

# Absolute filesystem path to the top-level project folder:
# (this is one directory up from the project config directory)
SITE_ROOT = dirname(DJANGO_PROJECT_CONFIG)

# Site name:
SITE_NAME = basename(SITE_ROOT)

# Add our project to our pythonpath, this way we don't need to type our project
# name in our dotted import paths:
path.append(SITE_ROOT)

# Quick-start development settings - unsuitable for production
# See https://docs.djangoproject.com/en/1.6/howto/deployment/checklist/

# SECURITY WARNING: keep the secret key used in production secret!
SECRET_KEY = SECURE_SETTINGS.get('django_secret_key')

DEBUG = SECURE_SETTINGS.get('enable_debug', False)

TEMPLATES = [
    {
        'BACKEND': 'django.template.backends.django.DjangoTemplates',
        'DIRS': [],
        'APP_DIRS': True,
        'OPTIONS': {
            'context_processors': [
                'django.template.context_processors.debug',
                'django.template.context_processors.request',
                'django.contrib.auth.context_processors.auth',
                'django.contrib.messages.context_processors.messages',
            ],
            'debug': DEBUG,
            'loaders': [
                'django.template.loaders.filesystem.Loader',
                'django.template.loaders.app_directories.Loader',
            ]
        },
    },
]

ALLOWED_HOSTS = []

# Application definition

INSTALLED_APPS = (
    'django.contrib.admin',
    'django.contrib.auth',
    'django.contrib.contenttypes',
    'django.contrib.sessions',
    'django.contrib.messages',
    'django.contrib.staticfiles',
    'crispy_forms',
    'edx2canvas',
)

MIDDLEWARE_CLASSES = (
    'django.contrib.sessions.middleware.SessionMiddleware',
    'django.middleware.common.CommonMiddleware',
    'cached_auth.Middleware',
    'django_auth_lti.middleware.LTIAuthMiddleware',
    'django.contrib.messages.middleware.MessageMiddleware',
)
AUTHENTICATION_BACKENDS = (
    'django_auth_lti.backends.LTIAuthBackend',
)

CRISPY_TEMPLATE_PACK = 'bootstrap3'

LOGIN_URL = reverse_lazy('lti_auth_error')

ROOT_URLCONF = 'edx_in_canvas.urls'

WSGI_APPLICATION = 'edx_in_canvas.wsgi.application'

# Database
# https://docs.djangoproject.com/en/1.6/ref/settings/#databases

DATABASES = {
    'default': {
        'ENGINE': 'django.db.backends.sqlite3',
        'NAME': os.path.join(BASE_DIR, 'db.sqlite3'),
    }
}

# Internationalization
# https://docs.djangoproject.com/en/1.6/topics/i18n/

LANGUAGE_CODE = 'en-us'

TIME_ZONE = 'UTC'

USE_I18N = True

USE_L10N = True

USE_TZ = True

# Static files (CSS, JavaScript, Images)
# https://docs.djangoproject.com/en/1.6/howto/static-files/

STATIC_URL = '/lti_tools/static/'

# Additional locations of static files
STATICFILES_DIRS = (
    # Put strings here, like "/home/html/static" or "C:/www/django/static".
    # Always use forward slashes, even on Windows.
    # Don't forget to use absolute paths, not relative paths.
    normpath(join(SITE_ROOT, 'static')),
)
STATIC_ROOT = normpath(join(SITE_ROOT, 'http_static'))

# Logging
_DEFAULT_LOG_LEVEL = SECURE_SETTINGS.get('log_level', 'DEBUG')
_LOG_ROOT = SECURE_SETTINGS.get('log_root', '')
# Turn off default Django logging
# https://docs.djangoproject.com/en/1.8/topics/logging/#disabling-logging-configuration
LOGGING_CONFIG = None
LOGGING = {
    'version': 1,
    'disable_existing_loggers': False,
    'formatters': {
        'verbose': {
            'format': '%(levelname)s\t%(asctime)s.%(msecs)03dZ\t%(name)s:%(lineno)s\t%(message)s',
            'datefmt': '%Y-%m-%dT%H:%M:%S'
        },
        'simple': {
            'format': '%(levelname)s\t%(name)s:%(lineno)s\t%(message)s',
        },
    },
    'handlers': {
        'default': {
            'class': 'logging.handlers.WatchedFileHandler',
            'level': _DEFAULT_LOG_LEVEL,
            'formatter': 'verbose',
            'filename': os.path.join(_LOG_ROOT, 'django-canvas_integration.log'),
        },
    },
    # This is the default logger for any apps or libraries that use the logger
    # package, but are not represented in the `loggers` dict below.  A level
    # must be set and handlers defined.  Setting this logger is equivalent to
    # setting and empty string logger in the loggers dict below, but the separation
    # here is a bit more explicit.  See link for more details:
    # https://docs.python.org/2.7/library/logging.config.html#dictionary-schema-details
    'root': {
        'level': _DEFAULT_LOG_LEVEL,
        'handlers': ['default'],
    },
    'loggers': {
        'edx2canvas': {
            'handlers': ['default'],
            'level': _DEFAULT_LOG_LEVEL,
            'propagate': False,
        },
    },
}

LTI_OAUTH_CREDENTIALS = SECURE_SETTINGS.get('lti_oauth_credentials', None)

CANVAS_DOMAIN = SECURE_SETTINGS.get('canvas_domain', None)
CANVAS_SDK_SETTINGS = {
    'auth_token': SECURE_SETTINGS.get('canvas_token', None),
    'base_api_url': '{}/api'.format(CANVAS_DOMAIN),
    'max_retries': 3,
    'per_page': 40,
}

EDX_URL_BASE = SECURE_SETTINGS.get('edx_url_base', None)
EXTERNAL_TOOL_DOMAIN = SECURE_SETTINGS.get('external_tool_domain', None)

EDX_LTI_PREVIEW_SETTINGS = {
    'url_base': '{}/lti_provider/courses'.format(EDX_URL_BASE),
    'key': SECURE_SETTINGS.get('edx_lti_preview_key', None),
    'secret': SECURE_SETTINGS.get('edx_lti_preview_secret', None),
    'tool_consumer_instance_guid': SECURE_SETTINGS.get('preview_tool_consumer_guid', None),
    'user_id': SECURE_SETTINGS.get('preview_user_id', None),
    'roles': u'[student]',
    'context_id': u'CanvasInstallationContext',
    'lti_version': u'LTI-1p0'
}

EDX_LTI_KEY = SECURE_SETTINGS.get('edx_lti_provider_key', None)
EDX_LTI_SECRET = SECURE_SETTINGS.get('edx_lti_provider_secret', None)

CANVAS_OAUTH_CLIENT_ID = SECURE_SETTINGS.get('canvas_oauth_client_id', None)
CANVAS_OAUTH_CLIENT_KEY = SECURE_SETTINGS.get('canvas_oauth_client_key', None)
<|endoftext|>


<commit_before>#!/usr/bin/env python

import sys, os, glob, string
import numpy as np
import astropy as ast
import matplotlib.pyplot as plt
from pyraf import iraf
from tqdm import tqdm
import odi_config as odi
import glob
import shutil
import pandas as pd
import time

try:
    object_str, filters, instrument, images, illcor_flag, skyflat_src, wcs_flag, reproject_flag, scale_flag, stack_flag, gaia_flag, cluster_flag, ra_center, dec_center, min_radius = odi.cfgparse('config.yaml')
except IOError:
    print 'config.yaml does not exist, quitting...'
    exit()

# for basic processing, filter shouldn't matter enough to consider separately
# or rather, this script already handles that just fine
# so just stick all the image names together into one long list
images_ = [img for sublist in images.values() for img in sublist]
# print images_

rad, decd = odi.get_targ_ra_dec(images_[0], 'OTA33.SCI')
if gaia_flag:
    source = 'gaia'
else:
    source = 'sdss'
inst = odi.instrument(images_[0])
#source = 'twomass'

#Create offline catalogs
for img in images_:
    print 'Retrieving QR SDSS and Gaia catalogs for:', img.stem()
    for key in tqdm(odi.OTA_dictionary):
        ota = odi.OTA_dictionary[key]
        outputsd = odi.sdsspath+'offline_'+ota+'.'+img.base()+'.sdss'
        if not os.path.isfile(outputsd):
            x,y = odi.get_sdss_coords_offline(img,ota,inst,output=outputsd)
            output2m = odi.twomasspath+'offline_'+ota+'.'+img.base()+'.mass'
            # x,y = odi.get_2mass_coords_offline(img,ota,inst,output=output2m)
        if gaia_flag == True:
            outputg = odi.gaiapath+'offline_'+ota+'.'+img.base()+'.gaia'
            if not os.path.isfile(outputg):
                if cluster_flag == True:
                    odi.get_gaia_coords(img,ota,inst,
                                        output=outputg,
                                        cluster=cluster_flag,
                                        racenter=float(ra_center),
                                        deccenter=float(dec_center),
                                        min_radius=float(min_radius),
                                        G_lim = 20.75)
                else:
                    odi.get_gaia_coords(img,ota,inst,
                                        output=outputg,
                                        cluster=cluster_flag)

# if illcor_flag:
listfiles = glob.glob('*.lis')
if len(listfiles) == 0:
    odi.imcombine_lists(images_, filters)
else:
    print 'imcombine lists done'

if not os.path.isfile('bpms.done'):
    for img in images_:
        print 'updating bpms for', img.stem()
        for key in tqdm(odi.OTA_dictionary):
            ota = odi.OTA_dictionary[key]
            odi.make_bpms(img, ota)
    with open('bpms.done', 'w+') as bpm:
        print >> bpm, 'bpms are done!'

listfiles = glob.glob(odi.skyflatpath+'*.med.fits')
if len(listfiles) == 0:
    for filter in filters:
        odi.dark_sky_flat(filter)
else:
    print 'dark sky flats done'

if not os.path.isfile('derived_props.txt'):
    f1 = open('derived_props.txt','w+')
    print >> f1, '# img  ota  filter fwhm  zp_med  zp_std  bg_mean  bg_med  bg_std'
    for img in images_:
        otalist = sorted(odi.OTA_dictionary.keys())
        for key in tqdm(otalist):
            ota = odi.OTA_dictionary[key]
            hdulist = odi.fits.open(img.f)
            hdr = hdulist[0].header
            filt = hdr['filter']
            image_to_correct = img.f+'['+ota+']'
            correction_image = ota+'.'+filt+'.med.fits'
            corrected_image = 'illcor_'+ota+'.'+img.stem()
            print corrected_image
            if not os.path.isfile(odi.illcorpath+corrected_image):
                odi.illumination_corrections(image_to_correct, correction_image, corrected_image, do_correction=illcor_flag)
            gaps = odi.get_gaps(img, ota)
            reprojed_image = 'reproj_'+ota+'.'+img.stem()
            if not os.path.isfile(odi.reprojpath+reprojed_image):
                if wcs_flag:
                    pixcrd3 = odi.list_wcs_coords(img, ota, gaps, inst,output=img.nofits()+'.'+ota+'.radec.coo', gmaglim=23., stars_only=True, offline = True, source = source)
                    try:
                        odi.fix_wcs(img, ota, coords=img.nofits()+'.'+ota+'.radec.coo', iters=1)
                    except:
                        try:
                            print 'msccmatch failed, wait a second and try again'
                            time.sleep(1.0)
                            odi.fix_wcs(img, ota, coords=img.nofits()+'.'+ota+'.radec.coo', iters=1)
                        except:
                            print 'there might be too few stars for msccmatch, just skip it.'
                if reproject_flag:
                    wcsref = odi.illcorpath+'illcor_OTA33.SCI.'+images_[0].stem()
                    odi.reproject_ota(img, ota, rad, decd, wcsref)
            gaps = odi.get_gaps_rep(img, ota)
            odi.refetch_sdss_coords(img, ota, gaps, inst,gmaglim=21.5,offline = True,source=source)
            #run an additional refetch to get the xy for 2mass so they can be used for scaling
            # odi.repoxy_offline(img, ota, gaps, inst,gmaglim=21.5,source='twomass')
            odi.repoxy_offline(img, ota, gaps, inst,gmaglim=21.5,source='gaia')
            fwhm = odi.getfwhm_ota(img, ota, gaia=gaia_flag)
            if 'odi_NB695' in filters:
                zp_med, zp_std = 99.99,99.99
            elif source == 'sdss':
                zp_med, zp_std = 99.99,99.99
            elif source == 'twomass':
                zp_med, zp_std = 99.99,99.99
            elif source == 'gaia':
                zp_med, zp_std = 99.99,99.99
            if not os.path.isfile(odi.bgsubpath+'bgsub_'+ota+'.'+img.stem()):
                bg_mean, bg_median, bg_std = odi.bgsub_ota(img, ota, apply=True)
            else:
                bg_mean, bg_median, bg_std = odi.bgsub_ota(img, ota, apply=False)
            print >> f1, img.dither(), ota, filt, fwhm, zp_med, zp_std, bg_mean, bg_median, bg_std
            dim_stats = odi.check_mask_dim(img,ota)
            if not dim_stats:
                print 'mask dimensions do not match image'
                print 'redo', img, ota
                raise ValueError
    f1.close()
else:
    imgnum,fwhm,zp_med, zp_std, bg_mean, bg_median, bg_std = np.loadtxt('derived_props.txt',usecols=(0,3,4,5,6,7,8),unpack=True)
    ota_d, filt_d = np.loadtxt('derived_props.txt',usecols=(1,2),unpack=True,dtype=str)
    finished = zip(imgnum,ota_d,filt_d)
    f1 = open('derived_props.txt','a+')
    for img in images_:
        for key in tqdm(odi.OTA_dictionary):
            ota = odi.OTA_dictionary[key]
            hdulist = odi.fits.open(img.f)
            hdr = hdulist[0].header
            filt = hdr['filter']
            finishcheck = (int(img.dither()),ota,filt)
            if finishcheck in finished:
                already = 0
            else:
                image_to_correct = img.f+'['+ota+']'
                correction_image = ota+'.'+filt+'.med.fits'
                corrected_image = 'illcor_'+ota+'.'+img.stem()
                if not os.path.isfile(odi.illcorpath+corrected_image):
                    odi.illumination_corrections(image_to_correct, correction_image, corrected_image, do_correction=illcor_flag)
                gaps = odi.get_gaps(img, ota)
                reprojed_image = 'reproj_'+ota+'.'+img.stem()
                if not os.path.isfile(odi.reprojpath+reprojed_image):
                    if wcs_flag:
                        pixcrd3 = odi.list_wcs_coords(img, ota, gaps, inst,output=img.nofits()+'.'+ota+'.radec.coo', gmaglim=23., stars_only=True, offline = True, source = source)
                        try:
                            odi.fix_wcs(img, ota, coords=img.nofits()+'.'+ota+'.radec.coo', iters=1)
                        except IndexError:
                            print 'not enough stars to fix wcs, skipping for this ota:', img, ota
                        except:
                            print 'msccmatch failed, wait a second and try again'
                            time.sleep(1.0)
                            odi.fix_wcs(img, ota, coords=img.nofits()+'.'+ota+'.radec.coo', iters=1)
                    if reproject_flag:
                        wcsref = odi.illcorpath+'illcor_OTA33.SCI.'+images_[0].stem()
                        odi.reproject_ota(img, ota, rad, decd, wcsref)
                gaps = odi.get_gaps_rep(img, ota)
                odi.refetch_sdss_coords(img, ota, gaps, inst,gmaglim=21.5,offline = True,source=source)
                #run an additional refetch to get the xy for 2mass so they can be used for scaling
                # odi.repoxy_offline(img, ota, gaps, inst,gmaglim=21.5,source='twomass')
                odi.repoxy_offline(img, ota, gaps, inst,gmaglim=21.5,source='gaia')
                fwhm = odi.getfwhm_ota(img, ota, gaia=gaia_flag)
                if source == 'sdss':
                    zp_med, zp_std = 99.99,99.99
                if source == 'twomass':
                    zp_med, zp_std = 99.99,99.99
                elif source == 'gaia':
                    zp_med, zp_std = 99.99,99.99
                if not os.path.isfile(odi.bgsubpath+'bgsub_'+ota+'.'+img.stem()):
                    bg_mean, bg_median, bg_std = odi.bgsub_ota(img, ota, apply=True)
                else:
                    bg_mean, bg_median, bg_std = odi.bgsub_ota(img, ota, apply=False)
                print >> f1, img.dither(), ota, filt, fwhm, zp_med, zp_std, bg_mean, bg_median, bg_std
                dim_stats = odi.check_mask_dim(img,ota)
                if not dim_stats:
                    print 'mask dimensions do not match image'
                    print 'redo', img.stem(), ota
                    raise ValueError
    f1.close()
<commit_msg>call reproj keyword fix function<commit_after>#!/usr/bin/env python

import sys, os, glob, string
import numpy as np
import astropy as ast
import matplotlib.pyplot as plt
from pyraf import iraf
from tqdm import tqdm
import odi_config as odi
import glob
import shutil
import pandas as pd
import time

try:
    object_str, filters, instrument, images, illcor_flag, skyflat_src, wcs_flag, reproject_flag, scale_flag, stack_flag, gaia_flag, cluster_flag, ra_center, dec_center, min_radius = odi.cfgparse('config.yaml')
except IOError:
    print 'config.yaml does not exist, quitting...'
    exit()

# for basic processing, filter shouldn't matter enough to consider separately
# or rather, this script already handles that just fine
# so just stick all the image names together into one long list
images_ = [img for sublist in images.values() for img in sublist]
# print images_

rad, decd = odi.get_targ_ra_dec(images_[0], 'OTA33.SCI')
if gaia_flag:
    source = 'gaia'
else:
    source = 'sdss'
inst = odi.instrument(images_[0])
#source = 'twomass'

#Create offline catalogs
for img in images_:
    print 'Retrieving QR SDSS and Gaia catalogs for:', img.stem()
    for key in tqdm(odi.OTA_dictionary):
        ota = odi.OTA_dictionary[key]
        outputsd = odi.sdsspath+'offline_'+ota+'.'+img.base()+'.sdss'
        if not os.path.isfile(outputsd):
            x,y = odi.get_sdss_coords_offline(img,ota,inst,output=outputsd)
            output2m = odi.twomasspath+'offline_'+ota+'.'+img.base()+'.mass'
            # x,y = odi.get_2mass_coords_offline(img,ota,inst,output=output2m)
        if gaia_flag == True:
            outputg = odi.gaiapath+'offline_'+ota+'.'+img.base()+'.gaia'
            if not os.path.isfile(outputg):
                if cluster_flag == True:
                    odi.get_gaia_coords(img,ota,inst,
                                        output=outputg,
                                        cluster=cluster_flag,
                                        racenter=float(ra_center),
                                        deccenter=float(dec_center),
                                        min_radius=float(min_radius),
                                        G_lim = 20.75)
                else:
                    odi.get_gaia_coords(img,ota,inst,
                                        output=outputg,
                                        cluster=cluster_flag)

# if illcor_flag:
# listfiles = glob.glob('*.lis')
# if len(listfiles) == 0:
odi.imcombine_lists(images_, filters)
# else:
#     print 'imcombine lists done'

if not os.path.isfile('bpms.done'):
    for img in images_:
        print 'updating bpms for', img.stem()
        for key in tqdm(odi.OTA_dictionary):
            ota = odi.OTA_dictionary[key]
            odi.make_bpms(img, ota)
    with open('bpms.done', 'w+') as bpm:
        print >> bpm, 'bpms are done!'

# listfiles = glob.glob(odi.skyflatpath+'*.med.fits')
# if len(listfiles) == 0:
for filter in filters:
    odi.dark_sky_flat(filter)
# else:
#     print 'dark sky flats done'

if not os.path.isfile('derived_props.txt'):
    f1 = open('derived_props.txt','w+')
    print >> f1, '# img  ota  filter fwhm  zp_med  zp_std  bg_mean  bg_med  bg_std'
    for img in images_:
        otalist = sorted(odi.OTA_dictionary.keys())
        for key in tqdm(otalist):
            ota = odi.OTA_dictionary[key]
            hdulist = odi.fits.open(img.f)
            hdr = hdulist[0].header
            filt = hdr['filter']
            image_to_correct = img.f+'['+ota+']'
            correction_image = ota+'.'+filt+'.med.fits'
            corrected_image = 'illcor_'+ota+'.'+img.stem()
            print corrected_image
            if not os.path.isfile(odi.illcorpath+corrected_image):
                odi.illumination_corrections(image_to_correct, correction_image, corrected_image, do_correction=illcor_flag)
            gaps = odi.get_gaps(img, ota)
            reprojed_image = 'reproj_'+ota+'.'+img.stem()
            if not os.path.isfile(odi.reprojpath+reprojed_image):
                if wcs_flag:
                    pixcrd3 = odi.list_wcs_coords(img, ota, gaps, inst,output=img.nofits()+'.'+ota+'.radec.coo', gmaglim=23., stars_only=True, offline = True, source = source)
                    try:
                        odi.fix_wcs(img, ota, coords=img.nofits()+'.'+ota+'.radec.coo', iters=1)
                    except:
                        try:
                            print 'msccmatch failed, wait a second and try again'
                            time.sleep(1.0)
                            odi.fix_wcs(img, ota, coords=img.nofits()+'.'+ota+'.radec.coo', iters=1)
                        except:
                            print 'there might be too few stars for msccmatch, just skip it.'
                if reproject_flag:
                    wcsref = odi.illcorpath+'illcor_OTA33.SCI.'+images_[0].stem()
                    odi.reproject_ota(img, ota, rad, decd, wcsref)
                    odi.tpv2tan_hdr(img, ota)

            gaps = odi.get_gaps_rep(img, ota)
            odi.refetch_sdss_coords(img, ota, gaps, inst,gmaglim=21.5,offline = True,source=source)
            #run an additional refetch to get the xy for 2mass so they can be used for scaling
            # odi.repoxy_offline(img, ota, gaps, inst,gmaglim=21.5,source='twomass')
            odi.repoxy_offline(img, ota, gaps, inst,gmaglim=21.5,source='gaia')
            fwhm = odi.getfwhm_ota(img, ota, gaia=gaia_flag)
            if 'odi_NB695' in filters:
                zp_med, zp_std = 99.99,99.99
            elif source == 'sdss':
                zp_med, zp_std = 99.99,99.99
            elif source == 'twomass':
                zp_med, zp_std = 99.99,99.99
            elif source == 'gaia':
                zp_med, zp_std = 99.99,99.99
            if not os.path.isfile(odi.bgsubpath+'bgsub_'+ota+'.'+img.stem()):
                bg_mean, bg_median, bg_std = odi.bgsub_ota(img, ota, apply=True)
            else:
                bg_mean, bg_median, bg_std = odi.bgsub_ota(img, ota, apply=False)
            print >> f1, img.dither(), ota, filt, fwhm, zp_med, zp_std, bg_mean, bg_median, bg_std
            dim_stats = odi.check_mask_dim(img,ota)
            if not dim_stats:
                print 'mask dimensions do not match image'
                print 'redo', img, ota
                raise ValueError
    f1.close()
else:
    imgnum,fwhm,zp_med, zp_std, bg_mean, bg_median, bg_std = np.loadtxt('derived_props.txt',usecols=(0,3,4,5,6,7,8),unpack=True)
    ota_d, filt_d = np.loadtxt('derived_props.txt',usecols=(1,2),unpack=True,dtype=str)
    finished = zip(imgnum,ota_d,filt_d)
    f1 = open('derived_props.txt','a+')
    for img in images_:
        for key in tqdm(odi.OTA_dictionary):
            ota = odi.OTA_dictionary[key]
            hdulist = odi.fits.open(img.f)
            hdr = hdulist[0].header
            filt = hdr['filter']
            finishcheck = (int(img.dither()),ota,filt)
            if finishcheck in finished:
                already = 0
            else:
                image_to_correct = img.f+'['+ota+']'
                correction_image = ota+'.'+filt+'.med.fits'
                corrected_image = 'illcor_'+ota+'.'+img.stem()
                if not os.path.isfile(odi.illcorpath+corrected_image):
                    odi.illumination_corrections(image_to_correct, correction_image, corrected_image, do_correction=illcor_flag)
                gaps = odi.get_gaps(img, ota)
                reprojed_image = 'reproj_'+ota+'.'+img.stem()
                if not os.path.isfile(odi.reprojpath+reprojed_image):
                    if wcs_flag:
                        pixcrd3 = odi.list_wcs_coords(img, ota, gaps, inst,output=img.nofits()+'.'+ota+'.radec.coo', gmaglim=23., stars_only=True, offline = True, source = source)
                        try:
                            odi.fix_wcs(img, ota, coords=img.nofits()+'.'+ota+'.radec.coo', iters=1)
                        except IndexError:
                            print 'not enough stars to fix wcs, skipping for this ota:', img, ota
                        except:
                            print 'msccmatch failed, wait a second and try again'
                            time.sleep(1.0)
                            odi.fix_wcs(img, ota, coords=img.nofits()+'.'+ota+'.radec.coo', iters=1)
                    if reproject_flag:
                        wcsref = odi.illcorpath+'illcor_OTA33.SCI.'+images_[0].stem()
                        odi.reproject_ota(img, ota, rad, decd, wcsref)
                gaps = odi.get_gaps_rep(img, ota)
                odi.refetch_sdss_coords(img, ota, gaps, inst,gmaglim=21.5,offline = True,source=source)
                #run an additional refetch to get the xy for 2mass so they can be used for scaling
                # odi.repoxy_offline(img, ota, gaps, inst,gmaglim=21.5,source='twomass')
                odi.repoxy_offline(img, ota, gaps, inst,gmaglim=21.5,source='gaia')
                fwhm = odi.getfwhm_ota(img, ota, gaia=gaia_flag)
                if source == 'sdss':
                    zp_med, zp_std = 99.99,99.99
                if source == 'twomass':
                    zp_med, zp_std = 99.99,99.99
                elif source == 'gaia':
                    zp_med, zp_std = 99.99,99.99
                if not os.path.isfile(odi.bgsubpath+'bgsub_'+ota+'.'+img.stem()):
                    bg_mean, bg_median, bg_std = odi.bgsub_ota(img, ota, apply=True)
                else:
                    bg_mean, bg_median, bg_std = odi.bgsub_ota(img, ota, apply=False)
                print >> f1, img.dither(), ota, filt, fwhm, zp_med, zp_std, bg_mean, bg_median, bg_std
                dim_stats = odi.check_mask_dim(img,ota)
                if not dim_stats:
                    print 'mask dimensions do not match image'
                    print 'redo', img.stem(), ota
                    raise ValueError
    f1.close()
<|endoftext|>