<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
# emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*-
# vi: set ft=python sts=4 ts=4 sw=4 et:
#
# @Author: oesteban
# @Date:   2016-01-05 11:33:39
# @Email:  code@oscaresteban.es
# @Last modified by:   oesteban
# @Last Modified time: 2016-01-18 18:05:18

import sys
import os
import os.path as op
import pandas as pd
import matplotlib
matplotlib.use('Agg')

import matplotlib.pyplot as plt
from matplotlib.backends.backend_pdf import PdfPages

from .interfaces.viz_utils import (plot_measures, plot_mosaic, plot_all,
                                   plot_fd, plot_dist)

# matplotlib.rc('figure', figsize=(11.69, 8.27))  # for DINA4 size

def workflow_report(in_csv, qap_type, sub_list=[], settings={}):
    import datetime
    import numpy as np

    out_dir = settings.get('output_dir', os.getcwd())
    work_dir = settings.get('work_dir', op.abspath('tmp'))
    out_file = op.join(out_dir, qap_type + '_%s.pdf')

    # Read csv file, sort and drop duplicates
    df = pd.read_csv(in_csv, dtype={'subject': str}).sort(
        columns=['subject', 'session', 'scan'])

    try:
        df.drop_duplicates(['subject', 'session', 'scan'], keep='last',
                           inplace=True)
    except TypeError:
        df.drop_duplicates(['subject', 'session', 'scan'], take_last=True,
                           inplace=True)

    subject_list = sorted(pd.unique(df.subject.ravel()))
    result = {}
    func = getattr(sys.modules[__name__], 'report_' + qap_type)

    # Identify failed subjects
    failed_str = "none"
    if sub_list:
        success = [tuple(x) for x in df[['subject', 'session', 'scan']].values]
        failed = set(sub_all) - set(success)
        if len(failed) > 0:
            failed_str = ', '.join(sorted(['%s_%s_%s' % f for f in failed]))

    pdf_group = []

    # Generate summary page
    out_sum = op.join(work_dir, 'summary_group.pdf')
    summary_cover(
        (qap_type, datetime.datetime.now().strftime("%Y-%m-%d, %H:%M"),
         failed_str), is_group=True, out_file=out_sum)
    pdf_group.append(out_sum)

    # Generate group report
    qc_group = op.join(work_dir, 'qc_measures_group.pdf')
    # Generate violinplots. If successfull, add documentation.
    func(df, out_file=qc_group)
    pdf_group.append(qc_group)

    # Generate documentation page
    doc = op.join(work_dir, '%s_doc.pdf' % qap_type)

    # Let documentation page fail
    get_documentation(qap_type, doc)
    if doc is not None:
        pdf_group.append(doc)

    if len(pdf_group) > 0:
        out_group_file = op.join(out_dir, '%s_group.pdf' % qap_type)
        # Generate final report with collected pdfs in plots
        concat_pdf(pdf_group, out_group_file)
        result['group'] = {'success': True, 'path': out_group_file}

    # Generate individual reports for subjects
    for subid in subject_list:
        # Get subject-specific info
        subdf = df.loc[df['subject'] == subid]
        sessions = sorted(pd.unique(subdf.session.ravel()))
        plots = []
        sess_scans = []
        # Re-build mosaic location
        for sesid in sessions:
            sesdf = subdf.loc[subdf['session'] == sesid]
            scans = sorted(pd.unique(sesdf.scan.ravel()))

            # Each scan has a volume and (optional) fd plot
            for scanid in scans:
                if 'anat' in qap_type:
                    m = op.join(work_dir, 'anatomical_%s_%s_%s.pdf' %
                                (subid, sesid, scanid))

                    if op.isfile(m):
                        plots.append(m)

                if 'func' in qap_type:
                    mepi = op.join(work_dir, 'meanepi_%s_%s_%s.pdf' %
                                   (subid, sesid, scanid))
                    if op.isfile(mepi):
                        plots.append(mepi)

                    tsnr = op.join(work_dir, 'tsnr_%s_%s_%s.pdf' %
                                   (subid, sesid, scanid))
                    if op.isfile(tsnr):
                        plots.append(tsnr)

                    fd = op.join(work_dir, 'fd_%s_%s_%s.pdf' %
                                 (subid, sesid, scanid))
                    if op.isfile(fd):
                        plots.append(fd)

            sess_scans.append('%s (%s)' % (sesid, ', '.join(scans)))

        # Summary cover
        sfailed = []
        if len(failed) > 0:
            sfailed = ['%s (%s)' % (s[1], s[2])
                       for s in failed if subid == s[0]]
        out_sum = op.join(work_dir, '%s_summary_%s.pdf' % (qap_type, subid))
        summary_cover(
            (subid, subid, qap_type,
             datetime.datetime.now().strftime("%Y-%m-%d, %H:%M"),
             ", ".join(sess_scans),
             ",".join(sfailed) if sfailed else "none"),
            out_file=out_sum)
        plots.insert(0, out_sum)

        # Summary (violinplots) of QC measures
        qc_ms = op.join(work_dir, '%s_measures_%s.pdf' % (qap_type, subid))

        func(df, subject=subid, out_file=qc_ms)
        plots.append(qc_ms)

        if len(plots) > 0:
            if doc is not None:
                plots.append(doc)

            # Generate final report with collected pdfs in plots
            sub_path = out_file % subid
            concat_pdf(plots, sub_path)
            result[subid] = {'success': True, 'path': sub_path}
    return result

def get_documentation(doc_type, out_file):
    import codecs
    import StringIO
    from xhtml2pdf import pisa
    # open output file for writing (truncated binary)
    result = open(out_file, "w+b")

    html_dir = op.abspath(
        op.join(op.dirname(__file__), 'html', '%s.html' % doc_type))

    with codecs.open(html_dir, mode='r', encoding='utf-8') as f:
        html = f.read()

    # convert HTML to PDF
    status = pisa.pisaDocument(html, result, encoding='UTF-8')
    result.close()

    # return True on success and False on errors
    return status.err

def summary_cover(data, is_group=False, out_file=None):
    import codecs
    import StringIO
    from xhtml2pdf import pisa
    # open output file for writing (truncated binary)
    result = open(out_file, "w+b")

    html_file = 'cover_group.html' if is_group else 'cover_subj.html'

    html_dir = op.abspath(
        op.join(op.dirname(__file__), 'html', html_file))

    with codecs.open(html_dir, mode='r', encoding='utf-8') as f:
        html = f.read()

    # convert HTML to PDF
    status = pisa.pisaDocument(html % data, result, encoding='UTF-8')
    result.close()

    # return True on success and False on errors
    return status.err

def concat_pdf(in_files, out_file='concatenated.pdf'):
    """
    Concatenate PDF list (http://stackoverflow.com/a/3444735)
    """
    from PyPDF2 import PdfFileWriter, PdfFileReader
    outpdf = PdfFileWriter()

    for in_file in in_files:
        inpdf = PdfFileReader(file(in_file, 'rb'))
        for p in range(inpdf.numPages):
            outpdf.addPage(inpdf.getPage(p))
    outpdf.write(file(out_file, 'wb'))
    return out_file

def _write_report(df, groups, sub_id=None, sc_split=False, condensed=True,
                  out_file='report.pdf'):
    columns = df.columns.ravel()
    headers = []
    for g in groups:
        rem = []
        for h in g:
            if h not in columns:
                rem.append(h)
            else:
                headers.append(h)
        for r in rem:
            g.remove(r)

    report = PdfPages(out_file)
    sessions = sorted(pd.unique(df.session.ravel()))
    for ss in sessions:
        sesdf = df.copy().loc[df['session'] == ss]
        scans = pd.unique(sesdf.scan.ravel())
        if sc_split:
            for sc in scans:
                subset = sesdf.loc[sesdf['scan'] == sc]
                if len(subset.index) > 1:
                    if sub_id is None:
                        subtitle = '(%s_%s)' % (ss, sc)
                    else:
                        subtitle = '(subject %s_%s_%s)' % (sub_id, ss, sc)
                    if condensed:
                        fig = plot_all(sesdf, groups, subject=sub_id,
                                       title='QC measures ' + subtitle)
                    else:
                        fig = plot_measures(
                            sesdf, headers, subject=sub_id,
                            title='QC measures ' + subtitle)
                    report.savefig(fig, dpi=300)
                    fig.clf()
        else:
            if len(sesdf.index) > 1:
                if sub_id is None:
                    subtitle = '(%s)' % (ss)
                else:
                    subtitle = '(subject %s_%s)' % (sub_id, ss)
                if condensed:
                    fig = plot_all(sesdf, groups, subject=sub_id,
                                   title='QC measures ' + subtitle)
                else:
                    fig = plot_measures(
                        sesdf, headers, subject=sub_id,
                        title='QC measures ' + subtitle)
                report.savefig(fig, dpi=300)
                fig.clf()

    report.close()
    plt.close()
    # print 'Written report file %s' % out_file
    return out_file

def _write_all_reports(df, groups, sc_split=False, condensed=True,
                       out_file='report.pdf'):

    outlist = []
    _write_report(
        df, groups, sc_split=sc_split, condensed=condensed, out_file=out_file)

    subject_list = sorted(pd.unique(df.subject.ravel()))
    for sub_id in subject_list:
        tpl, _ = op.splitext(op.basename(out_file))
        tpl = op.join(op.dirname(out_file), tpl) + '_%s.pdf'
        outlist.append(_write_report(
            df, groups, sub_id=sub_id, sc_split=sc_split, condensed=condensed,
            out_file=tpl % sub_id))
    return out_file, outlist

def all_anatomical(df, sc_split=False, condensed=True,
                   out_file='anatomical.pdf'):
    groups = [['bg_size', 'fg_size'],
              ['bg_mean', 'fg_mean'],
              ['bg_std', 'fg_std'],
              ['csf_size', 'gm_size', 'wm_size'],
              ['csf_mean', 'gm_mean', 'wm_mean'],
              ['csf_std', 'gm_std', 'wm_std'],
              ['cnr'],
              ['efc'],
              ['fber'],
              ['fwhm', 'fwhm_x', 'fwhm_y', 'fwhm_z'],
              ['qi1'],
              ['snr']]
    return _write_all_reports(
        df, groups, sc_split=sc_split,
        condensed=condensed, out_file=out_file)

def all_func_temporal(df, sc_split=False, condensed=True,
                      out_file='func_temporal.pdf'):
    groups = [['dvars'], ['gcor'], ['m_tsnr'], ['mean_fd'],
              ['num_fd'], ['outlier'], ['perc_fd'], ['quality']]
    return _write_all_reports(
        df, groups, sc_split=sc_split,
        condensed=condensed, out_file=out_file)

def all_func_spatial(df, sc_split=False, condensed=False,
                     out_file='func_spatial.pdf'):
    groups = [['bg_size', 'fg_size'],
              ['bg_mean', 'fg_mean'],
              ['bg_std', 'fg_std'],
              ['efc'],
              ['fber'],
              ['fwhm', 'fwhm_x', 'fwhm_y', 'fwhm_z'],
              ['ghost_%s' % a for a in ['x', 'y', 'z']],
              ['snr']]
    return _write_all_reports(
        df, groups, sc_split=sc_split,
        condensed=condensed, out_file=out_file)

def report_anatomical(
        df, subject=None, sc_split=False, condensed=True,
        out_file='anatomical.pdf'):
    groups = [['bg_size', 'fg_size'],
              ['bg_mean', 'fg_mean'],
              ['bg_std', 'fg_std'],
              ['csf_size', 'gm_size', 'wm_size'],
              ['csf_mean', 'gm_mean', 'wm_mean'],
              ['csf_std', 'gm_std', 'wm_std'],
              ['cnr'],
              ['efc'],
              ['fber'],
              ['fwhm', 'fwhm_x', 'fwhm_y', 'fwhm_z'],
              ['qi1'],
              ['snr']]
    return _write_report(
        df, groups, sub_id=subject, sc_split=sc_split, condensed=condensed,
        out_file=out_file)

def report_functional(
        df, subject=None, sc_split=False, condensed=True,
        out_file='functional.pdf'):
    from tempfile import mkdtemp

    wd = mkdtemp()
    groups = [['dvars'], ['gcor'], ['m_tsnr'], ['mean_fd'],
              ['num_fd'], ['outlier'], ['perc_fd'], ['quality']]
    fspatial = _write_report(
        df, groups, sub_id=subject, sc_split=sc_split, condensed=condensed,
        out_file=op.join(wd, 'fspatial.pdf'))

    groups = [['bg_size', 'fg_size'],
              ['bg_mean', 'fg_mean'],
              ['bg_std', 'fg_std'],
              ['efc'],
              ['fber'],
              ['fwhm', 'fwhm_x', 'fwhm_y', 'fwhm_z'],
              ['ghost_%s' % a for a in ['x', 'y', 'z']],
              ['snr']]
    ftemporal = _write_report(
        df, groups, sub_id=subject, sc_split=sc_split, condensed=condensed,
        out_file=op.join(wd, 'ftemporal.pdf'))

    concat_pdf([fspatial, ftemporal], out_file)
    return out_file<commit_msg>fixed error<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
# emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*-
# vi: set ft=python sts=4 ts=4 sw=4 et:
#
# @Author: oesteban
# @Date:   2016-01-05 11:33:39
# @Email:  code@oscaresteban.es
# @Last modified by:   oesteban
# @Last Modified time: 2016-01-18 18:05:18

import sys
import os
import os.path as op
import pandas as pd
import matplotlib
matplotlib.use('Agg')

import matplotlib.pyplot as plt
from matplotlib.backends.backend_pdf import PdfPages

from .interfaces.viz_utils import (plot_measures, plot_mosaic, plot_all,
                                   plot_fd, plot_dist)

# matplotlib.rc('figure', figsize=(11.69, 8.27))  # for DINA4 size

def workflow_report(in_csv, qap_type, sub_list=[], settings={}):
    import datetime
    import numpy as np

    out_dir = settings.get('output_dir', os.getcwd())
    work_dir = settings.get('work_dir', op.abspath('tmp'))
    out_file = op.join(out_dir, qap_type + '_%s.pdf')

    # Read csv file, sort and drop duplicates
    df = pd.read_csv(in_csv, dtype={'subject': str}).sort(
        columns=['subject', 'session', 'scan'])

    try:
        df.drop_duplicates(['subject', 'session', 'scan'], keep='last',
                           inplace=True)
    except TypeError:
        df.drop_duplicates(['subject', 'session', 'scan'], take_last=True,
                           inplace=True)

    subject_list = sorted(pd.unique(df.subject.ravel()))
    result = {}
    func = getattr(sys.modules[__name__], 'report_' + qap_type)

    # Identify failed subjects
    failed_str = "none"
    if sub_list:
        success = [tuple(x) for x in df[['subject', 'session', 'scan']].values]
        failed = set(sub_list) - set(success)
        if len(failed) > 0:
            failed_str = ', '.join(sorted(['%s_%s_%s' % f for f in failed]))

    pdf_group = []

    # Generate summary page
    out_sum = op.join(work_dir, 'summary_group.pdf')
    summary_cover(
        (qap_type, datetime.datetime.now().strftime("%Y-%m-%d, %H:%M"),
         failed_str), is_group=True, out_file=out_sum)
    pdf_group.append(out_sum)

    # Generate group report
    qc_group = op.join(work_dir, 'qc_measures_group.pdf')
    # Generate violinplots. If successfull, add documentation.
    func(df, out_file=qc_group)
    pdf_group.append(qc_group)

    # Generate documentation page
    doc = op.join(work_dir, '%s_doc.pdf' % qap_type)

    # Let documentation page fail
    get_documentation(qap_type, doc)
    if doc is not None:
        pdf_group.append(doc)

    if len(pdf_group) > 0:
        out_group_file = op.join(out_dir, '%s_group.pdf' % qap_type)
        # Generate final report with collected pdfs in plots
        concat_pdf(pdf_group, out_group_file)
        result['group'] = {'success': True, 'path': out_group_file}

    # Generate individual reports for subjects
    for subid in subject_list:
        # Get subject-specific info
        subdf = df.loc[df['subject'] == subid]
        sessions = sorted(pd.unique(subdf.session.ravel()))
        plots = []
        sess_scans = []
        # Re-build mosaic location
        for sesid in sessions:
            sesdf = subdf.loc[subdf['session'] == sesid]
            scans = sorted(pd.unique(sesdf.scan.ravel()))

            # Each scan has a volume and (optional) fd plot
            for scanid in scans:
                if 'anat' in qap_type:
                    m = op.join(work_dir, 'anatomical_%s_%s_%s.pdf' %
                                (subid, sesid, scanid))

                    if op.isfile(m):
                        plots.append(m)

                if 'func' in qap_type:
                    mepi = op.join(work_dir, 'meanepi_%s_%s_%s.pdf' %
                                   (subid, sesid, scanid))
                    if op.isfile(mepi):
                        plots.append(mepi)

                    tsnr = op.join(work_dir, 'tsnr_%s_%s_%s.pdf' %
                                   (subid, sesid, scanid))
                    if op.isfile(tsnr):
                        plots.append(tsnr)

                    fd = op.join(work_dir, 'fd_%s_%s_%s.pdf' %
                                 (subid, sesid, scanid))
                    if op.isfile(fd):
                        plots.append(fd)

            sess_scans.append('%s (%s)' % (sesid, ', '.join(scans)))

        # Summary cover
        sfailed = []
        if len(failed) > 0:
            sfailed = ['%s (%s)' % (s[1], s[2])
                       for s in failed if subid == s[0]]
        out_sum = op.join(work_dir, '%s_summary_%s.pdf' % (qap_type, subid))
        summary_cover(
            (subid, subid, qap_type,
             datetime.datetime.now().strftime("%Y-%m-%d, %H:%M"),
             ", ".join(sess_scans),
             ",".join(sfailed) if sfailed else "none"),
            out_file=out_sum)
        plots.insert(0, out_sum)

        # Summary (violinplots) of QC measures
        qc_ms = op.join(work_dir, '%s_measures_%s.pdf' % (qap_type, subid))

        func(df, subject=subid, out_file=qc_ms)
        plots.append(qc_ms)

        if len(plots) > 0:
            if doc is not None:
                plots.append(doc)

            # Generate final report with collected pdfs in plots
            sub_path = out_file % subid
            concat_pdf(plots, sub_path)
            result[subid] = {'success': True, 'path': sub_path}
    return result

def get_documentation(doc_type, out_file):
    import codecs
    import StringIO
    from xhtml2pdf import pisa
    # open output file for writing (truncated binary)
    result = open(out_file, "w+b")

    html_dir = op.abspath(
        op.join(op.dirname(__file__), 'html', '%s.html' % doc_type))

    with codecs.open(html_dir, mode='r', encoding='utf-8') as f:
        html = f.read()

    # convert HTML to PDF
    status = pisa.pisaDocument(html, result, encoding='UTF-8')
    result.close()

    # return True on success and False on errors
    return status.err

def summary_cover(data, is_group=False, out_file=None):
    import codecs
    import StringIO
    from xhtml2pdf import pisa
    # open output file for writing (truncated binary)
    result = open(out_file, "w+b")

    html_file = 'cover_group.html' if is_group else 'cover_subj.html'

    html_dir = op.abspath(
        op.join(op.dirname(__file__), 'html', html_file))

    with codecs.open(html_dir, mode='r', encoding='utf-8') as f:
        html = f.read()

    # convert HTML to PDF
    status = pisa.pisaDocument(html % data, result, encoding='UTF-8')
    result.close()

    # return True on success and False on errors
    return status.err

def concat_pdf(in_files, out_file='concatenated.pdf'):
    """
    Concatenate PDF list (http://stackoverflow.com/a/3444735)
    """
    from PyPDF2 import PdfFileWriter, PdfFileReader
    outpdf = PdfFileWriter()

    for in_file in in_files:
        inpdf = PdfFileReader(file(in_file, 'rb'))
        for p in range(inpdf.numPages):
            outpdf.addPage(inpdf.getPage(p))
    outpdf.write(file(out_file, 'wb'))
    return out_file

def _write_report(df, groups, sub_id=None, sc_split=False, condensed=True,
                  out_file='report.pdf'):
    columns = df.columns.ravel()
    headers = []
    for g in groups:
        rem = []
        for h in g:
            if h not in columns:
                rem.append(h)
            else:
                headers.append(h)
        for r in rem:
            g.remove(r)

    report = PdfPages(out_file)
    sessions = sorted(pd.unique(df.session.ravel()))
    for ss in sessions:
        sesdf = df.copy().loc[df['session'] == ss]
        scans = pd.unique(sesdf.scan.ravel())
        if sc_split:
            for sc in scans:
                subset = sesdf.loc[sesdf['scan'] == sc]
                if len(subset.index) > 1:
                    if sub_id is None:
                        subtitle = '(%s_%s)' % (ss, sc)
                    else:
                        subtitle = '(subject %s_%s_%s)' % (sub_id, ss, sc)
                    if condensed:
                        fig = plot_all(sesdf, groups, subject=sub_id,
                                       title='QC measures ' + subtitle)
                    else:
                        fig = plot_measures(
                            sesdf, headers, subject=sub_id,
                            title='QC measures ' + subtitle)
                    report.savefig(fig, dpi=300)
                    fig.clf()
        else:
            if len(sesdf.index) > 1:
                if sub_id is None:
                    subtitle = '(%s)' % (ss)
                else:
                    subtitle = '(subject %s_%s)' % (sub_id, ss)
                if condensed:
                    fig = plot_all(sesdf, groups, subject=sub_id,
                                   title='QC measures ' + subtitle)
                else:
                    fig = plot_measures(
                        sesdf, headers, subject=sub_id,
                        title='QC measures ' + subtitle)
                report.savefig(fig, dpi=300)
                fig.clf()

    report.close()
    plt.close()
    # print 'Written report file %s' % out_file
    return out_file

def _write_all_reports(df, groups, sc_split=False, condensed=True,
                       out_file='report.pdf'):

    outlist = []
    _write_report(
        df, groups, sc_split=sc_split, condensed=condensed, out_file=out_file)

    subject_list = sorted(pd.unique(df.subject.ravel()))
    for sub_id in subject_list:
        tpl, _ = op.splitext(op.basename(out_file))
        tpl = op.join(op.dirname(out_file), tpl) + '_%s.pdf'
        outlist.append(_write_report(
            df, groups, sub_id=sub_id, sc_split=sc_split, condensed=condensed,
            out_file=tpl % sub_id))
    return out_file, outlist

def all_anatomical(df, sc_split=False, condensed=True,
                   out_file='anatomical.pdf'):
    groups = [['bg_size', 'fg_size'],
              ['bg_mean', 'fg_mean'],
              ['bg_std', 'fg_std'],
              ['csf_size', 'gm_size', 'wm_size'],
              ['csf_mean', 'gm_mean', 'wm_mean'],
              ['csf_std', 'gm_std', 'wm_std'],
              ['cnr'],
              ['efc'],
              ['fber'],
              ['fwhm', 'fwhm_x', 'fwhm_y', 'fwhm_z'],
              ['qi1'],
              ['snr']]
    return _write_all_reports(
        df, groups, sc_split=sc_split,
        condensed=condensed, out_file=out_file)

def all_func_temporal(df, sc_split=False, condensed=True,
                      out_file='func_temporal.pdf'):
    groups = [['dvars'], ['gcor'], ['m_tsnr'], ['mean_fd'],
              ['num_fd'], ['outlier'], ['perc_fd'], ['quality']]
    return _write_all_reports(
        df, groups, sc_split=sc_split,
        condensed=condensed, out_file=out_file)

def all_func_spatial(df, sc_split=False, condensed=False,
                     out_file='func_spatial.pdf'):
    groups = [['bg_size', 'fg_size'],
              ['bg_mean', 'fg_mean'],
              ['bg_std', 'fg_std'],
              ['efc'],
              ['fber'],
              ['fwhm', 'fwhm_x', 'fwhm_y', 'fwhm_z'],
              ['ghost_%s' % a for a in ['x', 'y', 'z']],
              ['snr']]
    return _write_all_reports(
        df, groups, sc_split=sc_split,
        condensed=condensed, out_file=out_file)

def report_anatomical(
        df, subject=None, sc_split=False, condensed=True,
        out_file='anatomical.pdf'):
    groups = [['bg_size', 'fg_size'],
              ['bg_mean', 'fg_mean'],
              ['bg_std', 'fg_std'],
              ['csf_size', 'gm_size', 'wm_size'],
              ['csf_mean', 'gm_mean', 'wm_mean'],
              ['csf_std', 'gm_std', 'wm_std'],
              ['cnr'],
              ['efc'],
              ['fber'],
              ['fwhm', 'fwhm_x', 'fwhm_y', 'fwhm_z'],
              ['qi1'],
              ['snr']]
    return _write_report(
        df, groups, sub_id=subject, sc_split=sc_split, condensed=condensed,
        out_file=out_file)

def report_functional(
        df, subject=None, sc_split=False, condensed=True,
        out_file='functional.pdf'):
    from tempfile import mkdtemp

    wd = mkdtemp()
    groups = [['dvars'], ['gcor'], ['m_tsnr'], ['mean_fd'],
              ['num_fd'], ['outlier'], ['perc_fd'], ['quality']]
    fspatial = _write_report(
        df, groups, sub_id=subject, sc_split=sc_split, condensed=condensed,
        out_file=op.join(wd, 'fspatial.pdf'))

    groups = [['bg_size', 'fg_size'],
              ['bg_mean', 'fg_mean'],
              ['bg_std', 'fg_std'],
              ['efc'],
              ['fber'],
              ['fwhm', 'fwhm_x', 'fwhm_y', 'fwhm_z'],
              ['ghost_%s' % a for a in ['x', 'y', 'z']],
              ['snr']]
    ftemporal = _write_report(
        df, groups, sub_id=subject, sc_split=sc_split, condensed=condensed,
        out_file=op.join(wd, 'ftemporal.pdf'))

    concat_pdf([fspatial, ftemporal], out_file)
    return out_file<|endoftext|>


<commit_before>import logging

from django.contrib import messages
from django.template import RequestContext
from django.http import HttpResponse
from django.http import HttpResponseRedirect
from django.conf import settings
from django.shortcuts import render_to_response
from django.template.loader import render_to_string
from django.contrib.auth.decorators import login_required
from django.utils.datastructures import SortedDict

from physical.service.databaseinfra import DatabaseInfraService

LOG = logging.getLogger(__name__)

@login_required
def dashboard(request, *args, **kwargs):
    databaseinfra_service = DatabaseInfraService(request)
    databaseinfras = []

    for databaseinfra in databaseinfra_service.list():
        databaseinfra_status = databaseinfra_service.get_databaseinfra_status(databaseinfra)
        data = SortedDict()
        data["name"] = databaseinfra.name
        data["version"] = databaseinfra_status.version
        data["size"] = databaseinfra_status.size_in_mbytes
        data["databases"] = []

        for database_status in databaseinfra_status.databases_status.values():
            data["databases"].append({
                "name" : database_status.name,
                "size" : database_status.size_in_mbytes,
                "usage": round(100 * database_status.size_in_mbytes / databaseinfra_status.size_in_mbytes)
            })

        databaseinfras.append(data)
    return render_to_response("dashboard/dashboard.html", locals(), context_databaseinfra=RequestContext(request))
<commit_msg>fix on dashboard<commit_after>import logging

from django.contrib import messages
from django.template import RequestContext
from django.http import HttpResponse
from django.http import HttpResponseRedirect
from django.conf import settings
from django.shortcuts import render_to_response
from django.template.loader import render_to_string
from django.contrib.auth.decorators import login_required
from django.utils.datastructures import SortedDict

from physical.service.databaseinfra import DatabaseInfraService

LOG = logging.getLogger(__name__)

@login_required
def dashboard(request, *args, **kwargs):
    databaseinfra_service = DatabaseInfraService(request)
    databaseinfras = []

    for databaseinfra in databaseinfra_service.list():
        databaseinfra_status = databaseinfra_service.get_databaseinfra_status(databaseinfra)
        data = SortedDict()
        data["name"] = databaseinfra.name
        data["version"] = databaseinfra_status.version
        data["size"] = databaseinfra_status.size_in_mbytes
        data["databases"] = []

        for database_status in databaseinfra_status.databases_status.values():
            data["databases"].append({
                "name" : database_status.name,
                "size" : database_status.size_in_mbytes,
                "usage": round(100 * database_status.size_in_mbytes / databaseinfra_status.size_in_mbytes)
            })

        databaseinfras.append(data)
    return render_to_response("dashboard/dashboard.html", locals(), context_instance=RequestContext(request))
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

__title__ = 'djangorestframework-jwt'
__version__ = '1.9.0'
__author__ = 'José Padilla'
__license__ = 'MIT'
__copyright__ = 'Copyright 2014-2017 Blimp LLC'

# Version synonym
VERSION = __version__
<commit_msg>Bump up version<commit_after># -*- coding: utf-8 -*-

__title__ = 'djangorestframework-jwt'
__version__ = '1.10.0'
__author__ = 'José Padilla'
__license__ = 'MIT'
__copyright__ = 'Copyright 2014-2017 Blimp LLC'

# Version synonym
VERSION = __version__
<|endoftext|>


<commit_before>"""
The interface for interacting with Trumba web services.
"""

from restclients.dao import TrumbaBot_DAO, TrumbaSea_DAO, TrumbaTac_DAO
from restclients.util.timer import Timer
from restclients.util.log import log_info, log_err, null_handler
from lxml import etree
import logging
import json

class Trumba(object):
    """
    The Trumba object has methods for getting resources about calendar
    """
    logger = logging.getLogger(__name__)
    logger.addHandler(null_handler)

    @staticmethod
    def _log_xml_resp(campus, url, response, timer):
        if response.status == 200 and response.data is not None:
            root = etree.fromstring(response.data)
            resp_msg = ''
            for el in root.iterchildren():
                resp_msg = resp_msg + str(el.attrib)

            log_info(Trumba.logger,
                     "%s %s ==RETURNS==> %s %s" % (
                    campus, url, response.status, resp_msg),
                     timer)
        else:
            log_err(Trumba.logger,
                    "%s %s ==RETURNS==> %s %s" % (
                    campus, url, response.status, response.reason),
                    timer)

    @staticmethod
    def _log_json_resp(campus, url, body, response, timer):
        if response.status == 200 and response.data is not None:
            log_info(Trumba.logger,
                     "%s %s %s ==RETURNS==> %s %s" % (
                    campus, url, body, response.status, response.data),
                     timer)
        else:
            log_err(Trumba.logger,
                    "%s %s %s ==RETURNS==> %s %s" % (
                    campus, url, body, response.status, response.reason),
                    timer)

    @staticmethod
    def get_bot_resource(url):
        """
        Get the requested resource or update resource using Bothell account
        :returns: http response with content in xml
        """
        timer = Timer()
        response = TrumbaBot_DAO().getURL(url,
                                          {"Content-Type":"application/xml"})
        Trumba._log_xml_resp("Bothell", url, response, timer)
        return response

    @staticmethod
    def get_sea_resource(url):
        """
        Get the requested resource or update resource using Seattle account
        :returns: http response with content in xml
        """
        timer = Timer()
        response = TrumbaSea_DAO().getURL(url,
                                          {"Accept": "application/xml"})
        Trumba._log_xml_resp("Seattle", url, response, timer)
        return response

    @staticmethod
    def get_tac_resource(url):
        """
        Get the requested resource or update resource using Tacoma account
        :returns: http response with content in xml
        """
        timer = Timer()
        response = TrumbaTac_DAO().getURL(url,
                                          {"Accept": "application/xml"})
        Trumba._log_xml_resp("Tacoma", url, response, timer)
        return response

    @staticmethod
    def post_bot_resource(url, body):
        """
        Get the requested resource of Bothell calendars
        :returns: http response with content in json
        """
        timer = Timer()
        response = TrumbaBot_DAO().postURL(url,
                                           {"Content-Type": "application/json"},
                                           body)
        Trumba._log_json_resp("Bothell", url, body, response, timer)
        return response

    @staticmethod
    def post_sea_resource(url, body):
        """
        Get the requested resource using the Seattle account
        :returns: http response with content in json
        """
        timer = Timer()
        response = TrumbaSea_DAO().postURL(url,
                                           {"Content-Type": "application/json"},
                                           body)
        Trumba._log_json_resp("Seattle", url, body, response, timer)
        return response

    @staticmethod
    def post_tac_resource(url, body):
        """
        Get the requested resource of Tacoma calendars
        :returns: http response with content in json
        """
        timer = Timer()
        response = TrumbaTac_DAO().postURL(url,
                                           {"Content-Type": "application/json"},
                                           body)
        Trumba._log_json_resp("Tacoma", url, body, response, timer)
        return response

<commit_msg>Changed to log response data only when logging level is debug.<commit_after>"""
The interface for interacting with Trumba web services.
"""

from restclients.dao import TrumbaBot_DAO, TrumbaSea_DAO, TrumbaTac_DAO
from restclients.util.timer import Timer
from restclients.util.log import log_info, log_err, null_handler
from lxml import etree
import logging
import json

class Trumba(object):
    """
    The Trumba object has methods for getting resources about calendar
    """

    @staticmethod
    def _log_xml_resp(campus, url, response, timer):
        logger = logging.getLogger(__name__)
        logger.addHandler(null_handler)
        if response.status == 200 and response.data is not None:
            log_info(logger,
                     "%s %s ==status==> %s" % (
                    campus, url, response.status),
                     timer)
            if logger.getEffectiveLevel() == logging.DEBUG:
                root = etree.fromstring(response.data)
                resp_msg = ''
                for el in root.iterchildren():
                    resp_msg = resp_msg + str(el.attrib)
                logger.debug("%s %s ==data==> %s" % (
                        campus, url, resp_msg))
        else:
            log_err(logger,
                    "%s %s ==error==> %s %s" % (
                    campus, url, response.status, response.reason),
                    timer)

    @staticmethod
    def _log_json_resp(campus, url, body, response, timer):
        logger = logging.getLogger(__name__)
        logger.addHandler(null_handler)
        if response.status == 200 and response.data is not None:
            log_info(logger,
                     "%s %s %s ==status==> %s" % (
                    campus, url, body, response.status),
                     timer)
            logger.debug("%s %s %s ==data==> %s" % (
                    campus, url, body, response.data))
        else:
            log_err(logger,
                    "%s %s %s ==error==> %s %s" % (
                    campus, url, body, response.status, response.reason),
                    timer)

    @staticmethod
    def get_bot_resource(url):
        """
        Get the requested resource or update resource using Bothell account
        :returns: http response with content in xml
        """
        timer = Timer()
        response = TrumbaBot_DAO().getURL(url,
                                          {"Content-Type":"application/xml"})
        Trumba._log_xml_resp("Bothell", url, response, timer)
        return response

    @staticmethod
    def get_sea_resource(url):
        """
        Get the requested resource or update resource using Seattle account
        :returns: http response with content in xml
        """
        timer = Timer()
        response = TrumbaSea_DAO().getURL(url,
                                          {"Accept": "application/xml"})
        Trumba._log_xml_resp("Seattle", url, response, timer)
        return response

    @staticmethod
    def get_tac_resource(url):
        """
        Get the requested resource or update resource using Tacoma account
        :returns: http response with content in xml
        """
        timer = Timer()
        response = TrumbaTac_DAO().getURL(url,
                                          {"Accept": "application/xml"})
        Trumba._log_xml_resp("Tacoma", url, response, timer)
        return response

    @staticmethod
    def post_bot_resource(url, body):
        """
        Get the requested resource of Bothell calendars
        :returns: http response with content in json
        """
        timer = Timer()
        response = TrumbaBot_DAO().postURL(url,
                                           {"Content-Type": "application/json"},
                                           body)
        Trumba._log_json_resp("Bothell", url, body, response, timer)
        return response

    @staticmethod
    def post_sea_resource(url, body):
        """
        Get the requested resource using the Seattle account
        :returns: http response with content in json
        """
        timer = Timer()
        response = TrumbaSea_DAO().postURL(url,
                                           {"Content-Type": "application/json"},
                                           body)
        Trumba._log_json_resp("Seattle", url, body, response, timer)
        return response

    @staticmethod
    def post_tac_resource(url, body):
        """
        Get the requested resource of Tacoma calendars
        :returns: http response with content in json
        """
        timer = Timer()
        response = TrumbaTac_DAO().postURL(url,
                                           {"Content-Type": "application/json"},
                                           body)
        Trumba._log_json_resp("Tacoma", url, body, response, timer)
        return response

<|endoftext|>


<commit_before>"""Deep Dreaming using Caffe and Google's Inception convolutional neural network."""

# pylint: disable=invalid-name, wrong-import-position

from collections import namedtuple, OrderedDict
import os
from pathlib import Path
import re

os.environ['GLOG_minloglevel'] = '1'
import caffe
import numpy as np
from PIL import Image
from scipy import ndimage
from tqdm import tqdm

EPS = np.finfo(np.float32).eps

CNNData = namedtuple('CNNData', 'deploy model mean categories')
CNNData.__new__.__defaults__ = (None,)  # Make categories optional.

_BASE_DIR = Path(__file__).parent.parent
GOOGLENET_BVLC = CNNData(
    _BASE_DIR/'bvlc_googlenet/deploy.prototxt',
    _BASE_DIR/'bvlc_googlenet/bvlc_googlenet.caffemodel',
    (104, 117, 123),
    categories=_BASE_DIR/'bvlc_googlenet/categories.txt')
GOOGLENET_PLACES205 = CNNData(
    _BASE_DIR/'googlenet_places205/deploy_places205.prototxt',
    _BASE_DIR/'googlenet_places205/googlelet_places205_train_iter_2400000.caffemodel',
    (104.051, 112.514, 116.676),  # TODO: find the actual Places205 mean
    categories=_BASE_DIR/'googlenet_places205/categories.txt')
GOOGLENET_PLACES365 = CNNData(
    _BASE_DIR/'googlenet_places365/deploy_googlenet_places365.prototxt',
    _BASE_DIR/'googlenet_places365/googlenet_places365.caffemodel',
    (104.051, 112.514, 116.676),
    categories=_BASE_DIR/'googlenet_places365/categories_places365.txt')

def save_as_hdr(arr, filename, gamma=2.2, allow_negative=True):
    """Saves a float32 ndarray to a high dynamic range (OpenEXR or float32 TIFF) file.

    Args:
        arr (ndarray): The input array.
        filename (str | Path): The output filename.
        gamma (Optional[float]): The encoding gamma of arr.
        allow_negative (Optional[bool]): Clip negative values to zero if false."""
    arr = arr.astype(np.float32)/255
    if not allow_negative:
        arr[arr < 0] = 0
    if gamma != 1:
        arr = np.sign(arr)*np.abs(arr)**gamma
    filename = str(filename)
    extension = filename.rpartition('.')[2].lower()
    if extension == 'exr':
        import OpenEXR
        exr = OpenEXR.OutputFile(filename, OpenEXR.Header(arr.shape[1], arr.shape[0]))
        exr.writePixels({'R': arr[..., 0].tobytes(),
                         'G': arr[..., 1].tobytes(),
                         'B': arr[..., 2].tobytes()})
        exr.close()
    elif extension == 'tif' or extension == 'tiff':
        import tifffile
        tiff = tifffile.TiffWriter(filename)
        tiff.save(arr, photometric='rgb')
        tiff.close()
    else:
        raise Exception('Unknown HDR file format.')

def to_image(arr):
    """Clips the values in a float32 ndarray to 0-255 and converts it to a PIL image.

    Args:
        arr (ndarray): The input array."""
    return Image.fromarray(np.uint8(np.clip(np.round(arr), 0, 255)))

def _resize(arr, size, method=Image.BICUBIC):
    h, w = size
    arr = np.float32(arr)
    if arr.ndim == 3:
        planes = [arr[i, :, :] for i in range(arr.shape[0])]
    else:
        raise TypeError('Only 3D CxHxW arrays are supported')
    imgs = [Image.fromarray(plane) for plane in planes]
    imgs_resized = [img.resize((w, h), method) for img in imgs]
    return np.stack([np.array(img) for img in imgs_resized])

class ShapeError(Exception):
    """Raised by CNN when an invalid layer shape is requested which would otherwise crash Caffe."""
    def __str__(self):
        return 'bad shape %s' % self.args

class _LayerIndexer:
    def __init__(self, net, attr):
        self.net, self.attr = net, attr

    def __getitem__(self, key):
        return getattr(self.net.blobs[key], self.attr)[0]

    def __setitem__(self, key, value):
        getattr(self.net.blobs[key], self.attr)[0] = value

class CNN:
    """Represents an instance of a Caffe convolutional neural network."""

    def __init__(self, cnndata, gpu=None):
        """Initializes a CNN.

        Args:
            gpu (Optional[int]): If present, Caffe will use this GPU device number. On a typical
                system with one GPU, it should be 0. If not present Caffe will use the CPU.
        """
        self.net = caffe.Classifier(str(cnndata.deploy), str(cnndata.model),
                                    mean=np.float32(cnndata.mean), channel_swap=(2, 1, 0))
        self.data = _LayerIndexer(self.net, 'data')
        self.diff = _LayerIndexer(self.net, 'diff')
        self.categories = [str(i) for i in range(self.data['prob'].size)]
        if cnndata.categories is not None:
            self.categories = open(str(cnndata.categories)).read().splitlines()
        self.img = np.zeros_like(self.data['data'])
        self.total_px = 0
        self.progress_bar = None
        if gpu is not None:
            caffe.set_device(gpu)
            caffe.set_mode_gpu()
        else:
            caffe.set_mode_cpu()

    def _preprocess(self, img):
        return np.rollaxis(np.float32(img), 2)[::-1] - self.net.transformer.mean['data']

    def _deprocess(self, img):
        return np.dstack((img + self.net.transformer.mean['data'])[::-1])

    def get_features(self, input_img, layers, max_tile_size=512):
        """Retrieve feature maps from the classification (forward) phase of operation.

        Example:
            cnn.get_features(img, ['prob'])['prob'] classifies 'img' and returns the predicted
            probability distribution over the network's categories.

        Returns:
            A dict which maps each layer in layers to a retrieved feature map.
        """
        input_arr = self._preprocess(np.float32(input_img))
        h = min(max_tile_size, input_arr.shape[1])
        w = min(max_tile_size, input_arr.shape[2])
        if max(*input_arr.shape[1:]) > max_tile_size:
            input_arr = _resize(input_arr, (h, w))
        self.net.blobs['data'].reshape(1, 3, h, w)
        self.data['data'] = input_arr
        end = self.layers()[-1]
        self.net.forward(end=end)
        return {layer: self.data[layer].copy() for layer in layers}

    def _grad_tiled(self, layers, progress=False, max_tile_size=512, auto_weight=True):
        # pylint: disable=too-many-locals
        if progress:
            if not self.progress_bar:
                self.progress_bar = tqdm(
                    total=self.total_px, unit='pix', unit_scale=True, ncols=80, smoothing=0.1)

        h, w = self.img.shape[1:]  # Height and width of input image
        ny, nx = (h-1)//max_tile_size+1, (w-1)//max_tile_size+1  # Number of tiles per dimension
        g = np.zeros_like(self.img)
        for y in range(ny):
            th = h//ny
            if y == ny-1:
                th += h - th*ny
            for x in range(nx):
                tw = w//nx
                if x == nx-1:
                    tw += w - tw*nx
                self.net.blobs['data'].reshape(1, 3, th, tw)
                sy, sx = h//ny*y, w//nx*x
                self.data['data'] = self.img[:, sy:sy+th, sx:sx+tw]

                for layer in layers.keys():
                    self.diff[layer] = 0
                self.net.forward(end=next(iter(layers.keys())))
                layers_list = list(layers.keys())
                for i, layer in enumerate(layers_list):
                    if auto_weight:
                        self.diff[layer] += \
                            self.data[layer]*layers[layer]/np.abs(self.data[layer]).sum()
                    else:
                        self.diff[layer] += self.data[layer]*layers[layer]
                    if i+1 == len(layers):
                        self.net.backward(start=layer)
                    else:
                        self.net.backward(start=layer, end=layers_list[i+1])

                g[:, sy:sy+th, sx:sx+tw] = self.diff['data']

                if progress:
                    self.progress_bar.update(th*tw)
        return g

    def _step(self, n=1, step_size=1.5, jitter=32, seed=0, smooth=0, **kwargs):
        np.random.seed(self.img.size + seed)
        for _ in range(n):
            x, y = np.random.randint(-jitter, jitter+1, 2)
            self.img = np.roll(np.roll(self.img, x, 2), y, 1)
            g = self._grad_tiled(**kwargs)
            g /= np.mean(np.abs(g)) + EPS
            if smooth:
                kernel = np.float32([[[0, -1, 0], [-1, 4, -1], [0, -1, 0]]])
                g2 = ndimage.convolve(self.img, kernel)
                g2 /= np.mean(np.abs(g2)) + EPS
                g -= g2 * smooth
                self.img += step_size * g / (np.mean(np.abs(g)) + EPS)
            else:
                self.img += step_size * g
            self.img = np.roll(np.roll(self.img, -x, 2), -y, 1)

    def _octave_detail(self, base, scales=4, min_size=32, n=10, per_octave=2, **kwargs):
        if min(base.shape[1:]) < 32:
            raise ShapeError(base.shape)
        factor = 2**(1/per_octave)
        detail = np.zeros_like(base, dtype=np.float32)
        self.total_px += base.shape[1] * base.shape[2] * n
        if scales != 1:
            hf, wf = np.int32(np.round(np.array(base.shape)[-2:]/factor))
            if min(hf, wf) >= min_size:
                smaller_base = _resize(base, (hf, wf))
                smaller_detail = self._octave_detail(
                    smaller_base, scales-1, min_size, n, per_octave, **kwargs)
                detail = _resize(smaller_detail, base.shape[-2:])
        self.img = base + detail
        self._step(n, **kwargs)
        return self.img - base

    def layers(self, pattern='.*'):
        """Returns a list of layer names matching a regular expression."""
        layers = []
        for i, layer in enumerate(self.net.blobs.keys()):
            if i == 0 or layer.partition('_split_')[1]:
                continue
            if re.fullmatch(pattern, layer):
                layers.append(layer)
        return layers

    def classify(self, input_img, n=1, **kwargs):
        """Classifies the input image and returns the n most probable categories.

        Args:
            input_img: The image to process (PIL images or Numpy arrays are accepted).
            n: The n most probable categories to return.
            max_tile_size: Does not allow the image dimension to exceed this.

        Returns:
            A list containing the n most probable categories."""
        prob = self.get_features(input_img, ['prob'], **kwargs)['prob']
        indices = prob.argsort()[::-1][:n]
        return [(prob[i], self.categories[i]) for i in indices]

    def dream(self, input_img, layers, progress=True, **kwargs):
        """Runs the Deep Dream multiscale gradient ascent algorithm on the input image.

        Args:
            input_img: The image to process (PIL images or Numpy arrays are accepted)
            layers (dict): The layer/feature weights to use in the objective function for gradient
                ascent.
            progress (Optional[bool]): Display a progress bar while computing.
            scales (Optional[int]): The number of scales to process.
            min_size (Optional[int]): Don't permit the small edge of the image to go below this.
            per_octave (Optional[int]): Determines the difference between each scale; for instance,
                the default of 2 means that a 1000x1000 input image will get processed as 707x707
                and 500x500.
            n (Optional[int]): The number of gradient ascent steps per scale. Defaults to 10.
            step_size (Optional[float]): The strength of each individual gradient ascent step.
                Specifically, each step will change the image's pixel values by a median of
                step_size.
            smooth (Optional[float]): The factor to smooth by at each gradient ascent step.
                Try 0.1-0.25.
            max_tile_size (Optional[int]): Defaults to 512, suitable for a GPU with 2 GB RAM.
                Higher values perform better; if Caffe runs out of GPU memory and crashes then it
                should be lowered.

        Returns:
            The unclipped processed image as a float32 ndarray which has a valid range of 0-255 but
            which may contain components that are less than 0 or greater than 255.
            deep_dream.to_image() can be used to convert the ndarray to a PIL image.
        """
        if isinstance(layers, str):
            layers = [layers]
        if isinstance(layers, list):
            layers = {layer: 1 for layer in layers}
        _layers = OrderedDict()
        for layer in reversed(self.net.blobs.keys()):
            if layer in layers:
                _layers[layer] = layers[layer]

        for blob in self.net.blobs:
            self.diff[blob] = 0

        input_arr = self._preprocess(np.float32(input_img))
        self.total_px = 0
        self.progress_bar = None
        try:
            detail = self._octave_detail(input_arr, layers=_layers, progress=progress, **kwargs)
        finally:
            if self.progress_bar:
                self.progress_bar.close()
        return self._deprocess(detail + input_arr)

    def dream_guided(self, input_img, guide_img, layers, **kwargs):
        """Performs guided gradient ascent on input_img, weighted by the feature map channel sums
        of guide_img. This algorithm works best using a relatively large number of layers, such as
        (for googlenet) anything matching the regular expression 'inception_../output'. The
        relative weights of the layers are determined automatically."""
        if isinstance(layers, str):
            layers = [layers]
        guide_features = self.get_features(guide_img, layers)
        weights = {}
        for layer in layers:
            v = guide_features[layer].sum(1).sum(1)[:, None, None]
            weights[layer] = v/v.sum()**2
        return self.dream(input_img, weights, auto_weight=False, **kwargs)
<commit_msg>Add currently-undocumented max_guide_size option to dream_guided()<commit_after>"""Deep Dreaming using Caffe and Google's Inception convolutional neural network."""

# pylint: disable=invalid-name, wrong-import-position

from collections import namedtuple, OrderedDict
import os
from pathlib import Path
import re

os.environ['GLOG_minloglevel'] = '1'
import caffe
import numpy as np
from PIL import Image
from scipy import ndimage
from tqdm import tqdm

EPS = np.finfo(np.float32).eps

CNNData = namedtuple('CNNData', 'deploy model mean categories')
CNNData.__new__.__defaults__ = (None,)  # Make categories optional.

_BASE_DIR = Path(__file__).parent.parent
GOOGLENET_BVLC = CNNData(
    _BASE_DIR/'bvlc_googlenet/deploy.prototxt',
    _BASE_DIR/'bvlc_googlenet/bvlc_googlenet.caffemodel',
    (104, 117, 123),
    categories=_BASE_DIR/'bvlc_googlenet/categories.txt')
GOOGLENET_PLACES205 = CNNData(
    _BASE_DIR/'googlenet_places205/deploy_places205.prototxt',
    _BASE_DIR/'googlenet_places205/googlelet_places205_train_iter_2400000.caffemodel',
    (104.051, 112.514, 116.676),  # TODO: find the actual Places205 mean
    categories=_BASE_DIR/'googlenet_places205/categories.txt')
GOOGLENET_PLACES365 = CNNData(
    _BASE_DIR/'googlenet_places365/deploy_googlenet_places365.prototxt',
    _BASE_DIR/'googlenet_places365/googlenet_places365.caffemodel',
    (104.051, 112.514, 116.676),
    categories=_BASE_DIR/'googlenet_places365/categories_places365.txt')

def save_as_hdr(arr, filename, gamma=2.2, allow_negative=True):
    """Saves a float32 ndarray to a high dynamic range (OpenEXR or float32 TIFF) file.

    Args:
        arr (ndarray): The input array.
        filename (str | Path): The output filename.
        gamma (Optional[float]): The encoding gamma of arr.
        allow_negative (Optional[bool]): Clip negative values to zero if false."""
    arr = arr.astype(np.float32)/255
    if not allow_negative:
        arr[arr < 0] = 0
    if gamma != 1:
        arr = np.sign(arr)*np.abs(arr)**gamma
    filename = str(filename)
    extension = filename.rpartition('.')[2].lower()
    if extension == 'exr':
        import OpenEXR
        exr = OpenEXR.OutputFile(filename, OpenEXR.Header(arr.shape[1], arr.shape[0]))
        exr.writePixels({'R': arr[..., 0].tobytes(),
                         'G': arr[..., 1].tobytes(),
                         'B': arr[..., 2].tobytes()})
        exr.close()
    elif extension == 'tif' or extension == 'tiff':
        import tifffile
        tiff = tifffile.TiffWriter(filename)
        tiff.save(arr, photometric='rgb')
        tiff.close()
    else:
        raise Exception('Unknown HDR file format.')

def to_image(arr):
    """Clips the values in a float32 ndarray to 0-255 and converts it to a PIL image.

    Args:
        arr (ndarray): The input array."""
    return Image.fromarray(np.uint8(np.clip(np.round(arr), 0, 255)))

def _resize(arr, size, method=Image.BICUBIC):
    h, w = size
    arr = np.float32(arr)
    if arr.ndim == 3:
        planes = [arr[i, :, :] for i in range(arr.shape[0])]
    else:
        raise TypeError('Only 3D CxHxW arrays are supported')
    imgs = [Image.fromarray(plane) for plane in planes]
    imgs_resized = [img.resize((w, h), method) for img in imgs]
    return np.stack([np.array(img) for img in imgs_resized])

class ShapeError(Exception):
    """Raised by CNN when an invalid layer shape is requested which would otherwise crash Caffe."""
    def __str__(self):
        return 'bad shape %s' % self.args

class _LayerIndexer:
    def __init__(self, net, attr):
        self.net, self.attr = net, attr

    def __getitem__(self, key):
        return getattr(self.net.blobs[key], self.attr)[0]

    def __setitem__(self, key, value):
        getattr(self.net.blobs[key], self.attr)[0] = value

class CNN:
    """Represents an instance of a Caffe convolutional neural network."""

    def __init__(self, cnndata, gpu=None):
        """Initializes a CNN.

        Args:
            gpu (Optional[int]): If present, Caffe will use this GPU device number. On a typical
                system with one GPU, it should be 0. If not present Caffe will use the CPU.
        """
        self.net = caffe.Classifier(str(cnndata.deploy), str(cnndata.model),
                                    mean=np.float32(cnndata.mean), channel_swap=(2, 1, 0))
        self.data = _LayerIndexer(self.net, 'data')
        self.diff = _LayerIndexer(self.net, 'diff')
        self.categories = [str(i) for i in range(self.data['prob'].size)]
        if cnndata.categories is not None:
            self.categories = open(str(cnndata.categories)).read().splitlines()
        self.img = np.zeros_like(self.data['data'])
        self.total_px = 0
        self.progress_bar = None
        if gpu is not None:
            caffe.set_device(gpu)
            caffe.set_mode_gpu()
        else:
            caffe.set_mode_cpu()

    def _preprocess(self, img):
        return np.rollaxis(np.float32(img), 2)[::-1] - self.net.transformer.mean['data']

    def _deprocess(self, img):
        return np.dstack((img + self.net.transformer.mean['data'])[::-1])

    def get_features(self, input_img, layers, max_tile_size=512):
        """Retrieve feature maps from the classification (forward) phase of operation.

        Example:
            cnn.get_features(img, ['prob'])['prob'] classifies 'img' and returns the predicted
            probability distribution over the network's categories.

        Returns:
            A dict which maps each layer in layers to a retrieved feature map.
        """
        input_arr = self._preprocess(np.float32(input_img))
        h = min(max_tile_size, input_arr.shape[1])
        w = min(max_tile_size, input_arr.shape[2])
        if max(*input_arr.shape[1:]) > max_tile_size:
            input_arr = _resize(input_arr, (h, w))
        self.net.blobs['data'].reshape(1, 3, h, w)
        self.data['data'] = input_arr
        end = self.layers()[-1]
        self.net.forward(end=end)
        return {layer: self.data[layer].copy() for layer in layers}

    def _grad_tiled(self, layers, progress=False, max_tile_size=512, auto_weight=True):
        # pylint: disable=too-many-locals
        if progress:
            if not self.progress_bar:
                self.progress_bar = tqdm(
                    total=self.total_px, unit='pix', unit_scale=True, ncols=80, smoothing=0.1)

        h, w = self.img.shape[1:]  # Height and width of input image
        ny, nx = (h-1)//max_tile_size+1, (w-1)//max_tile_size+1  # Number of tiles per dimension
        g = np.zeros_like(self.img)
        for y in range(ny):
            th = h//ny
            if y == ny-1:
                th += h - th*ny
            for x in range(nx):
                tw = w//nx
                if x == nx-1:
                    tw += w - tw*nx
                self.net.blobs['data'].reshape(1, 3, th, tw)
                sy, sx = h//ny*y, w//nx*x
                self.data['data'] = self.img[:, sy:sy+th, sx:sx+tw]

                for layer in layers.keys():
                    self.diff[layer] = 0
                self.net.forward(end=next(iter(layers.keys())))
                layers_list = list(layers.keys())
                for i, layer in enumerate(layers_list):
                    if auto_weight:
                        self.diff[layer] += \
                            self.data[layer]*layers[layer]/np.abs(self.data[layer]).sum()
                    else:
                        self.diff[layer] += self.data[layer]*layers[layer]
                    if i+1 == len(layers):
                        self.net.backward(start=layer)
                    else:
                        self.net.backward(start=layer, end=layers_list[i+1])

                g[:, sy:sy+th, sx:sx+tw] = self.diff['data']

                if progress:
                    self.progress_bar.update(th*tw)
        return g

    def _step(self, n=1, step_size=1.5, jitter=32, seed=0, smooth=0, **kwargs):
        np.random.seed(self.img.size + seed)
        for _ in range(n):
            x, y = np.random.randint(-jitter, jitter+1, 2)
            self.img = np.roll(np.roll(self.img, x, 2), y, 1)
            g = self._grad_tiled(**kwargs)
            g /= np.mean(np.abs(g)) + EPS
            if smooth:
                kernel = np.float32([[[0, -1, 0], [-1, 4, -1], [0, -1, 0]]])
                g2 = ndimage.convolve(self.img, kernel)
                g2 /= np.mean(np.abs(g2)) + EPS
                g -= g2 * smooth
                self.img += step_size * g / (np.mean(np.abs(g)) + EPS)
            else:
                self.img += step_size * g
            self.img = np.roll(np.roll(self.img, -x, 2), -y, 1)

    def _octave_detail(self, base, scales=4, min_size=32, n=10, per_octave=2, **kwargs):
        if min(base.shape[1:]) < 32:
            raise ShapeError(base.shape)
        factor = 2**(1/per_octave)
        detail = np.zeros_like(base, dtype=np.float32)
        self.total_px += base.shape[1] * base.shape[2] * n
        if scales != 1:
            hf, wf = np.int32(np.round(np.array(base.shape)[-2:]/factor))
            if min(hf, wf) >= min_size:
                smaller_base = _resize(base, (hf, wf))
                smaller_detail = self._octave_detail(
                    smaller_base, scales-1, min_size, n, per_octave, **kwargs)
                detail = _resize(smaller_detail, base.shape[-2:])
        self.img = base + detail
        self._step(n, **kwargs)
        return self.img - base

    def layers(self, pattern='.*'):
        """Returns a list of layer names matching a regular expression."""
        layers = []
        for i, layer in enumerate(self.net.blobs.keys()):
            if i == 0 or layer.partition('_split_')[1]:
                continue
            if re.fullmatch(pattern, layer):
                layers.append(layer)
        return layers

    def classify(self, input_img, n=1, **kwargs):
        """Classifies the input image and returns the n most probable categories.

        Args:
            input_img: The image to process (PIL images or Numpy arrays are accepted).
            n: The n most probable categories to return.
            max_tile_size: Does not allow the image dimension to exceed this.

        Returns:
            A list containing the n most probable categories."""
        prob = self.get_features(input_img, ['prob'], **kwargs)['prob']
        indices = prob.argsort()[::-1][:n]
        return [(prob[i], self.categories[i]) for i in indices]

    def dream(self, input_img, layers, progress=True, **kwargs):
        """Runs the Deep Dream multiscale gradient ascent algorithm on the input image.

        Args:
            input_img: The image to process (PIL images or Numpy arrays are accepted)
            layers (dict): The layer/feature weights to use in the objective function for gradient
                ascent.
            progress (Optional[bool]): Display a progress bar while computing.
            scales (Optional[int]): The number of scales to process.
            min_size (Optional[int]): Don't permit the small edge of the image to go below this.
            per_octave (Optional[int]): Determines the difference between each scale; for instance,
                the default of 2 means that a 1000x1000 input image will get processed as 707x707
                and 500x500.
            n (Optional[int]): The number of gradient ascent steps per scale. Defaults to 10.
            step_size (Optional[float]): The strength of each individual gradient ascent step.
                Specifically, each step will change the image's pixel values by a median of
                step_size.
            smooth (Optional[float]): The factor to smooth by at each gradient ascent step.
                Try 0.1-0.25.
            max_tile_size (Optional[int]): Defaults to 512, suitable for a GPU with 2 GB RAM.
                Higher values perform better; if Caffe runs out of GPU memory and crashes then it
                should be lowered.

        Returns:
            The unclipped processed image as a float32 ndarray which has a valid range of 0-255 but
            which may contain components that are less than 0 or greater than 255.
            deep_dream.to_image() can be used to convert the ndarray to a PIL image.
        """
        if isinstance(layers, str):
            layers = [layers]
        if isinstance(layers, list):
            layers = {layer: 1 for layer in layers}
        _layers = OrderedDict()
        for layer in reversed(self.net.blobs.keys()):
            if layer in layers:
                _layers[layer] = layers[layer]

        for blob in self.net.blobs:
            self.diff[blob] = 0

        input_arr = self._preprocess(np.float32(input_img))
        self.total_px = 0
        self.progress_bar = None
        try:
            detail = self._octave_detail(input_arr, layers=_layers, progress=progress, **kwargs)
        finally:
            if self.progress_bar:
                self.progress_bar.close()
        return self._deprocess(detail + input_arr)

    def dream_guided(self, input_img, guide_img, layers, max_guide_size=512, **kwargs):
        """Performs guided gradient ascent on input_img, weighted by the feature map channel sums
        of guide_img. This algorithm works best using a relatively large number of layers, such as
        (for googlenet) anything matching the regular expression 'inception_../output'. The
        relative weights of the layers are determined automatically."""
        if isinstance(layers, str):
            layers = [layers]
        guide_features = self.get_features(guide_img, layers, max_tile_size=max_guide_size)
        weights = {}
        for layer in layers:
            v = guide_features[layer].sum(1).sum(1)[:, None, None]
            weights[layer] = v/v.sum()**2
        return self.dream(input_img, weights, auto_weight=False, **kwargs)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# Copyright (c) 2013, Mayo Clinic
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without modification,
# are permitted provided that the following conditions are met:
#
#     Redistributions of source code must retain the above copyright notice, this
#     list of conditions and the following disclaimer.
#
#     Redistributions in binary form must reproduce the above copyright notice,
#     this list of conditions and the following disclaimer in the documentation
#     and/or other materials provided with the distribution.
#
#     Neither the name of the <ORGANIZATION> nor the names of its contributors
#     may be used to endorse or promote products derived from this software
#     without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
# IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT,
# INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
# OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
# OF THE POSSIBILITY OF SUCH DAMAGE.
from rf2db.utils.check_digit import generate_verhoeff
from rf2db.utils.sctid import sctid

MAYO_Namespace = 1000134
CIMI_Namespace = 1000160

class sctid_generator(object):
    class partition(object):
        def __init__(self, id):
            self._id = id

    CONCEPT = partition(00)
    DESCRIPTION = partition(01)
    RELATIONSHIP = partition(02)

    def __init__(self, namespace, partition, start):
        """
        @param namespace: SNOMED Namespace identifier
        @param partition: Partition Identifier
        @param start:
        @return:
        """
        assert isinstance(partition, sctid_generator.partition)
        self._namespace = namespace
        self._partition = partition._id
        self._start = start

    def __iter__(self):
        return self

    def __next__(self):
        return self.next()

    def next(self):
        rval = generate_verhoeff(self._start * 10 ** 10 + self._namespace * 10 ** 3 + self._partition * 10)
        self._start += 1
        return sctid(rval)
<commit_msg>Turned into a stand alone module so we can grab ids if needed<commit_after># -*- coding: utf-8 -*-
# Copyright (c) 2013, Mayo Clinic
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without modification,
# are permitted provided that the following conditions are met:
#
#     Redistributions of source code must retain the above copyright notice, this
#     list of conditions and the following disclaimer.
#
#     Redistributions in binary form must reproduce the above copyright notice,
#     this list of conditions and the following disclaimer in the documentation
#     and/or other materials provided with the distribution.
#
#     Neither the name of the <ORGANIZATION> nor the names of its contributors
#     may be used to endorse or promote products derived from this software
#     without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
# IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT,
# INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
# OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
# OF THE POSSIBILITY OF SUCH DAMAGE.

import argparse

from check_digit import generate_verhoeff
from sctid import sctid

MAYO_Namespace = 1000134
CIMI_Namespace = 1000160

class sctid_generator(object):
    class partition(object):
        def __init__(self, id):
            self._id = id

    CONCEPT = partition(00)
    DESCRIPTION = partition(01)
    RELATIONSHIP = partition(02)

    def __init__(self, namespace, partition, start):
        """
        @param namespace: SNOMED Namespace identifier
        @param partition: Partition Identifier
        @param start: Starting concept number
        @return:
        """
        assert isinstance(partition, sctid_generator.partition)
        self._namespace = namespace
        self._partition = partition._id
        self._start = start

    def __iter__(self):
        return self

    def __next__(self):
        return self.next()

    def next(self):
        rval = generate_verhoeff(self._start * 10 ** 10 + self._namespace * 10 ** 3 + self._partition * 10)
        self._start += 1
        return sctid(rval)

nsMap = {'M': MAYO_Namespace, 'C': CIMI_Namespace}
partitionMap = {'C':sctid_generator.CONCEPT, 'D':sctid_generator.DESCRIPTION, 'R':sctid_generator.RELATIONSHIP}

def main():
    parser = argparse.ArgumentParser(description="SCTID Generator")
    parser.add_argument('-p', '--partition', choices=['C','D','R'], help="(C)oncept, (D)escription, (R)elationship", required=True)
    parser.add_argument('-n', '--namespace', choices=['M','C'], help="(M)ayo, (C)imi", required=True)
    parser.add_argument('-s', '--start', type=int, help="Starting identifier", required=True)
    opts = parser.parse_args()
    print sctid_generator(nsMap[opts.namespace], partitionMap[opts.partition], opts.start).next()

if __name__ == '__main__':
    main()<|endoftext|>


<commit_before>import logging
from testfixtures import LogCapture
from os import path
from easy_thumbnails import files, utils, signals, test, exceptions
from easy_thumbnails.conf import settings
try:
    from PIL import Image
except ImportError:
    import Image

class FilesTest(test.BaseTest):

    def setUp(self):
        super(FilesTest, self).setUp()
        self.storage = test.TemporaryStorage()
        self.remote_storage = test.FakeRemoteStorage()

        # Save a test image in both storages.
        filename = self.create_image(self.storage, 'test.jpg')
        self.thumbnailer = files.get_thumbnailer(self.storage, filename)
        self.thumbnailer.thumbnail_storage = self.storage

        filename = self.create_image(self.remote_storage, 'test.jpg')
        self.remote_thumbnailer = files.get_thumbnailer(
            self.remote_storage, filename)
        self.remote_thumbnailer.thumbnail_storage = self.remote_storage

        # Create another thumbnailer for extension test.
        self.ext_thumbnailer = files.get_thumbnailer(self.storage, filename)
        self.ext_thumbnailer.thumbnail_storage = self.storage

        # Generate test transparent images.
        filename = self.create_image(
            self.storage, 'transparent.png', image_mode='RGBA',
            image_format='PNG')
        self.transparent_thumbnailer = files.get_thumbnailer(
            self.storage, filename)
        self.transparent_thumbnailer.thumbnail_storage = self.storage

        filename = self.create_image(
            self.storage, 'transparent-greyscale.png', image_mode='LA',
            image_format='PNG')
        self.transparent_greyscale_thumbnailer = files.get_thumbnailer(
            self.storage, filename)
        self.transparent_greyscale_thumbnailer.thumbnail_storage = self.storage

    def tearDown(self):
        self.storage.delete_temporary_storage()
        self.remote_storage.delete_temporary_storage()
        super(FilesTest, self).tearDown()

    def test_tag(self):
        local = self.thumbnailer.get_thumbnail({'size': (100, 100)})
        remote = self.remote_thumbnailer.get_thumbnail({'size': (100, 100)})

        self.assertEqual(
            local.tag(), '' % local.url)
        self.assertEqual(
            local.tag(alt='A & B'), '' % local.url)

        # Can turn off dimensions.
        self.assertEqual(
            remote.tag(use_size=False), '' % remote.url)

        # Thumbnails on remote storage don't get dimensions...
        self.assertEqual(
            remote.tag(), '' % remote.url)
        # ...unless explicitly requested.
        self.assertEqual(
            remote.tag(use_size=True),
            '' % remote.url)

        # All other arguments are passed through as attributes.
        self.assertEqual(
            local.tag(**{'rel': 'A&B', 'class': 'fish'}),
            '' % local.url)

    def test_transparent_thumbnailing(self):
        thumb_file = self.thumbnailer.get_thumbnail(
            {'size': (100, 100)})
        thumb_file.seek(0)
        thumb = Image.open(thumb_file)
        self.assertFalse(
            utils.is_transparent(thumb),
            "%s shouldn't be transparent." % thumb_file.name)

        thumb_file = self.transparent_thumbnailer.get_thumbnail(
            {'size': (100, 100)})
        thumb_file.seek(0)
        thumb = Image.open(thumb_file)
        self.assertTrue(
            utils.is_transparent(thumb),
            "%s should be transparent." % thumb_file.name)

        thumb_file = self.transparent_greyscale_thumbnailer.get_thumbnail(
            {'size': (100, 100)})
        thumb_file.seek(0)
        thumb = Image.open(thumb_file)
        self.assertTrue(
            utils.is_transparent(thumb),
            "%s should be transparent." % thumb_file.name)

    def test_missing_thumb(self):
        opts = {'size': (100, 100)}
        thumb = self.thumbnailer.get_thumbnail(opts)
        thumb_cache = self.thumbnailer.get_thumbnail_cache(
            thumbnail_name=thumb.name)
        thumb_cache.delete()
        thumb.storage.delete(thumb.name)
        self.thumbnailer.get_thumbnail(opts)

    def test_missing_thumb_from_storage(self):
        opts = {'size': (100, 100)}
        thumb = self.thumbnailer.get_thumbnail(opts)
        thumb.storage.delete(thumb.name)
        new_thumb = self.thumbnailer.get_thumbnail(opts)
        self.assertEqual(thumb.name, new_thumb.name)
        self.assertTrue(thumb.storage.exists(new_thumb.name))

    def test_missing_remote_thumb(self):
        opts = {'size': (100, 100)}
        thumb = self.remote_thumbnailer.get_thumbnail(opts)
        thumb_cache = self.remote_thumbnailer.get_thumbnail_cache(
            thumbnail_name=thumb.name)
        thumb_cache.delete()
        thumb.storage.delete(thumb.name)
        self.remote_thumbnailer.get_thumbnail(opts)

    def test_missing_source(self):
        opts = {'size': (100, 100)}
        self.storage.delete(self.thumbnailer.name)
        self.assertRaises(
            exceptions.InvalidImageFormatError,
            self.thumbnailer.get_thumbnail, opts)

    def test_extensions(self):
        self.ext_thumbnailer.thumbnail_extension = 'png'
        thumb = self.ext_thumbnailer.get_thumbnail({'size': (100, 100)})
        self.assertEqual(path.splitext(thumb.name)[1], '.png')

        self.ext_thumbnailer.thumbnail_preserve_extensions = ('foo',)
        thumb = self.ext_thumbnailer.get_thumbnail({'size': (100, 100)})
        self.assertEqual(path.splitext(thumb.name)[1], '.png')

        self.ext_thumbnailer.thumbnail_preserve_extensions = True
        thumb = self.ext_thumbnailer.get_thumbnail({'size': (100, 100)})
        self.assertEqual(path.splitext(thumb.name)[1], '.jpg')

        self.ext_thumbnailer.thumbnail_preserve_extensions = ('foo', 'jpg')
        thumb = self.ext_thumbnailer.get_thumbnail({'size': (100, 100)})
        self.assertEqual(path.splitext(thumb.name)[1], '.jpg')

    def test_high_resolution(self):
        self.ext_thumbnailer.thumbnail_high_resolution = True
        thumb = self.ext_thumbnailer.get_thumbnail({'size': (100, 100)})
        base, ext = path.splitext(thumb.path)
        hires_thumb_file = ''.join([base + '@2x', ext])
        self.assertTrue(path.isfile(hires_thumb_file))
        thumb = Image.open(hires_thumb_file)
        self.assertEqual(thumb.size, (200, 150))

    def test_postprocessor(self):
        """use a mock image optimizing post processor doing nothing"""
        with self.settings(THUMBNAIL_OPTIMIZE_COMMAND={'png': 'easy_thumbnails/tests/mockoptim.py {filename}'}):
            with LogCapture() as logcap:
                self.ext_thumbnailer.thumbnail_extension = 'png'
                self.ext_thumbnailer.get_thumbnail({'size': (10, 10)})
                actual = tuple(logcap.actual())[0]
                self.assertEqual(actual[0], 'easy_thumbnails.optimize')
                self.assertEqual(actual[1], 'INFO')
                self.assertRegexpMatches(actual[2], '^easy_thumbnails/tests/mockoptim.py [^ ]+ returned nothing$')

    def test_postprocessor_fail(self):
        """use a mock image optimizing post processor doing nothing"""
        with self.settings(THUMBNAIL_OPTIMIZE_COMMAND={'png': 'easy_thumbnails/tests/mockoptim_fail.py {filename}'}):
            with LogCapture() as logcap:
                self.ext_thumbnailer.thumbnail_extension = 'png'
                self.ext_thumbnailer.get_thumbnail({'size': (10, 10)})
                actual = tuple(logcap.actual())[0]
                self.assertEqual(actual[0], 'easy_thumbnails.optimize')
                self.assertEqual(actual[1], 'ERROR')
                self.assertRegexpMatches(actual[2], r'^Command\ .+returned non-zero exit status 1$')

    def test_USE_TZ(self):
        settings.USE_TZ = True
        self.thumbnailer.get_thumbnail({'size': (10, 20)})

        settings.USE_TZ = False
        self.thumbnailer.get_thumbnail({'size': (20, 40)})

    def test_thumbnailfile_options(self):
        opts = {'size': (50, 50), 'crop': True, 'upscale': True}
        thumb = self.thumbnailer.get_thumbnail(opts)
        self.assertEqual(thumb.thumbnail_options, opts)

    def test_default_options_setting(self):
        settings.THUMBNAIL_DEFAULT_OPTIONS = {'crop': True}
        opts = {'size': (50, 50)}
        thumb = self.thumbnailer.get_thumbnail(opts)
        self.assertEqual((thumb.width, thumb.height), (50, 50))

    def test_dimensions_of_cached_image(self):
        opts = {'size': (50, 50)}
        thumb = self.thumbnailer.get_thumbnail(opts)
        self.assertEqual((thumb.width, thumb.height), (50, 38))
        # Now the thumb has been created, check that retrieving this still
        # gives access to the dimensions.
        thumb = self.thumbnailer.get_thumbnail(opts)
        self.assertEqual((thumb.width, thumb.height), (50, 38))

    def test_thumbnail_created_signal(self):

        def signal_handler(sender, **kwargs):
            sender.signal_received = True

        signals.thumbnail_created.connect(signal_handler)
        try:
            thumb = self.thumbnailer.get_thumbnail({'size': (10, 20)})
            self.assertTrue(hasattr(thumb, 'signal_received'))
        finally:
            signals.thumbnail_created.disconnect(signal_handler)

    def test_passive_thumbnailer(self):
        options = {'size': (10, 10)}

        # Explicitly using the generate=False option on get_thumbnail won't
        # generate a missing thumb.
        thumb = self.thumbnailer.get_thumbnail(options, generate=False)
        self.assertEqual(thumb, None)

        # If the thumbnailer has generate=False, get_thumbnail won't generate a
        # missing thumb by default.
        self.thumbnailer.generate = False
        thumb = self.thumbnailer.get_thumbnail(options)
        self.assertEqual(thumb, None)

        # If the thumbnailer has generate=False, get_thumbnail with
        # generate=True will stiff force generation a missing thumb.
        thumb = self.thumbnailer.get_thumbnail(options, generate=True)
        self.assertTrue(thumb)

        # If the thumbnailer has generate=False, get_thumbnail will still
        # return existing thumbnails.
        thumb = self.thumbnailer.get_thumbnail(options)
        self.assertTrue(thumb)

        # Explicitly using the generate=False option on get_thumbnail will
        # still return existing thumbnails.
        thumb = self.thumbnailer.get_thumbnail(options, generate=False)
        self.assertTrue(thumb)

    def test_thumbnail_missed_signal(self):

        def signal_handler(sender, **kwargs):
            sender.missed_signal = kwargs.get('options')

        signals.thumbnail_missed.connect(signal_handler)
        try:
            # Standard generation doesn't trigger signal.
            self.thumbnailer.get_thumbnail({'size': (100, 100)})
            self.assertFalse(hasattr(self.thumbnailer, 'missed_signal'))
            # Retrieval doesn't trigger signal.
            self.thumbnailer.get_thumbnail(
                {'size': (100, 100)}, generate=False)
            self.assertFalse(hasattr(self.thumbnailer, 'missed_signal'))
            # A thumbnail miss does trigger it.
            options = {'size': (10, 20)}
            thumb = self.thumbnailer.get_thumbnail(options, generate=False)
            self.assertEqual(thumb, None)
            self.assertEqual(self.thumbnailer.missed_signal, options)
        finally:
            signals.thumbnail_created.disconnect(signal_handler)
<commit_msg>Avoid Python 3.3 deprecation warning for assertRegexpMatches<commit_after>import logging
from testfixtures import LogCapture
from os import path
from easy_thumbnails import files, utils, signals, test, exceptions
from easy_thumbnails.conf import settings
try:
    from PIL import Image
except ImportError:
    import Image

class FilesTest(test.BaseTest):

    def setUp(self):
        super(FilesTest, self).setUp()
        self.storage = test.TemporaryStorage()
        self.remote_storage = test.FakeRemoteStorage()

        # Save a test image in both storages.
        filename = self.create_image(self.storage, 'test.jpg')
        self.thumbnailer = files.get_thumbnailer(self.storage, filename)
        self.thumbnailer.thumbnail_storage = self.storage

        filename = self.create_image(self.remote_storage, 'test.jpg')
        self.remote_thumbnailer = files.get_thumbnailer(
            self.remote_storage, filename)
        self.remote_thumbnailer.thumbnail_storage = self.remote_storage

        # Create another thumbnailer for extension test.
        self.ext_thumbnailer = files.get_thumbnailer(self.storage, filename)
        self.ext_thumbnailer.thumbnail_storage = self.storage

        # Generate test transparent images.
        filename = self.create_image(
            self.storage, 'transparent.png', image_mode='RGBA',
            image_format='PNG')
        self.transparent_thumbnailer = files.get_thumbnailer(
            self.storage, filename)
        self.transparent_thumbnailer.thumbnail_storage = self.storage

        filename = self.create_image(
            self.storage, 'transparent-greyscale.png', image_mode='LA',
            image_format='PNG')
        self.transparent_greyscale_thumbnailer = files.get_thumbnailer(
            self.storage, filename)
        self.transparent_greyscale_thumbnailer.thumbnail_storage = self.storage

    def tearDown(self):
        self.storage.delete_temporary_storage()
        self.remote_storage.delete_temporary_storage()
        super(FilesTest, self).tearDown()

    def assertRegex(self, *args, **kwargs):
        func = getattr(super(FilesTest, self), 'assertRegex', None)
        if func is None:
            func = self.assertRegexpMatches
        return func(*args, **kwargs)

    def test_tag(self):
        local = self.thumbnailer.get_thumbnail({'size': (100, 100)})
        remote = self.remote_thumbnailer.get_thumbnail({'size': (100, 100)})

        self.assertEqual(
            local.tag(), '' % local.url)
        self.assertEqual(
            local.tag(alt='A & B'), '' % local.url)

        # Can turn off dimensions.
        self.assertEqual(
            remote.tag(use_size=False), '' % remote.url)

        # Thumbnails on remote storage don't get dimensions...
        self.assertEqual(
            remote.tag(), '' % remote.url)
        # ...unless explicitly requested.
        self.assertEqual(
            remote.tag(use_size=True),
            '' % remote.url)

        # All other arguments are passed through as attributes.
        self.assertEqual(
            local.tag(**{'rel': 'A&B', 'class': 'fish'}),
            '' % local.url)

    def test_transparent_thumbnailing(self):
        thumb_file = self.thumbnailer.get_thumbnail(
            {'size': (100, 100)})
        thumb_file.seek(0)
        thumb = Image.open(thumb_file)
        self.assertFalse(
            utils.is_transparent(thumb),
            "%s shouldn't be transparent." % thumb_file.name)

        thumb_file = self.transparent_thumbnailer.get_thumbnail(
            {'size': (100, 100)})
        thumb_file.seek(0)
        thumb = Image.open(thumb_file)
        self.assertTrue(
            utils.is_transparent(thumb),
            "%s should be transparent." % thumb_file.name)

        thumb_file = self.transparent_greyscale_thumbnailer.get_thumbnail(
            {'size': (100, 100)})
        thumb_file.seek(0)
        thumb = Image.open(thumb_file)
        self.assertTrue(
            utils.is_transparent(thumb),
            "%s should be transparent." % thumb_file.name)

    def test_missing_thumb(self):
        opts = {'size': (100, 100)}
        thumb = self.thumbnailer.get_thumbnail(opts)
        thumb_cache = self.thumbnailer.get_thumbnail_cache(
            thumbnail_name=thumb.name)
        thumb_cache.delete()
        thumb.storage.delete(thumb.name)
        self.thumbnailer.get_thumbnail(opts)

    def test_missing_thumb_from_storage(self):
        opts = {'size': (100, 100)}
        thumb = self.thumbnailer.get_thumbnail(opts)
        thumb.storage.delete(thumb.name)
        new_thumb = self.thumbnailer.get_thumbnail(opts)
        self.assertEqual(thumb.name, new_thumb.name)
        self.assertTrue(thumb.storage.exists(new_thumb.name))

    def test_missing_remote_thumb(self):
        opts = {'size': (100, 100)}
        thumb = self.remote_thumbnailer.get_thumbnail(opts)
        thumb_cache = self.remote_thumbnailer.get_thumbnail_cache(
            thumbnail_name=thumb.name)
        thumb_cache.delete()
        thumb.storage.delete(thumb.name)
        self.remote_thumbnailer.get_thumbnail(opts)

    def test_missing_source(self):
        opts = {'size': (100, 100)}
        self.storage.delete(self.thumbnailer.name)
        self.assertRaises(
            exceptions.InvalidImageFormatError,
            self.thumbnailer.get_thumbnail, opts)

    def test_extensions(self):
        self.ext_thumbnailer.thumbnail_extension = 'png'
        thumb = self.ext_thumbnailer.get_thumbnail({'size': (100, 100)})
        self.assertEqual(path.splitext(thumb.name)[1], '.png')

        self.ext_thumbnailer.thumbnail_preserve_extensions = ('foo',)
        thumb = self.ext_thumbnailer.get_thumbnail({'size': (100, 100)})
        self.assertEqual(path.splitext(thumb.name)[1], '.png')

        self.ext_thumbnailer.thumbnail_preserve_extensions = True
        thumb = self.ext_thumbnailer.get_thumbnail({'size': (100, 100)})
        self.assertEqual(path.splitext(thumb.name)[1], '.jpg')

        self.ext_thumbnailer.thumbnail_preserve_extensions = ('foo', 'jpg')
        thumb = self.ext_thumbnailer.get_thumbnail({'size': (100, 100)})
        self.assertEqual(path.splitext(thumb.name)[1], '.jpg')

    def test_high_resolution(self):
        self.ext_thumbnailer.thumbnail_high_resolution = True
        thumb = self.ext_thumbnailer.get_thumbnail({'size': (100, 100)})
        base, ext = path.splitext(thumb.path)
        hires_thumb_file = ''.join([base + '@2x', ext])
        self.assertTrue(path.isfile(hires_thumb_file))
        thumb = Image.open(hires_thumb_file)
        self.assertEqual(thumb.size, (200, 150))

    def test_postprocessor(self):
        """use a mock image optimizing post processor doing nothing"""
        with self.settings(THUMBNAIL_OPTIMIZE_COMMAND={
                'png': 'easy_thumbnails/tests/mockoptim.py {filename}'}):
            with LogCapture() as logcap:
                self.ext_thumbnailer.thumbnail_extension = 'png'
                self.ext_thumbnailer.get_thumbnail({'size': (10, 10)})
                actual = tuple(logcap.actual())[0]
                self.assertEqual(actual[0], 'easy_thumbnails.optimize')
                self.assertEqual(actual[1], 'INFO')
                self.assertRegex(
                    actual[2],
                    '^easy_thumbnails/tests/mockoptim.py [^ ]+ returned '
                    'nothing$')

    def test_postprocessor_fail(self):
        """use a mock image optimizing post processor doing nothing"""
        with self.settings(THUMBNAIL_OPTIMIZE_COMMAND={
                'png': 'easy_thumbnails/tests/mockoptim_fail.py {filename}'}):
            with LogCapture() as logcap:
                self.ext_thumbnailer.thumbnail_extension = 'png'
                self.ext_thumbnailer.get_thumbnail({'size': (10, 10)})
                actual = tuple(logcap.actual())[0]
                self.assertEqual(actual[0], 'easy_thumbnails.optimize')
                self.assertEqual(actual[1], 'ERROR')
                self.assertRegex(
                    actual[2], r'^Command\ .+returned non-zero exit status 1$')

    def test_USE_TZ(self):
        settings.USE_TZ = True
        self.thumbnailer.get_thumbnail({'size': (10, 20)})

        settings.USE_TZ = False
        self.thumbnailer.get_thumbnail({'size': (20, 40)})

    def test_thumbnailfile_options(self):
        opts = {'size': (50, 50), 'crop': True, 'upscale': True}
        thumb = self.thumbnailer.get_thumbnail(opts)
        self.assertEqual(thumb.thumbnail_options, opts)

    def test_default_options_setting(self):
        settings.THUMBNAIL_DEFAULT_OPTIONS = {'crop': True}
        opts = {'size': (50, 50)}
        thumb = self.thumbnailer.get_thumbnail(opts)
        self.assertEqual((thumb.width, thumb.height), (50, 50))

    def test_dimensions_of_cached_image(self):
        opts = {'size': (50, 50)}
        thumb = self.thumbnailer.get_thumbnail(opts)
        self.assertEqual((thumb.width, thumb.height), (50, 38))
        # Now the thumb has been created, check that retrieving this still
        # gives access to the dimensions.
        thumb = self.thumbnailer.get_thumbnail(opts)
        self.assertEqual((thumb.width, thumb.height), (50, 38))

    def test_thumbnail_created_signal(self):

        def signal_handler(sender, **kwargs):
            sender.signal_received = True

        signals.thumbnail_created.connect(signal_handler)
        try:
            thumb = self.thumbnailer.get_thumbnail({'size': (10, 20)})
            self.assertTrue(hasattr(thumb, 'signal_received'))
        finally:
            signals.thumbnail_created.disconnect(signal_handler)

    def test_passive_thumbnailer(self):
        options = {'size': (10, 10)}

        # Explicitly using the generate=False option on get_thumbnail won't
        # generate a missing thumb.
        thumb = self.thumbnailer.get_thumbnail(options, generate=False)
        self.assertEqual(thumb, None)

        # If the thumbnailer has generate=False, get_thumbnail won't generate a
        # missing thumb by default.
        self.thumbnailer.generate = False
        thumb = self.thumbnailer.get_thumbnail(options)
        self.assertEqual(thumb, None)

        # If the thumbnailer has generate=False, get_thumbnail with
        # generate=True will stiff force generation a missing thumb.
        thumb = self.thumbnailer.get_thumbnail(options, generate=True)
        self.assertTrue(thumb)

        # If the thumbnailer has generate=False, get_thumbnail will still
        # return existing thumbnails.
        thumb = self.thumbnailer.get_thumbnail(options)
        self.assertTrue(thumb)

        # Explicitly using the generate=False option on get_thumbnail will
        # still return existing thumbnails.
        thumb = self.thumbnailer.get_thumbnail(options, generate=False)
        self.assertTrue(thumb)

    def test_thumbnail_missed_signal(self):

        def signal_handler(sender, **kwargs):
            sender.missed_signal = kwargs.get('options')

        signals.thumbnail_missed.connect(signal_handler)
        try:
            # Standard generation doesn't trigger signal.
            self.thumbnailer.get_thumbnail({'size': (100, 100)})
            self.assertFalse(hasattr(self.thumbnailer, 'missed_signal'))
            # Retrieval doesn't trigger signal.
            self.thumbnailer.get_thumbnail(
                {'size': (100, 100)}, generate=False)
            self.assertFalse(hasattr(self.thumbnailer, 'missed_signal'))
            # A thumbnail miss does trigger it.
            options = {'size': (10, 20)}
            thumb = self.thumbnailer.get_thumbnail(options, generate=False)
            self.assertEqual(thumb, None)
            self.assertEqual(self.thumbnailer.missed_signal, options)
        finally:
            signals.thumbnail_created.disconnect(signal_handler)
<|endoftext|>