<commit_before>#!/usr/bin/env python
#-*- coding:utf-8 -*-
'''
Date : 2017-11-13
@Author : Amy
Goal: 用于生成新增时d-v-h架构初始状态，并尽量保留mbbo数据类型，方便说明聚合时mbbo优于BFD算法
'''

import time
import random
import math
import json
import sys
import copy

def range2rect(size, num_var, type0):
    '''
    构造size * num_var 的矩阵
    ！！注意：python中list的引用可能造成全局参数的同时更改
    '''
    # 注意type0 是引用类型，若引用的是list，则更改type0的值导致所有通过type0进行list赋值的都会被更改
    res = [[type0 for j in xrange(num_var)] for i in xrange(size)]
    return res

def init_Docker(rp_u, rm_u, p, num_var):
    '''
    目标：随机生成容器cpu、mem二维尺寸
    参数：
    rp_u，rm_u指导对cpu，mem的最终需求量会以正态分布的形式落在以指导变量为期望的邻域附近
    p代表虚拟机的cpu和mem两种资源之间的相关系数，负责控制每台虚拟机对两种资源需求的关联程度
    num_var 初始状态中VM数量
    '''
    # print "进入 init_Docker"
    c_rp = range(num_var)                                                  # 记录每个虚拟机cpu,mem请求量
    c_rm = range(num_var)
    for i in range(num_var):
        c_rp[i] = random.uniform(0, 2*rp_u)
        c_rm[i] = random.uniform(0, rm_u)
    for i in range(num_var):
        r = random.random()
        if (r < p and c_rp[i] >= rp_u) or (r >= p and c_rp[i] < rp_u):     # p相关系数vm对cpu,mem请求量的相关性，值越大相关性越强
            c_rm[i] += rm_u
        if c_rm[i] >= 1.0:                                               # 控制mem请求量合理性
            c_rm[i] -= 1.0
    #print "len(c_rp)= %s,len(c_rm) = %s" %(len(c_rp),len(c_rm))
    return c_rp, c_rm

def init_VM(c_rp, c_rm, rp_option, rm_option, num_var):
    '''
    goal： 根据已有容器大小，初始化的d-v-h映射关系
    参数:
    rp_option和rm_option分别为VM可选的CPU、mem尺寸
    c_rp,c_rm为对应下标的容器尺寸
    num_var容器个数，初始假设vm数量与容器数量1：1
    返回：
    能够容纳对应编号容器的VM有效尺寸系列
    '''
    # print "进入init_VM"
    v_rp = []
    v_rm = []
    i = 0
    while True:
        if i == num_var:                        # 对num_var个docker选择随机的vm位置
            #print "len(v_rp)= %s,len(v_rm) = %s" %(len(v_rp),len(v_rm))
            return v_rp, v_rm
        rp = random.choice(rp_option)
        rm = random.choice(rm_option)
        if rp >= c_rp[i] and rm >= c_rm[i]:
            v_rp.append(rp)
            v_rm.append(rm)
            i += 1
        else:
            continue

def make_population(size, num_var, c_rp, c_rm, v_rp, v_rm): #    作为全局变量，按照需要传入各方法中 f, p_mutate, time_base, lambdaa):
    '''
    构造一个population，包含size个候选解chrom,每个chrom是num_var个分别记录该容器所在的vm和hm编号的元组
    '''
    # print "进入make_population"
    population0 = {
        'c_rp': c_rp,                                             # 每个容器的cpu请求
        'c_rm': c_rm,                                             # 每个容器的mem请求
        'v_rp': v_rp,                                             # 每个vm的cpu请求
        'v_rm': v_rm,                                             # 每个vm的mem请求
        'population': range2rect(size, num_var, [0, 0]),          # size个chrom，每个chrom有num_var个双元素list[vm,hm]对应每个容器放置的vm编号和物理机编号
        'init_save': range2rect(size, num_var, [0, 0]),           # 保存初始size个chroms
        'v_p_cost': range2rect(size, num_var, 0.0),               # size个num_var长list记录每个vm上所有容器的cpu总请求,初始为0
        'v_m_cost': range2rect(size, num_var, 0.0),               # 每个vm被容器请求的mem，初始为0
        'h_p_cost': range2rect(size, num_var, 0.0),               # 每个HM被请求的cpu，初始为0
        'h_m_cost': range2rect(size, num_var, 0.0),               # HM被请求的mem，初始为0
    }
    return population0

def initialize_population(popu1, size, num_var):
    '''
    初始化docker容器放置的vm编号，以及vm所能放置的hm编号，并计算初代种群的各项数值
    '''
    # print "进入initialize_population"
    # 为所有容器选择能够容纳其尺寸的vm_id
    for i in xrange(size):
        # 记录每个chrom中，vm-hm的映射，保证映射在相同vm的不同容器，所在的hm也相同
        vm_used_id = {}
        for j in xrange(num_var):
            # 为第j个容器选择要能容纳其尺寸的vm编号
            while True:
                tmp_v_id = random.randint(0, num_var-1)
                cpu_tmp_v = popu1['v_p_cost'][i][tmp_v_id] + popu1['c_rp'][j]
                mem_tmp_v = popu1['v_m_cost'][i][tmp_v_id] + popu1['c_rm'][j]
                if cpu_tmp_v <= popu1['v_rp'][tmp_v_id] and mem_tmp_v <= popu1['v_rm'][tmp_v_id]:
                    # popu1['population'][i][j]是第i个chrom的第j个容器放置的[vm_id,hm_id]
                    popu1['population'][i][j] = [tmp_v_id, 0]
                    popu1['v_p_cost'][i][tmp_v_id] = cpu_tmp_v
                    popu1['v_m_cost'][i][tmp_v_id] = mem_tmp_v

                    # 找到容器-vm的拓扑后，继续寻找能够满足该vm尺寸的hm
                    flag = True
                    # 对与已经安排过vm-hm映射位置的容器，直接进行对应vm,hm的赋值，并设while循环初始为False
                    if tmp_v_id in vm_used_id:
                        popu1['population'][i][j] = [tmp_v_id, vm_used_id[tmp_v_id]]
                        flag = False
                    while flag:
                        # 对未安排hm的容器所对应的vm，循环查找满足vm尺寸的h
                        tmp_h_id = random.randint(0, num_var-1)
                        cpu_tmp_h = popu1['h_p_cost'][i][tmp_h_id] + popu1['v_rp'][tmp_v_id]
                        mem_tmp_h = popu1['h_m_cost'][i][tmp_h_id] + popu1['v_rm'][tmp_v_id]
                        if cpu_tmp_h <= 1.0 and mem_tmp_h <= 1.0:
                            popu1['population'][i][j] = [tmp_v_id, tmp_h_id]
                            popu1['h_p_cost'][i][tmp_h_id] = cpu_tmp_h
                            popu1['h_m_cost'][i][tmp_h_id] = mem_tmp_h
                            vm_used_id[tmp_v_id] = tmp_h_id
                            break
                    break

    if check_effective(popu1, size, num_var):
        print "Initailization population is effective"
        return popu1
    else:
        sys.exit("failed in initializating population")

def check_effective(popu1, size, num_var):
    '''
    判断候选解的有效性
    '''
    # print "进入check_effective"
    # 先清空各vm,pm的资源使用率
    for i in xrange(size):
        for j in xrange(num_var):
            popu1['v_p_cost'][i][j] = 0
            popu1['v_m_cost'][i][j] = 0
            popu1['h_p_cost'][i][j] = 0
            popu1['h_m_cost'][i][j] = 0
    ## 逐一按照候选解，计算实际占用的vm,hm资源
    for i in xrange(size):
        vm_used_id = {}                                # 避免相同vm同时被映射的不同hm的错误
        for j in xrange(num_var):                      # 容器编号
            v_id = popu1['population'][i][j][0]
            h_id = popu1['population'][i][j][1]
            if v_id in vm_used_id:                     # 若vm已经被安排过
                if h_id == vm_used_id[v_id]:           # 且所在的hm与安排过的hm相同，则跳过该次循环，不需要重新计算资源占用
                    continue
                else:                                  # 若hm不同于安排的hm编号，说明出现同一个vm映射到不同hm的错误，直接返回
                    # print "in this chrom %s, a vm has been hosted on different hm, totally wrong!! " %popu1['population'][i]
                    return False
            else:                                      # 其他情况包括,包括多vm映射到1个hm,按正常情况计算
                popu1['v_p_cost'][i][v_id] += popu1['c_rp'][j]
                popu1['v_m_cost'][i][v_id] += popu1['c_rm'][j]
                popu1['h_p_cost'][i][h_id] += popu1['v_rp'][v_id]
                popu1['h_m_cost'][i][h_id] += popu1['v_rm'][v_id]
                vm_used_id[v_id] = h_id
        for x in xrange(num_var):                      # 只要超出资源限约束，即报错
            if popu1['v_p_cost'][i][x] > popu1['v_rp'][x] or popu1['v_m_cost'][i][x] > popu1['v_rm'][x] or popu1['h_p_cost'][i][x] > 1.0 or popu1['h_m_cost'][i][x] > 1.0:
                # print popu1['v_p_cost'][i][x], popu1['v_rp'][x]
                # print popu1['v_m_cost'][i][x], popu1['v_rm'][x]
                # print popu1['h_p_cost'][i][x]
                # print popu1['h_m_cost'][i][x]
                # print "find a error: VM or hm has overhold" , popu1['population'][i], x, vm_used_id, '\n'
                return False
    return True

def main(num_var,p):
    # 1.算法主要参数设置
    rp_u = 0.25                            # 容器请求CPU的指导变量
    rm_u = 0.25                            # 容器请求MEM的指导变量
    p = p                                  # 控制容器cpu,mem的资源相关度
    rp_option = [1.0]                      # vm可选的cpu尺寸
    rm_option = [1.0]                      # vm可选的mem尺寸

    size = 1                               # 新增阶段只需单解

    # 2.初始化num_var个容器和vm，以及计算迁移率
    c_rp, c_rm = init_Docker(rp_u, rm_u, p, num_var)
    v_rp, v_rm = init_VM(c_rp, c_rm, rp_option, rm_option, num_var)

    print "开始主流程"
    ## 以下程序的流程按照先按照串行结构书写
    # 3. 初代种群的生成、代价计算及排序
    init_popu = make_population(size, num_var, c_rp, c_rm, v_rp, v_rm)
    init_popu = initialize_population(init_popu, size, num_var)
    print init_popu

if __name__=='__main__':
    main(50,1.0)<commit_msg>初始化新增时d-v-h集群状态，并生成新增容器序列，即replicas<commit_after>#!/usr/bin/env python
#-*- coding:utf-8 -*-
'''
Date : 2017-11-13
@Author : Amy
Goal: 用于生成新增时d-v-h架构初始状态，并尽量保留mbbo数据类型，方便说明聚合时mbbo优于BFD算法
'''

import time
import random
import math
import json
import sys
import copy

def range2rect(size, num_var, type0):
    '''
    构造size * num_var 的矩阵
    ！！注意：python中list的引用可能造成全局参数的同时更改
    '''
    # 注意type0 是引用类型，若引用的是list，则更改type0的值导致所有通过type0进行list赋值的都会被更改
    res = [[type0 for j in xrange(num_var)] for i in xrange(size)]
    return res

def init_Docker(rp_u, rm_u, p, num_var):
    '''
    目标：随机生成容器cpu、mem二维尺寸
    参数：
    rp_u，rm_u指导对cpu，mem的最终需求量会以正态分布的形式落在以指导变量为期望的邻域附近
    p代表虚拟机的cpu和mem两种资源之间的相关系数，负责控制每台虚拟机对两种资源需求的关联程度
    num_var 初始状态中VM数量
    '''
    # print "进入 init_Docker"
    c_rp = range(num_var)                                                  # 记录每个虚拟机cpu,mem请求量
    c_rm = range(num_var)
    for i in range(num_var):
        c_rp[i] = random.uniform(0, 2*rp_u)
        c_rm[i] = random.uniform(0, rm_u)
    for i in range(num_var):
        r = random.random()
        if (r < p and c_rp[i] >= rp_u) or (r >= p and c_rp[i] < rp_u):     # p相关系数vm对cpu,mem请求量的相关性，值越大相关性越强
            c_rm[i] += rm_u
        if c_rm[i] >= 1.0:                                               # 控制mem请求量合理性
            c_rm[i] -= 1.0
    #print "len(c_rp)= %s,len(c_rm) = %s" %(len(c_rp),len(c_rm))
    return c_rp, c_rm

def init_VM(c_rp, c_rm, rp_option, rm_option, num_var):
    '''
    goal： 根据已有容器大小，初始化的d-v-h映射关系
    参数:
    rp_option和rm_option分别为VM可选的CPU、mem尺寸
    c_rp,c_rm为对应下标的容器尺寸
    num_var容器个数，初始假设vm数量与容器数量1：1
    返回：
    能够容纳对应编号容器的VM有效尺寸系列
    '''
    # print "进入init_VM"
    v_rp = []
    v_rm = []
    i = 0
    while True:
        if i == num_var:                        # 对num_var个docker选择随机的vm位置
            #print "len(v_rp)= %s,len(v_rm) = %s" %(len(v_rp),len(v_rm))
            return v_rp, v_rm
        rp = random.choice(rp_option)
        rm = random.choice(rm_option)
        if rp >= c_rp[i] and rm >= c_rm[i]:
            v_rp.append(rp)
            v_rm.append(rm)
            i += 1
        else:
            continue

def make_population(size, num_var, c_rp, c_rm, v_rp, v_rm): #    作为全局变量，按照需要传入各方法中 f, p_mutate, time_base, lambdaa):
    '''
    构造一个population，包含size个候选解chrom,每个chrom是num_var个分别记录该容器所在的vm和hm编号的元组
    '''
    # print "进入make_population"
    population0 = {
        'c_rp': c_rp,                                             # 每个容器的cpu请求
        'c_rm': c_rm,                                             # 每个容器的mem请求
        'v_rp': v_rp,                                             # 每个vm的cpu请求
        'v_rm': v_rm,                                             # 每个vm的mem请求
        'population': range2rect(size, num_var, [0, 0]),          # size个chrom，每个chrom有num_var个双元素list[vm,hm]对应每个容器放置的vm编号和物理机编号
        'v_p_cost': range2rect(size, num_var, 0.0),               # size个num_var长list记录每个vm上所有容器的cpu总请求,初始为0
        'v_m_cost': range2rect(size, num_var, 0.0),               # 每个vm被容器请求的mem，初始为0
        'h_p_cost': range2rect(size, num_var, 0.0),               # 每个HM被请求的cpu，初始为0
        'h_m_cost': range2rect(size, num_var, 0.0),               # HM被请求的mem，初始为0
    }
    return population0

def initialize_population(popu1, size, num_var):
    '''
    初始化docker容器放置的vm编号，以及vm所能放置的hm编号，并计算初代种群的各项数值
    '''
    # print "进入initialize_population"
    # 为所有容器选择能够容纳其尺寸的vm_id
    for i in xrange(size):
        # 记录每个chrom中，vm-hm的映射，保证映射在相同vm的不同容器，所在的hm也相同
        vm_used_id = {}
        for j in xrange(num_var):
            # 为第j个容器选择要能容纳其尺寸的vm编号
            while True:
                tmp_v_id = random.randint(0, num_var-1)
                cpu_tmp_v = popu1['v_p_cost'][i][tmp_v_id] + popu1['c_rp'][j]
                mem_tmp_v = popu1['v_m_cost'][i][tmp_v_id] + popu1['c_rm'][j]
                if cpu_tmp_v <= popu1['v_rp'][tmp_v_id] and mem_tmp_v <= popu1['v_rm'][tmp_v_id]:
                    # popu1['population'][i][j]是第i个chrom的第j个容器放置的[vm_id,hm_id]
                    popu1['population'][i][j] = [tmp_v_id, 0]
                    popu1['v_p_cost'][i][tmp_v_id] = cpu_tmp_v
                    popu1['v_m_cost'][i][tmp_v_id] = mem_tmp_v

                    # 找到容器-vm的拓扑后，继续寻找能够满足该vm尺寸的hm
                    flag = True
                    # 对与已经安排过vm-hm映射位置的容器，直接进行对应vm,hm的赋值，并设while循环初始为False
                    if tmp_v_id in vm_used_id:
                        popu1['population'][i][j] = [tmp_v_id, vm_used_id[tmp_v_id]]
                        flag = False
                    while flag:
                        # 对未安排hm的容器所对应的vm，循环查找满足vm尺寸的h
                        tmp_h_id = random.randint(0, num_var-1)
                        cpu_tmp_h = popu1['h_p_cost'][i][tmp_h_id] + popu1['v_rp'][tmp_v_id]
                        mem_tmp_h = popu1['h_m_cost'][i][tmp_h_id] + popu1['v_rm'][tmp_v_id]
                        if cpu_tmp_h <= 1.0 and mem_tmp_h <= 1.0:
                            popu1['population'][i][j] = [tmp_v_id, tmp_h_id]
                            popu1['h_p_cost'][i][tmp_h_id] = cpu_tmp_h
                            popu1['h_m_cost'][i][tmp_h_id] = mem_tmp_h
                            vm_used_id[tmp_v_id] = tmp_h_id
                            break
                    break

    if check_effective(popu1, size, num_var):
        print "Initailization population is effective"
        return popu1
    else:
        sys.exit("failed in initializating population")

def check_effective(popu1, size, num_var):

    '''
    判断候选解的有效性
    '''
    # print "进入check_effective"
    # 先清空各vm,pm的资源使用率
    for i in xrange(size):
        for j in xrange(num_var):
            popu1['v_p_cost'][i][j] = 0
            popu1['v_m_cost'][i][j] = 0
            popu1['h_p_cost'][i][j] = 0
            popu1['h_m_cost'][i][j] = 0
    ## 逐一按照候选解，计算实际占用的vm,hm资源
    for i in xrange(size):
        vm_used_id = {}                                # 避免相同vm同时被映射的不同hm的错误
        for j in xrange(num_var):                      # 容器编号
            v_id = popu1['population'][i][j][0]
            h_id = popu1['population'][i][j][1]
            if v_id in vm_used_id:                     # 若vm已经被安排过
                if h_id == vm_used_id[v_id]:           # 且所在的hm与安排过的hm相同，则跳过该次循环，不需要重新计算资源占用
                    continue
                else:                                  # 若hm不同于安排的hm编号，说明出现同一个vm映射到不同hm的错误，直接返回
                    # print "in this chrom %s, a vm has been hosted on different hm, totally wrong!! " %popu1['population'][i]
                    return False
            else:                                      # 其他情况包括,包括多vm映射到1个hm,按正常情况计算
                popu1['v_p_cost'][i][v_id] += popu1['c_rp'][j]
                popu1['v_m_cost'][i][v_id] += popu1['c_rm'][j]
                popu1['h_p_cost'][i][h_id] += popu1['v_rp'][v_id]
                popu1['h_m_cost'][i][h_id] += popu1['v_rm'][v_id]
                vm_used_id[v_id] = h_id
        for x in xrange(num_var):                      # 只要超出资源限约束，即报错
            if popu1['v_p_cost'][i][x] > popu1['v_rp'][x] or popu1['v_m_cost'][i][x] > popu1['v_rm'][x] or popu1['h_p_cost'][i][x] > 1.0 or popu1['h_m_cost'][i][x] > 1.0:
                return False
    return True

def create_addition_list(rp_u, rm_u, p, addtion_nums):
    '''
    生成新增序列，记录新增容器的尺寸，以及集群需要维持的各类容器replicas数（注： 同服务的各replicas不可放于同VM）
    '''
    c_rp, c_rm = init_Docker(rp_u, rm_u, p, addtion_nums)   # 生成新增容器尺寸

    addtion0 = {
        'c_rp':c_rp,                        # 新增容器cpu占用量
        'c_rm':c_rm,                        # 新增容器mem占用量
        'replicas':[ random.randint(0,5) for i in xrange(addtion_nums)]                     # 新增的该服务容器所需副本数
    }
    return addtion0

def main(num_var, p, addtion_nums):
    # 1.算法主要参数设置
    rp_u = 0.25                            # 容器请求CPU的指导变量
    rm_u = 0.25                            # 容器请求MEM的指导变量
    p = p                                  # 控制容器cpu,mem的资源相关度
    rp_option = [1.0]                      # vm可选的cpu尺寸
    rm_option = [1.0]                      # vm可选的mem尺寸

    size = 1                               # 新增阶段只需单解

    # 2.初始化num_var个容器和vm，以及计算迁移率
    c_rp, c_rm = init_Docker(rp_u, rm_u, p, num_var)
    v_rp, v_rm = init_VM(c_rp, c_rm, rp_option, rm_option, num_var)

    print "开始主流程"

    ## 以下程序的流程按照先按照串行结构书写
    # 3. 初代种群的生成
    init_popu = make_population(size, num_var, c_rp, c_rm, v_rp, v_rm)
    init_popu = initialize_population(init_popu, size, num_var)

    # 4. 生成新增容器序列
    addtion0 = create_addition_list(rp_u, rm_u, p, addtion_nums)

    print 'init_popu = {0}, \n addition0 = {1} \n'.format(init_popu, addtion0)

if __name__=='__main__':
    main(5, 1.0, 6)<|endoftext|>


<commit_before># vim: tabstop=4 shiftwidth=4 softtabstop=4

# Copyright 2010 United States Government as represented by the
# Administrator of the National Aeronautics and Space Administration.
# All Rights Reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

"""
Views for managing Nova images.
"""

import logging
import re

from django import http
from django import template
from django.conf import settings
from django.contrib import messages
from django.contrib.auth.decorators import login_required
from django.shortcuts import redirect, render_to_response
from django.utils.translation import ugettext as _
from django import shortcuts

from django_openstack import api
from django_openstack import forms
from openstackx.api import exceptions as api_exceptions
from glance.common import exception as glance_exception

LOG = logging.getLogger('django_openstack.dash')

class LaunchForm(forms.SelfHandlingForm):
    image_id = forms.CharField(widget=forms.HiddenInput())
    name = forms.CharField(max_length=80, label="Server Name")

    # make the dropdown populate when the form is loaded not when django is
    # started
    def __init__(self, *args, **kwargs):
        super(LaunchForm, self).__init__(*args, **kwargs)
        flavorlist = kwargs.get('initial', {}).get('flavorlist', [])
        self.fields['flavor'] = forms.ChoiceField(
                choices=flavorlist,
                label="Flavor",
                help_text="Size of Image to launch")

    def handle(self, request, data):
        image_id = data['image_id']
        try:
            image = api.compute_api(request).images.get(image_id)
            flavor = api.compute_api(request).flavors.get(data['flavor'])
            api.compute_api(request).servers.create(data['name'],
                                                    image,
                                                    flavor)
            messages.success(request, "Instance was successfully\
                                       launched.")
            return shortcuts.redirect(request.build_absolute_uri())

        except api_exceptions.ApiException, e:
            messages.error(request,
                           'Unable to launch instance: %s' % e.message)

@login_required
def index(request, tenant_id):
    tenant = api.get_tenant(request, request.user.tenant)
    images = api.glance_api(request).get_images_detailed()

    return render_to_response('dash_images.html', {
        'tenant': tenant,
        'images': images,
        #'image_lists': _image_lists(images, request.user.tenant),
    }, context_instance=template.RequestContext(request))

@login_required
def launch(request, tenant_id, image_id):
    def flavorlist():
        try:
            fl = api.extras_api(request).flavors.list()

            # TODO add vcpu count to flavors
            sel = [(f.id, '%s (%svcpu / %sGB Disk / %sMB Ram )' %
                   (f.name, f.vcpus, f.disk, f.ram)) for f in fl]
            return sorted(sel)
        except:
            return [(1, 'm1.tiny')]

    image = api.compute_api(request).images.get(image_id)
    tenant = api.get_tenant(request, request.user.tenant)

    form, handled = LaunchForm.maybe_handle(
            request, initial={'flavorlist': flavorlist(),
                              'image_id': image_id})
    if handled:
        return handled

    return render_to_response('dash_launch.html', {
        'tenant': tenant,
        'image': image,
        'form': form,
    }, context_instance=template.RequestContext(request))
<commit_msg>pull in code from trunk to only show amis<commit_after># vim: tabstop=4 shiftwidth=4 softtabstop=4

# Copyright 2010 United States Government as represented by the
# Administrator of the National Aeronautics and Space Administration.
# All Rights Reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

"""
Views for managing Nova images.
"""

import datetime
import logging
import re

from django import http
from django import template
from django.conf import settings
from django.contrib import messages
from django.contrib.auth.decorators import login_required
from django.shortcuts import redirect, render_to_response
from django.utils.translation import ugettext as _
from django import shortcuts

from django_openstack import api
from django_openstack import forms
from openstackx.api import exceptions as api_exceptions
from glance.common import exception as glance_exception

LOG = logging.getLogger('django_openstack.dash')

class LaunchForm(forms.SelfHandlingForm):
    image_id = forms.CharField(widget=forms.HiddenInput())
    name = forms.CharField(max_length=80, label="Server Name")

    # make the dropdown populate when the form is loaded not when django is
    # started
    def __init__(self, *args, **kwargs):
        super(LaunchForm, self).__init__(*args, **kwargs)
        flavorlist = kwargs.get('initial', {}).get('flavorlist', [])
        self.fields['flavor'] = forms.ChoiceField(
                choices=flavorlist,
                label="Flavor",
                help_text="Size of Image to launch")

    def handle(self, request, data):
        image_id = data['image_id']
        try:
            image = api.compute_api(request).images.get(image_id)
            flavor = api.compute_api(request).flavors.get(data['flavor'])
            api.compute_api(request).servers.create(data['name'],
                                                    image,
                                                    flavor)
            messages.success(request, "Instance was successfully\
                                       launched.")
            return shortcuts.redirect(request.build_absolute_uri())

        except api_exceptions.ApiException, e:
            messages.error(request,
                           'Unable to launch instance: %s' % e.message)

@login_required
def index(request, tenant_id):
    tenant = api.get_tenant(request, request.user.tenant)
    all_images = []
    try:
        all_images = api.glance_api(request).get_images_detailed()
        if not all_images:
            messages.info(request, "There are currently no images.")
    except GlanceClientConnectionError, e:
        messages.error(request, "Error connecting to glance: %s" % e.message)
    except glance_exception.Error, e:
        messages.error(request, "Error retrieving image list: %s" % e.message)

    images = []

    def convert_time(tstr):
        if tstr:
            return datetime.datetime.strptime(tstr, "%Y-%m-%dT%H:%M:%S.%f")
        else:
            return ''

    for im in all_images:
        im['created'] = convert_time(im['created_at'])
        im['updated'] = convert_time(im['updated_at'])
        if im['container_format'] not in ['aki', 'ari']:
            images.append(im)

    return render_to_response('dash_images.html', {
        'tenant': tenant,
        'images': images,
    }, context_instance=template.RequestContext(request))

@login_required
def launch(request, tenant_id, image_id):
    def flavorlist():
        try:
            fl = api.extras_api(request).flavors.list()

            # TODO add vcpu count to flavors
            sel = [(f.id, '%s (%svcpu / %sGB Disk / %sMB Ram )' %
                   (f.name, f.vcpus, f.disk, f.ram)) for f in fl]
            return sorted(sel)
        except:
            return [(1, 'm1.tiny')]

    image = api.compute_api(request).images.get(image_id)
    tenant = api.get_tenant(request, request.user.tenant)

    form, handled = LaunchForm.maybe_handle(
            request, initial={'flavorlist': flavorlist(),
                              'image_id': image_id})
    if handled:
        return handled

    return render_to_response('dash_launch.html', {
        'tenant': tenant,
        'image': image,
        'form': form,
    }, context_instance=template.RequestContext(request))
<|endoftext|>


<commit_before>#!/usr/bin/env python
# coding: utf-8
# test-py-parallel 0.0.1
# Generated by dx-app-wizard.
#
# Scatter-process-gather execution pattern: Your app will split its
# input into multiple pieces, each of which will be processed in
# parallel, after which they are gathered together in some final
# output.
#
# This pattern is very similar to the "parallelized" template.  What
# it does differently is that it formally breaks out the "scatter"
# phase as a separate black-box entry point in the app.  (As a side
# effect, this requires a "map" entry point to call "process" on each
# of the results from the "scatter" phase.)
#
# Note that you can also replace any entry point in this execution
# pattern with an API call to run a separate app or applet.
#
# The following is a Unicode art picture of the flow of execution.
# Each box is an entry point, and vertical lines indicate that the
# entry point connected at the top of the line calls the entry point
# connected at the bottom of the line.  The letters represent the
# different stages in which the input is transformed, e.g. the output
# of the "scatter" entry point ("array:B") is given to the "map" entry
# point as input.  The "map" entry point calls as many "process" entry
# points as there are elements in its array input and gathers the
# results in its array output.
#
#          ┌──────┐
#       A->│ main │->D (output from "postprocess")
#          └┬─┬─┬─┘
#           │ │ │
#          ┌┴──────┐
#       A->│scatter│->array:B
#          └───────┘
#             │ │
#            ┌┴──────────────┐
#   array:B->│      map      │->array:C
#            └─────────┬─┬─┬─┘
#               │      │ . .
#               │     ┌┴──────┐
#               │  B->│process│->C
#               │     └───────┘
#            ┌──┴────────┐
#   array:C->│postprocess│->D
#            └───────────┘
#
# A = original app input, split up by "scatter" into pieces of type B
# B = an input that will be provided to a "process" entry point
# C = the output of a "process" entry point
# D = app output aggregated from the outputs of the "process" entry points
#
# See https://wiki.dnanexus.com/Developer-Portal for documentation and
# tutorials on how to modify this file.
#
# DNAnexus Python Bindings (dxpy) documentation:
#   http://autodoc.dnanexus.com/bindings/python/current/

import os
import dxpy
import subprocess
import shlex
import glob
import logging

DEBUG = True

logger = logging.getLogger(__name__)
logger.addHandler(dxpy.DXLogHandler())
logger.propagate = False

if DEBUG:
    logger.setLevel(logging.DEBUG)
else:
    logger.setLevel(logging.INFO)

STRIP_EXTENSIONS = ['.gz', '.fq', '.fastq', '.fa', '.fasta']

def strip_extensions(filename, extensions):
    basename = filename
    for extension in extensions:
        basename = basename.rpartition(extension)[0] or basename
    return basename

def merge_bams(bam_files, bam_root, use_cat, use_sort, nthreads):

    fnames = []
    for bam in bam_files:
        dxbam = dxpy.DXFile(bam)
        dxfn = dxbam.describe()['name']
        logger.info("* Downloading %s... *" % dxfn)
        dxpy.download_dxfile(bam, dxfn)
        fnames.append(bam_root + '_bismark_techrep.bam')

    outfile_name = bam_root
    logger.info("* Merged alignments file will be: %s *" % outfile_name + '.bam')
    if len(fnames) == 1:
        rep_outfile_name = bam_root + '_bismark_biorep'
        os.rename('sofar.bam', rep_outfile_name + '.bam')
        logger.info("* Only one input file, no merging required.")

    else:
        if use_cat:
            for fn in fnames:
                if not os.path.isfile('sofar.bam'):
                    os.rename(fn, 'sofar.bam')
                else:
                    logger.info("* Merging...")
                    # NOTE: keeps the first header
                    catout = subprocess.check_output('samtools cat sofar.bam %s > merging.bam' % fn)
                    logger.info(catout)
                    os.rename('merging.bam', 'sofar.bam')

            # At this point there is a 'sofar.bam' with one or more input bams

            logger.info("* Files merged into %s (via cat) *" % outfile_name + '.bam')

        else:
            # use samtools merge
            filelist = " ".join(fnames)
            logger.info("Merging via merge %s " % filelist)
            mergeout = subprocess.check_output('samtools merge -nf %s %s' % ('merging.bam', filelist))
            logger.info(mergeout)

        if use_sort:
            # sorting needed due to samtools cat
            logger.info("* Sorting merged bam...")
            sortout = subprocess.check_output(['samtools sort -@', nthreads, '-m 6G -f sofar.bam sorted.bam'])
            logger.info(sortout)
            os.rename('sorted.bam', outfile_name + '.bam')
        else:
            os.rename('sofar.bam', outfile_name + '.bam')

    return outfile_name + '.bam'

def merge_reports():
    # dummy
    fh = open('fake_report', 'w')
    fh.write("test")
    fh.close()
    return 'fake_report'

def merge_qc():
    fh = open('fake_qc', 'w')
    fh.write("test qc")
    fh.close()
    return ('fake_qc', '444242', "{'key': 'value'}")

@dxpy.entry_point("postprocess")
def postprocess(bam_files, report_files, bam_root, nthreads=8, use_cat=False, use_sort=False):
    # This is the "gather" phase which aggregates and performs any
    # additional computation after the "map" (and therefore after all
    # the "process") jobs are done.

    if DEBUG:
        logger.setLevel(logging.DEBUG)
    else:
        logger.setLevel(logging.INFO)

    logger.debug("* In Postprocess - refactoed dme-merge-bams - *")

    merged_bam = merge_bams(bam_files, bam_root, use_cat, use_sort, nthreads)

    merged_report = merge_reports(report_files, bam_root)

    merged_qc, nreads, metadata = merge_qc()

    output = {
        "bam_techrep": dxpy.dxlink(dxpy.upload_local_file(merged_bam)),
        "bam_techrep_qc": dxpy.dxlink(merged_qc),
        "map_techrep": dxpy.dxlink(merged_report),
        "reads": nreads,
        "metadata": metadata
    }
    return output

@dxpy.entry_point("process")
def process(scattered_input, dme_ix, ncpus, reads_root):
    # Fill in code here to process the input and create output.

    ALIGN_SCRIPT = '/usr/bin/dname_align_se.sh'
    if DEBUG:
        logger.setLevel(logging.DEBUG)
    else:
        logger.setLevel(logging.INFO)

    dme_ix = dxpy.DXFile(dme_ix)

    # The following line(s) download your file inputs to the local file system
    # using variable names for the filenames.

    dxpy.download_dxfile(dme_ix.get_id(), "index.tgz")
    fq = dxpy.DXFile(scattered_input)
    name = fq.describe()['name']
    dxpy.download_dxfile(fq.get_id(), name)
    bam_root = name + '_techrep'

    logger.info("* === Calling DNAnexus and ENCODE independent script... ===")
    logger.debug("DIR: %s" % os.listdir('./'))
    logger.debug(subprocess.check_output(['head', name]))
    if os.path.isfile(ALIGN_SCRIPT):
        logger.debug("Executable %s exists" % ALIGN_SCRIPT)
    else:
        logger.debug("Executable %s DOES NOT exist" % ALIGN_SCRIPT)
        exit(1)
    logger.debug('command line: %s index.tgz %s %s %s' % (ALIGN_SCRIPT, name, ncpus, bam_root))
    map_out = subprocess.check_output([ALIGN_SCRIPT, 'index.tgz', name, str(ncpus), bam_root, 'no_stats'])
    logger.debug("* === Returned from dname_align_se  ===")

    # As always, you can choose not to return output if the
    # "postprocess" stage does not require any input, e.g. rows have
    # been added to a GTable that has been created in advance.  Just
    # make sure that the "postprocess" job does not run until all
    # "process" jobs have finished by making it wait for "map" to
    # finish using the depends_on argument (this is already done for
    # you in the invocation of the "postprocess" job in "main").

    logger.debug("DIR: %s" % os.listdir('./'))
    logger.debug("OURTPUR DIR: %s" % os.listdir('output/'))

    os.rename(name, bam_root+'.bam')
    return {
        "bam_file": dxpy.dxlink(dxpy.upload_local_file(bam_root+'.bam')),
        "report_file": dxpy.dxlink(dxpy.upload_local_file(bam_root+'_bismark_map_report.txt'))
    }

@dxpy.entry_point("map")
def map_entry_point(array_of_scattered_input, process_input):
    # The following calls "process" for each of the items in
    # *array_of_scattered_input*, using as input the item in the
    # array, as well as the rest of the fields in *process_input*.
    if DEBUG:
        logger.setLevel(logging.DEBUG)
    else:
        logger.setLevel(logging.INFO)

    logger.debug("* in map entry point with %s *" % process_input)
    process_jobs = []
    for item in array_of_scattered_input:
        logger.debug("* scattering: %s *" % item)
        process_input["scattered_input"] = item
        process_jobs.append(dxpy.new_dxjob(fn_input=process_input, fn_name="process"))

    logger.debug("* %s scatter jobs started *" % len(array_of_scattered_input))
    bams = []
    reports = []
    for subjob in process_jobs:
        bams.append(subjob.get_output_ref('bam_file'))
        reports.append(subjob.get_output_ref('report_file'))

    return {
        "bam_files": bams,
        "report_files": reports,
    }

@dxpy.entry_point("scatter")
def scatter(orig_reads, split_size):
    # Fill in code here to do whatever is necessary to scatter the
    # input.
    if DEBUG:
        logger.setLevel(logging.DEBUG)
    else:
        logger.setLevel(logging.INFO)

    splitsize = split_size * 1000000 * 4
    # each FQ read is 4 lines
    os.mkdir('splits')

    for f in orig_reads:
        reads_filename = dxpy.describe(f)['name']
        reads_basename = strip_extensions(reads_filename, STRIP_EXTENSIONS)
        dxpy.download_dxfile(dxpy.DXFile(f).get_id(), reads_filename)

        reads_root_name = simplify_name() or reads_basename

        logger.info('* RUNNING /bin/zcat %s | /usr/bin/split -l %d -d - %s * ' % (reads_filename, splitsize, 'splits/' + reads_root_name))
        subprocess.check_call('/bin/zcat %s | /usr/bin/split -l %d -d - %s ' % (reads_filename, splitsize, 'splits/' + reads_root_name), shell=True)

    splits = os.listdir('splits')
    logger.info("* Return from scatter: %s *" % splits)

    # SHould we gzip here?
    return {
        "array_of_scattered_input": [
            dxpy.dxlink(dxpy.upload_local_file('splits/' + split_file)) for split_file in splits]
        }

def simplify_name():
    # Try to simplify the names

    rep_root = ''
    if os.path.isfile('/usr/bin/parse_property.py'):
        rep_root = subprocess.check_output(['parse_property.py', '--job', os.environ['DX_JOB_ID'],'--root_name', '--quiet'], shell=True)

    return rep_root

@dxpy.entry_point("main")
def main(reads, dme_ix, ncpus, splitsize):

    # The following line(s) initialize your data object inputs on the platform
    # into dxpy.DXDataObject instances that you can start using immediately.

    #dx_reads = [dxpy.DXFile(item) for item in reads]

    # The following line(s) download your file inputs to the local file system
    # using variable names for the filenames.

    # We first create the "scatter" job which will scatter some input
    # (replace with your own input as necessary).
    logger.debug("* Start Scatter with %d files %sM read splits *" % (len(reads), splitsize))

    scatter_job = dxpy.new_dxjob(fn_input={
                                 'orig_reads': reads,
                                 'split_size': splitsize,
                                 },
                                 fn_name="scatter")

    # We will want to call "process" on each output of "scatter", so
    # we call the "map" entry point to do so.  We can also provide
    # here additional input that we want each "process" entry point to
    # receive, e.g. a GTable ID to which the "process" function should
    # add rows of data.

    reads_root = simplify_name() or strip_extensions(dxpy.describe(reads[0])['name'], STRIP_EXTENSIONS)

    map_input = {
        "array_of_scattered_input": scatter_job.get_output_ref("array_of_scattered_input"),
        "process_input": {
            "reads_root": reads_root,
            "ncpus": ncpus,
            "dme_ix": dme_ix
            }
        }
    logger.debug("* Start Map with: %s *" % map_input)
    map_job = dxpy.new_dxjob(fn_input=map_input, fn_name="map")

    # Finally, we want the "postprocess" job to run after "map" is
    # done calling "process" on each of its inputs.  Note that a job
    # is marked as "done" only after all of its child jobs are also
    # marked "done".
    logger.debug("* Waiting for map job to finish...")
    postprocess_input = {
        "bam_files": map_job.get_output_ref("bam_files"),
        "report_files": map_job.get_output_ref("report_files"),
        "bam_root": reads_root + '_techrep'
        }
    logger.debug("* Start Post process with: %s *" % postprocess_input)
    postprocess_job = dxpy.new_dxjob(fn_input=postprocess_input,
                                     fn_name="postprocess",
                                     depends_on=[map_job])

    # If you would like to include any of the output fields from the
    # postprocess_job as the output of your app, you should return it
    # here using a job-based object reference.
    #
    # return { "app_output_field": postprocess_job.get_output_ref("final_output"), ...}
    #
    # Tip: you can include in your output at this point any open
    # objects (such as gtables) which will be closed by a job that
    # finishes later.  The system will check to make sure that the
    # output object is closed and will attempt to clone it out as
    # output into the parent container only after all subjobs have
    # finished.

    output = {}
    output["bam_techrep"] = dxpy.dxlink(postprocess_job.get_output_ref("bam_techrep"))
    output["bam_techrep_qc"] = dxpy.dxlink(postprocess_job.get_output_ref("bam_techrep_qc"))
    output["map_techrep"] = dxpy.dxlink(postprocess_job.get_output_ref("map_techrep"))
    output["reads"] = postprocess_job.get_output_ref("reads")
    output["metadata"] = postprocess_job.get_output_ref("metadata")

    return output

dxpy.run()
<commit_msg>grinding through postprocess<commit_after>#!/usr/bin/env python
# coding: utf-8
# test-py-parallel 0.0.1
# Generated by dx-app-wizard.
#
# Scatter-process-gather execution pattern: Your app will split its
# input into multiple pieces, each of which will be processed in
# parallel, after which they are gathered together in some final
# output.
#
# This pattern is very similar to the "parallelized" template.  What
# it does differently is that it formally breaks out the "scatter"
# phase as a separate black-box entry point in the app.  (As a side
# effect, this requires a "map" entry point to call "process" on each
# of the results from the "scatter" phase.)
#
# Note that you can also replace any entry point in this execution
# pattern with an API call to run a separate app or applet.
#
# The following is a Unicode art picture of the flow of execution.
# Each box is an entry point, and vertical lines indicate that the
# entry point connected at the top of the line calls the entry point
# connected at the bottom of the line.  The letters represent the
# different stages in which the input is transformed, e.g. the output
# of the "scatter" entry point ("array:B") is given to the "map" entry
# point as input.  The "map" entry point calls as many "process" entry
# points as there are elements in its array input and gathers the
# results in its array output.
#
#          ┌──────┐
#       A->│ main │->D (output from "postprocess")
#          └┬─┬─┬─┘
#           │ │ │
#          ┌┴──────┐
#       A->│scatter│->array:B
#          └───────┘
#             │ │
#            ┌┴──────────────┐
#   array:B->│      map      │->array:C
#            └─────────┬─┬─┬─┘
#               │      │ . .
#               │     ┌┴──────┐
#               │  B->│process│->C
#               │     └───────┘
#            ┌──┴────────┐
#   array:C->│postprocess│->D
#            └───────────┘
#
# A = original app input, split up by "scatter" into pieces of type B
# B = an input that will be provided to a "process" entry point
# C = the output of a "process" entry point
# D = app output aggregated from the outputs of the "process" entry points
#
# See https://wiki.dnanexus.com/Developer-Portal for documentation and
# tutorials on how to modify this file.
#
# DNAnexus Python Bindings (dxpy) documentation:
#   http://autodoc.dnanexus.com/bindings/python/current/

import os
import dxpy
import subprocess
import shlex
import glob
import logging

DEBUG = True

logger = logging.getLogger(__name__)
logger.addHandler(dxpy.DXLogHandler())
logger.propagate = False

if DEBUG:
    logger.setLevel(logging.DEBUG)
else:
    logger.setLevel(logging.INFO)

STRIP_EXTENSIONS = ['.gz', '.fq', '.fastq', '.fa', '.fasta']

def strip_extensions(filename, extensions):
    basename = filename
    for extension in extensions:
        basename = basename.rpartition(extension)[0] or basename
    return basename

def merge_bams(bam_files, bam_root, use_cat, use_sort, nthreads):

    fnames = []
    for bam in bam_files:
        dxbam = dxpy.DXFile(bam)
        dxfn = dxbam.describe()['name']
        logger.info("* Downloading %s... *" % dxfn)
        dxpy.download_dxfile(bam, dxfn)
        fnames.append(dxfn)

    outfile_name = bam_root
    logger.info("* Merged alignments file will be: %s *" % outfile_name + '.bam')
    if len(fnames) == 1:
        # UNTESTED
        rep_outfile_name = bam_root + '_bismark_biorep'
        os.rename('sofar.bam', rep_outfile_name + '.bam')
        logger.info("* Only one input file, no merging required.")

    else:
        if use_cat:
            for fn in fnames:
                if not os.path.isfile('sofar.bam'):
                    os.rename(fn, 'sofar.bam')
                else:
                    logger.info("* Merging...")
                    # NOTE: keeps the first header
                    catout = subprocess.check_output(['samtools', 'cat', 'sofar.bam', fn, '> merging.bam'])
                    logger.info(catout)
                    os.rename('merging.bam', 'sofar.bam')

            # At this point there is a 'sofar.bam' with one or more input bams

            logger.info("* Files merged into %s (via cat) *" % outfile_name + '.bam')

        else:
            # use samtools merge
            # UNTESTED
            filelist = " ".join(fnames)
            logger.info("Merging via merge %s " % filelist)
            mergeout = subprocess.check_output(['samtools', 'merge', '-nf', 'merging.bam'] + fnames)
            logger.info(mergeout)

        if use_sort:
            # sorting needed due to samtools cat
            # UNTESTED
            logger.info("* Sorting merged bam...")
            sortout = subprocess.check_output(['samtools', 'sort', '-@', nthreads, '-m 6G', '-f,' 'sofar.bam', 'sorted.bam'])
            logger.info(sortout)
            os.rename('sorted.bam', outfile_name + '.bam')
        else:
            os.rename('sofar.bam', outfile_name + '.bam')

    return outfile_name + '.bam'

def merge_reports():
    # dummy
    fh = open('fake_report', 'w')
    fh.write("test")
    fh.close()
    return 'fake_report'

def merge_qc():
    fh = open('fake_qc', 'w')
    fh.write("test qc")
    fh.close()
    return ('fake_qc', '444242', "{'key': 'value'}")

@dxpy.entry_point("postprocess")
def postprocess(bam_files, report_files, bam_root, nthreads=8, use_cat=False, use_sort=False):
    # This is the "gather" phase which aggregates and performs any
    # additional computation after the "map" (and therefore after all
    # the "process") jobs are done.

    if DEBUG:
        logger.setLevel(logging.DEBUG)
    else:
        logger.setLevel(logging.INFO)

    logger.debug("* In Postprocess - refactoed dme-merge-bams - *")

    merged_bam = merge_bams(bam_files, bam_root, use_cat, use_sort, nthreads)

    merged_report = merge_reports(report_files, bam_root)

    merged_qc, nreads, metadata = merge_qc()

    output = {
        "bam_techrep": dxpy.dxlink(dxpy.upload_local_file(merged_bam)),
        "bam_techrep_qc": dxpy.dxlink(merged_qc),
        "map_techrep": dxpy.dxlink(merged_report),
        "reads": nreads,
        "metadata": metadata
    }
    return output

@dxpy.entry_point("process")
def process(scattered_input, dme_ix, ncpus, reads_root):
    # Fill in code here to process the input and create output.

    ALIGN_SCRIPT = '/usr/bin/dname_align_se.sh'
    if DEBUG:
        logger.setLevel(logging.DEBUG)
    else:
        logger.setLevel(logging.INFO)

    dme_ix = dxpy.DXFile(dme_ix)

    # The following line(s) download your file inputs to the local file system
    # using variable names for the filenames.

    dxpy.download_dxfile(dme_ix.get_id(), "index.tgz")
    fq = dxpy.DXFile(scattered_input)
    name = fq.describe()['name']
    dxpy.download_dxfile(fq.get_id(), name)
    bam_root = name + '_techrep'

    logger.info("* === Calling DNAnexus and ENCODE independent script... ===")
    logger.debug("DIR: %s" % os.listdir('./'))
    logger.debug(subprocess.check_output(['head', name]))
    if os.path.isfile(ALIGN_SCRIPT):
        logger.debug("Executable %s exists" % ALIGN_SCRIPT)
    else:
        logger.debug("Executable %s DOES NOT exist" % ALIGN_SCRIPT)
        exit(1)
    logger.debug('command line: %s index.tgz %s %s %s' % (ALIGN_SCRIPT, name, ncpus, bam_root))
    map_out = subprocess.check_output([ALIGN_SCRIPT, 'index.tgz', name, str(ncpus), bam_root, 'no_stats'])
    logger.debug("* === Returned from dname_align_se  ===")

    # As always, you can choose not to return output if the
    # "postprocess" stage does not require any input, e.g. rows have
    # been added to a GTable that has been created in advance.  Just
    # make sure that the "postprocess" job does not run until all
    # "process" jobs have finished by making it wait for "map" to
    # finish using the depends_on argument (this is already done for
    # you in the invocation of the "postprocess" job in "main").

    logger.debug("DIR: %s" % os.listdir('./'))
    logger.debug("OURTPUR DIR: %s" % os.listdir('output/'))

    os.rename(name, bam_root+'.bam')
    return {
        "bam_file": dxpy.dxlink(dxpy.upload_local_file(bam_root+'.bam')),
        "report_file": dxpy.dxlink(dxpy.upload_local_file(bam_root+'_bismark_map_report.txt'))
    }

@dxpy.entry_point("map")
def map_entry_point(array_of_scattered_input, process_input):
    # The following calls "process" for each of the items in
    # *array_of_scattered_input*, using as input the item in the
    # array, as well as the rest of the fields in *process_input*.
    if DEBUG:
        logger.setLevel(logging.DEBUG)
    else:
        logger.setLevel(logging.INFO)

    logger.debug("* in map entry point with %s *" % process_input)
    process_jobs = []
    for item in array_of_scattered_input:
        logger.debug("* scattering: %s *" % item)
        process_input["scattered_input"] = item
        process_jobs.append(dxpy.new_dxjob(fn_input=process_input, fn_name="process"))

    logger.debug("* %s scatter jobs started *" % len(array_of_scattered_input))
    bams = []
    reports = []
    for subjob in process_jobs:
        bams.append(subjob.get_output_ref('bam_file'))
        reports.append(subjob.get_output_ref('report_file'))

    return {
        "bam_files": bams,
        "report_files": reports,
    }

@dxpy.entry_point("scatter")
def scatter(orig_reads, split_size):
    # Fill in code here to do whatever is necessary to scatter the
    # input.
    if DEBUG:
        logger.setLevel(logging.DEBUG)
    else:
        logger.setLevel(logging.INFO)

    splitsize = split_size * 1000000 * 4
    # each FQ read is 4 lines
    os.mkdir('splits')

    for f in orig_reads:
        reads_filename = dxpy.describe(f)['name']
        reads_basename = strip_extensions(reads_filename, STRIP_EXTENSIONS)
        dxpy.download_dxfile(dxpy.DXFile(f).get_id(), reads_filename)

        reads_root_name = simplify_name() or reads_basename

        logger.info('* RUNNING /bin/zcat %s | /usr/bin/split -l %d -d - %s * ' % (reads_filename, splitsize, 'splits/' + reads_root_name))
        subprocess.check_call('/bin/zcat %s | /usr/bin/split -l %d -d - %s ' % (reads_filename, splitsize, 'splits/' + reads_root_name), shell=True)

    splits = os.listdir('splits')
    logger.info("* Return from scatter: %s *" % splits)

    # SHould we gzip here?
    return {
        "array_of_scattered_input": [
            dxpy.dxlink(dxpy.upload_local_file('splits/' + split_file)) for split_file in splits]
        }

def simplify_name():
    # Try to simplify the names

    rep_root = ''
    if os.path.isfile('/usr/bin/parse_property.py'):
        rep_root = subprocess.check_output(['parse_property.py', '--job', os.environ['DX_JOB_ID'],'--root_name', '--quiet'], shell=True)

    return rep_root

@dxpy.entry_point("main")
def main(reads, dme_ix, ncpus, splitsize):

    # The following line(s) initialize your data object inputs on the platform
    # into dxpy.DXDataObject instances that you can start using immediately.

    #dx_reads = [dxpy.DXFile(item) for item in reads]

    # The following line(s) download your file inputs to the local file system
    # using variable names for the filenames.

    # We first create the "scatter" job which will scatter some input
    # (replace with your own input as necessary).
    logger.debug("* Start Scatter with %d files %sM read splits *" % (len(reads), splitsize))

    scatter_job = dxpy.new_dxjob(fn_input={
                                 'orig_reads': reads,
                                 'split_size': splitsize,
                                 },
                                 fn_name="scatter")

    # We will want to call "process" on each output of "scatter", so
    # we call the "map" entry point to do so.  We can also provide
    # here additional input that we want each "process" entry point to
    # receive, e.g. a GTable ID to which the "process" function should
    # add rows of data.

    reads_root = simplify_name() or strip_extensions(dxpy.describe(reads[0])['name'], STRIP_EXTENSIONS)

    map_input = {
        "array_of_scattered_input": scatter_job.get_output_ref("array_of_scattered_input"),
        "process_input": {
            "reads_root": reads_root,
            "ncpus": ncpus,
            "dme_ix": dme_ix
            }
        }
    logger.debug("* Start Map with: %s *" % map_input)
    map_job = dxpy.new_dxjob(fn_input=map_input, fn_name="map")

    # Finally, we want the "postprocess" job to run after "map" is
    # done calling "process" on each of its inputs.  Note that a job
    # is marked as "done" only after all of its child jobs are also
    # marked "done".
    logger.debug("* Waiting for map job to finish...")
    postprocess_input = {
        "bam_files": map_job.get_output_ref("bam_files"),
        "report_files": map_job.get_output_ref("report_files"),
        "bam_root": reads_root + '_techrep'
        }
    logger.debug("* Start Post process with: %s *" % postprocess_input)
    postprocess_job = dxpy.new_dxjob(fn_input=postprocess_input,
                                     fn_name="postprocess",
                                     depends_on=[map_job])

    # If you would like to include any of the output fields from the
    # postprocess_job as the output of your app, you should return it
    # here using a job-based object reference.
    #
    # return { "app_output_field": postprocess_job.get_output_ref("final_output"), ...}
    #
    # Tip: you can include in your output at this point any open
    # objects (such as gtables) which will be closed by a job that
    # finishes later.  The system will check to make sure that the
    # output object is closed and will attempt to clone it out as
    # output into the parent container only after all subjobs have
    # finished.

    output = {}
    output["bam_techrep"] = dxpy.dxlink(postprocess_job.get_output_ref("bam_techrep"))
    output["bam_techrep_qc"] = dxpy.dxlink(postprocess_job.get_output_ref("bam_techrep_qc"))
    output["map_techrep"] = dxpy.dxlink(postprocess_job.get_output_ref("map_techrep"))
    output["reads"] = postprocess_job.get_output_ref("reads")
    output["metadata"] = postprocess_job.get_output_ref("metadata")

    return output

dxpy.run()
<|endoftext|>


<commit_before>import sys

from urlparse import urlparse
from httplib import HTTPConnection
from StringIO import StringIO

import PIL.Image
import TileStache

class WMS:

    def __init__(self, layer, url, layers):
        self.url = url
        self.layers = layers

    def renderArea(self, width, height, srs, xmin, ymin, xmax, ymax, zoom):
        s, host, path, p, query, f = urlparse(self.url)

        # http://geoint.lmic.state.mn.us/cgi-bin/wmsll?VERSION=1.1.1&SERVICE=WMS&REQUEST=GetMap&layers=msp2006&bbox=-93.2492355,44.9784920,-93.2457346,44.9809684&srs=EPSG:4326&width=500&height=500
        conn = HTTPConnection(host, 80)
        conn.request('GET', path+"?VERSION=1.1.1&SERVICE=WMS&REQUEST=GetMap&layers=%s&bbox=%f,%f,%f,%f&srs=102113&height=%d&width=%d" % (self.layers, xmin, ymin, xmax, ymax, height, width))

        body = conn.getresponse().read()
        tile = PIL.Image.open(StringIO(body)).convert('RGBA')

        return tile
<commit_msg>Adding some logging for error conditions.<commit_after>import sys

from urlparse import urlparse
from httplib import HTTPConnection
from StringIO import StringIO

import PIL.Image
import TileStache

class WMS:

    def __init__(self, layer, url, layers):
        self.url = url
        self.layers = layers

    def renderArea(self, width, height, srs, xmin, ymin, xmax, ymax, zoom):
        s, host, path, p, query, f = urlparse(self.url)

        # http://geoint.lmic.state.mn.us/cgi-bin/wmsll?VERSION=1.1.1&SERVICE=WMS&REQUEST=GetMap&layers=msp2006&bbox=-93.2492355,44.9784920,-93.2457346,44.9809684&srs=EPSG:4326&width=500&height=500
        conn = HTTPConnection(host, 80)
        fullpath = path+"?VERSION=1.1.1&SERVICE=WMS&REQUEST=GetMap&layers=%s&bbox=%f,%f,%f,%f&srs=EPSG:4326&height=%d&width=%d" % (self.layers, xmin, ymin, xmax, ymax, height, width)
        conn.request('GET', fullpath)

        response = conn.getresponse()
        if response.status != 200:
            raise Exception("Error %d on URL http://%s/%s" % (response.status, host, fullpath))
        body = response.read()
        tile = PIL.Image.open(StringIO(body)).convert('RGBA')

        return tile
<|endoftext|>


<commit_before>"""
==================================================
Explicit feature map approximation for RBF kernels
==================================================

An example illustrating the approximation of the feature map
of an RBF kernel.

.. currentmodule:: sklearn.kernel_approximation

It shows how to use :class:`Fastfood`, :class:`RBFSampler` and :class:`Nystroem` to
approximate the feature map of an RBF kernel for classification with an SVM on
the digits dataset. Results using a linear SVM in the original space, a linear
SVM using the approximate mappings and using a kernelized SVM are compared.
Timings and accuracy for varying amounts of Monte Carlo samplings (in the case
of :class:`RBFSampler`, which uses random Fourier features) and different sized
subsets of the training set (for :class:`Nystroem`) for the approximate mapping
are shown.

Please note that the dataset here is not large enough to show the benefits
of kernel approximation, as the exact SVM is still reasonably fast.

Sampling more dimensions clearly leads to better classification results, but
comes at a greater cost. This means there is a tradeoff between runtime and
accuracy, given by the parameter n_components. Note that solving the Linear
SVM and also the approximate kernel SVM could be greatly accelerated by using
stochastic gradient descent via :class:`sklearn.linear_model.SGDClassifier`.
This is not easily possible for the case of the kernelized SVM.

The second plot visualized the decision surfaces of the RBF kernel SVM and
the linear SVM with approximate kernel maps.
The plot shows decision surfaces of the classifiers projected onto
the first two principal components of the data. This visualization should
be taken with a grain of salt since it is just an interesting slice through
the decision surface in 64 dimensions. In particular note that
a datapoint (represented as a dot) does not necessarily be classified
into the region it is lying in, since it will not lie on the plane
that the first two principal components span.

The usage of :class:`Fastfood`, :class:`RBFSampler` and :class:`Nystroem` is described in detail
in :ref:`kernel_approximation`.

"""
print(__doc__)

# Author: Gael Varoquaux <gael dot varoquaux at normalesup dot org>
#         Andreas Mueller <amueller@ais.uni-bonn.de>
# License: BSD 3 clause

# Standard scientific Python imports
import matplotlib.pyplot as plt
import numpy as np
from time import time

# Import datasets, classifiers and performance metrics
from sklearn import datasets, svm, pipeline
from sklearn.kernel_approximation import (RBFSampler,
                                          Nystroem, Fastfood)
from sklearn.decomposition import PCA

# The digits dataset
digits = datasets.load_digits(n_class=9)

# To apply an classifier on this data, we need to flatten the image, to
# turn the data in a (samples, feature) matrix:
n_samples = len(digits.data)
data = digits.data / 16.
data -= data.mean(axis=0)

# We learn the digits on the first half of the digits
data_train, targets_train = data[:n_samples / 2], digits.target[:n_samples / 2]

# Now predict the value of the digit on the second half:
data_test, targets_test = data[n_samples / 2:], digits.target[n_samples / 2:]
#data_test = scaler.transform(data_test)

# fix model parameters:
GAMMA = .2
SIGMA = np.sqrt(1 / (2 * GAMMA))

# Create a classifier: a support vector classifier
kernel_svm = svm.SVC(gamma=GAMMA)
linear_svm = svm.LinearSVC()

# create pipeline from kernel approximation
# and linear svm
feature_map_fastfood = Fastfood(sigma=SIGMA, random_state=1)
feature_map_fourier = RBFSampler(gamma=GAMMA, random_state=1)
feature_map_nystroem = Nystroem(gamma=GAMMA, random_state=1)
fastfood_approx_svm = pipeline.Pipeline([("feature_map", feature_map_fastfood),
                                        ("svm", svm.LinearSVC())])

fourier_approx_svm = pipeline.Pipeline([("feature_map", feature_map_fourier),
                                        ("svm", svm.LinearSVC())])

nystroem_approx_svm = pipeline.Pipeline([("feature_map", feature_map_nystroem),
                                        ("svm", svm.LinearSVC())])

# fit and predict using linear and kernel svm:

kernel_svm_time = time()
kernel_svm.fit(data_train, targets_train)
kernel_svm_score = kernel_svm.score(data_test, targets_test)
kernel_svm_time = time() - kernel_svm_time

linear_svm_time = time()
linear_svm.fit(data_train, targets_train)
linear_svm_score = linear_svm.score(data_test, targets_test)
linear_svm_time = time() - linear_svm_time

sample_sizes = 30 * np.arange(1, 14)
fastfood_scores = []
fourier_scores = []
nystroem_scores = []
fastfood_times = []
fourier_times = []
nystroem_times = []

for D in sample_sizes:
    fastfood_approx_svm.set_params(feature_map__n_components=D)
    fourier_approx_svm.set_params(feature_map__n_components=D)
    nystroem_approx_svm.set_params(feature_map__n_components=D)
    start = time()
    fastfood_approx_svm.fit(data_train, targets_train)
    fastfood_times.append(time() - start)

    start = time()
    nystroem_approx_svm.fit(data_train, targets_train)
    nystroem_times.append(time() - start)

    start = time()
    fourier_approx_svm.fit(data_train, targets_train)
    fourier_times.append(time() - start)

    fastfood_score = fastfood_approx_svm.score(data_test, targets_test)
    fourier_score = fourier_approx_svm.score(data_test, targets_test)
    nystroem_score = nystroem_approx_svm.score(data_test, targets_test)
    fastfood_scores.append(fastfood_score)
    nystroem_scores.append(nystroem_score)
    fourier_scores.append(fourier_score)

# plot the results:
plt.figure(figsize=(8, 8))
accuracy = plt.subplot(211)
# second y axis for timeings
timescale = plt.subplot(212)

accuracy.plot(sample_sizes, nystroem_scores, label="Nystroem approx. kernel")
timescale.plot(sample_sizes, nystroem_times, '--',
               label='Nystroem approx. kernel')

accuracy.plot(sample_sizes, fourier_scores, label="Fourier approx. kernel")
timescale.plot(sample_sizes, fourier_times, '--',
               label='Fourier approx. kernel')

accuracy.plot(sample_sizes, fastfood_scores, label="Fastfood approx. kernel")
timescale.plot(sample_sizes, fastfood_times, '--',
               label='Fastfood approx. kernel')

# horizontal lines for exact rbf and linear kernels:
accuracy.plot([sample_sizes[0], sample_sizes[-1]],
              [linear_svm_score, linear_svm_score], label="linear svm")
timescale.plot([sample_sizes[0], sample_sizes[-1]],
               [linear_svm_time, linear_svm_time], '--', label='linear svm')

accuracy.plot([sample_sizes[0], sample_sizes[-1]],
              [kernel_svm_score, kernel_svm_score], label="rbf svm")
timescale.plot([sample_sizes[0], sample_sizes[-1]],
               [kernel_svm_time, kernel_svm_time], '--', label='rbf svm')

# vertical line for dataset dimensionality = 64
accuracy.plot([64, 64], [0.7, 1], label="n_features")

# legends and labels
accuracy.set_title("Classification accuracy")
timescale.set_title("Training times")
accuracy.set_xlim(sample_sizes[0], sample_sizes[-1])
accuracy.set_xticks(())
accuracy.set_ylim(np.min(fourier_scores), 1)
timescale.set_xlabel("Sampling steps = transformed feature dimension")
accuracy.set_ylabel("Classification accuracy")
timescale.set_ylabel("Training time in seconds")
accuracy.legend(loc='best')
timescale.legend(loc='best')

# visualize the decision surface, projected down to the first
# two principal components of the dataset
pca = PCA(n_components=8).fit(data_train)

X = pca.transform(data_train)

# Gemerate grid along first two principal components
multiples = np.arange(-2, 2, 0.1)
# steps along first component
first = multiples[:, np.newaxis] * pca.components_[0, :]
# steps along second component
second = multiples[:, np.newaxis] * pca.components_[1, :]
# combine
grid = first[np.newaxis, :, :] + second[:, np.newaxis, :]
flat_grid = grid.reshape(-1, data.shape[1])

# title for the plots
titles = ['SVC with rbf kernel',
          'SVC (linear kernel)\n with Fastfood rbf feature map\n'
          'n_components=100',
          'SVC (linear kernel)\n with Fourier rbf feature map\n'
          'n_components=100',
          'SVC (linear kernel)\n with Nystroem rbf feature map\n'
          'n_components=100']

plt.tight_layout()
plt.figure(figsize=(12, 5))

# predict and plot
for i, clf in enumerate((kernel_svm, fastfood_approx_svm, nystroem_approx_svm,
                         fourier_approx_svm)):
    # Plot the decision boundary. For that, we will assign a color to each
    # point in the mesh [x_min, m_max]x[y_min, y_max].
    plt.subplot(1, 4, i + 1)
    Z = clf.predict(flat_grid)

    # Put the result into a color plot
    Z = Z.reshape(grid.shape[:-1])
    plt.contourf(multiples, multiples, Z, cmap=plt.cm.Paired)
    plt.axis('off')

    # Plot also the training points
    plt.scatter(X[:, 0], X[:, 1], c=targets_train, cmap=plt.cm.Paired)

    plt.title(titles[i])
plt.tight_layout()
plt.show()
<commit_msg>Changed mode that Fastfood used from 'accuracy' to 'mem' to be more comparable with RKS<commit_after>"""
==================================================
Explicit feature map approximation for RBF kernels
==================================================

An example illustrating the approximation of the feature map
of an RBF kernel.

.. currentmodule:: sklearn.kernel_approximation

It shows how to use :class:`Fastfood`, :class:`RBFSampler` and :class:`Nystroem` to
approximate the feature map of an RBF kernel for classification with an SVM on
the digits dataset. Results using a linear SVM in the original space, a linear
SVM using the approximate mappings and using a kernelized SVM are compared.
Timings and accuracy for varying amounts of Monte Carlo samplings (in the case
of :class:`RBFSampler`, which uses random Fourier features) and different sized
subsets of the training set (for :class:`Nystroem`) for the approximate mapping
are shown.

Please note that the dataset here is not large enough to show the benefits
of kernel approximation, as the exact SVM is still reasonably fast.

Sampling more dimensions clearly leads to better classification results, but
comes at a greater cost. This means there is a tradeoff between runtime and
accuracy, given by the parameter n_components. Note that solving the Linear
SVM and also the approximate kernel SVM could be greatly accelerated by using
stochastic gradient descent via :class:`sklearn.linear_model.SGDClassifier`.
This is not easily possible for the case of the kernelized SVM.

The second plot visualized the decision surfaces of the RBF kernel SVM and
the linear SVM with approximate kernel maps.
The plot shows decision surfaces of the classifiers projected onto
the first two principal components of the data. This visualization should
be taken with a grain of salt since it is just an interesting slice through
the decision surface in 64 dimensions. In particular note that
a datapoint (represented as a dot) does not necessarily be classified
into the region it is lying in, since it will not lie on the plane
that the first two principal components span.

The usage of :class:`Fastfood`, :class:`RBFSampler` and :class:`Nystroem` is described in detail
in :ref:`kernel_approximation`.

"""
print(__doc__)

# Author: Gael Varoquaux <gael dot varoquaux at normalesup dot org>
#         Andreas Mueller <amueller@ais.uni-bonn.de>
# License: BSD 3 clause

# Standard scientific Python imports
import matplotlib.pyplot as plt
import numpy as np
from time import time

# Import datasets, classifiers and performance metrics
from sklearn import datasets, svm, pipeline
from sklearn.kernel_approximation import (RBFSampler,
                                          Nystroem, Fastfood)
from sklearn.decomposition import PCA

# The digits dataset
digits = datasets.load_digits(n_class=9)

# To apply an classifier on this data, we need to flatten the image, to
# turn the data in a (samples, feature) matrix:
n_samples = len(digits.data)
data = digits.data / 16.
data -= data.mean(axis=0)

# We learn the digits on the first half of the digits
data_train, targets_train = data[:n_samples / 2], digits.target[:n_samples / 2]

# Now predict the value of the digit on the second half:
data_test, targets_test = data[n_samples / 2:], digits.target[n_samples / 2:]
#data_test = scaler.transform(data_test)

# fix model parameters:
GAMMA = .2
SIGMA = np.sqrt(1 / (2 * GAMMA))

# Create a classifier: a support vector classifier
kernel_svm = svm.SVC(gamma=GAMMA)
linear_svm = svm.LinearSVC()

# create pipeline from kernel approximation
# and linear svm
feature_map_fastfood = Fastfood(sigma=SIGMA, tradeoff_mem_accuracy='mem', random_state=1)
feature_map_fourier = RBFSampler(gamma=GAMMA, random_state=1)
feature_map_nystroem = Nystroem(gamma=GAMMA, random_state=1)
fastfood_approx_svm = pipeline.Pipeline([("feature_map", feature_map_fastfood),
                                        ("svm", svm.LinearSVC())])

fourier_approx_svm = pipeline.Pipeline([("feature_map", feature_map_fourier),
                                        ("svm", svm.LinearSVC())])

nystroem_approx_svm = pipeline.Pipeline([("feature_map", feature_map_nystroem),
                                        ("svm", svm.LinearSVC())])

# fit and predict using linear and kernel svm:

kernel_svm_time = time()
kernel_svm.fit(data_train, targets_train)
kernel_svm_score = kernel_svm.score(data_test, targets_test)
kernel_svm_time = time() - kernel_svm_time

linear_svm_time = time()
linear_svm.fit(data_train, targets_train)
linear_svm_score = linear_svm.score(data_test, targets_test)
linear_svm_time = time() - linear_svm_time

sample_sizes = 30 * np.arange(1, 14)
fastfood_scores = []
fourier_scores = []
nystroem_scores = []
fastfood_times = []
fourier_times = []
nystroem_times = []

for D in sample_sizes:
    fastfood_approx_svm.set_params(feature_map__n_components=D)
    fourier_approx_svm.set_params(feature_map__n_components=D)
    nystroem_approx_svm.set_params(feature_map__n_components=D)
    start = time()
    fastfood_approx_svm.fit(data_train, targets_train)
    fastfood_times.append(time() - start)

    start = time()
    nystroem_approx_svm.fit(data_train, targets_train)
    nystroem_times.append(time() - start)

    start = time()
    fourier_approx_svm.fit(data_train, targets_train)
    fourier_times.append(time() - start)

    fastfood_score = fastfood_approx_svm.score(data_test, targets_test)
    fourier_score = fourier_approx_svm.score(data_test, targets_test)
    nystroem_score = nystroem_approx_svm.score(data_test, targets_test)
    fastfood_scores.append(fastfood_score)
    nystroem_scores.append(nystroem_score)
    fourier_scores.append(fourier_score)

# plot the results:
plt.figure(figsize=(8, 8))
accuracy = plt.subplot(211)
# second y axis for timeings
timescale = plt.subplot(212)

accuracy.plot(sample_sizes, nystroem_scores, label="Nystroem approx. kernel")
timescale.plot(sample_sizes, nystroem_times, '--',
               label='Nystroem approx. kernel')

accuracy.plot(sample_sizes, fourier_scores, label="Fourier approx. kernel")
timescale.plot(sample_sizes, fourier_times, '--',
               label='Fourier approx. kernel')

accuracy.plot(sample_sizes, fastfood_scores, label="Fastfood approx. kernel")
timescale.plot(sample_sizes, fastfood_times, '--',
               label='Fastfood approx. kernel')

# horizontal lines for exact rbf and linear kernels:
accuracy.plot([sample_sizes[0], sample_sizes[-1]],
              [linear_svm_score, linear_svm_score], label="linear svm")
timescale.plot([sample_sizes[0], sample_sizes[-1]],
               [linear_svm_time, linear_svm_time], '--', label='linear svm')

accuracy.plot([sample_sizes[0], sample_sizes[-1]],
              [kernel_svm_score, kernel_svm_score], label="rbf svm")
timescale.plot([sample_sizes[0], sample_sizes[-1]],
               [kernel_svm_time, kernel_svm_time], '--', label='rbf svm')

# vertical line for dataset dimensionality = 64
accuracy.plot([64, 64], [0.7, 1], label="n_features")

# legends and labels
accuracy.set_title("Classification accuracy")
timescale.set_title("Training times")
accuracy.set_xlim(sample_sizes[0], sample_sizes[-1])
accuracy.set_xticks(())
accuracy.set_ylim(np.min(fourier_scores), 1)
timescale.set_xlabel("Sampling steps = transformed feature dimension")
accuracy.set_ylabel("Classification accuracy")
timescale.set_ylabel("Training time in seconds")
accuracy.legend(loc='best')
timescale.legend(loc='best')

# visualize the decision surface, projected down to the first
# two principal components of the dataset
pca = PCA(n_components=8).fit(data_train)

X = pca.transform(data_train)

# Gemerate grid along first two principal components
multiples = np.arange(-2, 2, 0.1)
# steps along first component
first = multiples[:, np.newaxis] * pca.components_[0, :]
# steps along second component
second = multiples[:, np.newaxis] * pca.components_[1, :]
# combine
grid = first[np.newaxis, :, :] + second[:, np.newaxis, :]
flat_grid = grid.reshape(-1, data.shape[1])

# title for the plots
titles = ['SVC with rbf kernel',
          'SVC (linear kernel)\n with Fastfood rbf feature map\n'
          'n_components=100',
          'SVC (linear kernel)\n with Fourier rbf feature map\n'
          'n_components=100',
          'SVC (linear kernel)\n with Nystroem rbf feature map\n'
          'n_components=100']

plt.tight_layout()
plt.figure(figsize=(12, 5))

# predict and plot
for i, clf in enumerate((kernel_svm, fastfood_approx_svm, nystroem_approx_svm,
                         fourier_approx_svm)):
    # Plot the decision boundary. For that, we will assign a color to each
    # point in the mesh [x_min, m_max]x[y_min, y_max].
    plt.subplot(1, 4, i + 1)
    Z = clf.predict(flat_grid)

    # Put the result into a color plot
    Z = Z.reshape(grid.shape[:-1])
    plt.contourf(multiples, multiples, Z, cmap=plt.cm.Paired)
    plt.axis('off')

    # Plot also the training points
    plt.scatter(X[:, 0], X[:, 1], c=targets_train, cmap=plt.cm.Paired)

    plt.title(titles[i])
plt.tight_layout()
plt.show()
<|endoftext|>


<commit_before>#! /usr/bin/env python
# This file is part of the dune-hdd project:
#   http://users.dune-project.org/projects/dune-hdd
# Copyright holders: Felix Schindler
# License: BSD 2-Clause License (http://opensource.org/licenses/BSD-2-Clause)

import sys
from pybindgen import param, retval

from dune.pymor.core import prepare_python_bindings, inject_lib_dune_pymor, finalize_python_bindings
from dune.pymor.discretizations import inject_StationaryDiscretizationImplementation

def inject_Example(module, exceptions, interfaces, CONFIG_H):
    '''injects the user code into the module'''
    ssize_t = CONFIG_H['DUNE_STUFF_SSIZE_T']
    HAVE_DUNE_PDELAB = CONFIG_H['HAVE_DUNE_PDELAB']
    HAVE_DUNE_FEM    = CONFIG_H['HAVE_DUNE_FEM']
    HAVE_DUNE_ISTL   = CONFIG_H['HAVE_DUNE_ISTL']
    HAVE_EIGEN       = CONFIG_H['HAVE_EIGEN']
    HAVE_ALUGRID     = CONFIG_H['HAVE_ALUGRID']
    def add_example(GridType, space_backend, la_backend, name):
        # build all types needed for the discretization
        RangeFieldType = 'double'
        dimRange = '1'
        polOrder = '1'
        grid_layer = 'Dune::Stuff::Grid::ChooseLayer::leaf'
        MatrixType = 'Dune::Stuff::LA::'
        VectorType = 'Dune::Stuff::LA::'
        if 'eigen_sparse' in la_backend:
            MatrixType += 'EigenRowMajorSparseMatrix'
            VectorType += 'EigenDenseVector'
        elif 'istl_sparse' in la_backend:
            MatrixType += 'IstlRowMajorSparseMatrix'
            VectorType += 'IstlDenseVector'
        MatrixType += '< ' + RangeFieldType + ' >'
        VectorType += '< ' + RangeFieldType + ' >'
        OperatorType = 'Dune::Pymor::Operators::LinearAffinelyDecomposedContainerBased< ' + MatrixType + ', ' + VectorType + ' >'
        ProductType = OperatorType
        FunctionalType = 'Dune::Pymor::Functionals::LinearAffinelyDecomposedVectorBased< ' + VectorType + ' >'
        DiscretizationName = 'Dune::HDD::LinearElliptic::Discretizations::CG'
        DiscretizationType = (DiscretizationName + '< '
                              + GridType + ', ' + grid_layer + ', '
                              + RangeFieldType + ', '
                              + dimRange + ', ' + polOrder + ', '
                              + space_backend + ', ' + la_backend + '>')
        inject_StationaryDiscretizationImplementation(module, exceptions, interfaces, CONFIG_H,
                                                      DiscretizationName,
                                                      Traits={'VectorType': VectorType,
                                                              'OperatorType': OperatorType,
                                                              'FunctionalType': FunctionalType,
                                                              'ProductType': ProductType},
                                                      template_parameters=[GridType, grid_layer, RangeFieldType,
                                                                           dimRange, polOrder, space_backend, la_backend])
        # then create the example
        Example = module.add_class('PbCgExample', template_parameters=[GridType, space_backend, la_backend], custom_name=name)
        Example.add_constructor([param('const std::string&', 'num_blocks'),
                                 param('const std::string&', 'num_grid_elements'),
                                 param('const ' + ssize_t, 'info_log_levels'),
                                 param('const ' + ssize_t, 'debug_log_levels'),
                                 param('const bool', 'enable_warnings'),
                                 param('const bool', 'enable_colors'),
                                 param('const std::string', 'info_color'),
                                 param('const std::string', 'debug_color'),
                                 param('const std::string', 'warn_color')],
                                throw=exceptions)
        Example.add_constructor([param('const std::string&', 'num_blocks'),
                                 param('const std::string&', 'num_grid_elements'),
                                 param('const ' + ssize_t, 'info_log_levels'),
                                 param('const ' + ssize_t, 'debug_log_levels'),
                                 param('const bool', 'enable_warnings'),
                                 param('const bool', 'enable_colors'),
                                 param('const std::string', 'info_color'),
                                 param('const std::string', 'debug_color')],
                                throw=exceptions)
        Example.add_constructor([param('const std::string&', 'num_blocks'),
                                 param('const std::string&', 'num_grid_elements'),
                                 param('const ' + ssize_t, 'info_log_levels'),
                                 param('const ' + ssize_t, 'debug_log_levels'),
                                 param('const bool', 'enable_warnings'),
                                 param('const bool', 'enable_colors'),
                                 param('const std::string', 'info_color')],
                                throw=exceptions)
        Example.add_constructor([param('const std::string&', 'num_blocks'),
                                 param('const std::string&', 'num_grid_elements'),
                                 param('const ' + ssize_t, 'info_log_levels'),
                                 param('const ' + ssize_t, 'debug_log_levels'),
                                 param('const bool', 'enable_warnings'),
                                 param('const bool', 'enable_colors')],
                                throw=exceptions)
        Example.add_constructor([param('const std::string&', 'num_blocks'),
                                 param('const std::string&', 'num_grid_elements'),
                                 param('const ' + ssize_t, 'info_log_levels'),
                                 param('const ' + ssize_t, 'debug_log_levels'),
                                 param('const bool', 'enable_warnings')],
                                throw=exceptions)
        Example.add_constructor([param('const std::string&', 'num_blocks'),
                                 param('const std::string&', 'num_grid_elements'),
                                 param('const ' + ssize_t, 'info_log_levels'),
                                 param('const ' + ssize_t, 'debug_log_levels')],
                                throw=exceptions)
        Example.add_constructor([param('const std::string&', 'num_blocks'),
                                 param('const std::string&', 'num_grid_elements'),
                                 param('const ' + ssize_t, 'info_log_levels')],
                                throw=exceptions)
        Example.add_constructor([param('const std::string&', 'num_blocks'),
                                 param('const std::string&', 'num_grid_elements')],
                                throw=exceptions)
        Example.add_constructor([param('const std::string&', 'num_blocks')],
                                throw=exceptions)
        Example.add_constructor([],
                                throw=exceptions)
        Example.add_method('pb_discretization_and_return_ptr',
                           retval(DiscretizationType + ' *', caller_owns_return=True),
                           [], is_const=True, throw=exceptions,
                           custom_name='discretization')
    YaspGrid2d = 'Dune::YaspGrid< 2 >'
    YaspGrid3d = 'Dune::YaspGrid< 3 >'
    if HAVE_ALUGRID:
        AluGridConform2d = 'Dune::ALUGrid< 2, 2, Dune::simplex, Dune::conforming >'
    la_backend_eigen = 'Dune::Stuff::LA::ChooseBackend::eigen_sparse'
    la_backend_istl  = 'Dune::Stuff::LA::ChooseBackend::istl_sparse'
    space_backend_pdelab = 'Dune::GDT::ChooseSpaceBackend::pdelab'
    space_backend_fem    = 'Dune::GDT::ChooseSpaceBackend::fem'
    if HAVE_DUNE_PDELAB and HAVE_DUNE_ISTL:
        add_example(YaspGrid2d, space_backend_pdelab, la_backend_istl, 'CG_Thermalblock_2dYaspGrid_pdelab_istl')
        add_example(YaspGrid3d, space_backend_pdelab, la_backend_istl, 'CG_Thermalblock_3dYaspGrid_pdelab_istl')
        if HAVE_ALUGRID:
            add_example(AluGridConform2d, space_backend_pdelab, la_backend_istl,
                        'CG_Thermalblock_2dAluConformGrid_pdelab_istl')
    if HAVE_DUNE_PDELAB and HAVE_EIGEN:
        add_example(YaspGrid2d, space_backend_pdelab, la_backend_eigen, 'CG_Thermalblock_2dYaspGrid_pdelab_eigen')
        add_example(YaspGrid3d, space_backend_pdelab, la_backend_eigen, 'CG_Thermalblock_3dYaspGrid_pdelab_eigen')
        if HAVE_ALUGRID:
            add_example(AluGridConform2d, space_backend_pdelab, la_backend_eigen,
                        'CG_Thermalblock_2dAluConformGrid_pdelab_eigen')
    if HAVE_DUNE_FEM and HAVE_DUNE_ISTL:
        add_example(YaspGrid2d, space_backend_fem, la_backend_istl, 'CG_Thermalblock_2dYaspGrid_fem_istl')
        add_example(YaspGrid3d, space_backend_fem, la_backend_istl, 'CG_Thermalblock_3dYaspGrid_fem_istl')
        if HAVE_ALUGRID:
            add_example(AluGridConform2d, space_backend_fem, la_backend_istl,
                        'CG_Thermalblock_2dAluConformGrid_fem_istl')
    if HAVE_DUNE_FEM and HAVE_EIGEN:
        add_example(YaspGrid2d, space_backend_fem, la_backend_eigen, 'CG_Thermalblock_2dYaspGrid_fem_eigen')
        add_example(YaspGrid3d, space_backend_fem, la_backend_eigen, 'CG_Thermalblock_3dYaspGrid_fem_eigen')
        if HAVE_ALUGRID:
            add_example(AluGridConform2d, space_backend_fem, la_backend_fem,
                        'CG_Thermalblock_2dAluConformGrid_fem_eigen')

if __name__ == '__main__':
    # prepare the module
    module, pybindgen_filename, config_h_filename = prepare_python_bindings(sys.argv[1:])
    # add all of libdunepymor
    module, exceptions, interfaces, CONFIG_H = inject_lib_dune_pymor(module, config_h_filename)
    # add example user code (see above)
    inject_Example(module, exceptions, interfaces, CONFIG_H)
    # and finally write the pybindgen .cc file
    finalize_python_bindings(module, pybindgen_filename)
<commit_msg>[examples...thermalblock] do not bind YaspGrid variants<commit_after>#! /usr/bin/env python
# This file is part of the dune-hdd project:
#   http://users.dune-project.org/projects/dune-hdd
# Copyright holders: Felix Schindler
# License: BSD 2-Clause License (http://opensource.org/licenses/BSD-2-Clause)

import sys
from pybindgen import param, retval

from dune.pymor.core import prepare_python_bindings, inject_lib_dune_pymor, finalize_python_bindings
from dune.pymor.discretizations import inject_StationaryDiscretizationImplementation

def inject_Example(module, exceptions, interfaces, CONFIG_H):
    '''injects the user code into the module'''
    ssize_t = CONFIG_H['DUNE_STUFF_SSIZE_T']
    HAVE_DUNE_PDELAB = CONFIG_H['HAVE_DUNE_PDELAB']
    HAVE_DUNE_FEM    = CONFIG_H['HAVE_DUNE_FEM']
    HAVE_DUNE_ISTL   = CONFIG_H['HAVE_DUNE_ISTL']
    HAVE_EIGEN       = CONFIG_H['HAVE_EIGEN']
    HAVE_ALUGRID     = CONFIG_H['HAVE_ALUGRID']
    def add_example(GridType, space_backend, la_backend, name):
        # build all types needed for the discretization
        RangeFieldType = 'double'
        dimRange = '1'
        polOrder = '1'
        grid_layer = 'Dune::Stuff::Grid::ChooseLayer::leaf'
        MatrixType = 'Dune::Stuff::LA::'
        VectorType = 'Dune::Stuff::LA::'
        if 'eigen_sparse' in la_backend:
            MatrixType += 'EigenRowMajorSparseMatrix'
            VectorType += 'EigenDenseVector'
        elif 'istl_sparse' in la_backend:
            MatrixType += 'IstlRowMajorSparseMatrix'
            VectorType += 'IstlDenseVector'
        MatrixType += '< ' + RangeFieldType + ' >'
        VectorType += '< ' + RangeFieldType + ' >'
        OperatorType = 'Dune::Pymor::Operators::LinearAffinelyDecomposedContainerBased< ' + MatrixType + ', ' + VectorType + ' >'
        ProductType = OperatorType
        FunctionalType = 'Dune::Pymor::Functionals::LinearAffinelyDecomposedVectorBased< ' + VectorType + ' >'
        DiscretizationName = 'Dune::HDD::LinearElliptic::Discretizations::CG'
        DiscretizationType = (DiscretizationName + '< '
                              + GridType + ', ' + grid_layer + ', '
                              + RangeFieldType + ', '
                              + dimRange + ', ' + polOrder + ', '
                              + space_backend + ', ' + la_backend + '>')
        inject_StationaryDiscretizationImplementation(module, exceptions, interfaces, CONFIG_H,
                                                      DiscretizationName,
                                                      Traits={'VectorType': VectorType,
                                                              'OperatorType': OperatorType,
                                                              'FunctionalType': FunctionalType,
                                                              'ProductType': ProductType},
                                                      template_parameters=[GridType, grid_layer, RangeFieldType,
                                                                           dimRange, polOrder, space_backend, la_backend])
        # then create the example
        Example = module.add_class('PbCgExample', template_parameters=[GridType, space_backend, la_backend], custom_name=name)
        Example.add_constructor([param('const std::string&', 'num_blocks'),
                                 param('const std::string&', 'num_grid_elements'),
                                 param('const ' + ssize_t, 'info_log_levels'),
                                 param('const ' + ssize_t, 'debug_log_levels'),
                                 param('const bool', 'enable_warnings'),
                                 param('const bool', 'enable_colors'),
                                 param('const std::string', 'info_color'),
                                 param('const std::string', 'debug_color'),
                                 param('const std::string', 'warn_color')],
                                throw=exceptions)
        Example.add_constructor([param('const std::string&', 'num_blocks'),
                                 param('const std::string&', 'num_grid_elements'),
                                 param('const ' + ssize_t, 'info_log_levels'),
                                 param('const ' + ssize_t, 'debug_log_levels'),
                                 param('const bool', 'enable_warnings'),
                                 param('const bool', 'enable_colors'),
                                 param('const std::string', 'info_color'),
                                 param('const std::string', 'debug_color')],
                                throw=exceptions)
        Example.add_constructor([param('const std::string&', 'num_blocks'),
                                 param('const std::string&', 'num_grid_elements'),
                                 param('const ' + ssize_t, 'info_log_levels'),
                                 param('const ' + ssize_t, 'debug_log_levels'),
                                 param('const bool', 'enable_warnings'),
                                 param('const bool', 'enable_colors'),
                                 param('const std::string', 'info_color')],
                                throw=exceptions)
        Example.add_constructor([param('const std::string&', 'num_blocks'),
                                 param('const std::string&', 'num_grid_elements'),
                                 param('const ' + ssize_t, 'info_log_levels'),
                                 param('const ' + ssize_t, 'debug_log_levels'),
                                 param('const bool', 'enable_warnings'),
                                 param('const bool', 'enable_colors')],
                                throw=exceptions)
        Example.add_constructor([param('const std::string&', 'num_blocks'),
                                 param('const std::string&', 'num_grid_elements'),
                                 param('const ' + ssize_t, 'info_log_levels'),
                                 param('const ' + ssize_t, 'debug_log_levels'),
                                 param('const bool', 'enable_warnings')],
                                throw=exceptions)
        Example.add_constructor([param('const std::string&', 'num_blocks'),
                                 param('const std::string&', 'num_grid_elements'),
                                 param('const ' + ssize_t, 'info_log_levels'),
                                 param('const ' + ssize_t, 'debug_log_levels')],
                                throw=exceptions)
        Example.add_constructor([param('const std::string&', 'num_blocks'),
                                 param('const std::string&', 'num_grid_elements'),
                                 param('const ' + ssize_t, 'info_log_levels')],
                                throw=exceptions)
        Example.add_constructor([param('const std::string&', 'num_blocks'),
                                 param('const std::string&', 'num_grid_elements')],
                                throw=exceptions)
        Example.add_constructor([param('const std::string&', 'num_blocks')],
                                throw=exceptions)
        Example.add_constructor([],
                                throw=exceptions)
        Example.add_method('pb_discretization_and_return_ptr',
                           retval(DiscretizationType + ' *', caller_owns_return=True),
                           [], is_const=True, throw=exceptions,
                           custom_name='discretization')
    # YaspGrid2d = 'Dune::YaspGrid< 2 >'
    # YaspGrid3d = 'Dune::YaspGrid< 3 >'
    if HAVE_ALUGRID:
        AluGridConform2d = 'Dune::ALUGrid< 2, 2, Dune::simplex, Dune::conforming >'
    la_backend_eigen = 'Dune::Stuff::LA::ChooseBackend::eigen_sparse'
    la_backend_istl  = 'Dune::Stuff::LA::ChooseBackend::istl_sparse'
    space_backend_pdelab = 'Dune::GDT::ChooseSpaceBackend::pdelab'
    space_backend_fem    = 'Dune::GDT::ChooseSpaceBackend::fem'
    if HAVE_DUNE_PDELAB and HAVE_DUNE_ISTL:
        # add_example(YaspGrid2d, space_backend_pdelab, la_backend_istl, 'CG_Thermalblock_2dYaspGrid_pdelab_istl')
        # add_example(YaspGrid3d, space_backend_pdelab, la_backend_istl, 'CG_Thermalblock_3dYaspGrid_pdelab_istl')
        if HAVE_ALUGRID:
            add_example(AluGridConform2d, space_backend_pdelab, la_backend_istl,
                        'CG_Thermalblock_2dAluConformGrid_pdelab_istl')
    if HAVE_DUNE_PDELAB and HAVE_EIGEN:
        # add_example(YaspGrid2d, space_backend_pdelab, la_backend_eigen, 'CG_Thermalblock_2dYaspGrid_pdelab_eigen')
        # add_example(YaspGrid3d, space_backend_pdelab, la_backend_eigen, 'CG_Thermalblock_3dYaspGrid_pdelab_eigen')
        if HAVE_ALUGRID:
            add_example(AluGridConform2d, space_backend_pdelab, la_backend_eigen,
                        'CG_Thermalblock_2dAluConformGrid_pdelab_eigen')
    if HAVE_DUNE_FEM and HAVE_DUNE_ISTL:
        # add_example(YaspGrid2d, space_backend_fem, la_backend_istl, 'CG_Thermalblock_2dYaspGrid_fem_istl')
        # add_example(YaspGrid3d, space_backend_fem, la_backend_istl, 'CG_Thermalblock_3dYaspGrid_fem_istl')
        if HAVE_ALUGRID:
            add_example(AluGridConform2d, space_backend_fem, la_backend_istl,
                        'CG_Thermalblock_2dAluConformGrid_fem_istl')
    if HAVE_DUNE_FEM and HAVE_EIGEN:
        # add_example(YaspGrid2d, space_backend_fem, la_backend_eigen, 'CG_Thermalblock_2dYaspGrid_fem_eigen')
        # add_example(YaspGrid3d, space_backend_fem, la_backend_eigen, 'CG_Thermalblock_3dYaspGrid_fem_eigen')
        if HAVE_ALUGRID:
            add_example(AluGridConform2d, space_backend_fem, la_backend_fem,
                        'CG_Thermalblock_2dAluConformGrid_fem_eigen')

if __name__ == '__main__':
    # prepare the module
    module, pybindgen_filename, config_h_filename = prepare_python_bindings(sys.argv[1:])
    # add all of libdunepymor
    module, exceptions, interfaces, CONFIG_H = inject_lib_dune_pymor(module, config_h_filename)
    # add example user code (see above)
    inject_Example(module, exceptions, interfaces, CONFIG_H)
    # and finally write the pybindgen .cc file
    finalize_python_bindings(module, pybindgen_filename)
<|endoftext|>