<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

from django.contrib import admin
from django.contrib.auth.models import Group
from django.utils.translation import ugettext_lazy as _

from .models import *
from .forms import *

admin.site.unregister(Group)

class SocialNetworkAdmin(admin.ModelAdmin):
    model = SocialNetwork
    classes = ("grp-collapse grp-open",)
    inline_classes = ("grp-collapse grp-open",)
    fieldsets = (
        (
            _("Informations"), {
                "fields" : (
                    "type", "url",
                )
            }
        ),
    )
    list_display = ("type", "url")
    list_filter = ("type",)
    search_fields = ("type",)
    ordering = ("type", "url",)

admin.site.register(SocialNetwork, SocialNetworkAdmin)

class ContactAdmin(admin.ModelAdmin):
    model = Contact
    classes = ("grp-collapse grp-open",)
    inline_classes = ("grp-collapse grp-open",)
    fieldsets = (
        (
            _("Informations"), {
                "fields" : (
                    "type", "value",
                )
            }
        ),
    )
    list_display = ("type", "value")
    list_filter = ("type",)
    search_fields = ("type",)
    ordering = ("type", "value",)

admin.site.register(Contact, ContactAdmin)

class AddressAdmin(admin.ModelAdmin):
    model = Address
    classes = ("grp-collapse grp-open",)
    inline_classes = ("grp-collapse grp-open",)
    fieldsets = (
        (
            _("Informations"), {
                "fields" : (
                    ("street", "number",),
                    "complement",
                    ("district", "zip_code",),
                    ("city", "state",),
                )
            }
        ),
    )
    list_display = ("street",)

admin.site.register(Address, AddressAdmin)

class AboutAdmin(admin.ModelAdmin):
    model = About
    form = AboutAdminForm
    classes = ("grp-collapse grp-open",)
    inline_classes = ("grp-collapse grp-open",)
    fieldsets = (
        (
            _("Informations"), {
                "fields" : (
                    "body",
                )
            }
        ),
    )
    list_display = ("pk",)

admin.site.register(About, AboutAdmin)

class RoleAdmin(admin.ModelAdmin):
    model = Role

admin.site.register(Role, RoleAdmin)

class MemberAdmin(admin.ModelAdmin):
    model = Member
    form = MemberAdminForm
    classes = ("grp-collapse grp-open",)
    inline_classes = ("grp-collapse grp-open",)
    fieldsets = (
        (
            _("Informations"), {
                "fields" : (
                    ("photo", "photo_tag"),
                    ("name", "role",),
                    ("email", "phone",),
                    ("about",),
                )
            }
        ),
    )
    list_display = ("name", "role", "email", "phone",)
    list_filter = ("role",)
    search_fields = ("name", "role", "email",)
    ordering = ("name", "role",)
    readonly_fields = ("photo_tag",)

admin.site.register(Member, MemberAdmin)

class CurricularPracticeInline(admin.StackedInline):
    model = CurricularPractice
    extra = 1
    can_delete = True
    max_num = 1024
    classes = ("grp-collapse grp-open",)
    inline_classes = ("grp-collapse grp-open",)

class DisciplineAdmin(admin.ModelAdmin):
    model = Discipline
    inlines = (CurricularPracticeInline,)

admin.site.register(Discipline, DisciplineAdmin)

class EditorialAdmin(admin.ModelAdmin):
    model = Editorial

admin.site.register(Editorial, EditorialAdmin)

class AuthorAdmin(admin.ModelAdmin):
    model = Author
    form = AuthorAdminForm
    classes = ("grp-collapse grp-open",)
    inline_classes = ("grp-collapse grp-open",)
    fieldsets = (
        (
            _("Informations"), {
                "fields" : (
                    ("photo", "photo_tag"),
                    ("name",),
                    ("email", "phone",),
                    ("about",),
                )
            }
        ),
    )
    list_display = ("name", "email", "phone",)
    search_fields = ("name", "email",)
    ordering = ("name",)
    readonly_fields = ("photo_tag",)

admin.site.register(Author, AuthorAdmin)

class PhotographerAdmin(admin.ModelAdmin):
    model = Photographer
    form = PhotographerAdminForm
    classes = ("grp-collapse grp-open",)
    inline_classes = ("grp-collapse grp-open",)
    fieldsets = (
        (
            _("Informations"), {
                "fields" : (
                    ("photo", "photo_tag"),
                    ("name",),
                    ("email", "phone",),
                    ("about",),
                )
            }
        ),
    )
    list_display = ("name", "email", "phone",)
    search_fields = ("name", "email",)
    ordering = ("name",)
    readonly_fields = ("photo_tag",)

admin.site.register(Photographer, PhotographerAdmin)

class NoticeAdmin(admin.ModelAdmin):
    model = Notice
    form = NoticeAdminForm
    classes = ("grp-collapse grp-open",)
    inline_classes = ("grp-collapse grp-open",)
    fieldsets = (
        (
            _("Statistics"), {
                "fields" : (
                    ("views", "comments", "likes",),
                )
            }
        ),
        (
            _("Informations"), {
                "fields" : (
                    ("date", "active", "featured"),
                    ("editorial", "discipline", "curricular_practice"),
                    "author",
                )
            }
        ),
        (
            _("Description"), {
                "fields" : (
                    "title",
                    "subtitle",
                    "body",
                    ("photo", "photo_tag",),
                    "photographer",
                )
            }
        ),
    )
    list_display = ("title", "date", "date_modified", "editorial", "featured", "active",)
    list_filter = ("date", "editorial__name", "featured", "active",)
    search_fields = ("date", "date_modified", "editorial__name", "featured", "active",)
    ordering = ("date_modified", "date", "editorial", "active", "featured")
    readonly_fields = ("views", "comments", "likes", "photo_tag",)

    class Media:
        css = {
             "all": ("admin/datepicker.css",)
        }

admin.site.register(Notice, NoticeAdmin)

class PhotoInline(admin.StackedInline):
    model = Photo
    extra = 1
    can_delete = True
    max_num = 1024
    classes = ("grp-collapse grp-open",)
    inline_classes = ("grp-collapse grp-open",)
    readonly_fields = ("photo_tag",)

class PhotogalleryAdmin(admin.ModelAdmin):
    model = Photogallery
    form = PhotogalleryAdminForm
    inlines = (PhotoInline,)
    classes = ("grp-collapse grp-open",)
    inline_classes = ("grp-collapse grp-open",)
    fieldsets = (
        (
            _("Statistics"), {
                "fields" : (
                    ("views", "comments", "likes",),
                )
            }
        ),
        (
            _("Informations"), {
                "fields" : (
                    ("date", "active",),
                    ("editorial", "discipline", "curricular_practice"),
                    "author",
                )
            }
        ),
        (
            _("Description"), {
                "fields" : (
                    "title",
                    "subtitle",
                    "body",
                )
            }

        ),
    )
    list_display = ("title", "date", "date_modified", "active",)
    list_filter = ("date", "active",)
    search_fields = ("date", "date_modified", "active",)
    ordering = ("date_modified", "date", "active",)
    readonly_fields = ("views", "comments", "likes",)

    class Media:
        css = {
             "all": ("admin/datepicker.css",)
        }

admin.site.register(Photogallery, PhotogalleryAdmin)

class VideoInline(admin.StackedInline):
    model = Video
    extra = 1
    can_delete = True
    max_num = 1024
    classes = ("grp-collapse grp-open",)
    inline_classes = ("grp-collapse grp-open",)

class VideoLibraryAdmin(admin.ModelAdmin):
    model = VideoLibrary
    form = VideoLibraryAdminForm
    inlines = (VideoInline,)
    classes = ("grp-collapse grp-open",)
    inline_classes = ("grp-collapse grp-open",)
    fieldsets = (
        (
            _("Statistics"), {
                "fields" : (
                    ("views", "comments", "likes",),
                )
            }
        ),
        (
            _("Informations"), {
                "fields" : (
                    ("date", "active",),
                    ("editorial", "discipline", "curricular_practice"),
                    "aluno",
                )
            }
        ),
        (
            _("Description"), {
                "fields" : (
                    "title",
                    "subtitle",
                    "body",
                )
            }
        ),
    )
    list_display = ("title", "date", "date_modified", "active",)
    list_filter = ("date", "active",)
    search_fields = ("date", "date_modified", "active",)
    ordering = ("date_modified", "date", "active",)
    readonly_fields = ("views", "comments", "likes",)

    class Media:
        css = {
             "all": ("admin/datepicker.css",)
        }

admin.site.register(VideoLibrary, VideoLibraryAdmin)

class PodcastAdmin(admin.ModelAdmin):
    model = Podcast
    form = PodcastAdminForm
    classes = ("grp-collapse grp-open",)
    inline_classes = ("grp-collapse grp-open",)
    fieldsets = (
        (
            _("Statistics"), {
                "fields" : (
                    ("views", "comments", "likes",),
                )
            }
        ),
        (
            _("Informations"), {
                "fields" : (
                    ("date", "active",),
                )
            }
        ),
        (
            _("Description"), {
                "fields" : (
                    "title",
                    "subtitle",
                    "body",
                    "download_url",
                )
            }
        ),
    )
    list_display = ("title", "date", "date_modified", "active",)
    list_filter = ("date", "active",)
    search_fields = ("date", "date_modified", "active",)
    ordering = ("date_modified", "date", "active",)
    readonly_fields = ("views", "comments", "likes",)

    class Media:
        css = {
             "all": ("admin/datepicker.css",)
        }

admin.site.register(Podcast, PodcastAdmin)

class EventAdmin(admin.ModelAdmin):
    model = Event
    form = EventAdminForm
    fieldsets = (
        (
            _("Statistics"), {
                "fields" : (
                    ("views", "comments", "likes",),
                )
            }
        ),
        (
            _("Informations"), {
                "fields" : (
                    ("date", "active",),
                    "author",
                )
            }
        ),
        (
            _("Description"), {
                "fields" : (
                    "title",
                    "subtitle",
                    "body",
                )
            }
        ),
        (
            _("Medias"), {
                "fields" : (
                    "notices",
                    "photogalleries",
                    "video_libraries",
                    "podcasts",
                )
            }
        ),
    )
    filter_horizontal = ("notices", "photogalleries", "video_libraries", "podcasts",)
    list_display = ("title", "date", "date_modified", "active",)
    list_filter = ("date", "active",)
    search_fields = ("date", "date_modified", "active",)
    ordering = ("date_modified", "date", "active",)
    readonly_fields = ("views", "comments", "likes",)

    class Media:
        css = {
             "all": ("admin/datepicker.css",)
        }

admin.site.register(Event, EventAdmin)

<commit_msg>Issues.<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

from django.contrib import admin
from django.contrib.auth.models import Group
from django.utils.translation import ugettext_lazy as _

from .models import *
from .forms import *

admin.site.unregister(Group)

class SocialNetworkAdmin(admin.ModelAdmin):
    model = SocialNetwork
    classes = ("grp-collapse grp-open",)
    inline_classes = ("grp-collapse grp-open",)
    fieldsets = (
        (
            _("Informations"), {
                "fields" : (
                    "type", "url",
                )
            }
        ),
    )
    list_display = ("type", "url")
    list_filter = ("type",)
    search_fields = ("type",)
    ordering = ("type", "url",)

admin.site.register(SocialNetwork, SocialNetworkAdmin)

class ContactAdmin(admin.ModelAdmin):
    model = Contact
    classes = ("grp-collapse grp-open",)
    inline_classes = ("grp-collapse grp-open",)
    fieldsets = (
        (
            _("Informations"), {
                "fields" : (
                    "type", "value",
                )
            }
        ),
    )
    list_display = ("type", "value")
    list_filter = ("type",)
    search_fields = ("type",)
    ordering = ("type", "value",)

admin.site.register(Contact, ContactAdmin)

class AddressAdmin(admin.ModelAdmin):
    model = Address
    classes = ("grp-collapse grp-open",)
    inline_classes = ("grp-collapse grp-open",)
    fieldsets = (
        (
            _("Informations"), {
                "fields" : (
                    ("street", "number",),
                    "complement",
                    ("district", "zip_code",),
                    ("city", "state",),
                )
            }
        ),
    )
    list_display = ("street",)

admin.site.register(Address, AddressAdmin)

class AboutAdmin(admin.ModelAdmin):
    model = About
    form = AboutAdminForm
    classes = ("grp-collapse grp-open",)
    inline_classes = ("grp-collapse grp-open",)
    fieldsets = (
        (
            _("Informations"), {
                "fields" : (
                    "body",
                )
            }
        ),
    )
    list_display = ("pk",)

admin.site.register(About, AboutAdmin)

class RoleAdmin(admin.ModelAdmin):
    model = Role

admin.site.register(Role, RoleAdmin)

class MemberAdmin(admin.ModelAdmin):
    model = Member
    form = MemberAdminForm
    classes = ("grp-collapse grp-open",)
    inline_classes = ("grp-collapse grp-open",)
    fieldsets = (
        (
            _("Informations"), {
                "fields" : (
                    ("photo", "photo_tag"),
                    ("name", "role",),
                    ("email", "phone",),
                    ("about",),
                )
            }
        ),
    )
    list_display = ("name", "role", "email", "phone",)
    list_filter = ("role",)
    search_fields = ("name", "role", "email",)
    ordering = ("name", "role",)
    readonly_fields = ("photo_tag",)

admin.site.register(Member, MemberAdmin)

class CurricularPracticeInline(admin.StackedInline):
    model = CurricularPractice
    extra = 1
    can_delete = True
    max_num = 99
    classes = ("grp-collapse grp-open",)
    inline_classes = ("grp-collapse grp-open",)

class DisciplineAdmin(admin.ModelAdmin):
    model = Discipline
    inlines = (CurricularPracticeInline,)

admin.site.register(Discipline, DisciplineAdmin)

class EditorialAdmin(admin.ModelAdmin):
    model = Editorial

admin.site.register(Editorial, EditorialAdmin)

class AuthorAdmin(admin.ModelAdmin):
    model = Author
    form = AuthorAdminForm
    classes = ("grp-collapse grp-open",)
    inline_classes = ("grp-collapse grp-open",)
    fieldsets = (
        (
            _("Informations"), {
                "fields" : (
                    ("photo", "photo_tag"),
                    ("name",),
                    ("email", "phone",),
                    ("about",),
                )
            }
        ),
    )
    list_display = ("name", "email", "phone",)
    search_fields = ("name", "email",)
    ordering = ("name",)
    readonly_fields = ("photo_tag",)

admin.site.register(Author, AuthorAdmin)

class PhotographerAdmin(admin.ModelAdmin):
    model = Photographer
    form = PhotographerAdminForm
    classes = ("grp-collapse grp-open",)
    inline_classes = ("grp-collapse grp-open",)
    fieldsets = (
        (
            _("Informations"), {
                "fields" : (
                    ("photo", "photo_tag"),
                    ("name",),
                    ("email", "phone",),
                    ("about",),
                )
            }
        ),
    )
    list_display = ("name", "email", "phone",)
    search_fields = ("name", "email",)
    ordering = ("name",)
    readonly_fields = ("photo_tag",)

admin.site.register(Photographer, PhotographerAdmin)

class NoticeAdmin(admin.ModelAdmin):
    model = Notice
    form = NoticeAdminForm
    classes = ("grp-collapse grp-open",)
    inline_classes = ("grp-collapse grp-open",)
    fieldsets = (
        (
            _("Statistics"), {
                "fields" : (
                    ("views", "comments", "likes",),
                )
            }
        ),
        (
            _("Informations"), {
                "fields" : (
                    ("date", "active", "featured"),
                    ("editorial", "discipline", "curricular_practice"),
                    "author",
                )
            }
        ),
        (
            _("Description"), {
                "fields" : (
                    "title",
                    "subtitle",
                    "body",
                    ("photo", "photo_tag",),
                    "photographer",
                )
            }
        ),
    )
    list_display = ("title", "date", "date_modified", "editorial", "featured", "active",)
    list_filter = ("date", "editorial__name", "featured", "active",)
    search_fields = ("date", "date_modified", "editorial__name", "featured", "active",)
    ordering = ("date_modified", "date", "editorial", "active", "featured")
    readonly_fields = ("views", "comments", "likes", "photo_tag",)

    class Media:
        css = {
             "all": ("admin/datepicker.css",)
        }

admin.site.register(Notice, NoticeAdmin)

class PhotoInline(admin.StackedInline):
    model = Photo
    extra = 1
    can_delete = True
    max_num = 99
    classes = ("grp-collapse grp-open",)
    inline_classes = ("grp-collapse grp-open",)
    readonly_fields = ("photo_tag",)

class PhotogalleryAdmin(admin.ModelAdmin):
    model = Photogallery
    form = PhotogalleryAdminForm
    inlines = (PhotoInline,)
    classes = ("grp-collapse grp-open",)
    inline_classes = ("grp-collapse grp-open",)
    fieldsets = (
        (
            _("Statistics"), {
                "fields" : (
                    ("views", "comments", "likes",),
                )
            }
        ),
        (
            _("Informations"), {
                "fields" : (
                    ("date", "active",),
                    ("editorial", "discipline", "curricular_practice"),
                    "author",
                )
            }
        ),
        (
            _("Description"), {
                "fields" : (
                    "title",
                    "subtitle",
                    "body",
                )
            }

        ),
    )
    list_display = ("title", "date", "date_modified", "active",)
    list_filter = ("date", "active",)
    search_fields = ("date", "date_modified", "active",)
    ordering = ("date_modified", "date", "active",)
    readonly_fields = ("views", "comments", "likes",)

    class Media:
        css = {
             "all": ("admin/datepicker.css",)
        }

admin.site.register(Photogallery, PhotogalleryAdmin)

class VideoInline(admin.StackedInline):
    model = Video
    extra = 1
    can_delete = True
    max_num = 99
    classes = ("grp-collapse grp-open",)
    inline_classes = ("grp-collapse grp-open",)

class VideoLibraryAdmin(admin.ModelAdmin):
    model = VideoLibrary
    form = VideoLibraryAdminForm
    inlines = (VideoInline,)
    classes = ("grp-collapse grp-open",)
    inline_classes = ("grp-collapse grp-open",)
    fieldsets = (
        (
            _("Statistics"), {
                "fields" : (
                    ("views", "comments", "likes",),
                )
            }
        ),
        (
            _("Informations"), {
                "fields" : (
                    ("date", "active",),
                    ("editorial", "discipline", "curricular_practice"),
                    "author",
                )
            }
        ),
        (
            _("Description"), {
                "fields" : (
                    "title",
                    "subtitle",
                    "body",
                )
            }
        ),
    )
    list_display = ("title", "date", "date_modified", "active",)
    list_filter = ("date", "active",)
    search_fields = ("date", "date_modified", "active",)
    ordering = ("date_modified", "date", "active",)
    readonly_fields = ("views", "comments", "likes",)

    class Media:
        css = {
             "all": ("admin/datepicker.css",)
        }

admin.site.register(VideoLibrary, VideoLibraryAdmin)

class PodcastAdmin(admin.ModelAdmin):
    model = Podcast
    form = PodcastAdminForm
    classes = ("grp-collapse grp-open",)
    inline_classes = ("grp-collapse grp-open",)
    fieldsets = (
        (
            _("Statistics"), {
                "fields" : (
                    ("views", "comments", "likes",),
                )
            }
        ),
        (
            _("Informations"), {
                "fields" : (
                    ("date", "active",),
                )
            }
        ),
        (
            _("Description"), {
                "fields" : (
                    "title",
                    "subtitle",
                    "body",
                    "download_url",
                )
            }
        ),
    )
    list_display = ("title", "date", "date_modified", "active",)
    list_filter = ("date", "active",)
    search_fields = ("date", "date_modified", "active",)
    ordering = ("date_modified", "date", "active",)
    readonly_fields = ("views", "comments", "likes",)

    class Media:
        css = {
             "all": ("admin/datepicker.css",)
        }

admin.site.register(Podcast, PodcastAdmin)

class EventAdmin(admin.ModelAdmin):
    model = Event
    form = EventAdminForm
    fieldsets = (
        (
            _("Statistics"), {
                "fields" : (
                    ("views", "comments", "likes",),
                )
            }
        ),
        (
            _("Informations"), {
                "fields" : (
                    ("date", "active",),
                    "author",
                )
            }
        ),
        (
            _("Description"), {
                "fields" : (
                    "title",
                    "subtitle",
                    "body",
                )
            }
        ),
        (
            _("Medias"), {
                "fields" : (
                    "notices",
                    "photogalleries",
                    "video_libraries",
                    "podcasts",
                )
            }
        ),
    )
    filter_horizontal = ("notices", "photogalleries", "video_libraries", "podcasts",)
    list_display = ("title", "date", "date_modified", "active",)
    list_filter = ("date", "active",)
    search_fields = ("date", "date_modified", "active",)
    ordering = ("date_modified", "date", "active",)
    readonly_fields = ("views", "comments", "likes",)

    class Media:
        css = {
             "all": ("admin/datepicker.css",)
        }

admin.site.register(Event, EventAdmin)

<|endoftext|>


<commit_before>import time
import sys
import random
import subprocess
import logging
import json
import os
import glog

from emuvim.dcemulator.net import DCNetwork
from emuvim.api.rest.rest_api_endpoint import RestApiEndpoint
from emuvim.dcemulator.resourcemodel.upb.simple import UpbSimpleCloudDcRM

from mininet.node import RemoteController
from mininet.node import DefaultController
from mininet.clean import cleanup

def prepareDC(pn_fname, max_cu, max_mu, max_cu_net, max_mu_net):
    """ Prepares physical topology to place chains. """

    # TODO(nodir): explain compute and memory resource consumption models
    # according to https://github.com/sonata-nfv/son-emu/issues/238

    # We use Sonata data center construct to simulate physical servers (just
    # servers hereafter). The reason is that Sonata DC has CPU/RAM resource
    # constraints just like the servers. We also model the links between servers
    # with bandwidth constraints of Sonata switch-to-DC link.

    # The topology we create below is one rack with two servers. The rack has
    # ToR switches (Sonata switch called "tor1"), to place chain VNFs.

    # Similar to the paper story of middlebox-as-a-server, we will put client
    # and server (traffic source and sink) outside the DC.

    # Here is the reason why we do not use Sonata "host" to model the servers.
    # Sonata uses Mininet host construct as-is. Mininet "host" supports only CPU
    # resource constraint. Therefore, we do not use Sonata "host" construct.

    # Unless otherwise specified, we always use "server" for variables and
    # description instead of "DC". This should avoid confusion with terminology.

    # add resource model (rm) to limit cpu/ram available in each server. We
    # create one resource mode and use it for all servers, meaning all of our
    # servers are homogeneous. Create multiple RMs for heterogeneous servers
    # (with different amount of cpu,ram).

    # the cpu, ram resource above are consumed by VNFs with one of these
    # flavors. For some reason memory allocated for tiny flavor is 42 MB,
    # instead of 32 MB in this systems. Other flavors are multipliers of this
    # 42 MB (as expected).
    # "tiny",  {"compute": 0.5, "memory": 32, "disk": 1}
    # "small",  {"compute": 1.0, "memory": 128, "disk": 20}
    # "medium",  {"compute": 4.0, "memory": 256, "disk": 40}
    # "large",  {"compute": 8.0, "memory": 512, "disk": 80}
    # "xlarge",  {"compute": 16.0, "memory": 1024, "disk": 160}
    #
    # Note that all these container VNFs need at least 500 MB of memory to be
    # able to work. Firewall in particular, runs OVS, needs more than 1 GB to be
    # able to process packets. If you do not allocate sufficient CPU, system
    # behaves badly. In most cases all physical cores gets pinned (probably
    # because of the contention between OVS and cgroup mem limitation) and
    # Sonata VM OOM killer starts killing random processes.

    net = DCNetwork(controller=DefaultController, monitor=False,
                    dc_emulation_max_cpu=max_cu_net,
                    dc_emulation_max_mem=max_mu_net,
                    enable_learning=False)

    # Read physical topology from file. Note that cpu/ram/bandwidth numbers in
    # .pn file does not really matter. We just use server names and server to
    # ToR connectivity information from the .pn file.
    with open(pn_fname) as data_file:
        data = json.load(data_file)

    glog.info('read data center description from JSON file %s' % data['Servers'])

    dcs = {}
    for name, props in data['Servers'].iteritems():
        dcs[name] = net.addDatacenter(name)

    rms = {}
    for name, props in data['Servers'].iteritems():
        rms[name] = UpbSimpleCloudDcRM(max_cu, max_mu)

    for dc_name, dc_obj in dcs.iteritems():
        dc_obj.assignResourceModel(rms[dc_name])
        glog.info('assigned resource model %s to %s', id(rms[dc_name]), dc_name)

    glog.info('PN read from JSON file %s' % data['PN'])

    tors = {}

    # Extract ToR switch names. ToR are the ones not listed as 'Servers'. Note
    # that this code is generic for multi-rack topology, which can have fields
    # like
    # ['server-name', 'tor-name', bw], ['tor-name', 'aggr-sw-name', bw],
    # ['aggr-sw-name', 'gw-sw-name', bw] ...
    # From this example, we need to include 'tor-name', 'aggr-sw-name', and
    # 'gw-sw-name' to tors list.
    # iterate through each PN item and assign None object to the tor. Later,
    # None will be replaced with the ToR object (see net.addSwitch() below).
    for pn_item in data['PN']:
        # check if this item is not 'Server' and also is not already included to
        # tors list.
        if (pn_item[0] not in data['Servers'].keys()) and (
                pn_item[0] not in tors.keys()):
            # glog.info(pn_item[0])
            tors[pn_item[0]] = None

        # same comment as above, but for the second item
        if (pn_item[1] not in data['Servers'].keys()) and (
                pn_item[1] not in tors.keys()):
            # glog.info(pn_item[1])
            tors[pn_item[1]] = None

    # connect ToR switches and DC per PN topology
    for tor_name in tors.keys():
        tors[tor_name] = net.addSwitch(tor_name)

    glog.info('ToR switch name and objects: %s' % tors)

    for pn_item in data['PN']:
        net.addLink(dcs[pn_item[0]], tors[pn_item[1]])

    glog.info('added link from DCs to ToR')

    # create REST API endpoint
    api = RestApiEndpoint("0.0.0.0", 5001)

    # connect API endpoint to containernet
    api.connectDCNetwork(net)

    # connect data centers to the endpoint
    for dc_name, dc_obj in dcs.iteritems():
        api.connectDatacenter(dc_obj)

    return (net, api, dcs, tors)

def get_placement(pn_fname, vn_fname, algo):
    """ Does chain placement with NetSolver and returns the output. """

    if algo == 'netsolver':
        glog.info('using NetSolver for chain allocation')

        out_fname = '/tmp/ns_out.json'
        # cmd = "export PYTHONHASHSEED=1 && python3 %s %s %s --output %s --no-repeat" % (
        cmd = "export PYTHONHASHSEED=1 && python3 %s %s %s --output %s %s" % (
            "../../../monosat_datacenter/src/netsolver_nfv.py", pn_fname,
            vn_fname, out_fname, '--max-resource 4')
        execStatus = subprocess.call(cmd, shell=True)
        glog.info('returned %d from %s (0 is success)', execStatus, cmd)

        if execStatus == 1:
            glog.info("allocation failed")
            return execStatus

        glog.info("allocation succeeded")
        # Read physical topology from file.
        with open(out_fname) as data_file:
            allocs = json.load(data_file)
        return allocs
    elif algo == 'packing':
        glog.info('using packing for chain allocation')

        out_fname = '/tmp/ns_out.json'
        cmd = "export PYTHONHASHSEED=1 && python3 %s %s %s %s %s %s" % (
            "../../../monosat_datacenter/src/simple_nfv.py", pn_fname,
            vn_fname, "--output", out_fname, "--locality")
        execStatus = subprocess.call(cmd, shell=True)
        glog.info('returned %d from %s (0 is success)', execStatus, cmd)

        if execStatus == 1:
            glog.info("allocation failed")
            return execStatus

        glog.info("allocation succeeded")
        # Read physical topology from file.
        with open(out_fname) as data_file:
            allocs = json.load(data_file)
        return allocs

    elif algo == 'random':
        glog.info('using random for chain allocation')

        out_fname = '/tmp/ns_out.json'
        cmd = "export PYTHONHASHSEED=1 && python3 %s %s %s %s %s %s" % (
            "../../../monosat_datacenter/src/simple_nfv.py", pn_fname,
            vn_fname, "--output", out_fname, "--random")
        execStatus = subprocess.call(cmd, shell=True)
        glog.info('returned %d from %s (0 is success)', execStatus, cmd)

        if execStatus == 1:
            glog.info("allocation failed")
            return execStatus

        glog.info("allocation succeeded")
        # Read physical topology from file.
        with open(out_fname) as data_file:
            allocs = json.load(data_file)
        return allocs

    else:
        glog.error('ERROR: unsupported allocation algorithm: %s' % algo)
        sys.exit(1)

    # "allocs" has the following format
    # {'allocation_0':
    #   {'assignment': [['fw', 'chain-server0'],
    #                   ['ids', 'chain-server1']],
    #   {'bandwidth': []},
    # 'allocation_1': ...}
    # }

    # glog.info('allocations: %s' % allocations)
    return allocs

def allocate_chains(dcs, allocs):
    """ Create chains by assigning VNF to their respective server. """

    fl = "large"
    chain_index = 0
    # vnfs holds array of each VNF type
    vnfs = {'source': [], 'nat': [], 'fw': [], 'ids': [], 'vpn': [], 'sink': []}

    # iterate over each allocation and create each chain
    for alloc_name, chain_mapping in allocs.iteritems():
        if not alloc_name.startswith('allocation'):
            glog.info('not an allocation, but metadata: %s', alloc_name)
            continue
        glog.info('started allocating chain: %d', chain_index)
        try:
            glog.info('Chain Mapping: %s', chain_mapping['assignment'])
        except TypeError:
            glog.info('Chain Mapping: %f', chain_mapping)

        # iterate over each chain and create chain VNFs by placing it on an
        # appropriate server (such as chosen by NetSolver)..
        for vnf_mapping in chain_mapping['assignment']:
            glog.info('vnf_mapping = %s', vnf_mapping)
            vnf_name = vnf_mapping[0]
            server_name = vnf_mapping[1]
            vnf_prefix = 'chain%d' % chain_index
            vnf_id = '%s-%s' % (vnf_prefix, vnf_name)
            vnf_obj = None
            glog.info('creating %s on %s', vnf_id, server_name)

            if vnf_name == 'source':
                # create client with one interface
                vnf_obj = dcs[server_name].startCompute(vnf_id,
                                                        image='knodir/client', flavor_name="source",
                                                        network=[{'id': 'intf1', 'ip': '10.0.0.2/24'}])
                vnfs[vnf_name].append({vnf_id: vnf_obj})

            elif vnf_name == 'nat':
                # create NAT VNF with two interfaces. Its 'input'
                # interface faces the client and output interface the server VNF.
                vnf_obj = dcs[server_name].startCompute(vnf_id,
                                                        image='knodir/nat', flavor_name="nat",
                                                        network=[{'id': 'input', 'ip': '10.0.0.3/24'},
                                                                 {'id': 'output', 'ip': '10.0.1.4/24'}])
                vnfs[vnf_name].append({vnf_id: vnf_obj})

            elif vnf_name == 'fw':
                # create fw VNF with three interfaces. 'input' interface for
                # 'nat', 'output-ids' interface for 'ids' VNF, and 'output-vpn'
                # for VPN VNF. All three interfaces are bridged to ovs1 bridge.
                # knodir/sonata-fw-vnf:alloc image  has OVS and Ryu controller,
                # and is specifically designed for e2-allocations experiment.
                # node-upgrade experiment requires knodir/sonata-fw-vnf:upgrade
                # image as it has an additional interface for ids2.
                vnf_obj = dcs[server_name].startCompute(vnf_id,
                                                        image='knodir/sonata-fw-fixed', flavor_name="fw",
                                                        network=[{'id': 'input', 'ip': '10.0.1.5/24'},
                                                                 {'id': 'output-ids', 'ip': '10.0.1.61/24'},
                                                                 {'id': 'output-vpn', 'ip': '10.0.1.62/24'}])
                vnfs[vnf_name].append({vnf_id: vnf_obj})

            elif vnf_name == 'ids':
                # create ids VNF with two interfaces. 'input' interface for 'fw' and
                # 'output' interface for the 'server' VNF.
                vnf_obj = dcs[server_name].startCompute(vnf_id,
                                                        image='knodir/snort-trusty', flavor_name="ids",
                                                        network=[{'id': 'input', 'ip': '10.0.1.70/24'},
                                                                 {'id': 'output', 'ip': '10.0.1.80/24'}])
                vnfs[vnf_name].append({vnf_id: vnf_obj})

            elif vnf_name == 'vpn':
                # create VPN VNF with two interfaces. Its 'input'
                # interface faces the client and output interface the server VNF.
                vnf_obj = dcs[server_name].startCompute(vnf_id,
                                                        image='knodir/vpn-client', flavor_name="vpn",
                                                        network=[{'id': 'input-ids', 'ip': '10.0.1.91/24'},
                                                                 {'id': 'input-fw', 'ip': '10.0.1.92/24'},
                                                                 {'id': 'output', 'ip': '10.0.10.2/24'}])
                vnfs[vnf_name].append({vnf_id: vnf_obj})

            elif vnf_name == 'sink':
                # create server VNF with one interface. Do not change assigned 10.0.10.10/24
                # address of the server. It is the address VPN clients use to connect to the
                # server and this address is hardcoded inside client.ovpn of the vpn-client
                # Docker image. We also remove the injected routing table entry for this
                # address. So, if you change this address make sure it is changed inside
                # client.ovpn file as well as subprocess mn.vpn route injection call below.
                vnf_obj = dcs[server_name].startCompute(vnf_id,
                                                        image='knodir/vpn-server', flavor_name="sink",
                                                        network=[{'id': 'intf2', 'ip': '10.0.10.10/24'}])
                vnfs[vnf_name].append({vnf_id: vnf_obj})

            else:
                glog.error('ERROR: unknown VNF type: %s', vnf_name)
                sys.exit(1)

            glog.info('successfully created VNF: %s', vnf_id)

        glog.info('successfully created chain: %d', chain_index)
        chain_index += 1
    return vnfs

def plumb_chains(net, vnfs, num_of_chains):
    # vnfs have the following format:
    # {fw: [{chain0_fw: obj}, {chain1_fw: obj}, ...],
    #  nat: [{chain0_nat: obj}, {chain1_nat: obj}, ...],
    #  ...}

    # execute /start.sh script inside all firewalls. It starts Ryu
    # controller and OVS with proper configuration.
    vnf_index = 0
    for vnf_name_and_obj in vnfs['fw']:
        vnf_name = vnf_name_and_obj.keys()[0]
        cmd = 'sudo docker exec mn.%s /root/start.sh %s &' % (vnf_name, vnf_index)
        execStatus = subprocess.call(cmd, shell=True)
        glog.info('returned %d from %s (0 is success)', execStatus, cmd)
        vnf_index = vnf_index + 1

    # glog.info('> sleeping 10s to let ryu controller initialize properly')
    # time.sleep(10)
    # glog.info('< wait complete')
    # glog.info('fw start done')

    # execute /start.sh script inside ids image. It bridges input and output
    # interfaces with br0, and starts ids process listering on br0.
    for vnf_name_and_obj in vnfs['ids']:
        vnf_name = vnf_name_and_obj.keys()[0]
        cmd = 'sudo docker exec -i mn.%s /bin/bash -c "sh /start.sh"' % vnf_name
        execStatus = subprocess.call(cmd, shell=True)
        glog.info('returned %d from %s (0 is success)', execStatus, cmd)

    # execute /start.sh script inside nat image. It attaches both input
    # and output interfaces to OVS bridge to enable packet forwarding.
    for vnf_name_and_obj in vnfs['nat']:
        vnf_name = vnf_name_and_obj.keys()[0]
        cmd = 'sudo docker exec -i mn.%s /bin/bash /start.sh' % vnf_name
        execStatus = subprocess.call(cmd, shell=True)
        glog.info('returned %d from %s (0 is success)', execStatus, cmd)

    glog.info('> sleeping 5s to let fw, ids, nat initialize properly...')
    time.sleep(5)
    glog.info('< 5s wait complete')
    glog.info('start VNF chaining')

    # chain 'client <-> nat <-> fw <-> ids <-> vpn <-> server'
    for chain_index in range(num_of_chains):
        pair_src_name = vnfs['source'][chain_index].keys()[0]
        pair_dst_name = vnfs['nat'][chain_index].keys()[0]
        res = net.setChain(pair_src_name, pair_dst_name, 'intf1', 'input',
                           bidirectional=True, cmd='add-flow')
        glog.info('chain(%s, %s) output: %s', pair_src_name, pair_dst_name, res)

        pair_src_name = vnfs['nat'][chain_index].keys()[0]
        pair_dst_name = vnfs['fw'][chain_index].keys()[0]
        res = net.setChain(pair_src_name, pair_dst_name, 'output', 'input',
                           bidirectional=True, cmd='add-flow')
        glog.info('chain(%s, %s) output: %s', pair_src_name, pair_dst_name, res)

        pair_src_name = vnfs['fw'][chain_index].keys()[0]
        pair_dst_name = vnfs['ids'][chain_index].keys()[0]
        res = net.setChain(pair_src_name, pair_dst_name, 'output-ids', 'input',
                           bidirectional=True, cmd='add-flow')
        glog.info('chain(%s, %s) output: %s', pair_src_name, pair_dst_name, res)

        pair_src_name = vnfs['fw'][chain_index].keys()[0]
        pair_dst_name = vnfs['vpn'][chain_index].keys()[0]
        res = net.setChain(pair_src_name, pair_dst_name, 'output-vpn', 'input-fw',
                           bidirectional=True, cmd='add-flow')
        glog.info('chain(%s, %s) output: %s', pair_src_name, pair_dst_name, res)

        pair_src_name = vnfs['ids'][chain_index].keys()[0]
        pair_dst_name = vnfs['vpn'][chain_index].keys()[0]
        res = net.setChain(pair_src_name, pair_dst_name, 'output', 'input-ids',
                           bidirectional=True, cmd='add-flow')
        glog.info('chain(%s, %s) output: %s', pair_src_name, pair_dst_name, res)

        pair_src_name = vnfs['vpn'][chain_index].keys()[0]
        pair_dst_name = vnfs['sink'][chain_index].keys()[0]
        res = net.setChain(pair_src_name, pair_dst_name, 'output', 'intf2',
                           bidirectional=True, cmd='add-flow')
        glog.info('chain(%s, %s) output: %s', pair_src_name, pair_dst_name, res)

    cmds = []

    for vnf_name_and_obj in vnfs['sink']:
        vnf_name = vnf_name_and_obj.keys()[0]
        cmds.append('sudo docker exec -i mn.%s /bin/bash -c "route add -net 10.0.0.0/24 dev intf2"' % vnf_name)
        # start openvpn server and related services inside openvpn server
        cmds.append('sudo docker exec -i mn.%s /bin/bash -c "ufw enable"' % vnf_name)
        # open iperf3 port (5201) on firewall (ufw)
        cmds.append('sudo docker exec -i mn.%s /bin/bash -c "ufw allow 5201"' % vnf_name)
        cmds.append('sudo docker exec -i mn.%s /bin/bash -c "ufw status"' % vnf_name)
        cmds.append('sudo docker exec -i mn.%s /bin/bash -c "service openvpn start"' % vnf_name)
        cmds.append('sudo docker exec -i mn.%s /bin/bash -c "service openvpn status"' % vnf_name)
        cmds.append('sudo docker exec -i mn.%s /bin/bash -c "service rsyslog start"' % vnf_name)
        cmds.append('sudo docker exec -i mn.%s /bin/bash -c "service rsyslog status"' % vnf_name)

    for vnf_name_and_obj in vnfs['vpn']:
        vnf_name = vnf_name_and_obj.keys()[0]
        # execute /start.sh script inside VPN client to connect to VPN server.
        cmds.append('sudo docker exec -i mn.%s /bin/bash /start.sh &' % vnf_name)

    for cmd in cmds:
        execStatus = subprocess.call(cmd, shell=True)
        glog.info('returned %d from %s (0 is success)' % (execStatus, cmd))
    cmds[:] = []

    glog.info('> sleeping 5s to let VPN client initialize...')
    time.sleep(5)
    glog.info('< 5s wait complete')
    glog.info('VPN client VNF started')

    for vnf_name_and_obj in vnfs['nat']:
        vnf_name = vnf_name_and_obj.keys()[0]
        # rewrite NAT VNF MAC addresses for tcpreplay
        cmds.append('sudo docker exec -i mn.%s /bin/bash -c "ifconfig input hw ether 00:00:00:00:00:02"' % vnf_name)
        cmds.append('sudo docker exec -i mn.%s /bin/bash -c "route add -net 10.0.10.0/24 dev output"' % vnf_name)
        cmds.append('sudo docker exec -i mn.%s /bin/bash -c "ip route add 10.8.0.0/24 dev output"' % vnf_name)

    for vnf_name_and_obj in vnfs['vpn']:
        vnf_name = vnf_name_and_obj.keys()[0]
        cmds.append('sudo docker exec -i mn.%s /bin/bash -c "route add -net 10.0.0.0/24 dev input-ids"' % vnf_name)
        cmds.append('sudo docker exec -i mn.%s /bin/bash -c "ip route del 10.0.10.10/32"' % vnf_name)

    for vnf_name_and_obj in vnfs['source']:
        vnf_name = vnf_name_and_obj.keys()[0]
        # rewrite client VNF MAC addresses for tcpreplay
        cmds.append('sudo docker exec -i mn.%s /bin/bash -c "ifconfig intf1 hw ether 00:00:00:00:00:01"' % vnf_name)
        # manually chain routing table entries on VNFs
        cmds.append('sudo docker exec -i mn.%s /bin/bash -c "route add -net 10.0.0.0/16 dev intf1"' % vnf_name)
        cmds.append('sudo docker exec -i mn.%s /bin/bash -c "route add -net 10.8.0.0/24 dev intf1"' % vnf_name)
        cmds.append('sudo docker exec -i mn.%s /bin/bash -c "ping -i 0.1 -c 10 10.0.10.10"' % vnf_name)
        cmds.append('sudo docker exec -i mn.%s /bin/bash -c "ping -i 0.1 -c 10 10.8.0.1"' % vnf_name)

    for cmd in cmds:
        execStatus = subprocess.call(cmd, shell=True)
        glog.info('returned %d from %s (0 is success)' % (execStatus, cmd))
    cmds[:] = []

    for chain_index in range(num_of_chains):
        src_vnf_name = vnfs['source'][chain_index].keys()[0]
        dst_vnf_name = vnfs['sink'][chain_index].keys()[0]
        src_vnf_obj = vnfs['source'][chain_index][src_vnf_name]
        dst_vnf_obj = vnfs['sink'][chain_index][dst_vnf_name]

        ping_res = net.ping([src_vnf_obj, dst_vnf_obj], timeout=5)

        glog.info('ping %s -> %s. Packet drop %s%%',
                  src_vnf_name, dst_vnf_name, ping_res)

if __name__ == '__main__':
    random.seed(1)
    logging.basicConfig(level=logging.DEBUG)

    # vn_fname = "../topologies/e2-chain-4vnfs-8wa.vn.json"
    # e2-nss-1rack-8servers
    # pn_fname = "../topologies/e2-nss-1rack-8servers.pn.json"
    # net, api, dcs, tors = prepareDC(pn_fname, 8, 3584, 64, 28672)

    # vn_fname = "../topologies/e2-chain-4vnfs-8wa.vn.json"
    # e2-azure-1rack-24servers
    # pn_fname = "../topologies/e2-azure-1rack-24servers.pn.json"
    # net, api, dcs, tors = prepareDC(pn_fname, 20, 17408, 1200, 417792)

    # vn_fname = "../topologies/e2-chain-4vnfs-8wa.vn.json"
    # e2-azure-1rack-48servers (or 50 servers)
    # pn_fname = "../topologies/e2-azure-1rack-48servers.pn.json"
    # net, api, dcs, tors = prepareDC(pn_fname, 10, 8704, 600, 417792)
    # max_cu_net = 600 => 10 dc_cu x 60 physical cores

    if sys.argv[1] == "nss":
        # e2-nss-1rack-8servers
        pn_fname = "../topologies/e2-nss-1rack-8servers.pn.json"
        vn_fname = "../topologies/e2-chain-4vnfs-8wa.vn.json"
    else:
        # e2-azure-1rack-50servers
        vn_fname = "../topologies/e2-chain-4vnfs-50wa.vn.json"
        pn_fname = "../topologies/e2-azure-1rack-50servers.pn.json"
    algos = ['daisy', 'random', 'packing']
    bandwidths = [10]
    if sys.argv[1] == "nss":
        net, api, dcs, tors = prepareDC(pn_fname, 8, 3584, 64, 28672)
    else:
        net, api, dcs, tors = prepareDC(pn_fname, 10, 8704, 1000, 417792)

    # start API and containernet
    api.start()
    net.start()

    # allocs = get_placement(pn_fname, vn_fname, algos[0])  # netsolver
    # allocs = get_placement(pn_fname, vn_fname, algos[1])  # random
    allocs = get_placement(pn_fname, vn_fname, algos[2])  # packing
    num_of_chains = 0
    for alloc in allocs:
        if alloc.startswith('allocation'):
            num_of_chains += 1

    glog.info('allocs: %s; num_of_chains = %d', allocs, num_of_chains)
    # sys.exit(0)
    # allocate chains by placing them on appropriate servers
    vnfs = allocate_chains(dcs, allocs)
    # configure the datapath on chains to push packets through them
    plumb_chains(net, vnfs, num_of_chains)
    glog.info('successfully plumbed %d chains', num_of_chains)
    glog.info('Chain setup done. You should see the terminal now.')

    net.CLI()
    net.stop()

    cleanup()
    os.system("sudo ../clean-stale.sh")
<commit_msg>Delete e2-allocation<commit_after><|endoftext|>


<commit_before>from django.shortcuts import render
from django.http import HttpResponse

def groups_list(request):
    return render(request, 'students/groups_list.html', {})

def groups_add(request):
    return HttpResponse('
Group Add Form
')

def groups_edit(request, gid):
    return HttpResponse('
Edit Groups %s
' % gid)

def groups_delete(request, gid):
    return HttpResponse('
Delete Group %s
' % gid)<commit_msg>update groups list view<commit_after>from django.shortcuts import render
from django.http import HttpResponse
from django.core.paginator import Paginator, EmptyPage, PageNotAnInteger

from ..models import Group

def groups_list(request):
    groups = Group.objects.all()

    # try to order groups list
    order_by = request.GET.get('order_by', '')
    if order_by in ('id', 'title', 'leader'):
        groups = groups.order_by(order_by)
        if request.GET.get('reverse', '') == '1':
            groups = groups.reverse()

    # order by title if page loads first time
    else:
        groups = groups.order_by('title')

    # paginate groups
    paginator = Paginator(groups, 3)
    page = request.GET.get('page')
    try:
        groups = paginator.page(page)
    except PageNotAnInteger:
        # if page is not an integer, deliver first page
        groups = paginator.page(1)
    except EmptyPage:
        # if page is oout of range (e.g. 9999), deliver
        # last page of results
        groups = paginator.page(paginator.num_pages)

    return render(request, 'students/groups_list.html',
        {'groups_db': groups})

def groups_add(request):
    return HttpResponse('
Group Add Form
')

def groups_edit(request, gid):
    return HttpResponse('
Edit Groups %s
' % gid)

def groups_delete(request, gid):
    return HttpResponse('
Delete Group %s
' % gid)<|endoftext|>


<commit_before>import sys, os
stations = {}

stations["skai"] = "http://netradio.live24.gr/skai1003"
stations["somafm-sonicuniverse"] = "http://uwstream2.somafm.com:8604"

def recordStation(stationName, outputName):
#	command = "gst-launch-1.0 uridecodebin uri="+stations["skai"]+" ! tee name=t \
#	t. ! queue ! audioresample ! audio/x-raw, rate=16000 ! wavenc ! filesink location=" + outputName
	command = "gst-launch-1.0 uridecodebin uri=" + stations[stationName]+ " 	! tee name=t \ t. ! queue ! audioresample ! audio/x-raw, rate=16000 ! wavenc ! filesink location="+outputName
#	command = "gst-launch-1.0 souphttpsrc location=" + stations[stationName]+ " 	! tee name=t \ t. ! queue ! audioresample ! audio/x-raw, rate=16000 ! wavenc ! filesink location="+outputName
	os.system(command)

def main(argv):
	recordStation(argv[1], argv[2])
	return 0

if __name__ == '__main__':
	main(sys.argv)
<commit_msg>record changes<commit_after>import sys, os, gi, time
gi.require_version('Gst', '1.0')
from gi.repository import GObject, Gst

GObject.threads_init()
Gst.init(None)

stations = {}
stations["skai"] = "http://netradio.live24.gr/skai1003"
stations["somafm-sonicuniverse"] = "http://uwstream2.somafm.com:8604"

class StreamRecorder():

    def __init__(self):
        self._pipeline = Gst.Pipeline()
        self.uri        = None
        self.filename   = None
        self.samplerate = 44100
        self.listen     = False

    def _build_pipeline(self):

        if not self.uri:
            raise ValueError('Stream URI is not set')

        # 1. Create pipeline's elements
        source        = Gst.ElementFactory.make('uridecodebin', 'input_stream')
        splitter      = Gst.ElementFactory.make('tee', 'splitter')
        recording_bin = self._create_recording_bin() if self.filename else None
        listening_bin = self._create_listening_bin() if self.listen else None

        # 2. Set properties
        source.set_property('uri', self.uri)

        # 3. Add them to the pipeline
        self._pipeline.add(source)
        self._pipeline.add(splitter)
        if recording_bin: self._pipeline.add(recording_bin)
        if listening_bin: self._pipeline.add(listening_bin)

        # 4. Link elements
        # 4.a. uridecodebin has a "sometimes" pad (created after prerolling)
        source.connect('pad-added', lambda src, pad: src.link(splitter))

        # 4.b. tee has "request" pads (we have to ask for their creation)
        #
        # NOTE: Python bindings automatically request pads needed
        #splitter.get_request_pad('src_%u') #--> not needed.
        if recording_bin: splitter.link(recording_bin)
        if listening_bin: splitter.link(listening_bin)

    def _create_recording_bin(self):
        recording_bin = Gst.Bin.new('recording_bin')
        queue     = Gst.ElementFactory.make('queue', None)
        resampler = Gst.ElementFactory.make('audioresample', None)
        caps      = Gst.ElementFactory.make('capsfilter', None)
        encoder   = Gst.ElementFactory.make('wavenc', None)
        sink      = Gst.ElementFactory.make('filesink', None)

        caps.set_property('caps', Gst.Caps.from_string(
            'audio/x-raw, rate={0}'.format(self.samplerate)))
        sink.set_property('location', self.filename)

        recording_bin.add(queue, resampler, caps, encoder, sink)

        # Gst.Element.link_many() not implemented...
        queue.link(resampler)
        resampler.link(caps)
        caps.link(encoder)
        encoder.link(sink)

        recording_bin.add_pad(Gst.GhostPad.new('recording_sink', queue.get_static_pad('sink')))
        return recording_bin

    def _create_listening_bin(self):
        listening_bin = Gst.Bin.new('listening_bin')
        queue = Gst.ElementFactory.make('queue', None)
        sink  = Gst.ElementFactory.make('alsasink', None)

        #listening_bin.add(queue, sink)
        listening_bin.add(queue)
        listening_bin.add(sink)
        queue.link(sink)

        listening_bin.add_pad(Gst.GhostPad.new('listening_sink', queue.get_static_pad('sink')))
        return listening_bin

    def start(self):
        self._build_pipeline()
        self._pipeline.set_state(Gst.State.PLAYING)

    def stop(self):
        self._pipeline.set_state(Gst.State.NULL)

    def bus(self):
        return self._pipeline.get_bus()

def recordStation(stationName, outputName, sleepTime = -1, Listen = False):
#	command = "gst-launch-1.0 uridecodebin uri="+stations["skai"]+" ! tee name=t \
#	t. ! queue ! audioresample ! audio/x-raw, rate=16000 ! wavenc ! filesink location=" + outputName
#	command = "gst-launch-1.0 uridecodebin uri=" + stations[stationName]+ " 	! tee name=t \ t. ! queue ! audioresample ! audio/x-raw, rate=16000 ! wavenc ! filesink location="+outputName
#	command = "gst-launch-1.0 souphttpsrc location=" + stations[stationName]+ " 	! tee name=t \ t. ! queue ! audioresample ! audio/x-raw, rate=16000 ! wavenc ! filesink location="+outputName
#	os.system(command)

	r = StreamRecorder()
	r.uri = stations[stationName]
	tempName = outputName.replace(".wav", "_temp.wav")
	r.filename = tempName
	r.samplerate = 16000
	r.listen = Listen
	r.start()

	print r.bus()
	if sleepTime<=0:
		raw_input('Press [Enter] to stop')
	else:
		time.sleep(sleepTime)
	r.stop()
	os.system("avconv -i " + tempName + " -ar 16000 -ac 1 -y " + outputName)

def main(argv):
	recordStation(argv[1], argv[2], int(argv[3]), False)
	return 0

if __name__ == '__main__':
	main(sys.argv)
<|endoftext|>


<commit_before>import sys

import six
import subunit

class CallWhenProcFinishes(object):
    """Convert a process object to trigger a callback when returncode is set.

    This just wraps the entire object and when the returncode attribute access
    finds a set value, calls the callback.
    """

    def __init__(self, process, callback):
        """Adapt process

        :param process: A subprocess.Popen object.
        :param callback: The process to call when the process completes.
        """
        self._proc = process
        self._callback = callback
        self._done = False

    @property
    def stdin(self):
        return self._proc.stdin

    @property
    def stdout(self):
        return self._proc.stdout

    @property
    def stderr(self):
        return self._proc.stderr

    @property
    def returncode(self):
        result = self._proc.returncode
        if not self._done and result is not None:
            self._done = True
            self._callback()
        return result

    def wait(self):
        return self._proc.wait()

def _iter_streams(input_streams, stream_type, stdin=sys.stdin):
        # Only the first stream declared in a command can be accepted at the
        # moment - as there is only one stdin and alternate streams are not yet
        # configurable in the CLI.
        first_stream_type = input_streams[0]
        if (stream_type != first_stream_type and
            stream_type != first_stream_type[:-1]):
            return
        yield subunit.make_stream_binary(stdin)

def _iter_internal_streams(input_streams, stream_type):
    streams = []
    for in_stream in input_streams:
        if in_stream[0] == stream_type:
            streams.append(in_stream[1])
    for stream_value in streams:
        if getattr(stream_value, 'read', None):
            # NOTE(mtreinish): This is wrong it breaks real streaming. but
            # right now this is needed to workaround the lack of buffers
            yield six.BytesIO(stream_value.read())
        else:
            yield six.BytesIO(stream_value)

def iter_streams(input_streams, stream_type, internal=False):
    """Iterate over all the streams of type stream_type.

    :param stream_type: A simple string such as 'subunit' which matches
        one of the stream types defined for the cmd object this UI is
        being used with.
    :return: A generator of stream objects. stream objects have a read
        method and a close method which behave as for file objects.
    """
    for stream_spec in input_streams:
        if internal:
            _stream_spec = stream_spec[0]
        else:
            _stream_spec = stream_spec

        if '*' in _stream_spec or '?' in _stream_spec or '+' in _stream_spec:
            found = stream_type == _stream_spec[:-1]
        else:
            found = stream_type == _stream_spec
        if found:
            if internal:
                return _iter_internal_streams(input_streams, stream_type)
            return _iter_streams(input_streams, stream_type)
    raise KeyError(stream_type)
<commit_msg>Fix serialized subunit stream output from stestr run<commit_after>import sys

import six
import subunit

from stestr import output

class CallWhenProcFinishes(object):
    """Convert a process object to trigger a callback when returncode is set.

    This just wraps the entire object and when the returncode attribute access
    finds a set value, calls the callback.
    """

    def __init__(self, process, callback):
        """Adapt process

        :param process: A subprocess.Popen object.
        :param callback: The process to call when the process completes.
        """
        self._proc = process
        self._callback = callback
        self._done = False

    @property
    def stdin(self):
        return self._proc.stdin

    @property
    def stdout(self):
        return self._proc.stdout

    @property
    def stderr(self):
        return self._proc.stderr

    @property
    def returncode(self):
        result = self._proc.returncode
        if not self._done and result is not None:
            self._done = True
            self._callback()
        return result

    def wait(self):
        return self._proc.wait()

def _iter_streams(input_streams, stream_type, stdin=sys.stdin):
        # Only the first stream declared in a command can be accepted at the
        # moment - as there is only one stdin and alternate streams are not yet
        # configurable in the CLI.
        first_stream_type = input_streams[0]
        if (stream_type != first_stream_type and
            stream_type != first_stream_type[:-1]):
            return
        yield subunit.make_stream_binary(stdin)

def _iter_internal_streams(input_streams, stream_type):
    streams = []
    for in_stream in input_streams:
        if in_stream[0] == stream_type:
            streams.append(in_stream[1])
    for stream_value in streams:
        if isinstance(stream_value, output.ReturnCodeToSubunit):
            yield stream_value.source.detach()
        elif getattr(stream_value, 'read', None):
            yield stream_value
        else:
            yield six.BytesIO(stream_value)

def iter_streams(input_streams, stream_type, internal=False):
    """Iterate over all the streams of type stream_type.

    :param stream_type: A simple string such as 'subunit' which matches
        one of the stream types defined for the cmd object this UI is
        being used with.
    :return: A generator of stream objects. stream objects have a read
        method and a close method which behave as for file objects.
    """
    for stream_spec in input_streams:
        if internal:
            _stream_spec = stream_spec[0]
        else:
            _stream_spec = stream_spec

        if '*' in _stream_spec or '?' in _stream_spec or '+' in _stream_spec:
            found = stream_type == _stream_spec[:-1]
        else:
            found = stream_type == _stream_spec
        if found:
            if internal:
                return _iter_internal_streams(input_streams, stream_type)
            return _iter_streams(input_streams, stream_type)
    raise KeyError(stream_type)
<|endoftext|>


<commit_before>"""
Test functions for multivariate normal distributions.

"""
from __future__ import division, print_function, absolute_import

from numpy.testing import (assert_almost_equal,
        run_module_suite, assert_allclose, assert_equal, assert_raises)

import numpy
import numpy as np

import scipy.linalg
from scipy.stats._multivariate import _PSD
from scipy.stats import multivariate_normal
from scipy.stats import norm

from scipy.integrate import romb

def test_scalar_values():
    np.random.seed(1234)

    # When evaluated on scalar data, the pdf should return a scalar
    x, mean, cov = 1.5, 1.7, 2.5
    pdf = multivariate_normal.pdf(x, mean, cov)
    assert_equal(pdf.ndim, 0)

    # When evaluated on a single vector, the pdf should return a scalar
    x = np.random.randn(5)
    mean = np.random.randn(5)
    cov = np.abs(np.random.randn(5))  # Diagonal values for cov. matrix
    pdf = multivariate_normal.pdf(x, mean, cov)
    assert_equal(pdf.ndim, 0)

def test_logpdf():
    # Check that the log of the pdf is in fact the logpdf
    np.random.seed(1234)
    x = np.random.randn(5)
    mean = np.random.randn(5)
    cov = np.abs(np.random.randn(5))
    d1 = multivariate_normal.logpdf(x, mean, cov)
    d2 = multivariate_normal.pdf(x, mean, cov)
    assert_allclose(d1, np.log(d2))

def test_rank():
    # Check that the rank is detected correctly.
    np.random.seed(1234)
    n = 4
    mean = np.random.randn(n)
    x = np.random.randn(n)
    for expected_rank in range(1, n+1):
        s = np.random.randn(n, expected_rank)
        cov = np.dot(s, s.T)
        distn = multivariate_normal(mean, cov, allow_singular=True)
        assert_equal(distn.cov_info.rank, expected_rank)

def test_degenerate_distributions():
    for n in range(1, 5):
        mu = np.zeros(n)
        x = np.random.randn(n)
        for k in range(1, n+1):
            s = np.random.randn(k, k)
            cov_kk = np.dot(s, s.T)
            cov_nn = np.zeros((n, n))
            cov_nn[:k, :k] = cov_kk
            distn_kk = multivariate_normal(np.zeros(k), cov_kk,
                    allow_singular=True)
            distn_nn = multivariate_normal(np.zeros(n), cov_nn,
                    allow_singular=True)
            assert_equal(distn_kk.cov_info.rank, k)
            assert_equal(distn_nn.cov_info.rank, k)
            pdf_kk = distn_kk.pdf(x[:k])
            pdf_nn = distn_nn.pdf(x[:n])
            assert_allclose(pdf_kk, pdf_nn)
            logpdf_kk = distn_kk.logpdf(x[:k])
            logpdf_nn = distn_nn.logpdf(x[:n])
            assert_allclose(logpdf_kk, logpdf_nn)

def test_large_pseudo_determinant():
    # Check that large pseudo-determinants are handled appropriately.

    # Construct a singular diagonal covariance matrix
    # whose pseudo determinant overflows double precision.
    large_total_log = 1000.0
    npos = 100
    nzero = 2
    large_entry = np.exp(large_total_log / npos)
    n = npos + nzero
    cov = np.zeros((n, n), dtype=float)
    np.fill_diagonal(cov, large_entry)
    cov[-nzero:, -nzero:] = 0

    # Check some determinants.
    assert_equal(scipy.linalg.det(cov), 0)
    assert_equal(scipy.linalg.det(cov[:npos, :npos]), np.inf)

    # np.linalg.slogdet is only available in numpy 1.6+
    # but scipy currently supports numpy 1.5.1.
    #assert_allclose(np.linalg.slogdet(cov[:npos, :npos]), (1, large_total_log))

    # Check the pseudo-determinant.
    psd = _PSD(cov)
    assert_allclose(psd.log_pdet, large_total_log)

def test_broadcasting():
    np.random.seed(1234)
    n = 4

    # Construct a random covariance matrix.
    data = np.random.randn(n, n)
    cov = np.dot(data, data.T)
    mean = np.random.randn(n)

    # Construct an ndarray which can be interpreted as
    # a 2x3 array whose elements are random data vectors.
    X = np.random.randn(2, 3, n)

    # Check that multiple data points can be evaluated at once.
    for i in range(2):
        for j in range(3):
            actual = multivariate_normal.pdf(X[i, j], mean, cov)
            desired = multivariate_normal.pdf(X, mean, cov)[i, j]
            assert_allclose(actual, desired)

def test_normal_1D():
    # The probability density function for a 1D normal variable should
    # agree with the standard normal distribution in scipy.stats.distributions
    x = np.linspace(0, 2, 10)
    mean, cov = 1.2, 0.9
    scale = cov**0.5
    d1 = norm.pdf(x, mean, scale)
    d2 = multivariate_normal.pdf(x, mean, cov)
    assert_allclose(d1, d2)

def test_marginalization():
    # Integrating out one of the variables of a 2D Gaussian should
    # yield a 1D Gaussian
    mean = np.array([2.5, 3.5])
    cov = np.array([[.5, 0.2], [0.2, .6]])
    n = 2**8 + 1  # Number of samples
    delta = 6 / (n - 1)  # Grid spacing

    v = np.linspace(0, 6, n)
    xv, yv = np.meshgrid(v, v)
    pos = np.empty((n, n, 2))
    pos[:, :, 0] = xv
    pos[:, :, 1] = yv
    pdf = multivariate_normal.pdf(pos, mean, cov)

    # Marginalize over x and y axis
    margin_x = romb(pdf, delta, axis=0)
    margin_y = romb(pdf, delta, axis=1)

    # Compare with standard normal distribution
    gauss_x = norm.pdf(v, loc=mean[0], scale=cov[0, 0]**0.5)
    gauss_y = norm.pdf(v, loc=mean[1], scale=cov[1, 1]**0.5)
    assert_allclose(margin_x, gauss_x, rtol=1e-2, atol=1e-2)
    assert_allclose(margin_y, gauss_y, rtol=1e-2, atol=1e-2)

def test_frozen():
    # The frozen distribution should agree with the regular one
    np.random.seed(1234)
    x = np.random.randn(5)
    mean = np.random.randn(5)
    cov = np.abs(np.random.randn(5))
    norm_frozen = multivariate_normal(mean, cov)
    assert_allclose(norm_frozen.pdf(x), multivariate_normal.pdf(x, mean, cov))
    assert_allclose(norm_frozen.logpdf(x),
                    multivariate_normal.logpdf(x, mean, cov))

def test_pseudodet_pinv():
    # Make sure that pseudo-inverse and pseudo-det agree on cutoff

    # Assemble random covariance matrix with large and small eigenvalues
    np.random.seed(1234)
    n = 7
    x = np.random.randn(n, n)
    cov = np.dot(x, x.T)
    s, u = scipy.linalg.eigh(cov)
    s = 0.5 * np.ones(n)
    s[0] = 1.0
    s[-1] = 1e-7
    cov = np.dot(u, np.dot(np.diag(s), u.T))

    # Set cond so that the lowest eigenvalue is below the cutoff
    cond = 1e-5
    psd = _PSD(cov, cond=cond)
    psd_pinv = _PSD(psd.pinv, cond=cond)

    # Check that the log pseudo-determinant agrees with the sum
    # of the logs of all but the smallest eigenvalue
    assert_allclose(psd.log_pdet, np.sum(np.log(s[:-1])))
    # Check that the pseudo-determinant of the pseudo-inverse
    # agrees with 1 / pseudo-determinant
    assert_allclose(-psd.log_pdet, psd_pinv.log_pdet)

def test_exception_nonsquare_cov():
    cov = [[1, 2, 3], [4, 5, 6]]
    assert_raises(ValueError, _PSD, cov)

def test_exception_nonfinite_cov():
    cov_nan = [[1, 0], [0, np.nan]]
    assert_raises(ValueError, _PSD, cov_nan)
    cov_inf = [[1, 0], [0, np.inf]]
    assert_raises(ValueError, _PSD, cov_inf)

def test_exception_non_psd_cov():
    cov = [[1, 0], [0, -1]]
    assert_raises(ValueError, _PSD, cov)

def test_exception_singular_cov():
    np.random.seed(1234)
    x = np.random.randn(5)
    mean = np.random.randn(5)
    cov = np.ones((5, 5))
    e = np.linalg.LinAlgError
    assert_raises(e, multivariate_normal, mean, cov)
    assert_raises(e, multivariate_normal.pdf, x, mean, cov)
    assert_raises(e, multivariate_normal.logpdf, x, mean, cov)

def test_R_values():
    # Compare the multivariate pdf with some values precomputed
    # in R version 3.0.1 (2013-05-16) on Mac OS X 10.6.

    # The values below were generated by the following R-script:
    # > library(mnormt)
    # > x <- seq(0, 2, length=5)
    # > y <- 3*x - 2
    # > z <- x + cos(y)
    # > mu <- c(1, 3, 2)
    # > Sigma <- matrix(c(1,2,0,2,5,0.5,0,0.5,3), 3, 3)
    # > r_pdf <- dmnorm(cbind(x,y,z), mu, Sigma)
    r_pdf = np.array([0.0002214706, 0.0013819953, 0.0049138692,
                      0.0103803050, 0.0140250800])

    x = np.linspace(0, 2, 5)
    y = 3 * x - 2
    z = x + np.cos(y)
    r = np.array([x, y, z]).T

    mean = np.array([1, 3, 2], 'd')
    cov = np.array([[1, 2, 0], [2, 5, .5], [0, .5, 3]], 'd')

    pdf = multivariate_normal.pdf(r, mean, cov)
    assert_allclose(pdf, r_pdf, atol=1e-10)

def test_rvs_shape():
    # Check that rvs parses the mean and covariance correctly, and returns
    # an array of the right shape
    N = 300
    d = 4
    sample = multivariate_normal.rvs(mean=np.zeros(d), cov=1, size=N)
    assert_equal(sample.shape, (N, d))

    sample = multivariate_normal.rvs(mean=None,
                                     cov=np.array([[2, .1], [.1, 1]]),
                                     size=N)
    assert_equal(sample.shape, (N, 2))

    u = multivariate_normal(mean=0, cov=1)
    sample = u.rvs(N)
    assert_equal(sample.shape, (N, ))

def test_large_sample():
    # Generate large sample and compare sample mean and sample covariance
    # with mean and covariance matrix.

    np.random.seed(2846)

    n = 3
    mean = np.random.randn(n)
    M = np.random.randn(n, n)
    cov = np.dot(M, M.T)
    size = 5000

    sample = multivariate_normal.rvs(mean, cov, size)

    assert_allclose(numpy.cov(sample.T), cov, rtol=1e-1)
    assert_allclose(sample.mean(0), mean, rtol=1e-1)

def test_entropy():
    np.random.seed(2846)

    n = 3
    mean = np.random.randn(n)
    M = np.random.randn(n, n)
    cov = np.dot(M, M.T)

    rv = multivariate_normal(mean, cov)

    # Check that frozen distribution agrees with entropy function
    assert_almost_equal(rv.entropy(), multivariate_normal.entropy(mean, cov))
    # Compare entropy with manually computed expression involving
    # the sum of the logs of the eigenvalues of the covariance matrix
    eigs = np.linalg.eig(cov)[0]
    desired = 1/2 * (n * (np.log(2*np.pi) + 1) + np.sum(np.log(eigs)))
    assert_almost_equal(desired, rv.entropy())

if __name__ == "__main__":
    run_module_suite()
<commit_msg>TST: more tests for singular covariance matrices<commit_after>"""
Test functions for multivariate normal distributions.

"""
from __future__ import division, print_function, absolute_import

from numpy.testing import (assert_almost_equal,
        run_module_suite, assert_allclose, assert_equal, assert_raises)

import numpy
import numpy as np

import scipy.linalg
from scipy.stats._multivariate import _PSD
from scipy.stats import multivariate_normal
from scipy.stats import norm

from scipy.integrate import romb

def test_scalar_values():
    np.random.seed(1234)

    # When evaluated on scalar data, the pdf should return a scalar
    x, mean, cov = 1.5, 1.7, 2.5
    pdf = multivariate_normal.pdf(x, mean, cov)
    assert_equal(pdf.ndim, 0)

    # When evaluated on a single vector, the pdf should return a scalar
    x = np.random.randn(5)
    mean = np.random.randn(5)
    cov = np.abs(np.random.randn(5))  # Diagonal values for cov. matrix
    pdf = multivariate_normal.pdf(x, mean, cov)
    assert_equal(pdf.ndim, 0)

def test_logpdf():
    # Check that the log of the pdf is in fact the logpdf
    np.random.seed(1234)
    x = np.random.randn(5)
    mean = np.random.randn(5)
    cov = np.abs(np.random.randn(5))
    d1 = multivariate_normal.logpdf(x, mean, cov)
    d2 = multivariate_normal.pdf(x, mean, cov)
    assert_allclose(d1, np.log(d2))

def test_rank():
    # Check that the rank is detected correctly.
    np.random.seed(1234)
    n = 4
    mean = np.random.randn(n)
    x = np.random.randn(n)
    for expected_rank in range(1, n+1):
        s = np.random.randn(n, expected_rank)
        cov = np.dot(s, s.T)
        distn = multivariate_normal(mean, cov, allow_singular=True)
        assert_equal(distn.cov_info.rank, expected_rank)

def _sample_orthonormal_matrix(n):
    M = np.random.randn(n, n)
    u, s, v = scipy.linalg.svd(M)
    return u

def test_degenerate_distributions():
    for n in range(1, 5):
        mu = np.zeros(n)
        x = np.random.randn(n)
        for k in range(1, n+1):

            # Sample a small covariance matrix.
            s = np.random.randn(k, k)
            cov_kk = np.dot(s, s.T)

            # Embed the small covariance matrix into a larger low rank matrix.
            cov_nn = np.zeros((n, n))
            cov_nn[:k, :k] = cov_kk

            # Define a rotation of the larger low rank matrix.
            u = _sample_orthonormal_matrix(n)
            cov_rr = np.dot(u, np.dot(cov_nn, u.T))
            y = np.dot(u, x)

            # Check some identities.
            distn_kk = multivariate_normal(np.zeros(k), cov_kk,
                    allow_singular=True)
            distn_nn = multivariate_normal(np.zeros(n), cov_nn,
                    allow_singular=True)
            distn_rr = multivariate_normal(np.zeros(n), cov_rr,
                    allow_singular=True)
            assert_equal(distn_kk.cov_info.rank, k)
            assert_equal(distn_nn.cov_info.rank, k)
            assert_equal(distn_rr.cov_info.rank, k)
            pdf_kk = distn_kk.pdf(x[:k])
            pdf_nn = distn_nn.pdf(x)
            pdf_rr = distn_rr.pdf(y)
            assert_allclose(pdf_kk, pdf_nn)
            assert_allclose(pdf_kk, pdf_rr)
            logpdf_kk = distn_kk.logpdf(x[:k])
            logpdf_nn = distn_nn.logpdf(x)
            logpdf_rr = distn_rr.logpdf(y)
            assert_allclose(logpdf_kk, logpdf_nn)
            assert_allclose(logpdf_kk, logpdf_rr)

def test_large_pseudo_determinant():
    # Check that large pseudo-determinants are handled appropriately.

    # Construct a singular diagonal covariance matrix
    # whose pseudo determinant overflows double precision.
    large_total_log = 1000.0
    npos = 100
    nzero = 2
    large_entry = np.exp(large_total_log / npos)
    n = npos + nzero
    cov = np.zeros((n, n), dtype=float)
    np.fill_diagonal(cov, large_entry)
    cov[-nzero:, -nzero:] = 0

    # Check some determinants.
    assert_equal(scipy.linalg.det(cov), 0)
    assert_equal(scipy.linalg.det(cov[:npos, :npos]), np.inf)

    # np.linalg.slogdet is only available in numpy 1.6+
    # but scipy currently supports numpy 1.5.1.
    #assert_allclose(np.linalg.slogdet(cov[:npos, :npos]), (1, large_total_log))

    # Check the pseudo-determinant.
    psd = _PSD(cov)
    assert_allclose(psd.log_pdet, large_total_log)

def test_broadcasting():
    np.random.seed(1234)
    n = 4

    # Construct a random covariance matrix.
    data = np.random.randn(n, n)
    cov = np.dot(data, data.T)
    mean = np.random.randn(n)

    # Construct an ndarray which can be interpreted as
    # a 2x3 array whose elements are random data vectors.
    X = np.random.randn(2, 3, n)

    # Check that multiple data points can be evaluated at once.
    for i in range(2):
        for j in range(3):
            actual = multivariate_normal.pdf(X[i, j], mean, cov)
            desired = multivariate_normal.pdf(X, mean, cov)[i, j]
            assert_allclose(actual, desired)

def test_normal_1D():
    # The probability density function for a 1D normal variable should
    # agree with the standard normal distribution in scipy.stats.distributions
    x = np.linspace(0, 2, 10)
    mean, cov = 1.2, 0.9
    scale = cov**0.5
    d1 = norm.pdf(x, mean, scale)
    d2 = multivariate_normal.pdf(x, mean, cov)
    assert_allclose(d1, d2)

def test_marginalization():
    # Integrating out one of the variables of a 2D Gaussian should
    # yield a 1D Gaussian
    mean = np.array([2.5, 3.5])
    cov = np.array([[.5, 0.2], [0.2, .6]])
    n = 2**8 + 1  # Number of samples
    delta = 6 / (n - 1)  # Grid spacing

    v = np.linspace(0, 6, n)
    xv, yv = np.meshgrid(v, v)
    pos = np.empty((n, n, 2))
    pos[:, :, 0] = xv
    pos[:, :, 1] = yv
    pdf = multivariate_normal.pdf(pos, mean, cov)

    # Marginalize over x and y axis
    margin_x = romb(pdf, delta, axis=0)
    margin_y = romb(pdf, delta, axis=1)

    # Compare with standard normal distribution
    gauss_x = norm.pdf(v, loc=mean[0], scale=cov[0, 0]**0.5)
    gauss_y = norm.pdf(v, loc=mean[1], scale=cov[1, 1]**0.5)
    assert_allclose(margin_x, gauss_x, rtol=1e-2, atol=1e-2)
    assert_allclose(margin_y, gauss_y, rtol=1e-2, atol=1e-2)

def test_frozen():
    # The frozen distribution should agree with the regular one
    np.random.seed(1234)
    x = np.random.randn(5)
    mean = np.random.randn(5)
    cov = np.abs(np.random.randn(5))
    norm_frozen = multivariate_normal(mean, cov)
    assert_allclose(norm_frozen.pdf(x), multivariate_normal.pdf(x, mean, cov))
    assert_allclose(norm_frozen.logpdf(x),
                    multivariate_normal.logpdf(x, mean, cov))

def test_pseudodet_pinv():
    # Make sure that pseudo-inverse and pseudo-det agree on cutoff

    # Assemble random covariance matrix with large and small eigenvalues
    np.random.seed(1234)
    n = 7
    x = np.random.randn(n, n)
    cov = np.dot(x, x.T)
    s, u = scipy.linalg.eigh(cov)
    s = 0.5 * np.ones(n)
    s[0] = 1.0
    s[-1] = 1e-7
    cov = np.dot(u, np.dot(np.diag(s), u.T))

    # Set cond so that the lowest eigenvalue is below the cutoff
    cond = 1e-5
    psd = _PSD(cov, cond=cond)
    psd_pinv = _PSD(psd.pinv, cond=cond)

    # Check that the log pseudo-determinant agrees with the sum
    # of the logs of all but the smallest eigenvalue
    assert_allclose(psd.log_pdet, np.sum(np.log(s[:-1])))
    # Check that the pseudo-determinant of the pseudo-inverse
    # agrees with 1 / pseudo-determinant
    assert_allclose(-psd.log_pdet, psd_pinv.log_pdet)

def test_exception_nonsquare_cov():
    cov = [[1, 2, 3], [4, 5, 6]]
    assert_raises(ValueError, _PSD, cov)

def test_exception_nonfinite_cov():
    cov_nan = [[1, 0], [0, np.nan]]
    assert_raises(ValueError, _PSD, cov_nan)
    cov_inf = [[1, 0], [0, np.inf]]
    assert_raises(ValueError, _PSD, cov_inf)

def test_exception_non_psd_cov():
    cov = [[1, 0], [0, -1]]
    assert_raises(ValueError, _PSD, cov)

def test_exception_singular_cov():
    np.random.seed(1234)
    x = np.random.randn(5)
    mean = np.random.randn(5)
    cov = np.ones((5, 5))
    e = np.linalg.LinAlgError
    assert_raises(e, multivariate_normal, mean, cov)
    assert_raises(e, multivariate_normal.pdf, x, mean, cov)
    assert_raises(e, multivariate_normal.logpdf, x, mean, cov)

def test_R_values():
    # Compare the multivariate pdf with some values precomputed
    # in R version 3.0.1 (2013-05-16) on Mac OS X 10.6.

    # The values below were generated by the following R-script:
    # > library(mnormt)
    # > x <- seq(0, 2, length=5)
    # > y <- 3*x - 2
    # > z <- x + cos(y)
    # > mu <- c(1, 3, 2)
    # > Sigma <- matrix(c(1,2,0,2,5,0.5,0,0.5,3), 3, 3)
    # > r_pdf <- dmnorm(cbind(x,y,z), mu, Sigma)
    r_pdf = np.array([0.0002214706, 0.0013819953, 0.0049138692,
                      0.0103803050, 0.0140250800])

    x = np.linspace(0, 2, 5)
    y = 3 * x - 2
    z = x + np.cos(y)
    r = np.array([x, y, z]).T

    mean = np.array([1, 3, 2], 'd')
    cov = np.array([[1, 2, 0], [2, 5, .5], [0, .5, 3]], 'd')

    pdf = multivariate_normal.pdf(r, mean, cov)
    assert_allclose(pdf, r_pdf, atol=1e-10)

def test_rvs_shape():
    # Check that rvs parses the mean and covariance correctly, and returns
    # an array of the right shape
    N = 300
    d = 4
    sample = multivariate_normal.rvs(mean=np.zeros(d), cov=1, size=N)
    assert_equal(sample.shape, (N, d))

    sample = multivariate_normal.rvs(mean=None,
                                     cov=np.array([[2, .1], [.1, 1]]),
                                     size=N)
    assert_equal(sample.shape, (N, 2))

    u = multivariate_normal(mean=0, cov=1)
    sample = u.rvs(N)
    assert_equal(sample.shape, (N, ))

def test_large_sample():
    # Generate large sample and compare sample mean and sample covariance
    # with mean and covariance matrix.

    np.random.seed(2846)

    n = 3
    mean = np.random.randn(n)
    M = np.random.randn(n, n)
    cov = np.dot(M, M.T)
    size = 5000

    sample = multivariate_normal.rvs(mean, cov, size)

    assert_allclose(numpy.cov(sample.T), cov, rtol=1e-1)
    assert_allclose(sample.mean(0), mean, rtol=1e-1)

def test_entropy():
    np.random.seed(2846)

    n = 3
    mean = np.random.randn(n)
    M = np.random.randn(n, n)
    cov = np.dot(M, M.T)

    rv = multivariate_normal(mean, cov)

    # Check that frozen distribution agrees with entropy function
    assert_almost_equal(rv.entropy(), multivariate_normal.entropy(mean, cov))
    # Compare entropy with manually computed expression involving
    # the sum of the logs of the eigenvalues of the covariance matrix
    eigs = np.linalg.eig(cov)[0]
    desired = 1/2 * (n * (np.log(2*np.pi) + 1) + np.sum(np.log(eigs)))
    assert_almost_equal(desired, rv.entropy())

if __name__ == "__main__":
    run_module_suite()
<|endoftext|>