<commit_before>import asyncio

import discord
import youtube_dl

from discord.ext import commands

# Suppress noise about console usage from errors
youtube_dl.utils.bug_reports_message = lambda: ''

ytdl_format_options = {
    'format': 'bestaudio/best',
    'outtmpl': '%(extractor)s-%(id)s-%(title)s.%(ext)s',
    'restrictfilenames': True,
    'noplaylist': True,
    'nocheckcertificate': True,
    'ignoreerrors': False,
    'logtostderr': False,
    'quiet': True,
    'no_warnings': True,
    'default_search': 'auto',
    'source_address': '0.0.0.0' # bind to ipv4 since ipv6 addresses cause issues sometimes
}

ffmpeg_options = {
    'before_options': '-nostdin',
    'options': '-vn'
}

ytdl = youtube_dl.YoutubeDL(ytdl_format_options)

class YTDLSource(discord.PCMVolumeTransformer):
    def __init__(self, source, *, data, volume=0.5):
        super().__init__(source, volume)

        self.data = data

        self.title = data.get('title')
        self.url = data.get('url')

    @classmethod
    async def from_url(cls, url, *, loop=None, stream=False):
        loop = loop or asyncio.get_event_loop()
        data = await loop.run_in_executor(None, lambda: ytdl.extract_info(url, download=not stream))

        if 'entries' in data:
            # take first item from a playlist
            data = data['entries'][0]

        filename = data['url'] if stream else ytdl.prepare_filename(data)
        return cls(discord.FFmpegPCMAudio(filename, **ffmpeg_options), data=data)

class Music:
    def __init__(self, bot):
        self.bot = bot

    @commands.command()
    async def join(self, ctx, *, channel: discord.VoiceChannel):
        """Joins a voice channel"""

        if ctx.voice_client is not None:
            return await ctx.voice_client.move_to(channel)

        await channel.connect()

    @commands.command()
    async def play(self, ctx, *, query):
        """Plays a file from the local filesystem"""

        source = discord.PCMVolumeTransformer(discord.FFmpegPCMAudio(query))
        ctx.voice_client.play(source, after=lambda e: print('Player error: %s' % e) if e else None)

        await ctx.send('Now playing: {}'.format(query))

    @commands.command()
    async def yt(self, ctx, *, url):
        """Plays from a url (almost anything youtube_dl supports)"""

        async with ctx.typing():
            player = await YTDLSource.from_url(url, loop=self.bot.loop)
            ctx.voice_client.play(player, after=lambda e: print('Player error: %s' % e) if e else None)

        await ctx.send('Now playing: {}'.format(player.title))

    @commands.command()
    async def stream(self, ctx, *, url):
        """Streams from a url (same as yt, but doesn't predownload)"""

        async with ctx.typing():
            player = await YTDLSource.from_url(url, loop=self.bot.loop, stream=True)
            ctx.voice_client.play(player, after=lambda e: print('Player error: %s' % e) if e else None)

        await ctx.send('Now playing: {}'.format(player.title))

    @commands.command()
    async def volume(self, ctx, volume: int):
        """Changes the player's volume"""

        if ctx.voice_client is None:
            return await ctx.send("Not connected to a voice channel.")

        ctx.voice_client.source.volume = volume
        await ctx.send("Changed volume to {}%".format(volume))

    @commands.command()
    async def stop(self, ctx):
        """Stops and disconnects the bot from voice"""

        await ctx.voice_client.disconnect()

    @play.before_invoke
    @yt.before_invoke
    @stream.before_invoke
    async def ensure_voice(self, ctx):
        if ctx.voice_client is None:
            if ctx.author.voice:
                await ctx.author.voice.channel.connect()
            else:
                await ctx.send("You are not connected to a voice channel.")
                raise commands.CommandError("Author not connected to a voice channel.")
        elif ctx.voice_client.is_playing():
            ctx.voice_client.stop()

bot = commands.Bot(command_prefix=commands.when_mentioned_or("!"),
                   description='Relatively simple music bot example')

@bot.event
async def on_ready():
    print('Logged in as {0} ({0.id})'.format(bot.user))
    print('------')

bot.add_cog(Music(bot))
bot.run('token')
<commit_msg>Remove deprecated ffmpeg/avconv flag from basic_voice example<commit_after>import asyncio

import discord
import youtube_dl

from discord.ext import commands

# Suppress noise about console usage from errors
youtube_dl.utils.bug_reports_message = lambda: ''

ytdl_format_options = {
    'format': 'bestaudio/best',
    'outtmpl': '%(extractor)s-%(id)s-%(title)s.%(ext)s',
    'restrictfilenames': True,
    'noplaylist': True,
    'nocheckcertificate': True,
    'ignoreerrors': False,
    'logtostderr': False,
    'quiet': True,
    'no_warnings': True,
    'default_search': 'auto',
    'source_address': '0.0.0.0' # bind to ipv4 since ipv6 addresses cause issues sometimes
}

ffmpeg_options = {
    'options': '-vn'
}

ytdl = youtube_dl.YoutubeDL(ytdl_format_options)

class YTDLSource(discord.PCMVolumeTransformer):
    def __init__(self, source, *, data, volume=0.5):
        super().__init__(source, volume)

        self.data = data

        self.title = data.get('title')
        self.url = data.get('url')

    @classmethod
    async def from_url(cls, url, *, loop=None, stream=False):
        loop = loop or asyncio.get_event_loop()
        data = await loop.run_in_executor(None, lambda: ytdl.extract_info(url, download=not stream))

        if 'entries' in data:
            # take first item from a playlist
            data = data['entries'][0]

        filename = data['url'] if stream else ytdl.prepare_filename(data)
        return cls(discord.FFmpegPCMAudio(filename, **ffmpeg_options), data=data)

class Music:
    def __init__(self, bot):
        self.bot = bot

    @commands.command()
    async def join(self, ctx, *, channel: discord.VoiceChannel):
        """Joins a voice channel"""

        if ctx.voice_client is not None:
            return await ctx.voice_client.move_to(channel)

        await channel.connect()

    @commands.command()
    async def play(self, ctx, *, query):
        """Plays a file from the local filesystem"""

        source = discord.PCMVolumeTransformer(discord.FFmpegPCMAudio(query))
        ctx.voice_client.play(source, after=lambda e: print('Player error: %s' % e) if e else None)

        await ctx.send('Now playing: {}'.format(query))

    @commands.command()
    async def yt(self, ctx, *, url):
        """Plays from a url (almost anything youtube_dl supports)"""

        async with ctx.typing():
            player = await YTDLSource.from_url(url, loop=self.bot.loop)
            ctx.voice_client.play(player, after=lambda e: print('Player error: %s' % e) if e else None)

        await ctx.send('Now playing: {}'.format(player.title))

    @commands.command()
    async def stream(self, ctx, *, url):
        """Streams from a url (same as yt, but doesn't predownload)"""

        async with ctx.typing():
            player = await YTDLSource.from_url(url, loop=self.bot.loop, stream=True)
            ctx.voice_client.play(player, after=lambda e: print('Player error: %s' % e) if e else None)

        await ctx.send('Now playing: {}'.format(player.title))

    @commands.command()
    async def volume(self, ctx, volume: int):
        """Changes the player's volume"""

        if ctx.voice_client is None:
            return await ctx.send("Not connected to a voice channel.")

        ctx.voice_client.source.volume = volume
        await ctx.send("Changed volume to {}%".format(volume))

    @commands.command()
    async def stop(self, ctx):
        """Stops and disconnects the bot from voice"""

        await ctx.voice_client.disconnect()

    @play.before_invoke
    @yt.before_invoke
    @stream.before_invoke
    async def ensure_voice(self, ctx):
        if ctx.voice_client is None:
            if ctx.author.voice:
                await ctx.author.voice.channel.connect()
            else:
                await ctx.send("You are not connected to a voice channel.")
                raise commands.CommandError("Author not connected to a voice channel.")
        elif ctx.voice_client.is_playing():
            ctx.voice_client.stop()

bot = commands.Bot(command_prefix=commands.when_mentioned_or("!"),
                   description='Relatively simple music bot example')

@bot.event
async def on_ready():
    print('Logged in as {0} ({0.id})'.format(bot.user))
    print('------')

bot.add_cog(Music(bot))
bot.run('token')
<|endoftext|>


<commit_before>#!/usr/bin/env python

import os,sys

build_root = os.getcwd()

gp_cmd = 'gaffer python'
ad_cmd = 'asciidoc'
ag_cmd = 'gaffer screenGrabber'
pdf_cmd = '/home/btoogood/Tools/wkhtmltopdf-amd64'

'''
#build the node reference
os.chdir( os.path.join (build_root, 'GafferNodeReference' ))

## extract the info from the node metadata
os.system( gp_cmd + " extractoMatic.py" )

## run asciidoc then convert the result into a pdf
os.system( ad_cmd + " GafferNodeReference.txt" )
os.system( pdf_cmd + " GafferNodeReference.html GafferNodeReference.pdf" )
'''

#build the user guide
os.chdir( os.path.join( build_root, 'GafferUserGuide' ))

##generate images from reference scripts
source_dir = os.path.join( build_root, 'GafferUserGuide', 'images', 'autoGenerated_source' )
target_dir = os.path.join( build_root, 'GafferUserGuide', 'images', 'autoGenerated_target' )
for source in os.listdir( source_dir ):
    print( "Processing [ %s ]" % ( source ) )
    target_img = source[:-4] + ".png"
    ag_cmd_full = "%s -source %s -image %s" % \
                  (ag_cmd, os.path.join( source_dir, source ), os.path.join( target_dir, target_img ) )
    print( ag_cmd_full )

## run asciidoc then convert the result into a pdf
os.system( ad_cmd + " GafferUserGuide.txt" )
os.system( pdf_cmd + " GafferUserGuide.html GafferUserGuide.pdf" )

<commit_msg>un-commenting<commit_after>#!/usr/bin/env python

import os,sys

build_root = os.getcwd()

gp_cmd = 'gaffer python'
ad_cmd = 'asciidoc'
ag_cmd = 'gaffer screenGrabber'
pdf_cmd = '/home/btoogood/Tools/wkhtmltopdf-amd64'

#build the node reference
os.chdir( os.path.join (build_root, 'GafferNodeReference' ))

## extract the info from the node metadata
os.system( gp_cmd + " extractoMatic.py" )

## run asciidoc then convert the result into a pdf
os.system( ad_cmd + " GafferNodeReference.txt" )
os.system( pdf_cmd + " GafferNodeReference.html GafferNodeReference.pdf" )

#build the user guide
os.chdir( os.path.join( build_root, 'GafferUserGuide' ))

##generate images from reference scripts
source_dir = os.path.join( build_root, 'GafferUserGuide', 'images', 'autoGenerated_source' )
target_dir = os.path.join( build_root, 'GafferUserGuide', 'images', 'autoGenerated_target' )
for source in os.listdir( source_dir ):
    print( "Processing [ %s ]" % ( source ) )
    target_img = source[:-4] + ".png"
    ag_cmd_full = "%s -source %s -image %s" % \
                  (ag_cmd, os.path.join( source_dir, source ), os.path.join( target_dir, target_img ) )
    print( ag_cmd_full )

## run asciidoc then convert the result into a pdf
os.system( ad_cmd + " GafferUserGuide.txt" )
os.system( pdf_cmd + " GafferUserGuide.html GafferUserGuide.pdf" )

<|endoftext|>


<commit_before>#!/bin/env python
# -*- coding:utf-8 -*-
from __future__ import with_statement

import os.path
import time

import fabric.operations
from fabric.operations import put, get
from fabric.api import *
from fabric.colors import cyan
from fabric.contrib.files import *

def reloadapp():
    """
    Touch the wsgi
    """
    print(cyan('Reloading the application'))
    venvcmd('touch apache/%(wsginame)s' % env)

def venvcmd(cmd, shell=True, user=None, pty=False, subdir=""):
    if not user:
        user = env.user

    with cd(env.venvfullpath + '/' + env.projectname + '/' + subdir):
        return sudo('source %(venvfullpath)s/bin/activate && ' % env + cmd, shell=shell, user=user, pty=pty)

def printenv():
    """
    Print shell env
    """
    venvcmd('env')

## Server scenarios
def commonenv():
    """
    Base environment
    """
    env.venvname = "imaginationforpeople.org"
    env.projectname = "imaginationforpeople"
    env.gitrepo = "ssh://webapp@i4p-dev.imaginationforpeople.org/var/repositories/imaginationforpeople.git"
    env.gitbranch = "master"

def prodenv():
    """
    production environment - Will need some work when moving to seperate server
    """
    commonenv()
    env.venvname = "imaginationforpeople.org"
    env.wsginame = "prod.wsgi"
    env.urlhost = "www.imaginationforpeople.org"
    env.user = "web"
    env.home = "www"
    require('venvname', provided_by=('commonenv',))
    env.hosts = ['i4p-prod.imaginationforpeople.org']

    env.gitbranch = "master"

    env.venvbasepath = os.path.join("/home", env.home, "virtualenvs")
    env.venvfullpath = env.venvbasepath + '/' + env.venvname + '/'

def stagenv():
    """
    Stagging environment
    """
    commonenv()
    env.wsginame = "staging.wsgi"
    env.urlhost = "staging.imaginationforpeople.org"
    env.user = "webapp"
    env.home = "webapp"
    require('venvname', provided_by=('commonenv',))
    env.hosts = ['i4p-dev.imaginationforpeople.org']

    env.gitrepo = "/var/repositories/imaginationforpeople.git"
    env.gitbranch = "release/multisite"

    env.venvbasepath = os.path.join("/home", env.home, "virtualenvs")
    env.venvfullpath = env.venvbasepath + '/' + env.venvname + '/'

## Virtualenv
def build_virtualenv():
    """
    Build the virtualenv
    """
    print(cyan('Creating a fresh virtualenv'))
    require('venvfullpath', provided_by=('stagenv', 'prodenv'))
    sudo('rm /tmp/distribute* || echo "ok"') # clean after hudson
    run('virtualenv --no-site-packages --distribute %(venvfullpath)s' % env)
    sudo('rm /tmp/distribute* || echo "ok"') # clean after myself

def update_requirements():
    """
    update external dependencies on remote host
    """
    print(cyan('Updating requirements using PIP'))
    run("yes w | pip install -E %(venvfullpath)s -Ur %(venvfullpath)s/%(projectname)s/requirements.txt" % env)

def fixperms():
    """
    fix permissions
    """
    with cd(env.venvfullpath):
        # sudo('chown :www-data -R %(projectname)s && chmod g+rw -R %(projectname)s' % env)
        # FIXME: This SUCKS
        pass

## Django
def syncdb():
    print(cyan('Synching Django database'))
    venvcmd('./manage.py syncdb --noinput')
    venvcmd('./manage.py migrate')

def collect_static_files():
    """
    Collect static files such as pictures
    """
    print(cyan('Collecting static files'))
    venvcmd('./manage.py collectstatic --noinput')

def compile_messages():
    """
    Run compile messages and reload the app
    """
    apps = venvcmd('ls -d */', subdir="apps").split("\n")
    cmd = "django-admin.py compilemessages -v0"
    print(cyan('Compiling i18 messages for the following apps: %s' % apps))
    for app in apps:
        appsubdir = 'apps/%s' % app
        cwd = venvcmd('pwd', subdir=appsubdir)
        if exists(cwd + '/locale'):
            print(cyan('\t * %s' % appsubdir))
            venvcmd(cmd, subdir=appsubdir)
    fixperms()
    reloadapp()

def tests():
    """
    Run all tests on remote
    """
    print(cyan('Running TDD tests'))
    venvcmd('./manage.py test')

    print(cyan('Running BDD tests'))
    venvcmd('./manage.py harvest --verbosity=2')

def deploy_bootstrap():
    """
    Deploy the project the first time.
    """
    build_virtualenv()

    print(cyan('Cloning Git repository'))
    with cd(env.venvfullpath):
        run("git clone %(gitrepo)s %(projectname)s" % env)
        with cd("%(projectname)s" % env):
            run("git fetch origin %s" % env.gitbranch)

    fullupdate()

    # Fix perms
    with cd(env.venvfullpath):
        with cd("%(projectname)s" % env):
            run('mkdir media/uploads media/cache static/CACHE media/mugshots -p')
            sudo('chown www-data -R media/uploads media/cache media/mugshots static/CACHE')

def _updatemaincode():
    """
    Private : we don't want people updating the code without running
    tests
    """
    print(cyan('Updating Git repository'))
    with cd(env.venvfullpath + '/%(projectname)s/' % env):
        run('git fetch')
        run('git checkout %s' % env.gitbranch)
        run('git pull origin %s' % env.gitbranch)

def fullupdate():
    """
    Full Update the maincode and the deps
    """
    _updatemaincode()
    update_requirements()
    compile_messages()
    syncdb()
    collect_static_files()
    # tests()
    reloadapp()

def update():
    """
    Fast Update : project maincode
    """
    _updatemaincode()
    compile_messages()
    syncdb()
    collect_static_files()
    # tests()
    reloadapp()

## Webserver
def configure_webservers():
    """
    Configure the webserver stack.
    """
    # apache
    print(cyan('Configuring Apache'))
    fullprojectpath = env.venvfullpath + '/%(projectname)s/' % env
    sudo('cp %sapache/%s /etc/apache2/sites-available/%s' % (fullprojectpath, env.urlhost, env.urlhost))
    sudo('a2ensite %s' % env.urlhost)

    # nginx
    print(cyan('Configuring Nginx'))
    sudo('cp %snginx/%s /etc/nginx/sites-available/%s' % (fullprojectpath, env.urlhost, env.urlhost))
    with cd('/etc/nginx/sites-enabled/'):
        sudo('ln -sf ../sites-available/%s .' % env.urlhost)

    # Fix log dir
    check_or_install_logdir()

def install_webservers():
    """
    Install the webserver stack
    """
    print(cyan('Installing web servers'))
    sudo('apt-get install apache2-mpm-prefork libapache2-mod-wsgi -y')
    sudo('apt-get install nginx -y')

def reload_webservers():
    """
    Reload the webserver stack.
    """
    print(cyan("Reloading apache"))
    # Apache
    sudo('apache2ctl -k graceful')

    # Nginx
    print(cyan("Reloading nginx"))
    sudo('/etc/init.d/nginx restart')

def check_or_install_logdir():
    """
    Make sure the log directory exists and has right mode and ownership
    """
    print(cyan('Installing a log dir'))
    with cd(env.venvfullpath + '/'):
        sudo('mkdir -p logs/ ; chown www-data logs; chmod o+rwx logs ; pwd')

## Database server
def install_database_server():
    """
    Install a postgresql DB
    """
    print(cyan('Installing Postgresql'))
    sudo('apt-get install -y postgresql-8.4 postgresql-8.4')

def setup_database():
    """
    Create a user and a DB for the project
    """
    # FIXME: pg_hba has to be changed by hand (see doc)
    # FIXME: Password has to be set by hand (see doc)
    sudo('createuser %s -D -R -S' % env.projectname, user='postgres')
    sudo('createdb -O%s %s' % (env.projectname, env.projectname), user='postgres')

## Server packages
def install_basetools():
    """
    Install required base tools
    """
    print(cyan('Installing base tools'))
    sudo('apt-get install -y python-virtualenv python-pip')
    sudo('apt-get install -y git mercurial subversion')
    sudo('apt-get install -y gettext')

def install_builddeps():
    """
    Will install commonly needed build deps for pip django virtualenvs.
    """
    print(cyan('Installing compilers and required libraries'))
    sudo('apt-get install -y build-essential python-dev libjpeg62-dev libpng12-dev zlib1g-dev libfreetype6-dev liblcms-dev libpq-dev libxslt1-dev libxml2-dev')

def meta_full_bootstrap():
    """
    For use on new, empty environnements
    """
    sudo('apt-get update')
    install_basetools()
    install_database_server()
    install_webservers()
    install_builddeps()

    deploy_bootstrap()

    configure_webservers()
    reload_webservers()

def mirror_prod_to_staging():
    """
    Mirror the content of the production server to the staging one
    """
    assert(env.wsginame == 'staging.wsgi')

    # Files
    with cd(os.path.join(env.venvfullpath, env.projectname, 'media')):
        sudo('rm -rf mugshots')
        run('scp -r web@i4p-prod.imaginationforpeople.org:/home/www/virtualenvs/imaginationforpeople.org/imaginationforpeople/media/mugshots .')

        sudo('rm -rf uploads')
        run('scp -r web@i4p-prod.imaginationforpeople.org:/home/www/virtualenvs/imaginationforpeople.org/imaginationforpeople/media/uploads .')

        sudo('rm -rf cache')
        run('scp -r web@i4p-prod.imaginationforpeople.org:/home/www/virtualenvs/imaginationforpeople.org/imaginationforpeople/media/cache .')

        sudo('chown www-data -R mugshots uploads cache')
        sudo('chmod u+rw -R mugshots uploads cache')

def dump_database():
    assert(env.wsginame == 'prod.wsgi')
    run('pg_dump -Uimaginationforpeople imaginationforpeople > ~/i4p_db_%s.sql' % time.strftime('%Y%m%d'))

def get_database():
    dump_database()
    with cd(os.path.join('/home', env.home)):
        filename = 'i4p_db_%s.sql' % time.strftime('%Y%m%d')
        compressed_filename = '%s.bz2' % filename
        run('bzip2 -9 -c %s > %s' % (filename, compressed_filename))
        get(compressed_filename, 'current_database.sql.bz2')

def restore_database():
    assert(env.wsginame == 'staging.wsgi')
    put('current_database.sql.bz2', 'current_database.sql.bz2')
    run('bunzip2 -c current_database.sql.bz2 > current_database.sql')
    sudo('sudo su - postgres -c "dropdb imaginationforpeople"')
    sudo('sudo su - postgres -c "createdb -E UNICODE -Ttemplate0 -Oimaginationforpeople imaginationforpeople"')
    run('psql -Uimaginationforpeople imaginationforpeople < ~/current_database.sql')
<commit_msg>use develop as staging branch<commit_after>#!/bin/env python
# -*- coding:utf-8 -*-
from __future__ import with_statement

import os.path
import time

import fabric.operations
from fabric.operations import put, get
from fabric.api import *
from fabric.colors import cyan
from fabric.contrib.files import *

def reloadapp():
    """
    Touch the wsgi
    """
    print(cyan('Reloading the application'))
    venvcmd('touch apache/%(wsginame)s' % env)

def venvcmd(cmd, shell=True, user=None, pty=False, subdir=""):
    if not user:
        user = env.user

    with cd(env.venvfullpath + '/' + env.projectname + '/' + subdir):
        return sudo('source %(venvfullpath)s/bin/activate && ' % env + cmd, shell=shell, user=user, pty=pty)

def printenv():
    """
    Print shell env
    """
    venvcmd('env')

## Server scenarios
def commonenv():
    """
    Base environment
    """
    env.venvname = "imaginationforpeople.org"
    env.projectname = "imaginationforpeople"
    env.gitrepo = "ssh://webapp@i4p-dev.imaginationforpeople.org/var/repositories/imaginationforpeople.git"
    env.gitbranch = "master"

def prodenv():
    """
    production environment - Will need some work when moving to seperate server
    """
    commonenv()
    env.venvname = "imaginationforpeople.org"
    env.wsginame = "prod.wsgi"
    env.urlhost = "www.imaginationforpeople.org"
    env.user = "web"
    env.home = "www"
    require('venvname', provided_by=('commonenv',))
    env.hosts = ['i4p-prod.imaginationforpeople.org']

    env.gitbranch = "master"

    env.venvbasepath = os.path.join("/home", env.home, "virtualenvs")
    env.venvfullpath = env.venvbasepath + '/' + env.venvname + '/'

def stagenv():
    """
    Stagging environment
    """
    commonenv()
    env.wsginame = "staging.wsgi"
    env.urlhost = "staging.imaginationforpeople.org"
    env.user = "webapp"
    env.home = "webapp"
    require('venvname', provided_by=('commonenv',))
    env.hosts = ['i4p-dev.imaginationforpeople.org']

    env.gitrepo = "/var/repositories/imaginationforpeople.git"
    env.gitbranch = "develop"

    env.venvbasepath = os.path.join("/home", env.home, "virtualenvs")
    env.venvfullpath = env.venvbasepath + '/' + env.venvname + '/'

## Virtualenv
def build_virtualenv():
    """
    Build the virtualenv
    """
    print(cyan('Creating a fresh virtualenv'))
    require('venvfullpath', provided_by=('stagenv', 'prodenv'))
    sudo('rm /tmp/distribute* || echo "ok"') # clean after hudson
    run('virtualenv --no-site-packages --distribute %(venvfullpath)s' % env)
    sudo('rm /tmp/distribute* || echo "ok"') # clean after myself

def update_requirements():
    """
    update external dependencies on remote host
    """
    print(cyan('Updating requirements using PIP'))
    run("yes w | pip install -E %(venvfullpath)s -Ur %(venvfullpath)s/%(projectname)s/requirements.txt" % env)

def fixperms():
    """
    fix permissions
    """
    with cd(env.venvfullpath):
        # sudo('chown :www-data -R %(projectname)s && chmod g+rw -R %(projectname)s' % env)
        # FIXME: This SUCKS
        pass

## Django
def syncdb():
    print(cyan('Synching Django database'))
    venvcmd('./manage.py syncdb --noinput')
    venvcmd('./manage.py migrate')

def collect_static_files():
    """
    Collect static files such as pictures
    """
    print(cyan('Collecting static files'))
    venvcmd('./manage.py collectstatic --noinput')

def compile_messages():
    """
    Run compile messages and reload the app
    """
    apps = venvcmd('ls -d */', subdir="apps").split("\n")
    cmd = "django-admin.py compilemessages -v0"
    print(cyan('Compiling i18 messages for the following apps: %s' % apps))
    for app in apps:
        appsubdir = 'apps/%s' % app
        cwd = venvcmd('pwd', subdir=appsubdir)
        if exists(cwd + '/locale'):
            print(cyan('\t * %s' % appsubdir))
            venvcmd(cmd, subdir=appsubdir)
    fixperms()
    reloadapp()

def tests():
    """
    Run all tests on remote
    """
    print(cyan('Running TDD tests'))
    venvcmd('./manage.py test')

    print(cyan('Running BDD tests'))
    venvcmd('./manage.py harvest --verbosity=2')

def deploy_bootstrap():
    """
    Deploy the project the first time.
    """
    build_virtualenv()

    print(cyan('Cloning Git repository'))
    with cd(env.venvfullpath):
        run("git clone %(gitrepo)s %(projectname)s" % env)
        with cd("%(projectname)s" % env):
            run("git fetch origin %s" % env.gitbranch)

    fullupdate()

    # Fix perms
    with cd(env.venvfullpath):
        with cd("%(projectname)s" % env):
            run('mkdir media/uploads media/cache static/CACHE media/mugshots -p')
            sudo('chown www-data -R media/uploads media/cache media/mugshots static/CACHE')

def _updatemaincode():
    """
    Private : we don't want people updating the code without running
    tests
    """
    print(cyan('Updating Git repository'))
    with cd(env.venvfullpath + '/%(projectname)s/' % env):
        run('git fetch')
        run('git checkout %s' % env.gitbranch)
        run('git pull origin %s' % env.gitbranch)

def fullupdate():
    """
    Full Update the maincode and the deps
    """
    _updatemaincode()
    update_requirements()
    compile_messages()
    syncdb()
    collect_static_files()
    # tests()
    reloadapp()

def update():
    """
    Fast Update : project maincode
    """
    _updatemaincode()
    compile_messages()
    syncdb()
    collect_static_files()
    # tests()
    reloadapp()

## Webserver
def configure_webservers():
    """
    Configure the webserver stack.
    """
    # apache
    print(cyan('Configuring Apache'))
    fullprojectpath = env.venvfullpath + '/%(projectname)s/' % env
    sudo('cp %sapache/%s /etc/apache2/sites-available/%s' % (fullprojectpath, env.urlhost, env.urlhost))
    sudo('a2ensite %s' % env.urlhost)

    # nginx
    print(cyan('Configuring Nginx'))
    sudo('cp %snginx/%s /etc/nginx/sites-available/%s' % (fullprojectpath, env.urlhost, env.urlhost))
    with cd('/etc/nginx/sites-enabled/'):
        sudo('ln -sf ../sites-available/%s .' % env.urlhost)

    # Fix log dir
    check_or_install_logdir()

def install_webservers():
    """
    Install the webserver stack
    """
    print(cyan('Installing web servers'))
    sudo('apt-get install apache2-mpm-prefork libapache2-mod-wsgi -y')
    sudo('apt-get install nginx -y')

def reload_webservers():
    """
    Reload the webserver stack.
    """
    print(cyan("Reloading apache"))
    # Apache
    sudo('apache2ctl -k graceful')

    # Nginx
    print(cyan("Reloading nginx"))
    sudo('/etc/init.d/nginx restart')

def check_or_install_logdir():
    """
    Make sure the log directory exists and has right mode and ownership
    """
    print(cyan('Installing a log dir'))
    with cd(env.venvfullpath + '/'):
        sudo('mkdir -p logs/ ; chown www-data logs; chmod o+rwx logs ; pwd')

## Database server
def install_database_server():
    """
    Install a postgresql DB
    """
    print(cyan('Installing Postgresql'))
    sudo('apt-get install -y postgresql-8.4 postgresql-8.4')

def setup_database():
    """
    Create a user and a DB for the project
    """
    # FIXME: pg_hba has to be changed by hand (see doc)
    # FIXME: Password has to be set by hand (see doc)
    sudo('createuser %s -D -R -S' % env.projectname, user='postgres')
    sudo('createdb -O%s %s' % (env.projectname, env.projectname), user='postgres')

## Server packages
def install_basetools():
    """
    Install required base tools
    """
    print(cyan('Installing base tools'))
    sudo('apt-get install -y python-virtualenv python-pip')
    sudo('apt-get install -y git mercurial subversion')
    sudo('apt-get install -y gettext')

def install_builddeps():
    """
    Will install commonly needed build deps for pip django virtualenvs.
    """
    print(cyan('Installing compilers and required libraries'))
    sudo('apt-get install -y build-essential python-dev libjpeg62-dev libpng12-dev zlib1g-dev libfreetype6-dev liblcms-dev libpq-dev libxslt1-dev libxml2-dev')

def meta_full_bootstrap():
    """
    For use on new, empty environnements
    """
    sudo('apt-get update')
    install_basetools()
    install_database_server()
    install_webservers()
    install_builddeps()

    deploy_bootstrap()

    configure_webservers()
    reload_webservers()

def mirror_prod_to_staging():
    """
    Mirror the content of the production server to the staging one
    """
    assert(env.wsginame == 'staging.wsgi')

    # Files
    with cd(os.path.join(env.venvfullpath, env.projectname, 'media')):
        sudo('rm -rf mugshots')
        run('scp -r web@i4p-prod.imaginationforpeople.org:/home/www/virtualenvs/imaginationforpeople.org/imaginationforpeople/media/mugshots .')

        sudo('rm -rf uploads')
        run('scp -r web@i4p-prod.imaginationforpeople.org:/home/www/virtualenvs/imaginationforpeople.org/imaginationforpeople/media/uploads .')

        sudo('rm -rf cache')
        run('scp -r web@i4p-prod.imaginationforpeople.org:/home/www/virtualenvs/imaginationforpeople.org/imaginationforpeople/media/cache .')

        sudo('chown www-data -R mugshots uploads cache')
        sudo('chmod u+rw -R mugshots uploads cache')

def dump_database():
    assert(env.wsginame == 'prod.wsgi')
    run('pg_dump -Uimaginationforpeople imaginationforpeople > ~/i4p_db_%s.sql' % time.strftime('%Y%m%d'))

def get_database():
    dump_database()
    with cd(os.path.join('/home', env.home)):
        filename = 'i4p_db_%s.sql' % time.strftime('%Y%m%d')
        compressed_filename = '%s.bz2' % filename
        run('bzip2 -9 -c %s > %s' % (filename, compressed_filename))
        get(compressed_filename, 'current_database.sql.bz2')

def restore_database():
    assert(env.wsginame == 'staging.wsgi')
    put('current_database.sql.bz2', 'current_database.sql.bz2')
    run('bunzip2 -c current_database.sql.bz2 > current_database.sql')
    sudo('sudo su - postgres -c "dropdb imaginationforpeople"')
    sudo('sudo su - postgres -c "createdb -E UNICODE -Ttemplate0 -Oimaginationforpeople imaginationforpeople"')
    run('psql -Uimaginationforpeople imaginationforpeople < ~/current_database.sql')
<|endoftext|>


<commit_before>#!/usr/bin/python

from fabric.api import *
from fabric.colors import red, green

import cloubed
import logging
import os

# use ~/.ssh/config
env.use_ssh_config = True
env.ssh_config_path = "http/ssh_config"

NODES = { 'admin': ['10.5.0.1', '10.11.0.1'],
          'login': ['10.5.0.2', '10.11.0.2'],
          'cn1':   ['10.11.0.11'],
          'cn2':   ['10.11.0.12'],
          'cn3':   ['10.11.0.13'], }

def __gen_ssh_keypair():

    type = 'rsa'
    key_dir = "ansible/roles/ssh/files"
    priv_key = "{key_dir}/id_{type}".format(key_dir=key_dir,type=type)
    if not os.path.exists(priv_key):
        print(green("generating root ssh keypair"))
        if not os.path.isdir(key_dir): os.makedirs(key_dir)
        local("ssh-keygen -t {type} -N '' -C 'root@cluster' " \
              "-f {priv_key}".format(priv_key=priv_key))

def __gen_ssh_hosts_keys():

    for node in NODES.iterkeys():
        key_dir = "ansible/roles/ssh/files/{node}".format(node=node)
        if not os.path.isdir(key_dir): os.makedirs(key_dir)
        for type in ["rsa", "dsa", "ecdsa"]:
            key_path = "{dir}/ssh_host_{type}_key".format(dir=key_dir, type=type)
            if not os.path.exists(key_path):
                print(green("generating host keys type {type} for node {node}" \
                            .format(type=type, node=node)))
                local("ssh-keygen -t {type} -N '' -C 'root@{node}' " \
                      "-f {key}" \
                      .format(type=type,
                              node=node,
                              key=key_path))

    # generate known hosts file
    known_hosts = ""
    for node in NODES.iterkeys():
        key_dir = "ansible/roles/ssh/files/{node}".format(node=node)
        pubkey_path = "{dir}/ssh_host_ecdsa_key.pub".format(dir=key_dir)
        with open(pubkey_path,'r') as pubkey_file:
            # remove comment after public key
            pubkey = ' '.join(pubkey_file.readline().split(' ')[:2])
            known_hosts += "{node} {pubkey}\n".format(node=node, pubkey=pubkey)
            for ip in NODES[node]:
                known_hosts += "{ip} {pubkey}\n".format(ip=ip, pubkey=pubkey)
    known_hosts_path = "ansible/roles/ssh/files/known_hosts"
    with open(known_hosts_path,'w') as known_hosts_file:
        known_hosts_file.write(known_hosts)

def __gen_munge_key():

    key_path="ansible/roles/slurm/files/munge.key"
    local("dd if=/dev/urandom bs=1 count=1024 > {key_path}" \
          .format(key_path=key_path))

def __dl_extract_netboot():

    netboot_url = "http://ftp.fr.debian.org/debian/dists/wheezy/main/" \
                  "installer-amd64/current/images/netboot/netboot.tar.gz"
    netboot_lpath = "/tmp/netboot.tar.gz"
    if not os.path.exists('http/debian-installer/amd64/linux'):
        local("wget {url} -O {lpath}" \
              .format(url=netboot_url,
                      lpath=netboot_lpath))
        local("tar -xzf {lpath} -C http/".format(lpath=netboot_lpath))
        local("rm {lpath}".format(lpath=netboot_lpath))

def __create_needed_dirs():

    needed_dirs = [ 'pool', 'http' ]
    for dir in needed_dir:
        if not os.path.isdir(pool_dir):
            os.makedirs(pool_dir)

@task
def install_admin():

    node = "admin"

    __gen_ssh_keypair()
    __gen_ssh_hosts_keys()
    __gen_munge_key()
    __create_needed_dirs()
    __dl_extract_netboot()
    __pack_config()

    # generate the config files based on templates
    cloubed.gen_file(domain_name=node, template_name="ipxe")
    cloubed.gen_file(domain_name=node, template_name="preseed")
    cloubed.gen_file(domain_name=node, template_name="late-command")
    cloubed.gen_file(domain_name=node, template_name="ssh-config")

    # boot admin on network device
    cloubed.boot_vm(domain_name=node,
                    bootdev="network",
                    overwrite_disks=True,
                    recreate_networks=True)

    # wait admin shutdown
    cloubed.wait_event(node, "STOPPED", "SHUTDOWN")

    # boot admin on disk device
    cloubed.boot_vm(node)

def __pack_config():

    tar_file_name = "config.tar.gz"
    # generate tarball
    local("tar -czf http/{tarball} ansible".format(tarball=tar_file_name))

@task
@hosts('admin')
def run_ansible():

    __pack_config()
    put('http/config.tar.gz', '/tmp/config.tar.gz')
    run('tar -xzf /tmp/config.tar.gz -C /opt')

    with cd("/opt/ansible"):
        run("ansible-playbook cluster.yml")

@task
def install_login():

    node = "login"

    # boot node on network device
    cloubed.boot_vm(domain_name=node,
                    bootdev="network",
                    overwrite_disks=True)

    # wait node shutdown
    cloubed.wait_event(node, "STOPPED", "SHUTDOWN")

    # boot node on disk device
    cloubed.boot_vm(node)

@task
def install_node():

    node = env.host

    # boot node on network device
    cloubed.boot_vm(domain_name=node,
                    bootdev="network",
                    overwrite_disks=True)

    # wait node shutdown
    cloubed.wait_event(node, "STOPPED", "SHUTDOWN")

    # boot node on disk device
    cloubed.boot_vm(node)
<commit_msg>Fix some errors in fabfile<commit_after>#!/usr/bin/python

from fabric.api import *
from fabric.colors import red, green

import cloubed
import logging
import os

# use ~/.ssh/config
env.use_ssh_config = True
env.ssh_config_path = "http/ssh_config"

NODES = { 'admin': ['10.5.0.1', '10.11.0.1'],
          'login': ['10.5.0.2', '10.11.0.2'],
          'cn1':   ['10.11.0.11'],
          'cn2':   ['10.11.0.12'],
          'cn3':   ['10.11.0.13'], }

def __gen_ssh_keypair():

    type = 'rsa'
    key_dir = "ansible/roles/ssh/files"
    priv_key = "{key_dir}/id_{type}".format(key_dir=key_dir,type=type)
    if not os.path.exists(priv_key):
        print(green("generating root ssh keypair"))
        if not os.path.isdir(key_dir): os.makedirs(key_dir)
        local("ssh-keygen -t {type} -N '' -C 'root@cluster' " \
              "-f {priv_key}".format(type=type,priv_key=priv_key))

def __gen_ssh_hosts_keys():

    for node in NODES.iterkeys():
        key_dir = "ansible/roles/ssh/files/{node}".format(node=node)
        if not os.path.isdir(key_dir): os.makedirs(key_dir)
        for type in ["rsa", "dsa", "ecdsa"]:
            key_path = "{dir}/ssh_host_{type}_key".format(dir=key_dir, type=type)
            if not os.path.exists(key_path):
                print(green("generating host keys type {type} for node {node}" \
                            .format(type=type, node=node)))
                local("ssh-keygen -t {type} -N '' -C 'root@{node}' " \
                      "-f {key}" \
                      .format(type=type,
                              node=node,
                              key=key_path))

    # generate known hosts file
    known_hosts = ""
    for node in NODES.iterkeys():
        key_dir = "ansible/roles/ssh/files/{node}".format(node=node)
        pubkey_path = "{dir}/ssh_host_ecdsa_key.pub".format(dir=key_dir)
        with open(pubkey_path,'r') as pubkey_file:
            # remove comment after public key
            pubkey = ' '.join(pubkey_file.readline().split(' ')[:2])
            known_hosts += "{node} {pubkey}\n".format(node=node, pubkey=pubkey)
            for ip in NODES[node]:
                known_hosts += "{ip} {pubkey}\n".format(ip=ip, pubkey=pubkey)
    known_hosts_path = "ansible/roles/ssh/files/known_hosts"
    with open(known_hosts_path,'w') as known_hosts_file:
        known_hosts_file.write(known_hosts)

def __gen_munge_key():

    key_path="ansible/roles/slurm/files/munge.key"
    local("dd if=/dev/urandom bs=1 count=1024 > {key_path}" \
          .format(key_path=key_path))

def __dl_extract_netboot():

    netboot_url = "http://ftp.fr.debian.org/debian/dists/wheezy/main/" \
                  "installer-amd64/current/images/netboot/netboot.tar.gz"
    netboot_lpath = "/tmp/netboot.tar.gz"
    if not os.path.exists('http/debian-installer/amd64/linux'):
        local("wget {url} -O {lpath}" \
              .format(url=netboot_url,
                      lpath=netboot_lpath))
        local("tar -xzf {lpath} -C http/".format(lpath=netboot_lpath))
        local("rm {lpath}".format(lpath=netboot_lpath))

def __create_needed_dirs():

    needed_dirs = [ 'pool', 'http' ]
    for dir in needed_dirs:
        if not os.path.isdir(dir):
            os.makedirs(dir)

@task
def install_admin():

    node = "admin"

    __gen_ssh_keypair()
    __gen_ssh_hosts_keys()
    __gen_munge_key()
    __create_needed_dirs()
    __dl_extract_netboot()
    __pack_config()

    # generate the config files based on templates
    cloubed.gen_file(domain_name=node, template_name="ipxe")
    cloubed.gen_file(domain_name=node, template_name="preseed")
    cloubed.gen_file(domain_name=node, template_name="late-command")
    cloubed.gen_file(domain_name=node, template_name="ssh-config")

    # boot admin on network device
    cloubed.boot_vm(domain_name=node,
                    bootdev="network",
                    overwrite_disks=True,
                    recreate_networks=True)

    # wait admin shutdown
    cloubed.wait_event(node, "STOPPED", "SHUTDOWN")

    # boot admin on disk device
    cloubed.boot_vm(node)

def __pack_config():

    tar_file_name = "config.tar.gz"
    # generate tarball
    local("tar -czf http/{tarball} ansible".format(tarball=tar_file_name))

@task
@hosts('admin')
def run_ansible():

    __pack_config()
    put('http/config.tar.gz', '/tmp/config.tar.gz')
    run('tar -xzf /tmp/config.tar.gz -C /opt')

    with cd("/opt/ansible"):
        run("ansible-playbook cluster.yml")

@task
def install_login():

    node = "login"

    # boot node on network device
    cloubed.boot_vm(domain_name=node,
                    bootdev="network",
                    overwrite_disks=True)

    # wait node shutdown
    cloubed.wait_event(node, "STOPPED", "SHUTDOWN")

    # boot node on disk device
    cloubed.boot_vm(node)

@task
def install_node():

    node = env.host

    # boot node on network device
    cloubed.boot_vm(domain_name=node,
                    bootdev="network",
                    overwrite_disks=True)

    # wait node shutdown
    cloubed.wait_event(node, "STOPPED", "SHUTDOWN")

    # boot node on disk device
    cloubed.boot_vm(node)
<|endoftext|>


<commit_before>import sys
sys.path.append('../../')
from SimPEG import TensorMesh
import numpy as np
import unittest

class OrderTest(unittest.TestCase):
    """

    OrderTest is a base class for testing convergence orders with respect to mesh
    sizes of integral/differential operators.

    Mathematical Problem:

        Given are an operator A and its discretization A[h]. For a given test function f
        and h --> 0  we compare:

        error(h) = \| A[h](f) - A(f) \|_{\infty}

        Note that you can provide any norm.

        Test is passed when estimated rate order of convergence is  at least 90% of the
        estimated rate supplied by the user.

    Minimal example for a curl operator:

    class TestCURL(OrderTest):
        name = "Curl"

        def getError(self):
            # For given Mesh, generate A[h], f and A(f) and return norm of error.

            fun  = lambda x: np.cos(x)  # i (cos(y)) + j (cos(z)) + k (cos(x))
            sol = lambda x: np.sin(x)  # i (sin(z)) + j (sin(x)) + k (sin(y))

            Ex = fun(self.M.gridEx[:, 1])
            Ey = fun(self.M.gridEy[:, 2])
            Ez = fun(self.M.gridEz[:, 0])
            f = np.concatenate((Ex, Ey, Ez))

            Fx = sol(self.M.gridFx[:, 2])
            Fy = sol(self.M.gridFy[:, 0])
            Fz = sol(self.M.gridFz[:, 1])
            Af = np.concatenate((Fx, Fy, Fz))

            # Generate DIV matrix
            Ah = self.M.edgeCurl

            curlE = Ah*E
            err = np.linalg.norm((Ah*f -Af), np.inf)
            return err

        def test_order(self):
            # runs the test
            self.orderTest()

    See also: test_operatorOrder.py

    """

    name = "Order Test"
    expectedOrder = 2
    meshSizes = [4, 8, 16, 32]

    def setupMesh(self, nc):
        """
        For a given number of cells nc, generate a TensorMesh with uniform cells with edge length h=1/nc.
        """
        h1 = np.ones(nc)/nc
        h2 = np.ones(nc)/nc
        h3 = np.ones(nc)/nc
        h = [h1, h2, h3]
        self.M = TensorMesh(h)

    def getError(self):
        """For given h, generate A[h], f and A(f) and return norm of error."""
        return 1.

    def orderTest(self):
        """
        For number of cells specified in meshSizes setup mesh, call getError
        and prints mesh size, error, ratio between current and previous error,
        and estimated order of convergence.

        """
        order = []
        err_old = 0.
        nc_old = 0.
        for ii, nc in enumerate(self.meshSizes):
            self.setupMesh(nc)
            err = self.getError()
            if ii == 0:
                print ''
                print 'Testing order of:  ' + self.name
                print '_____________________________________________'
                print '   h  |    error    | e(i-1)/e(i) |  order'
                print '~~~~~~|~~~~~~~~~~~~~|~~~~~~~~~~~~~|~~~~~~~~~~'
                print '%4i  |  %8.2e   |' % (nc, err)
            else:
                order.append(np.log(err/err_old)/np.log(float(nc_old)/float(nc)))
                print '%4i  |  %8.2e   |   %6.4f    |  %6.4f' % (nc, err, err_old/err, order[-1])
            err_old = err
            nc_old = nc
        print '---------------------------------------------'
        self.assertTrue(len(np.where(np.array(order) > 0.9*self.expectedOrder)[0]) > np.floor(0.75*len(order)))

if __name__ == '__main__':
    unittest.main()
<commit_msg>Updates to OrderTest so that it runs random meshes.<commit_after>import sys
sys.path.append('../../')
from SimPEG import TensorMesh
import numpy as np
import unittest

class OrderTest(unittest.TestCase):
    """

    OrderTest is a base class for testing convergence orders with respect to mesh
    sizes of integral/differential operators.

    Mathematical Problem:

        Given are an operator A and its discretization A[h]. For a given test function f
        and h --> 0  we compare:

        error(h) = \| A[h](f) - A(f) \|_{\infty}

        Note that you can provide any norm.

        Test is passed when estimated rate order of convergence is  at least within the specified tolerance of the
        estimated rate supplied by the user.

    Minimal example for a curl operator:

    class TestCURL(OrderTest):
        name = "Curl"

        def getError(self):
            # For given Mesh, generate A[h], f and A(f) and return norm of error.

            fun  = lambda x: np.cos(x)  # i (cos(y)) + j (cos(z)) + k (cos(x))
            sol = lambda x: np.sin(x)  # i (sin(z)) + j (sin(x)) + k (sin(y))

            Ex = fun(self.M.gridEx[:, 1])
            Ey = fun(self.M.gridEy[:, 2])
            Ez = fun(self.M.gridEz[:, 0])
            f = np.concatenate((Ex, Ey, Ez))

            Fx = sol(self.M.gridFx[:, 2])
            Fy = sol(self.M.gridFy[:, 0])
            Fz = sol(self.M.gridFz[:, 1])
            Af = np.concatenate((Fx, Fy, Fz))

            # Generate DIV matrix
            Ah = self.M.edgeCurl

            curlE = Ah*E
            err = np.linalg.norm((Ah*f -Af), np.inf)
            return err

        def test_order(self):
            # runs the test
            self.orderTest()

    See also: test_operatorOrder.py

    """

    name = "Order Test"
    expectedOrder = 2
    tolerance = 0.85
    meshSizes = [4, 8, 16, 32, 64]
    meshType = 'uniformTensorMesh'
    meshDimension = 3

    def setupMesh(self, nc):
        """
        For a given number of cells nc, generate a TensorMesh with uniform cells with edge length h=1/nc.
        """
        if 'TensorMesh' in self.meshType:
            if 'uniform' in self.meshType:
                h1 = np.ones(nc)/nc
                h2 = np.ones(nc)/nc
                h3 = np.ones(nc)/nc
                h = [h1, h2, h3]
            elif 'random' in self.meshType:
                h1 = np.random.rand(nc)
                h2 = np.random.rand(nc)
                h3 = np.random.rand(nc)
                h = [hi/np.sum(hi) for hi in [h1, h2, h3]]  # normalize
            else:
                raise Exception('Unexpected meshType')

            self.M = TensorMesh(h[:self.meshDimension])
            max_h = max([np.max(hi) for hi in self.M.h])
            return max_h

    def getError(self):
        """For given h, generate A[h], f and A(f) and return norm of error."""
        return 1.

    def orderTest(self):
        """
        For number of cells specified in meshSizes setup mesh, call getError
        and prints mesh size, error, ratio between current and previous error,
        and estimated order of convergence.

        """
        order = []
        err_old = 0.
        max_h_old = 0.
        for ii, nc in enumerate(self.meshSizes):
            max_h = self.setupMesh(nc)
            err = self.getError()
            if ii == 0:
                print ''
                print 'Testing order of:  ' + self.name
                print '_____________________________________________'
                print '   h  |    error    | e(i-1)/e(i) |  order'
                print '~~~~~~|~~~~~~~~~~~~~|~~~~~~~~~~~~~|~~~~~~~~~~'
                print '%4i  |  %8.2e   |' % (nc, err)
            else:
                order.append(np.log(err/err_old)/np.log(max_h/max_h_old))
                print '%4i  |  %8.2e   |   %6.4f    |  %6.4f' % (nc, err, err_old/err, order[-1])
            err_old = err
            max_h_old = max_h
        print '---------------------------------------------'
        passTest = np.mean(np.array(order)) > self.tolerance*self.expectedOrder
        # passTest = len(np.where(np.array(order) > self.tolerance*self.expectedOrder)[0]) > np.floor(0.75*len(order))
        self.assertTrue(passTest)

if __name__ == '__main__':
    unittest.main()
<|endoftext|>


<commit_before>"""Implements spectral biclustering algorithms.

Authors : Kemal Eren
License: BSD 3 clause

"""
from ..base import BaseEstimator, BiclusterMixin
from ..cluster.k_means_ import k_means
from sklearn.utils.extmath import randomized_svd
from sklearn.utils.arpack import svds
from sklearn.utils.validation import assert_all_finite, check_arrays

import numpy as np
from scipy.sparse import csr_matrix, lil_matrix, issparse
from ..externals import six
from abc import ABCMeta, abstractmethod

def _sparse_min(X):
    """Returns the minimum value of a dense or a CSR/CSC matrix.

    Adapated from http://stackoverflow.com/q/13426580

    """
    if issparse(X):
        if len(X.data) == 0:
            return 0
        m = X.data.min()
        return m if X.getnnz() == X.size else min(m, 0)
    else:
        return X.min()

def _make_nonnegative(X, min_value=0):
    """Ensure `X.min()` >= `min_value`.

    Converts X to a dense matrix if necessary.

    """
    min_ = _sparse_min(X)
    if min_ < min_value:
        if issparse(X):
            X = np.asarray(X.todense())
        X = X + (min_value - min_)
    return X

def _scale_preprocess(X):
    """Normalize `X` by scaling rows and columns independently.

    Returns the normalized matrix and the row and column scaling
    factors.

    """
    X = _make_nonnegative(X)
    row_diag = np.asarray(1.0 / np.sqrt(X.sum(axis=1))).squeeze()
    col_diag = np.asarray(1.0 / np.sqrt(X.sum(axis=0))).squeeze()
    row_diag = np.where(np.isnan(row_diag), 0, row_diag)
    col_diag = np.where(np.isnan(col_diag), 0, col_diag)
    if issparse(X):
        n_rows, n_cols = X.shape
        r = lil_matrix((n_rows, n_rows))
        c = lil_matrix((n_cols, n_cols))
        r.setdiag(row_diag)
        c.setdiag(col_diag)
        an = r * X * c
    else:
        an = row_diag[:, np.newaxis] * X * col_diag
    return an, row_diag, col_diag

def _bistochastic_preprocess(X, maxiter=1000, tol=1e-5):
    """Normalize rows and columns of `X` simultaneously so that all
    rows sum to one constant and all columns sum to a different
    constant.

    """
    # According to paper, this can also be done more efficiently with
    # deviation reduction and balancing algorithms.
    X = _make_nonnegative(X)
    X_scaled = X
    dist = None
    for _ in range(maxiter):
        X_new, _, _ = _scale_preprocess(X_scaled)
        if issparse(X):
            dist = np.linalg.norm(X_scaled.data - X.data)
        else:
            dist = np.linalg.norm(X_scaled - X_new)
        X_scaled = X_new
        if dist is not None and dist < tol:
            break
    return X_scaled

def _log_preprocess(X):
    """Normalize `X` according to Kluger's log-interactions scheme.

    Converts X to a dense matrix if necessary.

    """
    X = _make_nonnegative(X, min_value=1)
    if issparse(X):
        X = np.asarray(X.todense())
    L = np.log(X)
    row_avg = L.mean(axis=1)[:, np.newaxis]
    col_avg = L.mean(axis=0)
    avg = L.mean()
    return L - row_avg - col_avg + avg

def _svd(array, n_components, method, kwargs, random_state):
    """Returns first `n_components` left and right singular
    vectors u and v.

    """
    if method == 'randomized':
        u, _, vt = randomized_svd(array, n_components,
                                 random_state=random_state, **kwargs)

    elif method == 'arpack':
        u, _, vt = svds(array, k=n_components, **kwargs)

    assert_all_finite(u)
    assert_all_finite(vt)
    return u, vt.T

def _fit_best_piecewise(vectors, k, n_clusters, random_state,
                        kmeans_kwargs):
    """Find the `k` vectors that are best approximated by piecewise
    constant vectors.

    The piecewise vectors are found by k-means; the best is chosen
    according to Euclidean distance.

    """
    def make_piecewise(v):
        centroid, labels, _ = k_means(v.reshape(-1, 1), n_clusters,
                                      random_state=random_state,
                                      **kmeans_kwargs)
        return centroid[labels].reshape(-1)
    piecewise_vectors = np.apply_along_axis(make_piecewise, 1,
                                            vectors)
    dists = np.apply_along_axis(np.linalg.norm, 1,
                                vectors - piecewise_vectors)
    result = vectors[np.argsort(dists)[:k]]
    return result

def _project_and_cluster(data, vectors, n_clusters, random_state,
                         kmeans_kwargs):
    """Project `data` to `vectors` and cluster the result."""
    if issparse(data):
        projected = data * vectors
    else:
        projected = np.dot(data, vectors)
    _, labels, _ = k_means(projected, n_clusters,
                           random_state=random_state,
                           **kmeans_kwargs)
    return labels

class BaseSpectral(six.with_metaclass(ABCMeta, BaseEstimator,
                                      BiclusterMixin)):
    """Base class for spectral biclustering."""

    @abstractmethod
    def __init__(self, n_clusters, svd_method, svd_kwargs,
                 kmeans_kwargs, random_state):
        self.n_clusters = n_clusters
        self.svd_method = svd_method
        self.svd_kwargs = svd_kwargs
        self.kmeans_kwargs = kmeans_kwargs
        self.random_state = random_state

    def fit(self, X):
        """Creates a biclustering for X.

        Parameters
        ----------
        X : array-like, shape (n_samples, n_features)

        """
        X = check_arrays(X, sparse_format='csr', dtype=np.float64)[0]
        if X.ndim != 2:
            raise ValueError("Argument `X` has the wrong dimensionality."
                             " It must have exactly two dimensions.")
        self._fit(X)

class SpectralCoclustering(BaseSpectral):
    """Spectral Co-Clustering algorithm (Dhillon, 2001).

    Clusters rows and columns of an array `X` to solve the relaxed
    normalized cut of the bipartite graph created from `X` as follows:
    the edge between row vertex `i` and column vertex `j` has weight
    `X[i, j]`.

    The resulting bicluster structure is block-diagonal, since each
    row and each column belongs to exactly one bicluster.

    Supports sparse matrices, as long as they are nonnegative. If
    there are negative values, it will be converted to a dense array.

    Parameters
    ----------
    n_clusters : integer
        The number of biclusters to find.

    svd_method : string, optional, default: 'randomized'
        Selects the algorithm for finding singular vectors. May be
        'randomized' or 'arpack'. If 'randomized', use
        `sklearn.utils.extmath.randomized_svd`, which may be faster
        for large matrices. If 'arpack', use
        `sklearn.utils.arpack.svds`, which is more accurate, but
        possibly slower in some cases.

    svd_kwargs : dictionary, optional, default: {}
        Keyword arguments to pass to the svd function.

    kmeans_kwargs : dictionary, optinal, default: {}
        Keyword arguments to pass to sklearn.cluster.k_means_.

    random_state : int seed, RandomState instance, or None (default)
        A pseudo random number generator used by the K-Means
        initialization.

    Attributes
    ----------
    `rows_` : array-like, shape (n_row_clusters, n_rows)
        Results of the clustering. `rows[i, r]` is True if cluster `i`
        contains row `r`. Available only after calling ``fit``.

    `columns_` : array-like, shape (n_column_clusters, n_columns)
        Results of the clustering, like `rows`.

    `row_labels_` : array-like, shape (n_rows,)
        The bicluster label of each row.

    `column_labels_` : array-like, shape (n_cols,)
        The bicluster label of each column.

    References
    ----------

    - Co-clustering documents and words using
      bipartite spectral graph partitioning, 2001
      Dhillon, Inderjit S.
      http://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.140.3011

    """
    def __init__(self, n_clusters=3, svd_method='randomized',
                 svd_kwargs={}, kmeans_kwargs={}, random_state=None):
        super(SpectralCoclustering, self).__init__(n_clusters,
                                                   svd_method,
                                                   svd_kwargs,
                                                   kmeans_kwargs,
                                                   random_state)

    def _fit(self, X):
        normalized_data, row_diag, col_diag = _scale_preprocess(X)
        n_singular_vals = 1 + int(np.ceil(np.log2(self.n_clusters)))
        u, v = _svd(normalized_data, n_singular_vals, self.svd_method,
                    self.svd_kwargs, self.random_state)
        z = np.vstack((row_diag[:, np.newaxis] * u[:, 1:],
                       col_diag[:, np.newaxis] * v[:, 1:]))
        _, labels, _ = k_means(z, self.n_clusters,
                               random_state=self.random_state,
                               **self.kmeans_kwargs)

        n_rows = X.shape[0]
        self.row_labels_ = labels[0:n_rows]
        self.column_labels_ = labels[n_rows:]

        self.rows_ = np.vstack(self.row_labels_ == c
                               for c in range(self.n_clusters))
        self.columns_ = np.vstack(self.column_labels_ == c
                                  for c in range(self.n_clusters))

class SpectralBiclustering(BaseSpectral):
    """Spectral biclustering (Kluger, 2003).

    Partitions rows and columns under the assumption that the data has
    an underlying checkerboard structure. For instance, if there are
    two row partitions and three column partitions, each row will
    belong to three biclusters, and each column will belong to two
    biclusters. The outer product of the corresponding row and column
    label vectors gives this checkerboard structure.

    Parameters
    ----------
    n_clusters : integer or tuple (n_row_clusters, n_column_clusters)
        The number of row and column clusters in the checkerboard
        structure.

    method : string
        Method of normalizing and converting singular vectors into
        biclusters. May be one of 'scale', 'bistochastic', or 'log'.
        CAUTION: if `method='log'`, the data must be converted to a
        dense array.

    n_components : integer
        Number of singular vectors to check.

    n_best : integer
        Number of best singular vectors to which to project the data
        for clustering.

    svd_method : string, optional, default: 'randomized'
        Selects the algorithm for finding singular vectors. May be
        'randomized' or 'arpack'. If 'randomized', uses
        `sklearn.utils.extmath.randomized_svd`, which may be faster
        for large matrices. If 'arpack', uses
        `sklearn.utils.arpack.svds`, which is more accurate, but
        possibly slower in some cases.

    svd_kwargs : dictionary, optional, default: {}
        Keyword arguments to pass to the svd function.

    kmeans_kwargs : dictionary, optinal, default: {}
        Keyword arguments to pass to sklearn.cluster.k_means_.

    random_state : int seed, RandomState instance, or None (default)
        A pseudo random number generator used by the K-Means
        initialization.

    Attributes
    ----------
    `rows_` : array-like, shape (n_row_clusters, n_rows)
        Results of the clustering. `rows[i, r]` is True if cluster `i`
        contains row `r`. Available only after calling ``fit``.

    `columns_` : array-like, shape (n_column_clusters, n_columns)
        Results of the clustering, like `rows`.

    `row_labels_` : array-like, shape (n_rows,)
        Row partition labels.

    `column_labels_` : array-like, shape (n_cols,)
        Column partition labels.

    References
    ----------

    - Spectral biclustering of microarray data:
      coclustering genes and conditions, 2003
      Kluger, Yuval, et al.
      http://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.135.1608

    """
    def __init__(self, n_clusters=3, method='bistochastic',
                 n_components=6, n_best=3,
                 svd_method='randomized', svd_kwargs={}, kmeans_kwargs={},
                 random_state=None):
        super(SpectralBiclustering, self).__init__(n_clusters,
                                                   svd_method,
                                                   svd_kwargs,
                                                   kmeans_kwargs,
                                                   random_state)
        legal_methods = ('bistochastic', 'scale', 'log')
        if method not in legal_methods:
            raise ValueError("Unknown method: '{}'. `method` must be"
                             " one of {}.".format(method, legal_methods))
        legal_svd_methods = ('randomized', 'arpack')
        if svd_method not in legal_svd_methods:
            raise ValueError("Unknown SVD method: '{}'. `svd_method` must be"
                             " one of {}.".format(svd_method,
                                                  legal_svd_methods))

        try:
            int(n_clusters)
        except TypeError:
            try:
                r, c = n_clusters
                int(r)
                int(c)
            except (ValueError, TypeError):
                raise ValueError("The parameter `n_clusters` is incorrect."
                                 " It should either be a single integer"
                                 " or an iterable with two integers:"
                                 " `(n_row_clusters, n_column_clusters)`")
        if n_best > n_components:
            raise ValueError("`n_best` cannot be larger than"
                             " `n_components`.")

        self.method = method
        self.n_components = n_components
        self.n_best = n_best

    def _fit(self, X):
        n_sv = self.n_components
        if self.method == 'bistochastic':
            normalized_data = _bistochastic_preprocess(X)
            n_sv += 1
        elif self.method == 'scale':
            normalized_data, _, _ = _scale_preprocess(X)
            n_sv += 1
        elif self.method == 'log':
            normalized_data = _log_preprocess(X)
        u, v = _svd(normalized_data, n_sv, self.svd_method,
                    self.svd_kwargs, self.random_state)
        ut = u.T
        vt = v.T
        if self.method != 'log':
            ut = ut[1:]
            vt = vt[1:]

        try:
            n_row_clusters, n_col_clusters = self.n_clusters
        except ValueError:
            n_row_clusters = n_col_clusters = self.n_clusters

        best_ut = _fit_best_piecewise(ut, self.n_best,
                                      n_row_clusters,
                                      self.random_state,
                                      self.kmeans_kwargs)

        best_vt = _fit_best_piecewise(vt, self.n_best,
                                      n_col_clusters,
                                      self.random_state,
                                      self.kmeans_kwargs)

        self.row_labels_ = _project_and_cluster(X, best_vt.T,
                                                n_row_clusters,
                                                self.random_state,
                                                self.kmeans_kwargs)

        self.column_labels_ = _project_and_cluster(X.T, best_ut.T,
                                                   n_col_clusters,
                                                   self.random_state,
                                                   self.kmeans_kwargs)

        self.rows_ = np.vstack(self.row_labels_ == label
                               for label in range(n_row_clusters)
                               for _ in range(n_col_clusters))
        self.columns_ = np.vstack(self.column_labels_ == label
                                  for _ in range(n_row_clusters)
                                  for label in range(n_col_clusters))
<commit_msg>fixed exception type<commit_after>"""Implements spectral biclustering algorithms.

Authors : Kemal Eren
License: BSD 3 clause

"""
from ..base import BaseEstimator, BiclusterMixin
from ..cluster.k_means_ import k_means
from sklearn.utils.extmath import randomized_svd
from sklearn.utils.arpack import svds
from sklearn.utils.validation import assert_all_finite, check_arrays

import numpy as np
from scipy.sparse import csr_matrix, lil_matrix, issparse
from ..externals import six
from abc import ABCMeta, abstractmethod

def _sparse_min(X):
    """Returns the minimum value of a dense or a CSR/CSC matrix.

    Adapated from http://stackoverflow.com/q/13426580

    """
    if issparse(X):
        if len(X.data) == 0:
            return 0
        m = X.data.min()
        return m if X.getnnz() == X.size else min(m, 0)
    else:
        return X.min()

def _make_nonnegative(X, min_value=0):
    """Ensure `X.min()` >= `min_value`.

    Converts X to a dense matrix if necessary.

    """
    min_ = _sparse_min(X)
    if min_ < min_value:
        if issparse(X):
            X = np.asarray(X.todense())
        X = X + (min_value - min_)
    return X

def _scale_preprocess(X):
    """Normalize `X` by scaling rows and columns independently.

    Returns the normalized matrix and the row and column scaling
    factors.

    """
    X = _make_nonnegative(X)
    row_diag = np.asarray(1.0 / np.sqrt(X.sum(axis=1))).squeeze()
    col_diag = np.asarray(1.0 / np.sqrt(X.sum(axis=0))).squeeze()
    row_diag = np.where(np.isnan(row_diag), 0, row_diag)
    col_diag = np.where(np.isnan(col_diag), 0, col_diag)
    if issparse(X):
        n_rows, n_cols = X.shape
        r = lil_matrix((n_rows, n_rows))
        c = lil_matrix((n_cols, n_cols))
        r.setdiag(row_diag)
        c.setdiag(col_diag)
        an = r * X * c
    else:
        an = row_diag[:, np.newaxis] * X * col_diag
    return an, row_diag, col_diag

def _bistochastic_preprocess(X, maxiter=1000, tol=1e-5):
    """Normalize rows and columns of `X` simultaneously so that all
    rows sum to one constant and all columns sum to a different
    constant.

    """
    # According to paper, this can also be done more efficiently with
    # deviation reduction and balancing algorithms.
    X = _make_nonnegative(X)
    X_scaled = X
    dist = None
    for _ in range(maxiter):
        X_new, _, _ = _scale_preprocess(X_scaled)
        if issparse(X):
            dist = np.linalg.norm(X_scaled.data - X.data)
        else:
            dist = np.linalg.norm(X_scaled - X_new)
        X_scaled = X_new
        if dist is not None and dist < tol:
            break
    return X_scaled

def _log_preprocess(X):
    """Normalize `X` according to Kluger's log-interactions scheme.

    Converts X to a dense matrix if necessary.

    """
    X = _make_nonnegative(X, min_value=1)
    if issparse(X):
        X = np.asarray(X.todense())
    L = np.log(X)
    row_avg = L.mean(axis=1)[:, np.newaxis]
    col_avg = L.mean(axis=0)
    avg = L.mean()
    return L - row_avg - col_avg + avg

def _svd(array, n_components, method, kwargs, random_state):
    """Returns first `n_components` left and right singular
    vectors u and v.

    """
    if method == 'randomized':
        u, _, vt = randomized_svd(array, n_components,
                                 random_state=random_state, **kwargs)

    elif method == 'arpack':
        u, _, vt = svds(array, k=n_components, **kwargs)

    assert_all_finite(u)
    assert_all_finite(vt)
    return u, vt.T

def _fit_best_piecewise(vectors, k, n_clusters, random_state,
                        kmeans_kwargs):
    """Find the `k` vectors that are best approximated by piecewise
    constant vectors.

    The piecewise vectors are found by k-means; the best is chosen
    according to Euclidean distance.

    """
    def make_piecewise(v):
        centroid, labels, _ = k_means(v.reshape(-1, 1), n_clusters,
                                      random_state=random_state,
                                      **kmeans_kwargs)
        return centroid[labels].reshape(-1)
    piecewise_vectors = np.apply_along_axis(make_piecewise, 1,
                                            vectors)
    dists = np.apply_along_axis(np.linalg.norm, 1,
                                vectors - piecewise_vectors)
    result = vectors[np.argsort(dists)[:k]]
    return result

def _project_and_cluster(data, vectors, n_clusters, random_state,
                         kmeans_kwargs):
    """Project `data` to `vectors` and cluster the result."""
    if issparse(data):
        projected = data * vectors
    else:
        projected = np.dot(data, vectors)
    _, labels, _ = k_means(projected, n_clusters,
                           random_state=random_state,
                           **kmeans_kwargs)
    return labels

class BaseSpectral(six.with_metaclass(ABCMeta, BaseEstimator,
                                      BiclusterMixin)):
    """Base class for spectral biclustering."""

    @abstractmethod
    def __init__(self, n_clusters, svd_method, svd_kwargs,
                 kmeans_kwargs, random_state):
        self.n_clusters = n_clusters
        self.svd_method = svd_method
        self.svd_kwargs = svd_kwargs
        self.kmeans_kwargs = kmeans_kwargs
        self.random_state = random_state

    def fit(self, X):
        """Creates a biclustering for X.

        Parameters
        ----------
        X : array-like, shape (n_samples, n_features)

        """
        X = check_arrays(X, sparse_format='csr', dtype=np.float64)[0]
        if X.ndim != 2:
            raise ValueError("Argument `X` has the wrong dimensionality."
                             " It must have exactly two dimensions.")
        self._fit(X)

class SpectralCoclustering(BaseSpectral):
    """Spectral Co-Clustering algorithm (Dhillon, 2001).

    Clusters rows and columns of an array `X` to solve the relaxed
    normalized cut of the bipartite graph created from `X` as follows:
    the edge between row vertex `i` and column vertex `j` has weight
    `X[i, j]`.

    The resulting bicluster structure is block-diagonal, since each
    row and each column belongs to exactly one bicluster.

    Supports sparse matrices, as long as they are nonnegative. If
    there are negative values, it will be converted to a dense array.

    Parameters
    ----------
    n_clusters : integer
        The number of biclusters to find.

    svd_method : string, optional, default: 'randomized'
        Selects the algorithm for finding singular vectors. May be
        'randomized' or 'arpack'. If 'randomized', use
        `sklearn.utils.extmath.randomized_svd`, which may be faster
        for large matrices. If 'arpack', use
        `sklearn.utils.arpack.svds`, which is more accurate, but
        possibly slower in some cases.

    svd_kwargs : dictionary, optional, default: {}
        Keyword arguments to pass to the svd function.

    kmeans_kwargs : dictionary, optinal, default: {}
        Keyword arguments to pass to sklearn.cluster.k_means_.

    random_state : int seed, RandomState instance, or None (default)
        A pseudo random number generator used by the K-Means
        initialization.

    Attributes
    ----------
    `rows_` : array-like, shape (n_row_clusters, n_rows)
        Results of the clustering. `rows[i, r]` is True if cluster `i`
        contains row `r`. Available only after calling ``fit``.

    `columns_` : array-like, shape (n_column_clusters, n_columns)
        Results of the clustering, like `rows`.

    `row_labels_` : array-like, shape (n_rows,)
        The bicluster label of each row.

    `column_labels_` : array-like, shape (n_cols,)
        The bicluster label of each column.

    References
    ----------

    - Co-clustering documents and words using
      bipartite spectral graph partitioning, 2001
      Dhillon, Inderjit S.
      http://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.140.3011

    """
    def __init__(self, n_clusters=3, svd_method='randomized',
                 svd_kwargs={}, kmeans_kwargs={}, random_state=None):
        super(SpectralCoclustering, self).__init__(n_clusters,
                                                   svd_method,
                                                   svd_kwargs,
                                                   kmeans_kwargs,
                                                   random_state)

    def _fit(self, X):
        normalized_data, row_diag, col_diag = _scale_preprocess(X)
        n_singular_vals = 1 + int(np.ceil(np.log2(self.n_clusters)))
        u, v = _svd(normalized_data, n_singular_vals, self.svd_method,
                    self.svd_kwargs, self.random_state)
        z = np.vstack((row_diag[:, np.newaxis] * u[:, 1:],
                       col_diag[:, np.newaxis] * v[:, 1:]))
        _, labels, _ = k_means(z, self.n_clusters,
                               random_state=self.random_state,
                               **self.kmeans_kwargs)

        n_rows = X.shape[0]
        self.row_labels_ = labels[0:n_rows]
        self.column_labels_ = labels[n_rows:]

        self.rows_ = np.vstack(self.row_labels_ == c
                               for c in range(self.n_clusters))
        self.columns_ = np.vstack(self.column_labels_ == c
                                  for c in range(self.n_clusters))

class SpectralBiclustering(BaseSpectral):
    """Spectral biclustering (Kluger, 2003).

    Partitions rows and columns under the assumption that the data has
    an underlying checkerboard structure. For instance, if there are
    two row partitions and three column partitions, each row will
    belong to three biclusters, and each column will belong to two
    biclusters. The outer product of the corresponding row and column
    label vectors gives this checkerboard structure.

    Parameters
    ----------
    n_clusters : integer or tuple (n_row_clusters, n_column_clusters)
        The number of row and column clusters in the checkerboard
        structure.

    method : string
        Method of normalizing and converting singular vectors into
        biclusters. May be one of 'scale', 'bistochastic', or 'log'.
        CAUTION: if `method='log'`, the data must be converted to a
        dense array.

    n_components : integer
        Number of singular vectors to check.

    n_best : integer
        Number of best singular vectors to which to project the data
        for clustering.

    svd_method : string, optional, default: 'randomized'
        Selects the algorithm for finding singular vectors. May be
        'randomized' or 'arpack'. If 'randomized', uses
        `sklearn.utils.extmath.randomized_svd`, which may be faster
        for large matrices. If 'arpack', uses
        `sklearn.utils.arpack.svds`, which is more accurate, but
        possibly slower in some cases.

    svd_kwargs : dictionary, optional, default: {}
        Keyword arguments to pass to the svd function.

    kmeans_kwargs : dictionary, optinal, default: {}
        Keyword arguments to pass to sklearn.cluster.k_means_.

    random_state : int seed, RandomState instance, or None (default)
        A pseudo random number generator used by the K-Means
        initialization.

    Attributes
    ----------
    `rows_` : array-like, shape (n_row_clusters, n_rows)
        Results of the clustering. `rows[i, r]` is True if cluster `i`
        contains row `r`. Available only after calling ``fit``.

    `columns_` : array-like, shape (n_column_clusters, n_columns)
        Results of the clustering, like `rows`.

    `row_labels_` : array-like, shape (n_rows,)
        Row partition labels.

    `column_labels_` : array-like, shape (n_cols,)
        Column partition labels.

    References
    ----------

    - Spectral biclustering of microarray data:
      coclustering genes and conditions, 2003
      Kluger, Yuval, et al.
      http://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.135.1608

    """
    def __init__(self, n_clusters=3, method='bistochastic',
                 n_components=6, n_best=3,
                 svd_method='randomized', svd_kwargs={}, kmeans_kwargs={},
                 random_state=None):
        super(SpectralBiclustering, self).__init__(n_clusters,
                                                   svd_method,
                                                   svd_kwargs,
                                                   kmeans_kwargs,
                                                   random_state)
        legal_methods = ('bistochastic', 'scale', 'log')
        if method not in legal_methods:
            raise ValueError("Unknown method: '{}'. `method` must be"
                             " one of {}.".format(method, legal_methods))
        legal_svd_methods = ('randomized', 'arpack')
        if svd_method not in legal_svd_methods:
            raise ValueError("Unknown SVD method: '{}'. `svd_method` must be"
                             " one of {}.".format(svd_method,
                                                  legal_svd_methods))

        try:
            int(n_clusters)
        except TypeError:
            try:
                r, c = n_clusters
                int(r)
                int(c)
            except (ValueError, TypeError):
                raise ValueError("The parameter `n_clusters` is incorrect."
                                 " It should either be a single integer"
                                 " or an iterable with two integers:"
                                 " `(n_row_clusters, n_column_clusters)`")
        if n_best > n_components:
            raise ValueError("`n_best` cannot be larger than"
                             " `n_components`.")

        self.method = method
        self.n_components = n_components
        self.n_best = n_best

    def _fit(self, X):
        n_sv = self.n_components
        if self.method == 'bistochastic':
            normalized_data = _bistochastic_preprocess(X)
            n_sv += 1
        elif self.method == 'scale':
            normalized_data, _, _ = _scale_preprocess(X)
            n_sv += 1
        elif self.method == 'log':
            normalized_data = _log_preprocess(X)
        u, v = _svd(normalized_data, n_sv, self.svd_method,
                    self.svd_kwargs, self.random_state)
        ut = u.T
        vt = v.T
        if self.method != 'log':
            ut = ut[1:]
            vt = vt[1:]

        try:
            n_row_clusters, n_col_clusters = self.n_clusters
        except TypeError:
            n_row_clusters = n_col_clusters = self.n_clusters

        best_ut = _fit_best_piecewise(ut, self.n_best,
                                      n_row_clusters,
                                      self.random_state,
                                      self.kmeans_kwargs)

        best_vt = _fit_best_piecewise(vt, self.n_best,
                                      n_col_clusters,
                                      self.random_state,
                                      self.kmeans_kwargs)

        self.row_labels_ = _project_and_cluster(X, best_vt.T,
                                                n_row_clusters,
                                                self.random_state,
                                                self.kmeans_kwargs)

        self.column_labels_ = _project_and_cluster(X.T, best_ut.T,
                                                   n_col_clusters,
                                                   self.random_state,
                                                   self.kmeans_kwargs)

        self.rows_ = np.vstack(self.row_labels_ == label
                               for label in range(n_row_clusters)
                               for _ in range(n_col_clusters))
        self.columns_ = np.vstack(self.column_labels_ == label
                                  for _ in range(n_row_clusters)
                                  for label in range(n_col_clusters))
<|endoftext|>