<commit_before>import numpy as np

from Interpolation import *

class NewtonInterpolation(Interpolation):

    def __init__(self, x, y):
        Interpolation.__init__(self, x, y)
        self.diffQuotient(x, y)

    def diffQuotient(self, x, y):
        self.dquo = np.zeros(self.size - 1)
        self.dquo = self.dquo.reshape(self.size - 1, 1)
        for k in range(1, self.size):
            for j in range(k + 1):
                self.dquo[k - 1][0] += self.y[0][j] / self.omega(j, k)

        return 0

    def omega(self, j, k):
        xi = np.empty(k)
        if j != 0:
            xi[ : j] = self.x[0][ : j]
        if j != k:
            xi[j : ] = self.x[0][j + 1 : k + 1]

        xi = np.linspace(self.x[0][j], self.x[0][j], k) - xi

        return xi.cumprod()[-1]

    def predict(self, x):
        if not isinstance(x, np.ndarray):
            raise TypeError("Type of Parameter should be 1 dimension numpy.ndarray")
        else:
            if len(x.shape) != 1:
                raise TypeError("Type of Parameter should be 1 dimension numpy.ndarray")

        y = []
        for xi in x:
            xmult = np.linspace(xi, xi, self.size - 1) - self.y[0][self.size - 1]
            xmult = xmult.cumprod().reshape(1, self.size - 1)
            y.append(np.dot(xmult, self.dquo)[0][0] + self.y[0][0])

        return np.array(y)
<commit_msg>Fix Bugs<commit_after>import numpy as np

from Interpolation import *

class NewtonInterpolation(Interpolation):

    def __init__(self, x, y):
        Interpolation.__init__(self, x, y)
        self.diffQuotient(x, y)

    def diffQuotient(self, x, y):
        self.dquo = np.zeros(self.size - 1)
        self.dquo = self.dquo.reshape(self.size - 1, 1)
        for k in range(1, self.size):
            for j in range(k + 1):
                self.dquo[k - 1][0] += self.y[0][j] / self.omega(j, k)

        return 0

    def omega(self, j, k):
        xi = np.empty(k)
        if j != 0:
            xi[ : j] = self.x[0][ : j]
        if j != k:
            xi[j : ] = self.x[0][j + 1 : k + 1]

        xi = np.linspace(self.x[0][j], self.x[0][j], k) - xi

        return xi.cumprod()[-1]

    def predict(self, x):
        if not isinstance(x, np.ndarray):
            raise TypeError("Type of Parameter should be 1 dimension numpy.ndarray")
        else:
            if len(x.shape) != 1:
                raise TypeError("Type of Parameter should be 1 dimension numpy.ndarray")

        y = []
        for xi in x:
            xmult = np.linspace(xi, xi, self.size - 1) - self.x[0][ : -1]
            xmult = xmult.cumprod().reshape(1, self.size - 1)
            y.append(np.dot(xmult, self.dquo)[0][0] + self.y[0][0])

        return np.array(y)
<|endoftext|>


<commit_before>from .irc_message import IRCMessage
from .irc_level import IRCLevel, get_userlevels
from .database import Database
from socket import create_connection
from importlib import import_module, reload
from os import listdir
from time import strftime, localtime
import re
import traceback

class IRCBot:
    plugins = {}
    commands = {}
    patterns = []
    sinks = []

    def __init__(self, nick, server, port=6667, command_prefix='.'):
        """ Create a new instance of IRCBot. """
        self.nick = nick
        self.server = server
        self.port = port
        self.command_prefix = command_prefix

        self.socket = None
        self.running = self.connected = self.identified = False
        self.read_buffer = ''

        self.database = None
        self.channels = []
        self.ignore_list = []
        self.userlevels = {}

        self.database = Database()

    def run(self):
        """ Run the bot.

        If an exception is raised during message handing
        or recv, it is eaten, and the traceback is printed.

        """
        self.running = True
        while self.running:
            self.__connect()
            while self.connected:
                try:
                    for msg in self.__recv():
                        message = IRCMessage(msg)
                        self.__handle_message(message)
                except:
                    traceback.print_exc()

    @staticmethod
    def load_plugins(folder):
        """ Load or reload plugins from folder. """
        IRCBot.commands = {}
        IRCBot.patterns = []
        IRCBot.sinks = []

        for file in listdir(folder):
            if file.endswith('.py'):
                module_name = folder + '.' + file[:-3]
                if module_name not in IRCBot.plugins:
                    print("Loading {}".format(module_name))
                    module = import_module(module_name)
                    IRCBot.plugins[module_name] = module
                else:
                    print("Reloading {}".format(module_name))
                    reload(IRCBot.plugins[module_name])

    @staticmethod
    def reload_plugins():
        """ Reloads all loaded plugins. """
        IRCBot.commands = {}
        IRCBot.patterns = []
        IRCBot.sinks = []

        for module_name, module in IRCBot.plugins.items():
            print("Reloading {}".format(module_name))
            reload(module)

    def load_database(self, path):
        self.database = Database(path)

    def join(self, channel):
        """ Join a channel. """
        if self.connected:
            self.send('JOIN ' + channel)
        self.channels.append(channel)

    def ignore(self, hostmask):
        """ Ignore a user with the given hostmask. """
        pattern = re.compile(hostmask.replace('*', '.*'), re.IGNORECASE)
        self.ignore_list.append(pattern)

    def __ignored(self, host):
        """ Test if a given user is ignored or not. """
        if host is None:
            return False

        for pattern in self.ignore_list:
            if pattern.match(host):
                return True
        else:
            return False

    def __connect(self):
        """ Connect the socket. """
        self.socket = create_connection((self.server, self.port))
        self.connected = True
        self.identified = False

    def disconnect(self):
        """ Disconnect the bot. """
        self.send('QUIT :BAI!')
        self.running = self.connected = self.identified = False

    def __recv(self):
        """ Receive messages from the socket. """
        self.read_buffer += str(self.socket.recv(512), 'ASCII', 'ignore')
        self.read_buffer = strip_control_codes(self.read_buffer)
        msgs = self.read_buffer.split('\r\n')
        self.read_buffer = msgs.pop()
        return msgs

    def send(self, msg):
        """ Send a message to the socket. """
        if msg is not None:
            self.socket.send(bytes(msg + '\r\n', 'UTF-8'))
            print("Sent: {}".format(msg))

    def __handle_message(self, message):
        """ Handle an IRCMessage object with the appropriate handler.

        Will ignore a user if their mask is on the ignore list.

        """
        print(message)

        if not self.__ignored(message.sender):
            mapping = {
                'PING': IRCBot.__handle_ping,
                'PRIVMSG': IRCBot.__handle_privmsg,
                'NOTICE': IRCBot.__handle_notice,
                'INVITE': IRCBot.__handle_invite,
                'JOIN': IRCBot.__handle_join,
                'PART': IRCBot.__handle_leave,
                'QUIT': IRCBot.__handle_leave,
                '353': IRCBot.__handle_names,
                'MODE': IRCBot.__handle_mode,
                'ERROR': IRCBot.__handle_error
            }
            if message.command.upper() in mapping:
                self.send(mapping[message.command.upper()](self, message))
            else:
                print("Unknown command: {}".format(message.command))

    def __handle_ping(self, message):
        """ Handle the server's pings. """
        self.send('PONG :' + message.message)

    def __handle_privmsg(self, message):
        """ Handle the privmsg commands.

        Will send the reply back to the channel the command was sent from,
        or back to the user whom sent it in the case of a private message.
        Commands (prefixed with command_prefix) are executed, CTCP is handled,
        and the matches are checked.

        """
        response = None

        target = message.channel \
            if is_channel(message.channel) \
            else message.nick

        if message.message.startswith(self.command_prefix):
            command = message.message.split(' ')[0][len(self.command_prefix):]
            response = IRCBot.commands[command](self, message, self.database)
            if response is not None:
                response = 'PRIVMSG {} :{}'.format(target, response)

        elif is_ctcp(message):
            response = ctcp_response(message.message[1:-1])
            if response is not None:
                response = 'NOTICE {} :\u0001{}\u0001'.format(target, response)

        else:
            for pattern, func in IRCBot.patterns:
                if pattern.search(message.message):
                    response = func(self, message, self.database)
                    if response is not None:
                        response = 'PRIVMSG {} :{}'.format(target, response)

            if response is None:
                for sink in IRCBot.sinks:
                    response = sink(message, self.database)
                    if response is not None:
                        response = 'PRIVMSG {} :{}'.format(target, response)
                        break

        return response

    def __handle_notice(self, message):
        """ Use the notice message to identify and register to the server. """
        if not self.identified:
            self.send('USER MotoBot localhost localhost MotoBot')
            self.send('NICK ' + self.nick)
            for channel in self.channels:
                self.send('JOIN ' + channel)
            self.identified = True

    def __handle_invite(self, message):
        """ Join a channel when invited. """
        self.join(message.message)

    def __handle_names(self, message):
        """ Parse the name command and record the userlevels of users. """
        channel = message.channel.split(' ')[-1]
        for nick in message.message.split(' '):
            self.userlevels[(nick.lstrip('+%@&~'), channel)] = \
                get_userlevels(nick)

    def __handle_join(self, message):
        """ Handle the join of a user. """
        self.userlevels[(message.nick, message.message)] = [0]

    def __handle_leave(self, message):
        """ Handle the part or quit of a user. """
        pass

    def __handle_mode(self, message):
        """ Handle the mode command and update userlevels accordingly. """
        pass

    def __handle_error(self, message):
        """ Handle an error message from the server. """
        self.connected = self.identified = False

def userlevel_wrapper(func, level):
    """ Modify a plugin to only take users above a certain userlevel. """
    def wrapped(bot, message):
        if max(bot.userlevels[(message.nick, message.channel)]) >= level:
            return func(message)
    return wrapped

def command(name, level=IRCLevel.user):
    """ Decorator to add a command to the bot. """
    def register_command(func):
        wrapped = userlevel_wrapper(func, level)
        IRCBot.commands[name] = wrapped
        return func
    return register_command

def match(pattern, level=IRCLevel.user):
    """ Decorator to add a regex pattern to the bot. """
    def register_pattern(func):
        wrapped = userlevel_wrapper(func, level)
        IRCBot.patterns.append((re.compile(pattern, re.IGNORECASE), wrapped))
        return func
    return register_pattern

def sink(func):
    """ Decorator to add sink to the bot. """
    IRCBot.sinks.append(func)
    return func

def action(message):
    """ Make the message an action. """
    return '\u0001ACTION {}\u0001'.format(message)

def is_channel(name):
    """ Check if a name is a valid channel name or not. """
    valid = ['&', '#', '+', '!']
    invalid = [' ', ',', '\u0007']
    return (name[0] in valid) and all(c not in invalid for c in name)

def is_ctcp(message):
    """ Check if a message object is a ctcp message or not. """
    return message.message.startswith('\u0001') and \
        message.message.endswith('\u0001')

def ctcp_response(message):
    """ Return the appropriate response to a CTCP request. """
    mapping = {
        'VERSION': 'MotoBot Version 2.0',
        'FINGER': 'Oh you dirty man!',
        'TIME': strftime('%a %b %d %H:%M:%S', localtime()),
        'PING': message
    }
    return mapping.get(message.split(' ')[0].upper(), None)

def strip_control_codes(input):
    """ Strip the control codes from the input. """
    pattern = re.compile(r'\x03[0-9]{0,2},?[0-9]{0,2}|\x02|\x1D|\x1F|\x16|\x0F')
    output = pattern.sub('', input)
    return output
<commit_msg>Change decorator to accept stuff<commit_after>from .irc_message import IRCMessage
from .irc_level import IRCLevel, get_userlevels
from .database import Database
from socket import create_connection
from importlib import import_module, reload
from os import listdir
from time import strftime, localtime
import re
import traceback

class IRCBot:
    plugins = {}
    commands = {}
    patterns = []
    sinks = []

    def __init__(self, nick, server, port=6667, command_prefix='.'):
        """ Create a new instance of IRCBot. """
        self.nick = nick
        self.server = server
        self.port = port
        self.command_prefix = command_prefix

        self.socket = None
        self.running = self.connected = self.identified = False
        self.read_buffer = ''

        self.database = None
        self.channels = []
        self.ignore_list = []
        self.userlevels = {}

        self.database = Database()

    def run(self):
        """ Run the bot.

        If an exception is raised during message handing
        or recv, it is eaten, and the traceback is printed.

        """
        self.running = True
        while self.running:
            self.__connect()
            while self.connected:
                try:
                    for msg in self.__recv():
                        message = IRCMessage(msg)
                        self.__handle_message(message)
                except:
                    traceback.print_exc()

    @staticmethod
    def load_plugins(folder):
        """ Load or reload plugins from folder. """
        IRCBot.commands = {}
        IRCBot.patterns = []
        IRCBot.sinks = []

        for file in listdir(folder):
            if file.endswith('.py'):
                module_name = folder + '.' + file[:-3]
                if module_name not in IRCBot.plugins:
                    print("Loading {}".format(module_name))
                    module = import_module(module_name)
                    IRCBot.plugins[module_name] = module
                else:
                    print("Reloading {}".format(module_name))
                    reload(IRCBot.plugins[module_name])

    @staticmethod
    def reload_plugins():
        """ Reloads all loaded plugins. """
        IRCBot.commands = {}
        IRCBot.patterns = []
        IRCBot.sinks = []

        for module_name, module in IRCBot.plugins.items():
            print("Reloading {}".format(module_name))
            reload(module)

    def load_database(self, path):
        self.database = Database(path)

    def join(self, channel):
        """ Join a channel. """
        if self.connected:
            self.send('JOIN ' + channel)
        self.channels.append(channel)

    def ignore(self, hostmask):
        """ Ignore a user with the given hostmask. """
        pattern = re.compile(hostmask.replace('*', '.*'), re.IGNORECASE)
        self.ignore_list.append(pattern)

    def __ignored(self, host):
        """ Test if a given user is ignored or not. """
        if host is None:
            return False

        for pattern in self.ignore_list:
            if pattern.match(host):
                return True
        else:
            return False

    def __connect(self):
        """ Connect the socket. """
        self.socket = create_connection((self.server, self.port))
        self.connected = True
        self.identified = False

    def disconnect(self):
        """ Disconnect the bot. """
        self.send('QUIT :BAI!')
        self.running = self.connected = self.identified = False

    def __recv(self):
        """ Receive messages from the socket. """
        self.read_buffer += str(self.socket.recv(512), 'ASCII', 'ignore')
        self.read_buffer = strip_control_codes(self.read_buffer)
        msgs = self.read_buffer.split('\r\n')
        self.read_buffer = msgs.pop()
        return msgs

    def send(self, msg):
        """ Send a message to the socket. """
        if msg is not None:
            self.socket.send(bytes(msg + '\r\n', 'UTF-8'))
            print("Sent: {}".format(msg))

    def __handle_message(self, message):
        """ Handle an IRCMessage object with the appropriate handler.

        Will ignore a user if their mask is on the ignore list.

        """
        print(message)

        if not self.__ignored(message.sender):
            mapping = {
                'PING': IRCBot.__handle_ping,
                'PRIVMSG': IRCBot.__handle_privmsg,
                'NOTICE': IRCBot.__handle_notice,
                'INVITE': IRCBot.__handle_invite,
                'JOIN': IRCBot.__handle_join,
                'PART': IRCBot.__handle_leave,
                'QUIT': IRCBot.__handle_leave,
                '353': IRCBot.__handle_names,
                'MODE': IRCBot.__handle_mode,
                'ERROR': IRCBot.__handle_error
            }
            if message.command.upper() in mapping:
                self.send(mapping[message.command.upper()](self, message))
            else:
                print("Unknown command: {}".format(message.command))

    def __handle_ping(self, message):
        """ Handle the server's pings. """
        self.send('PONG :' + message.message)

    def __handle_privmsg(self, message):
        """ Handle the privmsg commands.

        Will send the reply back to the channel the command was sent from,
        or back to the user whom sent it in the case of a private message.
        Commands (prefixed with command_prefix) are executed, CTCP is handled,
        and the matches are checked.

        """
        response = None

        target = message.channel \
            if is_channel(message.channel) \
            else message.nick

        if message.message.startswith(self.command_prefix):
            command = message.message.split(' ')[0][len(self.command_prefix):]
            response = IRCBot.commands[command](self, message, self.database)
            if response is not None:
                response = 'PRIVMSG {} :{}'.format(target, response)

        elif is_ctcp(message):
            response = ctcp_response(message.message[1:-1])
            if response is not None:
                response = 'NOTICE {} :\u0001{}\u0001'.format(target, response)

        else:
            for pattern, func in IRCBot.patterns:
                if pattern.search(message.message):
                    response = func(self, message, self.database)
                    if response is not None:
                        response = 'PRIVMSG {} :{}'.format(target, response)

            if response is None:
                for sink in IRCBot.sinks:
                    response = sink(message, self.database)
                    if response is not None:
                        response = 'PRIVMSG {} :{}'.format(target, response)
                        break

        return response

    def __handle_notice(self, message):
        """ Use the notice message to identify and register to the server. """
        if not self.identified:
            self.send('USER MotoBot localhost localhost MotoBot')
            self.send('NICK ' + self.nick)
            for channel in self.channels:
                self.send('JOIN ' + channel)
            self.identified = True

    def __handle_invite(self, message):
        """ Join a channel when invited. """
        self.join(message.message)

    def __handle_names(self, message):
        """ Parse the name command and record the userlevels of users. """
        channel = message.channel.split(' ')[-1]
        for nick in message.message.split(' '):
            self.userlevels[(nick.lstrip('+%@&~'), channel)] = \
                get_userlevels(nick)

    def __handle_join(self, message):
        """ Handle the join of a user. """
        self.userlevels[(message.nick, message.message)] = [0]

    def __handle_leave(self, message):
        """ Handle the part or quit of a user. """
        pass

    def __handle_mode(self, message):
        """ Handle the mode command and update userlevels accordingly. """
        pass

    def __handle_error(self, message):
        """ Handle an error message from the server. """
        self.connected = self.identified = False

def userlevel_wrapper(func, level):
    """ Modify a plugin to only take users above a certain userlevel. """
    def wrapped(bot, message, *args, **kwargs):
        if max(bot.userlevels[(message.nick, message.channel)]) >= level:
            return func(message, *args, **kwargs)
    return wrapped

def command(name, level=IRCLevel.user):
    """ Decorator to add a command to the bot. """
    def register_command(func):
        wrapped = userlevel_wrapper(func, level)
        IRCBot.commands[name] = wrapped
        return func
    return register_command

def match(pattern, level=IRCLevel.user):
    """ Decorator to add a regex pattern to the bot. """
    def register_pattern(func):
        wrapped = userlevel_wrapper(func, level)
        IRCBot.patterns.append((re.compile(pattern, re.IGNORECASE), wrapped))
        return func
    return register_pattern

def sink(func):
    """ Decorator to add sink to the bot. """
    IRCBot.sinks.append(func)
    return func

def action(message):
    """ Make the message an action. """
    return '\u0001ACTION {}\u0001'.format(message)

def is_channel(name):
    """ Check if a name is a valid channel name or not. """
    valid = ['&', '#', '+', '!']
    invalid = [' ', ',', '\u0007']
    return (name[0] in valid) and all(c not in invalid for c in name)

def is_ctcp(message):
    """ Check if a message object is a ctcp message or not. """
    return message.message.startswith('\u0001') and \
        message.message.endswith('\u0001')

def ctcp_response(message):
    """ Return the appropriate response to a CTCP request. """
    mapping = {
        'VERSION': 'MotoBot Version 2.0',
        'FINGER': 'Oh you dirty man!',
        'TIME': strftime('%a %b %d %H:%M:%S', localtime()),
        'PING': message
    }
    return mapping.get(message.split(' ')[0].upper(), None)

def strip_control_codes(input):
    """ Strip the control codes from the input. """
    pattern = re.compile(r'\x03[0-9]{0,2},?[0-9]{0,2}|\x02|\x1D|\x1F|\x16|\x0F')
    output = pattern.sub('', input)
    return output
<|endoftext|>


<commit_before>def replace_directory_entries(directory_entries, replacements):
    replaced_entries = []
    replace_keys = ['path', 'content']

    for directory_entry in directory_entries:
        entry = dict(directory_entry)

        for search, replace in replacements.items():
            entry['path'] = entry['path'].replace(search, replace)

        for file_record in entry['files']:
            for search, replace in replacements.items():
                for replace_key in replace_keys:
                    replace_candidate = file_record[replace_key]
                    if replace_candidate:
                        file_record[replace_key] = replace_candidate.replace(
                            search,
                            replace
                        )

        replaced_entries.append(entry)

    return replaced_entries
<commit_msg>Explicitly replace file entries: content only if not binary<commit_after>def replace_directory_entries(directory_entries, replacements):
    replaced_entries = []

    for directory_entry in directory_entries:
        entry = dict(directory_entry)

        for search, replace in replacements.items():
            entry['path'] = entry['path'].replace(search, replace)

        for file_record in entry['files']:
            for search, replace in replacements.items():
                file_record['path'] = file_record['path'].replace(
                    search,
                    replace
                )

                if not file_record['binary']:
                    file_record['content'] = file_record['content'].replace(
                        search,
                        replace
                    )

        replaced_entries.append(entry)

    return replaced_entries
<|endoftext|>


<commit_before>#!/usr/bin/python

import os,sys
import lcm
import time
from lcm import LCM
import math

home_dir =os.getenv("HOME")
#print home_dir
sys.path.append(home_dir + "/drc/software/build/lib/python2.7/site-packages")
sys.path.append(home_dir + "/drc/software/build/lib/python2.7/dist-packages")

from drc.utime_t import utime_t
from drc.twist_timed_t import twist_timed_t
from drc.vector_3d_t import vector_3d_t

from drc.data_request_t import data_request_t
from drc.data_request_list_t import data_request_list_t
########################################################################################
def timestamp_now (): return int (time.time () * 1000000)

def on_utime(channel, data):
  m = utime_t.decode(data)
  t = m.utime
  #print t
  if ( abs(t - 300000) < 5000): # after .3secs send a spin rate
    print "spin for bloody laser already - %d" %(t)
    o = twist_timed_t()
    o.utime = t
    l = vector_3d_t()
    a = vector_3d_t()
    a.x = 0.733038
    o.linear_velocity =a
    o.angular_velocity =a
    lc.publish("SCAN_RATE_CMD", o.encode())
  if (abs(t - 3000000) < 5000): # after 4secs send the data request
    print "gimmie the data or else - %d" %(t)
    o = data_request_list_t()
    o.utime = t
    r = data_request_t()
    r.type = 1
    r.period = 10
    o.num_requests= 3
    o.requests.append(r)
    r.type =2
    o.requests.append(r)
    r.type =6
    o.requests.append(r)
    lc.publish("DATA_REQUEST", o.encode())

####################################################################
lc = lcm.LCM()
print "started"

sub1 = lc.subscribe("ROBOT_UTIME", on_utime)

while True:
  ## Handle LCM if new messages have arrived.
  lc.handle()

lc.unsubscribe(sub)
<commit_msg>rem<commit_after><|endoftext|>


<commit_before>import datetime

from pyramid.view import view_config, view_defaults, forbidden_view_config
from pyramid.httpexceptions import HTTPFound
from pyramid.response import Response
from pyramid.renderers import render
from pyramid.security import remember, forget, authenticated_userid
from deform import Form, ValidationFailure
from authomatic import Authomatic
from authomatic.adapters import WebObAdapter

from phoenix.views import MyView
from phoenix.security import Admin, Guest, admin_users, ESGF_Provider

import logging
logger = logging.getLogger(__name__)

from phoenix import config_public as config
authomatic = Authomatic(config=config.config,
                        secret=config.SECRET,
                        report_errors=True,
                        logging_level=logging.DEBUG)

@forbidden_view_config(renderer='phoenix:templates/account/forbidden.pt', layout="frontpage")
def forbidden(request):
    request.response.status = 403
    return dict()

@view_config(route_name='account_register', renderer='phoenix:templates/account/register.pt',
             permission='view', layout="frontpage")
def register(request):
    return dict()

@view_defaults(permission='view', layout='default')
class Account(MyView):
    def __init__(self, request):
        super(Account, self).__init__(request, name="account", title='Account')

    def appstruct(self):
        return dict(provider='DKRZ')

    def generate_form(self, protocol):
        from phoenix.schema import OpenIDSchema, ESGFOpenIDSchema
        schema = None
        if protocol == 'esgf':
            schema = ESGFOpenIDSchema()
        else:
            schema = OpenIDSchema()
        form = Form(schema=schema, buttons=('submit',), formid='deform')
        return form

    def process_form(self, form, protocol):
        try:
            controls = self.request.POST.items()
            appstruct = form.validate(controls)
        except ValidationFailure, e:
            logger.exception('validation of form failed.')
            return dict(active=protocol, form=e.render())
        else:
            logger.debug('openid route = %s', self.request.route_path('account_openid', _query=appstruct.items()))
            return HTTPFound(location=self.request.route_path('account_openid', _query=appstruct.items()))

    def send_notification(self, email, subject, message):
        """Sends email notification to admins.

        Sends email with the pyramid_mailer module.
        For configuration look at documentation http://pythonhosted.org//pyramid_mailer/
        """
        from pyramid_mailer import get_mailer
        mailer = get_mailer(self.request)

        sender = "noreply@%s" % (self.request.server_name)

        from phoenix.security import admin_users
        recipients = admin_users(self.request)

        from pyramid_mailer.message import Message
        message = Message(subject=subject,
                          sender=sender,
                          recipients=recipients,
                          body=message)
        mailer.send_immediately(message, fail_silently=True)

    def login_success(self, email, openid=None, name="Unknown"):
        from phoenix.models import add_user
        logger.debug('login success: email=%s', email)
        user = self.get_user(email)
        if user is None:
            logger.warn("openid login: new user %s", email)
            user = add_user(self.request, email=email, group=Guest)
            subject = 'New user %s logged in on %s' % (email, self.request.server_name)
            message = 'Please check the activation of the user %s on the Phoenix host %s' % (email, self.request.server_name)
            self.send_notification(email, subject, message)
        if email in admin_users(self.request):
            user['group'] = Admin
        user['last_login'] = datetime.datetime.now()
        if openid is not None:
            user['openid'] = openid
        user['name'] = name
        self.userdb.update({'email':email}, user)
        self.session.flash("Welcome %s (%s)." % (name, email), queue='success')
        if user.get('group') == Guest:
            self.session.flash("You are logged in as guest. You are not allowed to submit any process.", queue='danger')

    @view_config(route_name='account_login', renderer='phoenix:templates/account/login.pt')
    def login(self):
        protocol = self.request.matchdict.get('protocol', 'esgf')
        form = self.generate_form(protocol)
        if 'submit' in self.request.POST:
            return self.process_form(form, protocol)
        return dict(active=protocol, title="ESGF OpenID", form=form.render( self.appstruct() ))

    @view_config(route_name='account_logout', permission='edit')
    def logout(self):
        headers = forget(self.request)
        return HTTPFound(location = self.request.route_path('home'), headers = headers)

    @view_config(route_name='account_openid')
    def openid(self):
        """authomatic openid login"""
        username = self.request.params.get('username')
        # esgf openid login with username and provider
        if username is not None:
            provider = self.request.params.get('provider')
            logger.debug('username=%s, provider=%s', username, provider)
            openid = ESGF_Provider.get(provider) % username
            self.request.GET['id'] = openid
            del self.request.GET['username']
            del self.request.GET['provider']

        # Start the login procedure.
        response = Response()
        #response = request.response
        result = authomatic.login(WebObAdapter(self.request, response), "openid")

        #logger.debug('authomatic login result: %s', result)

        # TODO: refactor handling of result and response
        if result:
            if result.error:
                # Login procedure finished with an error.
                self.session.flash('Sorry, login failed: %s', queue='danger' % (result.error.message))
                logger.warn('openid login failed: %s', result.error.message)
                response.text = render('phoenix:templates/account/forbidden.pt', request=self.request)
            elif result.user:
                # Hooray, we have the user!
                logger.info("openid login successful for user %s", result.user.email)
                #logger.debug("user=%s, id=%s, email=%s, credentials=%s",
                #          result.user.name, result.user.id, result.user.email, result.user.credentials)
                #logger.debug("provider=%s", result.provider.name )
                #logger.debug("response headers=%s", response.headers.keys())
                #logger.debug("response cookie=%s", response.headers['Set-Cookie'])
                self.login_success(email=result.user.email, openid=result.user.id, name=result.user.name)
                response.text = render('phoenix:templates/account/openid_success.pt', {'result': result}, request=self.request)
                # Add the headers required to remember the user to the response
                response.headers.extend(remember(self.request, result.user.email))

        #logger.debug('response: %s', response)

        return response

<commit_msg>fixed some syntactical errors<commit_after>import datetime

from pyramid.view import view_config, view_defaults, forbidden_view_config
from pyramid.httpexceptions import HTTPFound
from pyramid.response import Response
from pyramid.renderers import render
from pyramid.security import remember, forget, authenticated_userid
from deform import Form, ValidationFailure
from authomatic import Authomatic
from authomatic.adapters import WebObAdapter

from phoenix.views import MyView
from phoenix.security import Admin, Guest, admin_users, ESGF_Provider

import logging
logger = logging.getLogger(__name__)

from phoenix import config_public as config
authomatic = Authomatic(config=config.config,
                        secret=config.SECRET,
                        report_errors=True,
                        logging_level=logging.DEBUG)

@forbidden_view_config(renderer='phoenix:templates/account/forbidden.pt', layout="frontpage")
def forbidden(request):
    request.response.status = 403
    return dict()

@view_config(route_name='account_register', renderer='phoenix:templates/account/register.pt',
             permission='view', layout="frontpage")
def register(request):
    return dict()

@view_defaults(permission='view', layout='default')
class Account(MyView):
    def __init__(self, request):
        super(Account, self).__init__(request, name="account", title='Account')

    def appstruct(self):
        return dict(provider='DKRZ')

    def generate_form(self, protocol):
        from phoenix.schema import OpenIDSchema, ESGFOpenIDSchema
        schema = None
        if protocol == 'esgf':
            schema = ESGFOpenIDSchema()
        else:
            schema = OpenIDSchema()
        form = Form(schema=schema, buttons=('submit',), formid='deform')
        return form

    def process_form(self, form, protocol):
        try:
            controls = self.request.POST.items()
            appstruct = form.validate(controls)
        except ValidationFailure, e:
            logger.exception('validation of form failed.')
            return dict(active=protocol, form=e.render())
        else:
            logger.debug('openid route = %s', self.request.route_path('account_openid', _query=appstruct.items()))
            return HTTPFound(location=self.request.route_path('account_openid', _query=appstruct.items()))

    def send_notification(self, email, subject, message):
        """Sends email notification to admins.

        Sends email with the pyramid_mailer module.
        For configuration look at documentation http://pythonhosted.org//pyramid_mailer/
        """
        from pyramid_mailer import get_mailer
        mailer = get_mailer(self.request)

        sender = "noreply@%s" % (self.request.server_name)

        from phoenix.security import admin_users
        recipients = admin_users(self.request)

        from pyramid_mailer.message import Message
        message = Message(subject=subject,
                          sender=sender,
                          recipients=recipients,
                          body=message)
        mailer.send_immediately(message, fail_silently=True)

    def login_success(self, email, openid=None, name="Unknown"):
        from phoenix.models import add_user
        logger.debug('login success: email=%s', email)
        user = self.get_user(email)
        if user is None:
            logger.warn("openid login: new user %s", email)
            user = add_user(self.request, email=email, group=Guest)
            subject = 'New user %s logged in on %s' % (email, self.request.server_name)
            message = 'Please check the activation of the user %s on the Phoenix host %s' % (email, self.request.server_name)
            self.send_notification(email, subject, message)
        if email in admin_users(self.request):
            user['group'] = Admin
        user['last_login'] = datetime.datetime.now()
        if openid is not None:
            user['openid'] = openid
        user['name'] = name
        self.userdb.update({'email':email}, user)
        self.session.flash("Welcome %s (%s)." % (name, email), queue='success')
        if user.get('group') == Guest:
            self.session.flash("You are logged in as guest. You are not allowed to submit any process.", queue='danger')

    @view_config(route_name='account_login', renderer='phoenix:templates/account/login.pt')
    def login(self):
        protocol = self.request.matchdict.get('protocol', 'esgf')
        form = self.generate_form(protocol)
        if 'submit' in self.request.POST:
            return self.process_form(form, protocol)
        return dict(active=protocol, title="ESGF OpenID", form=form.render( self.appstruct() ))

    @view_config(route_name='account_logout', permission='edit')
    def logout(self):
        headers = forget(self.request)
        return HTTPFound(location = self.request.route_path('home'), headers = headers)

    @view_config(route_name='account_openid')
    def openid(self):
        """authomatic openid login"""
        username = self.request.params.get('username')
        # esgf openid login with username and provider
        if username is not None:
            provider = self.request.params.get('provider')
            logger.debug('username=%s, provider=%s', username, provider)
            openid = ESGF_Provider.get(provider) % username
            self.request.GET['id'] = openid
            del self.request.GET['username']
            del self.request.GET['provider']

        # Start the login procedure.
        response = Response()
        #response = request.response
        result = authomatic.login(WebObAdapter(self.request, response), "openid")

        #logger.debug('authomatic login result: %s', result)

        # TODO: refactor handling of result and response
        if result:
            if result.error:
                # Login procedure finished with an error.
                self.session.flash('Sorry, login failed: %s' % (result.error.message), queue='danger')
                logger.warn('openid login failed: %s', result.error.message)
                response.text = render('phoenix:templates/account/forbidden.pt', dict(), self.request)
            elif result.user:
                # Hooray, we have the user!
                logger.info("openid login successful for user %s", result.user.email)
                #logger.debug("user=%s, id=%s, email=%s, credentials=%s",
                #          result.user.name, result.user.id, result.user.email, result.user.credentials)
                #logger.debug("provider=%s", result.provider.name )
                #logger.debug("response headers=%s", response.headers.keys())
                #logger.debug("response cookie=%s", response.headers['Set-Cookie'])
                self.login_success(email=result.user.email, openid=result.user.id, name=result.user.name)
                response.text = render('phoenix:templates/account/openid_success.pt', {'result': result}, request=self.request)
                # Add the headers required to remember the user to the response
                response.headers.extend(remember(self.request, result.user.email))

        #logger.debug('response: %s', response)

        return response

<|endoftext|>


<commit_before># Copyright 2021, The TensorFlow Federated Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Utilities for saving and loading program state in a federated program."""

import abc
from typing import Any, List, Optional, Tuple

class ProgramStateManagerStateAlreadyExistsError(Exception):
  pass

class ProgramStateManagerStateNotFoundError(Exception):
  pass

class ProgramStateManagerStructureError(Exception):
  pass

class ProgramStateManager(metaclass=abc.ABCMeta):
  """An interface for saving and loading program state in a federated program.

  A `tff.program.ProgramStateManager` is used to implement fault tolerance in a
  federated program. The structure or type of the program state that is saved is
  unknown at construction time and can change as the program runs.
  """

  @abc.abstractmethod
  def versions(self) -> Optional[List[int]]:
    """Returns a list of saved versions or `None`.

    Returns:
      A list of saved versions or `None` if there is no saved program state.
    """
    raise NotImplementedError

  @abc.abstractmethod
  def load(self, version: int, structure: Any) -> Any:
    """Returns the saved program state for the given `version`.

    Args:
      version: A integer representing the version of a saved program state.
      structure: The nested structure of the saved program state for the given
        `version` used to support serialization and deserailization of
        user-defined classes in the structure.

    Raises:
      ProgramStateManagerStateNotFoundError: If there is no program state for
        the given `version`.
    """
    raise NotImplementedError

  def load_latest(self, structure: Any) -> Tuple[Any, int]:
    """Returns the latest saved program state and version or (`None`, 0).

    Args:
      structure: The nested structure of the saved program state for the given
        `version` used to support serialization and deserailization of
        user-defined classes in the structure.

    Returns:
      A tuple of the latest saved (program state, version) or (`None`, 0) if
      there is no latest saved program state.
    """
    versions = self.versions()
    if versions is None:
      return None, 0
    latest_version = max(versions)
    try:
      return self.load(latest_version, structure), latest_version
    except ProgramStateManagerStateNotFoundError:
      return None, 0

  @abc.abstractmethod
  def save(self, program_state: Any, version: int):
    """Saves `program_state` for the given `version`.

    Args:
      program_state: A materialized value, a value reference, or a structure of
        materialized values and value references representing the program state
        to save.
      version: A strictly increasing integer representing the version of a saved
        `program_state`.

    Raises:
      ProgramStateManagerStateAlreadyExistsError: If there is already program
        state for the given `version`.
    """
    raise NotImplementedError
<commit_msg>Add `ProgramStateManager` documentation for when to raise an error.<commit_after># Copyright 2021, The TensorFlow Federated Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Utilities for saving and loading program state in a federated program."""

import abc
from typing import Any, List, Optional, Tuple

class ProgramStateManagerStateAlreadyExistsError(Exception):
  pass

class ProgramStateManagerStateNotFoundError(Exception):
  pass

class ProgramStateManagerStructureError(Exception):
  pass

class ProgramStateManager(metaclass=abc.ABCMeta):
  """An interface for saving and loading program state in a federated program.

  A `tff.program.ProgramStateManager` is used to implement fault tolerance in a
  federated program. The structure or type of the program state that is saved is
  unknown at construction time and can change as the program runs.
  """

  @abc.abstractmethod
  def versions(self) -> Optional[List[int]]:
    """Returns a list of saved versions or `None`.

    Returns:
      A list of saved versions or `None` if there is no saved program state.
    """
    raise NotImplementedError

  @abc.abstractmethod
  def load(self, version: int, structure: Any) -> Any:
    """Returns the saved program state for the given `version`.

    Args:
      version: A integer representing the version of a saved program state.
      structure: The nested structure of the saved program state for the given
        `version` used to support serialization and deserailization of
        user-defined classes in the structure.

    Raises:
      ProgramStateManagerStateNotFoundError: If there is no program state for
        the given `version`.
      ProgramStateManagerStructureError: If `structure` does not match the value
        loaded for the given `version`.
    """
    raise NotImplementedError

  def load_latest(self, structure: Any) -> Tuple[Any, int]:
    """Returns the latest saved program state and version or (`None`, 0).

    Args:
      structure: The nested structure of the saved program state for the given
        `version` used to support serialization and deserailization of
        user-defined classes in the structure.

    Returns:
      A tuple of the latest saved (program state, version) or (`None`, 0) if
      there is no latest saved program state.
    """
    versions = self.versions()
    if versions is None:
      return None, 0
    latest_version = max(versions)
    try:
      return self.load(latest_version, structure), latest_version
    except ProgramStateManagerStateNotFoundError:
      return None, 0

  @abc.abstractmethod
  def save(self, program_state: Any, version: int):
    """Saves `program_state` for the given `version`.

    Args:
      program_state: A materialized value, a value reference, or a structure of
        materialized values and value references representing the program state
        to save.
      version: A strictly increasing integer representing the version of a saved
        `program_state`.

    Raises:
      ProgramStateManagerStateAlreadyExistsError: If there is already program
        state for the given `version`.
    """
    raise NotImplementedError
<|endoftext|>


<commit_before>import h5py
import time
import numpy as np
import tensorflow as tf
from tensorflow.contrib import rnn
import sys
import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt

# usage: python exp6a.py 256 0 0 0
# system arg
n_hidden = 128 # hidden layer num of features
SMALL_FLAG = 1
FAST_FLAG = 1
SYS_FLAG = 0 # 0 for bridges, 1 for supermic
try:
    n_hidden = int(sys.argv[1])
    SMALL_FLAG = int(sys.argv[2])
    FAST_FLAG = int(sys.argv[3])
    SYS_FLAG = int(sys.argv[4])
except Exception, e:
    print('-- {}'.format(e))

print('-- LSTM size: {}'.format(n_hidden))
print('-- SMALL FLAG: {}'.format(SMALL_FLAG))
print('-- FAST FLAG: {}'.format(FAST_FLAG))
print('-- SYS FLAG: {}'.format(SYS_FLAG))

print('==> Experiment 6a: LSTM memory sizes')
sys_path = '/pylon2/ci560sp/haunter/'
if SYS_FLAG:
    sys_path = '/scratch/zwang3/'
filename = 'exp3_taylorswift_d15_1s_C1C8.mat'
if SMALL_FLAG:
    filename = 'exp3_small.mat'
filepath = sys_path + filename
print('==> Loading data from {}...'.format(filepath))

# ==============================================
#               reading data
# ==============================================
# benchmark
t_start = time.time()
f = h5py.File(filepath)
X_train = np.array(f.get('trainingFeatures'))
y_train = np.array(f.get('trainingLabels'))
X_val = np.array(f.get('validationFeatures'))
y_val = np.array(f.get('validationLabels'))

t_end = time.time()
print('--Time elapsed for loading data: {t:.2f} \
        seconds'.format(t = t_end - t_start))
del f
print('-- Number of training samples: {}'.format(X_train.shape[0]))
print('-- Number of validation samples: {}'.format(X_val.shape[0]))

# ==============================================
#               RNN configs
# ==============================================
# Network Parameters
num_training_vec, total_features = X_train.shape
num_freq = 169
num_frames = int(total_features / num_freq)

max_iter = 300
print_freq = 5
if FAST_FLAG:
    max_iter = 5
    print_freq = 1

batch_size = 1000
learning_rate = 0.001
n_input = num_freq # number of sequences (rows)
n_steps = num_frames # size of each sequence (number of columns), timesteps

n_classes = int(max(y_train.max(), y_val.max()) + 1)

# ==============================================
#               RNN architecture
# ==============================================
# Transform labels into on-hot encoding form
y_train_OHEnc = tf.one_hot(y_train.copy(), n_classes)
y_val_OHEnc = tf.one_hot(y_val.copy(), n_classes)

# tf Graph input
x = tf.placeholder("float", [None, n_steps, n_input])
y = tf.placeholder("float", [None, n_classes])

# Define weights
weights = tf.Variable(tf.random_normal([n_hidden, n_classes]))
biases = tf.Variable(tf.random_normal([n_classes]))

def RNN(x, weights, biases):

    # Prepare data shape to match `rnn` function requirements
    # Current data input shape: (batch_size, n_steps, n_input)
    # Required shape: 'n_steps' tensors list of shape (batch_size, n_input)

    # Unstack to get a list of 'n_steps' tensors of shape (batch_size, n_input)
    x = tf.unstack(x, n_steps, 1)

    # Define a lstm cell with tensorflow
    lstm_cell = rnn.BasicLSTMCell(n_hidden, forget_bias=1.0)

    # Get lstm cell output
    outputs, states = rnn.static_rnn(lstm_cell, x, dtype=tf.float32)

    # Linear activation, using rnn inner loop last output
    return tf.matmul(outputs[-1], weights) + biases

pred = RNN(x, weights, biases)

# Define loss and optimizer
cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=pred, labels=y))
optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(cost)

# Evaluate model
correct_pred = tf.equal(tf.argmax(pred,1), tf.argmax(y,1))
accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))

# Initializing the variables
init = tf.global_variables_initializer()

# evaluation metrics
train_acc_list = []
val_acc_list = []
train_err_list = []
val_err_list = []

# saver setup
varsave_list = [weights, biases]
saver = tf.train.Saver(varsave_list)
save_path = './out/6amodel_{}.ckpt'.format(n_hidden)
opt_val_err = np.inf
opt_epoch = -1
step_counter = 0
max_counter = 50

# ==============================================
#               RNN training
# ==============================================
# Launch the graph
with tf.InteractiveSession() as sess:
    sess.run(init)
    y_train = sess.run(y_train_OHEnc)[:, 0, :]
    y_val = sess.run(y_val_OHEnc)[:, 0, :]

    print('==> Training the full network...')
    t_start = time.time()
    # Keep training until reach max iterations
    for epoch in range(max_iter):
        for i in range(0, num_training_vec, batch_size):
            end_ind = min(i + batch_size, num_training_vec)
            batch_x = X_train[i : end_ind]
            batch_y = y_train[i : end_ind]
            # Reshape data to get 28 seq of 28 elements
            batch_x = batch_x.reshape((-1, n_steps, n_input))
            # Run optimization op (backprop)
            sess.run(optimizer, feed_dict={x: batch_x, y: batch_y})
        if (epoch + 1) % print_freq == 0:
            train_acc = accuracy.eval(feed_dict={x: batch_x,\
             y: batch_y})
            train_acc_list.append(train_acc)
            val_acc = accuracy.eval(feed_dict={x: X_val.reshape((-1, n_steps, n_input)),\
             y: y_val})
            val_acc_list.append(val_acc)
            train_err = cost.eval(feed_dict={x: batch_x,\
             y: batch_y})
            train_err_list.append(train_err)
            val_err = cost.eval(feed_dict={x: X_val.reshape((-1, n_steps, n_input)),\
             y: y_val})
            val_err_list.append(val_err)
            print("-- epoch: %d, training error %g, validation error %g"%(epoch + 1, train_err, val_err))
            # save screenshot of the model
            if val_err < opt_val_err:
                step_counter = 0
                saver.save(sess, save_path)
                print('==> New optimal validation error found. Model saved.')
                opt_val_err, opt_epoch = val_err, epoch + 1
        if step_counter > max_counter:
            print('==> Step counter exceeds maximum value. Stop training at epoch {}.'.format(epoch + 1))
            break
        step_counter += 1

    t_end = time.time()
    print('--Time elapsed for training: {t:.2f} \
        seconds'.format(t = t_end - t_start))
    # ==============================================
    # Restore model & Evaluations
    # ==============================================
    saver.restore(sess, save_path)
    print('==> Model restored to epoch {}'.format(opt_epoch))
    train_acc = accuracy.eval(feed_dict={x:X_train.reshape((-1, n_steps, n_input)), y: y_train})
    val_acc = accuracy.eval(feed_dict={x: X_val.reshape((-1, n_steps, n_input)), y: y_val})
    train_err = cost.eval(feed_dict={x: X_train.reshape((-1, n_steps, n_input)), y: y_train})
    val_err = cost.eval(feed_dict={x: X_val.reshape((-1, n_steps, n_input)), y: y_val})
    print('-- Training accuracy: {:.4f}'.format(train_acc))
    print('-- Validation accuracy: {:.4f}'.format(val_acc))
    print('-- Training error: {:.4E}'.format(train_err))
    print('-- Validation error: {:.4E}'.format(val_err))

print('-- Training accuracy --')
print([float('{:.4f}'.format(x)) for x in train_acc_list])
print('-- Validation accuracy --')
print([float('{:.4f}'.format(x)) for x in val_acc_list])
print('-- Training error --')
print([float('{:.4E}'.format(x)) for x in train_err_list])
print('-- Validation error --')
print([float('{:.4E}'.format(x)) for x in val_err_list])

print('==> Generating error plot...')
x_list = range(0, print_freq * len(train_acc_list), print_freq)
train_err_plot = plt.plot(x_list, train_err_list, 'b-', label='training')
val_err_plot = plt.plot(x_list, val_err_list, '-', color='orange', label='validation')
plt.xlabel('Number of epochs')
plt.ylabel('Cross-Entropy Error')
plt.title('Error vs Number of Epochs with {} Hidden Units'.format(n_hidden))
plt.legend(loc='best')
plt.savefig('./out/exp6a_{}.png'.format(n_hidden), format='png')
plt.close()

print('==> Finished!')
<commit_msg>fixing session<commit_after>import h5py
import time
import numpy as np
import tensorflow as tf
from tensorflow.contrib import rnn
import sys
import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt

# usage: python exp6a.py 256 0 0 0
# system arg
n_hidden = 128 # hidden layer num of features
SMALL_FLAG = 1
FAST_FLAG = 1
SYS_FLAG = 0 # 0 for bridges, 1 for supermic
try:
    n_hidden = int(sys.argv[1])
    SMALL_FLAG = int(sys.argv[2])
    FAST_FLAG = int(sys.argv[3])
    SYS_FLAG = int(sys.argv[4])
except Exception, e:
    print('-- {}'.format(e))

print('-- LSTM size: {}'.format(n_hidden))
print('-- SMALL FLAG: {}'.format(SMALL_FLAG))
print('-- FAST FLAG: {}'.format(FAST_FLAG))
print('-- SYS FLAG: {}'.format(SYS_FLAG))

print('==> Experiment 6a: LSTM memory sizes')
sys_path = '/pylon2/ci560sp/haunter/'
if SYS_FLAG:
    sys_path = '/scratch/zwang3/'
filename = 'exp3_taylorswift_d15_1s_C1C8.mat'
if SMALL_FLAG:
    filename = 'exp3_small.mat'
filepath = sys_path + filename
print('==> Loading data from {}...'.format(filepath))

# ==============================================
#               reading data
# ==============================================
# benchmark
t_start = time.time()
f = h5py.File(filepath)
X_train = np.array(f.get('trainingFeatures'))
y_train = np.array(f.get('trainingLabels'))
X_val = np.array(f.get('validationFeatures'))
y_val = np.array(f.get('validationLabels'))

t_end = time.time()
print('--Time elapsed for loading data: {t:.2f} \
        seconds'.format(t = t_end - t_start))
del f
print('-- Number of training samples: {}'.format(X_train.shape[0]))
print('-- Number of validation samples: {}'.format(X_val.shape[0]))

# ==============================================
#               RNN configs
# ==============================================
# Network Parameters
num_training_vec, total_features = X_train.shape
num_freq = 169
num_frames = int(total_features / num_freq)

max_iter = 300
print_freq = 5
if FAST_FLAG:
    max_iter = 5
    print_freq = 1

batch_size = 1000
learning_rate = 0.001
n_input = num_freq # number of sequences (rows)
n_steps = num_frames # size of each sequence (number of columns), timesteps

n_classes = int(max(y_train.max(), y_val.max()) + 1)

# ==============================================
#               RNN architecture
# ==============================================
# Transform labels into on-hot encoding form
y_train_OHEnc = tf.one_hot(y_train.copy(), n_classes)
y_val_OHEnc = tf.one_hot(y_val.copy(), n_classes)

# tf Graph input
x = tf.placeholder("float", [None, n_steps, n_input])
y = tf.placeholder("float", [None, n_classes])

# Define weights
weights = tf.Variable(tf.random_normal([n_hidden, n_classes]))
biases = tf.Variable(tf.random_normal([n_classes]))

def RNN(x, weights, biases):

    # Prepare data shape to match `rnn` function requirements
    # Current data input shape: (batch_size, n_steps, n_input)
    # Required shape: 'n_steps' tensors list of shape (batch_size, n_input)

    # Unstack to get a list of 'n_steps' tensors of shape (batch_size, n_input)
    x = tf.unstack(x, n_steps, 1)

    # Define a lstm cell with tensorflow
    lstm_cell = rnn.BasicLSTMCell(n_hidden, forget_bias=1.0)

    # Get lstm cell output
    outputs, states = rnn.static_rnn(lstm_cell, x, dtype=tf.float32)

    # Linear activation, using rnn inner loop last output
    return tf.matmul(outputs[-1], weights) + biases

pred = RNN(x, weights, biases)

# Define loss and optimizer
cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=pred, labels=y))
optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(cost)

# Evaluate model
correct_pred = tf.equal(tf.argmax(pred,1), tf.argmax(y,1))
accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))

# Initializing the variables
init = tf.global_variables_initializer()

# ==============================================
#               RNN training
# ==============================================
# Launch the graph
sess = tf.InteractiveSession()
sess.run(init)
y_train = sess.run(y_train_OHEnc)[:, 0, :]
y_val = sess.run(y_val_OHEnc)[:, 0, :]

# evaluation metrics
train_acc_list = []
val_acc_list = []
train_err_list = []
val_err_list = []

# saver setup
varsave_list = [weights, biases]
saver = tf.train.Saver(varsave_list)
save_path = './out/6amodel_{}.ckpt'.format(n_hidden)
opt_val_err = np.inf
opt_epoch = -1
step_counter = 0
max_counter = 50

print('==> Training the full network...')
t_start = time.time()
# Keep training until reach max iterations
for epoch in range(max_iter):
    for i in range(0, num_training_vec, batch_size):
        end_ind = min(i + batch_size, num_training_vec)
        batch_x = X_train[i : end_ind]
        batch_y = y_train[i : end_ind]
        # Reshape data to get 28 seq of 28 elements
        batch_x = batch_x.reshape((-1, n_steps, n_input))
        # Run optimization op (backprop)
        sess.run(optimizer, feed_dict={x: batch_x, y: batch_y})
    if (epoch + 1) % print_freq == 0:
        train_acc = accuracy.eval(feed_dict={x: batch_x,\
         y: batch_y})
        train_acc_list.append(train_acc)
        val_acc = accuracy.eval(feed_dict={x: X_val.reshape((-1, n_steps, n_input)),\
         y: y_val})
        val_acc_list.append(val_acc)
        train_err = cost.eval(feed_dict={x: batch_x,\
         y: batch_y})
        train_err_list.append(train_err)
        val_err = cost.eval(feed_dict={x: X_val.reshape((-1, n_steps, n_input)),\
         y: y_val})
        val_err_list.append(val_err)
        print("-- epoch: %d, training error %g, validation error %g"%(epoch + 1, train_err, val_err))
        # save screenshot of the model
        if val_err < opt_val_err:
            step_counter = 0
            saver.save(sess, save_path)
            print('==> New optimal validation error found. Model saved.')
            opt_val_err, opt_epoch = val_err, epoch + 1
    if step_counter > max_counter:
        print('==> Step counter exceeds maximum value. Stop training at epoch {}.'.format(epoch + 1))
        break
    step_counter += 1

t_end = time.time()
print('--Time elapsed for training: {t:.2f} \
    seconds'.format(t = t_end - t_start))
# ==============================================
# Restore model & Evaluations
# ==============================================
saver.restore(sess, save_path)
print('==> Model restored to epoch {}'.format(opt_epoch))
train_acc = accuracy.eval(feed_dict={x:X_train.reshape((-1, n_steps, n_input)), y: y_train})
val_acc = accuracy.eval(feed_dict={x: X_val.reshape((-1, n_steps, n_input)), y: y_val})
train_err = cost.eval(feed_dict={x: X_train.reshape((-1, n_steps, n_input)), y: y_train})
val_err = cost.eval(feed_dict={x: X_val.reshape((-1, n_steps, n_input)), y: y_val})
print('-- Training accuracy: {:.4f}'.format(train_acc))
print('-- Validation accuracy: {:.4f}'.format(val_acc))
print('-- Training error: {:.4E}'.format(train_err))
print('-- Validation error: {:.4E}'.format(val_err))

print('-- Training accuracy --')
print([float('{:.4f}'.format(x)) for x in train_acc_list])
print('-- Validation accuracy --')
print([float('{:.4f}'.format(x)) for x in val_acc_list])
print('-- Training error --')
print([float('{:.4E}'.format(x)) for x in train_err_list])
print('-- Validation error --')
print([float('{:.4E}'.format(x)) for x in val_err_list])

print('==> Generating error plot...')
x_list = range(0, print_freq * len(train_acc_list), print_freq)
train_err_plot = plt.plot(x_list, train_err_list, 'b-', label='training')
val_err_plot = plt.plot(x_list, val_err_list, '-', color='orange', label='validation')
plt.xlabel('Number of epochs')
plt.ylabel('Cross-Entropy Error')
plt.title('Error vs Number of Epochs with {} Hidden Units'.format(n_hidden))
plt.legend(loc='best')
plt.savefig('./out/exp6a_{}.png'.format(n_hidden), format='png')
plt.close()

print('==> Finished!')
<|endoftext|>


<commit_before># Copyright 2020 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""
"""

import sys
import os
import errno
import tensorflow as tf
import soundfile as sf
import numpy as np
import matplotlib.pyplot as plt
import pesq
from audio_synthesis.datasets import maestro_dataset
from audio_synthesis.utils import spectral

def mkdir(dirname):
    try:
        os.mkdir(dirname)
    except OSError as exc:
        if exc.errno != errno.EEXIST:
            raise
        pass

def add_noise_at_snr_channel(channel_in, snr):
    RMS = np.sqrt(np.mean(channel_in ** 2))
    RMS_noise = RMS / np.sqrt(10 ** (snr/10))
    noise_std = RMS_noise

    return channel_in + np.random.normal(size=channel_in.shape, scale=noise_std)

def distort_two_channel_representation(representation, snr):
    distorted_channels = []
    orig_channels = []
    for channel in range(representation.shape[-1]):
        input_channel = representation[:, :, channel]
        orig_channels.append(np.expand_dims(input_channel, 2))
        distorted_channel = add_noise_at_snr_channel(input_channel, snr)
        distorted_channels.append(np.expand_dims(distorted_channel, 2))

    magnitude_only = np.concatenate([distorted_channels[0], orig_channels[1]], axis=-1)
    phase_only = np.concatenate([orig_channels[0], distorted_channels[1]], axis=-1)
    return magnitude_only, phase_only

def distort_one_channel_representation(representation, snr):
    return np.expand_dims(add_noise_at_snr_channel(representation, snr), 0)

def perceptual_error(audio_hat, audio):
    audio_hat = np.array(audio_hat)
    audio = np.array(audio)
    return pesq.pesq(SAMPLE_RATE, audio, audio_hat, 'wb')

MAX_SNR = 80
MIN_SNR = 0
SNR_STEP = 2
SNRS = list(range(MIN_SNR, MAX_SNR+SNR_STEP, SNR_STEP))
SAMPLE_RATE = 16000
LOG_MAGNITUDE = False
DATASET_PATH = './data/SpeechMNIST_1850.npz'
RESULTS_PATH = './_results/snr_vs_pesq/'
LINE_STYLES = ['--', ':']

REPRESENTATIONS = {
    'STFT': {
        'requires_fft_params': True,
        'waveform_2_representation': lambda waveform, window_length, window_step: spectral.waveform_2_stft(
            waveform, window_length, window_step
        )[0],
        'representation_2_waveform': lambda representation, window_length, window_step: spectral.stft_2_waveform(
            representation, window_length, window_step
        )[0],
        'distort_representation': distort_two_channel_representation,
    },
    'Mag': {
        'requires_fft_params': True,
        'waveform_2_representation': lambda waveform, window_length, window_step: spectral.waveform_2_magnitude(
            waveform, window_length, window_step, log_magnitude=LOG_MAGNITUDE
        )[0],
        'representation_2_waveform': lambda representation, window_length, window_step: spectral.magnitude_2_waveform(
            representation, 32, window_length, window_step, log_magnitude=LOG_MAGNITUDE
        )[0],
        'distort_representation': distort_one_channel_representation,
    },
    'Mag + Phase': {
        'requires_fft_params': True,
        'waveform_2_representation': lambda waveform, window_length, window_step: spectral.waveform_2_spectogram(
            waveform, window_length, window_step, log_magnitude=LOG_MAGNITUDE, instantaneous_frequency=False
        )[0],
        'representation_2_waveform': lambda representation, window_length, window_step: spectral.spectogram_2_waveform(
            representation, window_length, window_step, log_magnitude=LOG_MAGNITUDE, instantaneous_frequency=False
        )[0],
        'distort_representation': distort_two_channel_representation,
    },
    'Mag + IF': {
        'requires_fft_params': True,
        'waveform_2_representation': lambda waveform, window_length, window_step: spectral.waveform_2_spectogram(
            waveform, window_length, window_step, log_magnitude=LOG_MAGNITUDE, instantaneous_frequency=True
        )[0],
        'representation_2_waveform': lambda representation, window_length, window_step: spectral.spectogram_2_waveform(
            representation, window_length, window_step, log_magnitude=LOG_MAGNITUDE, instantaneous_frequency=True
        )[0],
        'distort_representation': distort_two_channel_representation,
    },
    'Waveform': {
        'requires_fft_params': False,
        'waveform_2_representation': lambda x: x,
        'representation_2_waveform': lambda x: x,
        'distort_representation': distort_one_channel_representation,
    }
}

def main(fft_window_size, fft_window_step):
    os.environ['CUDA_VISIBLE_DEVICES'] = ''

    raw_maestro = maestro_dataset.get_maestro_waveform_dataset(DATASET_PATH)
    origonal_audio = np.array(raw_maestro[0]).astype(np.float32)

    for representation in REPRESENTATIONS:
        REPRESENTATIONS[representation]['perceptual_errors'] = []
        REPRESENTATIONS[representation]['waveforms'] = []

    for snr in SNRS:
        for representation in REPRESENTATIONS:
            if REPRESENTATIONS[representation]['requires_fft_params']:
                audio_representation = REPRESENTATIONS[representation]['waveform_2_representation'](
                    origonal_audio, fft_window_size, fft_window_step
                )
            else:
                audio_representation = REPRESENTATIONS[representation]['waveform_2_representation'](origonal_audio)

            noisy_representations = REPRESENTATIONS[representation]['distort_representation'](audio_representation, snr)

            perceptual_errors = []
            audio_hats = []
            for noisy_representation in noisy_representations:
                if REPRESENTATIONS[representation]['requires_fft_params']:
                    audio_hat = REPRESENTATIONS[representation]['representation_2_waveform'](
                        noisy_representation, fft_window_size, fft_window_step
                    )
                else:
                    audio_hat = REPRESENTATIONS[representation]['representation_2_waveform'](noisy_representation)

                audio_hat = audio_hat[0:len(origonal_audio)]

                audio_hats.append(audio_hat)
                perceptual_errors.append(perceptual_error(origonal_audio, audio_hat))

            REPRESENTATIONS[representation]['perceptual_errors'].append(perceptual_errors)
            REPRESENTATIONS[representation]['waveforms'].append(audio_hats)

    # Plot the graph
    for representation in REPRESENTATIONS:
        perceptual_errors = REPRESENTATIONS[representation]['perceptual_errors']
        perceptual_errors = np.array(perceptual_errors)

        color = None
        label = representation

        p = plt.plot(SNRS, perceptual_errors[:,0], label=representation)

        for i in range(perceptual_errors.shape[-1] - 1):
            plt.plot(SNRS, perceptual_errors[:,i+1], color=p[0].get_color(), linestyle=LINE_STYLES[i])

    plt.xlabel('SNR')
    plt.ylabel('PESQ')
    plt.legend()

    file_name = 'pesq_vs_snr__{}ws_{}s'.format(fft_window_size, fft_window_step)
    plt.savefig(os.path.join(RESULTS_PATH, file_name))
    plt.clf()

    # Save the audio files
    setup = 'audio_{}ws_{}s'.format(fft_window_size, fft_window_step)
    base_audio_dir = os.path.join(RESULTS_PATH, setup)
    mkdir(base_audio_dir)
    for representation in REPRESENTATIONS:
        audio_dir = os.path.join(base_audio_dir, representation)
        mkdir(audio_dir)
        for i, audio in enumerate(REPRESENTATIONS[representation]['waveforms']):
            for j, channel in enumerate(audio):
                file_path = os.path.join(audio_dir, '{}_{}db_{}.wav'.format(representation, SNRS[i], j))
                sf.write(file_path, audio[j], SAMPLE_RATE)

if __name__ == '__main__':
    main(512, 64)
    main(512, 128)
    main(512, 256)
    main(256, 128)
    main(1024, 128)<commit_msg>Update the Graph Generator Include additional diagnostic plots<commit_after># Copyright 2020 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""
"""

import sys
import os
import errno
import tensorflow as tf
import soundfile as sf
import numpy as np
import matplotlib.pyplot as plt
import pesq
from audio_synthesis.datasets import maestro_dataset
from audio_synthesis.utils import spectral

def mkdir(dirname):
    try:
        os.mkdir(dirname)
    except OSError as exc:
        if exc.errno != errno.EEXIST:
            raise
        pass

def add_noise_at_snr_channel_with_average(channel_in, snr, n_avg):
    noisy_channel_in = add_noise_at_snr_channel(channel_in, snr)

    for i in range(n_avg-1):
        noisy_channel_in = noisy_channel_in + add_noise_at_snr_channel(channel_in, snr)

    return noisy_channel_in / n_avg

def add_noise_at_snr_channel(channel_in, snr):
    RMS = np.sqrt(np.mean(channel_in ** 2))
    RMS_noise = RMS / np.sqrt(10 ** (snr/10))
    noise_std = RMS_noise

    return channel_in + np.random.normal(size=channel_in.shape, scale=noise_std)

def distort_two_channel_representation(representation, snr):
    distorted_channels = []
    orig_channels = []
    for channel in range(representation.shape[-1]):
        input_channel = representation[:, :, channel]
        orig_channels.append(np.expand_dims(input_channel, 2))
        distorted_channel = add_noise_at_snr_channel(input_channel, snr)
        distorted_channels.append(np.expand_dims(distorted_channel, 2))

    magnitude_only = np.concatenate([distorted_channels[0], orig_channels[1]], axis=-1)
    phase_only = np.concatenate([orig_channels[0], distorted_channels[1]], axis=-1)
    return magnitude_only, phase_only

def distort_one_channel_representation(representation, snr, n_avg=1):
    return np.expand_dims(add_noise_at_snr_channel_with_average(representation, snr, n_avg), 0)

def perceptual_error(audio_hat, audio):
    audio_hat = np.array(audio_hat)
    audio = np.array(audio)
    return pesq.pesq(SAMPLE_RATE, audio, audio_hat, 'wb')

MAX_SNR = 45
MIN_SNR = 0
SNR_STEP = 2
SNRS = list(range(MIN_SNR, MAX_SNR+SNR_STEP, SNR_STEP))
SAMPLE_RATE = 16000
N_GL_REPEATS = 20
LOG_MAGNITUDE = False
N_MEL_BINS = 256
MEL_LOWER_HERTZ_EDGE = 0.0
MEL_UPPER_HERTZ_EDGE = 8000.0
WAVEFORM_PATH = './data/speech.wav'
RESULTS_PATH = './_results/snr_vs_pesq/'
LINE_STYLES = ['--', ':']

REPRESENTATIONS = {
    'STFT': {
        'requires_fft_params': True,
        'waveform_2_representation': lambda waveform, window_length, window_step: spectral.waveform_2_stft(
            waveform, window_length, window_step
        )[0],
        'representation_2_waveform': lambda representation, window_length, window_step: spectral.stft_2_waveform(
            representation, window_length, window_step
        )[0],
        'distort_representation': distort_two_channel_representation,
    },
    'Mag': {
        'average': True,
        'requires_fft_params': True,
        'waveform_2_representation': lambda waveform, window_length, window_step: spectral.waveform_2_magnitude(
            waveform, window_length, window_step, log_magnitude=LOG_MAGNITUDE
        )[0],
        'representation_2_waveform': lambda representation, window_length, window_step: [spectral.magnitude_2_waveform(
            representation, 32, window_length, window_step, log_magnitude=LOG_MAGNITUDE
        ) for i in range(N_GL_REPEATS)],
        'distort_representation': distort_one_channel_representation,
    },
    'Mag + Phase': {
        'requires_fft_params': True,
        'waveform_2_representation': lambda waveform, window_length, window_step: spectral.waveform_2_spectogram(
            waveform, window_length, window_step, log_magnitude=LOG_MAGNITUDE, instantaneous_frequency=False
        )[0],
        'representation_2_waveform': lambda representation, window_length, window_step: spectral.spectogram_2_waveform(
            representation, window_length, window_step, log_magnitude=LOG_MAGNITUDE, instantaneous_frequency=False
        )[0],
        'distort_representation': distort_two_channel_representation,
    },
#    '(mel) Mag + Phase': {
#        'requires_fft_params': True,
#        'waveform_2_representation': lambda waveform, window_length, window_step: spectral.waveform_2_spectogram(
#            waveform, window_length, window_step, log_magnitude=LOG_MAGNITUDE, instantaneous_frequency=False,
#            n_mel_bins=N_MEL_BINS, mel_lower_hertz_edge=MEL_LOWER_HERTZ_EDGE, mel_upper_hertz_edge=MEL_UPPER_HERTZ_EDGE
#        )[0],
#        'representation_2_waveform': lambda representation, window_length, window_step: #spectral.spectogram_2_waveform(
#            representation, window_length, window_step, log_magnitude=LOG_MAGNITUDE, instantaneous_frequency=False,
#            n_mel_bins=N_MEL_BINS, mel_lower_hertz_edge=MEL_LOWER_HERTZ_EDGE, mel_upper_hertz_edge=MEL_UPPER_HERTZ_EDGE
#        )[0],
#        'distort_representation': distort_two_channel_representation,
#    },
    'Mag + IF': {
        'requires_fft_params': True,
        'waveform_2_representation': lambda waveform, window_length, window_step: spectral.waveform_2_spectogram(
            waveform, window_length, window_step, log_magnitude=LOG_MAGNITUDE, instantaneous_frequency=True
        )[0],
        'representation_2_waveform': lambda representation, window_length, window_step: spectral.spectogram_2_waveform(
            representation, window_length, window_step, log_magnitude=LOG_MAGNITUDE, instantaneous_frequency=True
        )[0],
        'distort_representation': distort_two_channel_representation,
    },
    'Waveform': {
        'requires_fft_params': False,
        'waveform_2_representation': lambda x: x,
        'representation_2_waveform': lambda x: x,
        'distort_representation': distort_one_channel_representation,
    },
    '(Avg) Waveform': {
        'requires_fft_params': False,
        'distort_with_fft_average': True,
        'waveform_2_representation': lambda x: x,
        'representation_2_waveform': lambda x: x,
        'distort_representation': distort_one_channel_representation,
    }
}

def main(fft_window_size, fft_window_step):
    os.environ['CUDA_VISIBLE_DEVICES'] = ''
    print(fft_window_size, ' ', fft_window_step)

    origonal_audio, sr = sf.read(WAVEFORM_PATH)
    origonal_audio = origonal_audio.astype(np.float32)

    for representation in REPRESENTATIONS:
        REPRESENTATIONS[representation]['perceptual_errors'] = []
        REPRESENTATIONS[representation]['waveforms'] = []

    for snr in SNRS:
        for representation in REPRESENTATIONS:
            print(representation)
            if REPRESENTATIONS[representation]['requires_fft_params']:
                audio_representation = REPRESENTATIONS[representation]['waveform_2_representation'](
                    origonal_audio, fft_window_size, fft_window_step
                )
            else:
                audio_representation = REPRESENTATIONS[representation]['waveform_2_representation'](origonal_audio)

            if 'distort_with_fft_average' in REPRESENTATIONS[representation]:
                noisy_representations = REPRESENTATIONS[representation]['distort_representation'](audio_representation, snr, (fft_window_size // fft_window_step))
            else:
                noisy_representations = REPRESENTATIONS[representation]['distort_representation'](audio_representation, snr)

            #print('NOISY')

            perceptual_errors = []
            audio_hats = []
            for noisy_representation in noisy_representations:
                if REPRESENTATIONS[representation]['requires_fft_params']:
                    audio_hat = REPRESENTATIONS[representation]['representation_2_waveform'](
                        noisy_representation, fft_window_size, fft_window_step
                    )
                else:
                    audio_hat = REPRESENTATIONS[representation]['representation_2_waveform'](noisy_representation)

                #print('RECONSTRUCTED')

                if 'average' in REPRESENTATIONS[representation]:
                    audio_hats.append(np.squeeze(audio_hat[0])[0:len(origonal_audio)])
                    all_perceptual_errors = [
                        perceptual_error(origonal_audio, np.squeeze(audio_hat_i)[0:len(origonal_audio)]) for audio_hat_i in audio_hat
                    ]
                    perceptual_errors.append(np.mean(all_perceptual_errors))
                else:
                    audio_hat = audio_hat[0:len(origonal_audio)]
                    audio_hats.append(audio_hat)
                    perceptual_errors.append(perceptual_error(origonal_audio, audio_hat))

            REPRESENTATIONS[representation]['perceptual_errors'].append(perceptual_errors)
            REPRESENTATIONS[representation]['waveforms'].append(audio_hats)

    # Plot the graph
    for representation in REPRESENTATIONS:
        perceptual_errors = REPRESENTATIONS[representation]['perceptual_errors']
        perceptual_errors = np.array(perceptual_errors)

        color = None
        label = representation

        p = plt.plot(SNRS, perceptual_errors[:,0], label=representation)

        for i in range(perceptual_errors.shape[-1] - 1):
            plt.plot(SNRS, perceptual_errors[:,i+1], color=p[0].get_color(), linestyle=LINE_STYLES[i])

    plt.xlabel('SNR')
    plt.ylabel('PESQ')
    plt.legend()

    file_name = 'pesq_vs_snr__{}ws_{}s'.format(fft_window_size, fft_window_step)
    plt.savefig(os.path.join(RESULTS_PATH, file_name), bbox_inches='tight', dpi=920)
    plt.clf()

    # Save the audio files
    setup = 'audio_{}ws_{}s'.format(fft_window_size, fft_window_step)
    base_audio_dir = os.path.join(RESULTS_PATH, setup)
    mkdir(base_audio_dir)
    for representation in REPRESENTATIONS:
        audio_dir = os.path.join(base_audio_dir, representation)
        mkdir(audio_dir)
        for i, audio in enumerate(REPRESENTATIONS[representation]['waveforms']):
            for j, channel in enumerate(audio):
                file_path = os.path.join(audio_dir, '{}_{}db_{}.wav'.format(representation, SNRS[i], j))
                sf.write(file_path, audio[j], SAMPLE_RATE)

if __name__ == '__main__':
    main(512, 64)
    main(512, 128)
    main(512, 256)
    main(256, 128)
    main(1024, 128)<|endoftext|>


<commit_before># -- coding: utf-8 --

import logging
import pickle
import time
import datetime

import equipment
import regulator
import pid
import templogger

class ExperimentException(Exception):
    def __init__(self, value):
        self.value = value

    def __str__(self):
        return self.value

class Experiment:
    _CFG_SECTION = 'experiment'

    def __init__(self, args, cfg, result_dir):
        self._args = args
        self._cfg = cfg
        self._result_dir = result_dir

        self._logger = logging.getLogger(__name__)

        self._experiment_loops = self._cfg.getint(Experiment._CFG_SECTION, 'max_loops')

    def step(self):
        raise NotImplementedError()

    def stop(self):
        raise NotImplementedError()

    def is_running(self):
        return self._experiment_loops > 0

    def get_remaining_loops(self):
        return self._experiment_loops

    def set_remaining_loops(self, remaining_loops):
        self._experiment_loops = remaining_loops

    def finish_loop(self):
        self._experiment_loops -= 1

    def get_result_key(self):
        raise NotImplementedError()

    def get_result_key_name(self):
        raise NotImplementedError()

    def get_state(self, capture_id):
        state = {
            'experiment': self.__class__.__name__,
            'capture_id': capture_id,
            'capture_time': time.strftime('%a, %d %b %Y %H:%M:%S +0000', time.gmtime()),
            'capture_timestamp': time.time()
        }

        return state

class TemperatureExperiment(Experiment):
    _CFG_SECTION = 'temperature'
    _STATE_FILE = 'temperature.pickle'

    def __init__(self, args, cfg, result_dir):
        Experiment.__init__(self, args, cfg, result_dir)

        # Experiment parameters and hardware addresses
        self._temperature_min = self._cfg.getfloat(self._CFG_SECTION, 'temperature_min')
        self._temperature_max = self._cfg.getfloat(self._CFG_SECTION, 'temperature_max')
        self._temperature_step = self._cfg.getfloat(self._CFG_SECTION, 'temperature_step')

        self._step_time = self._cfg.getint(self._CFG_SECTION, 'step_time')

        logger_port = self._cfg.get('temperature', 'logger_port')
        logger_ambient_channel = self._cfg.getint(self._CFG_SECTION, 'logger_ambient_channel')
        logger_sensor_channel = self._cfg.getint(self._CFG_SECTION, 'logger_sensor_channel')

        supply_address = self._cfg.get(self._CFG_SECTION, 'supply_address')
        supply_bus_id = self._cfg.getint(self._CFG_SECTION, 'supply_bus_id')
        supply_limit = pid.Limit(self._cfg.getfloat(self._CFG_SECTION, 'voltage_min'), self._cfg.getfloat(self._CFG_SECTION, 'voltage_max'))

        pid_param = pid.ControllerParameters(self._cfg.getfloat(self._CFG_SECTION, 'pid_p'), self._cfg.getfloat(self._CFG_SECTION, 'pid_i'), self._cfg.getfloat(self._CFG_SECTION, 'pid_d'))
        pid_period = self._cfg.getfloat(self._CFG_SECTION, 'pid_period')

        # Reload experiment state
        try:
            with open(self._STATE_FILE, 'r') as f:
                self._temperature, self._temperature_step = pickle.load(f)
            self._logger.info("Loaded existing experiment state from file")
        except:
            self._temperature = self._temperature_min

        self._logger.info(u"Initial temperature: {}°C, step: {}°C".format(self._temperature, self._temperature_step))

        # Setup temperature regulation hardware
        logger = templogger.TemperatureLogger(logger_port)

        supply_connector = equipment.VISAConnector(supply_address, term_chars='\r', use_bus_address=True)
        supply = equipment.PowerSupply(supply_connector, supply_bus_id)

        self._logger_ambient_channel = logger_ambient_channel
        self._logger_sensor_channel = logger_sensor_channel
        self._temperature_regulator = regulator.TemperatureRegulator(logger, logger_sensor_channel, supply, pid_param, pid_period, supply_limit)

    def step(self):
        # Set temperature
        self._logger.info(u"Target temperature: {}°C".format(self._temperature))
        self._temperature_regulator.set_target(self._temperature)
        self._temperature_regulator.start()

        # Save current state
        with open(self._STATE_FILE, 'w') as f:
            pickle.dump((self._temperature, self._temperature_step), f)

        # Wait for temperature to stabilize
        resume_time = datetime.datetime.now() + datetime.timedelta(seconds=self._step_time)
        self._logger.info("Wait to {:%H:%M:%S}".format(resume_time))

        try:
            time.sleep(self._step_time)
        except (KeyboardInterrupt):
            self._logger.info("Wait interrupted")
            user_input = raw_input("Continue? ")
            if not user_input.lower() in ['y', 'yes', 'true', '1']:
                raise

        # Check that the regulator is still working
        if not self._temperature_regulator.is_running():
            self._logger.error('Temperature regulator is not running')
            raise self._temperature_regulator.get_controller_exception()

        # Update for next step
        if (self._temperature >= self._temperature_max and self._temperature_step > 0) or (self._temperature <= self._temperature_min and self._temperature_step < 0):
            self._temperature_step = -self._temperature_step

        self._temperature += self._temperature_step

    def stop(self):
        self._logger.info("Stopping temperature regulator")
        self._temperature_regulator.stop()

    def get_result_key(self, state=None):
        if not state:
            self._temperature_regulator.lock_acquire()

            key = (self._temperature_regulator.get_temperature())

            self._temperature_regulator.lock_release()
        else:
            key = (state['sensor_temperature'],)

        return key

    def get_result_key_name(self):
        return ('result_sensor_temperature',)

    def get_state(self, capture_id):
        parent_state = Experiment.get_state(self, capture_id)

        self._temperature_regulator.lock_acquire()

        state = {
            'target_temperature': self._temperature_regulator.get_target(),
            'ambient_temperature': self._temperature_regulator.get_temperature(self._logger_ambient_channel),
            'sensor_temperature': self._temperature_regulator.get_temperature(),
            'supply_voltage': self._temperature_regulator.get_voltage(),
            'supply_current': self._temperature_regulator.get_current()
        }

        self._temperature_regulator.lock_release()

        return dict(state.items() + parent_state.items())

class HumidityExperiment(Experiment):
    def get_result_key_name(self):
        return ('sensor_temperature', 'sensor_humidity')

class ExposureExperiment(Experiment):
    def get_result_key_name(self):
        return ('sensor_temperature', 'sensor_analyte')
<commit_msg>Added experiment with no control<commit_after># -- coding: utf-8 --

import logging
import pickle
import time
import datetime

import equipment
import regulator
import pid
import templogger

class ExperimentException(Exception):
    def __init__(self, value):
        self.value = value

    def __str__(self):
        return self.value

class Experiment:
    _CFG_SECTION = 'experiment'

    def __init__(self, args, cfg, result_dir):
        self._args = args
        self._cfg = cfg
        self._result_dir = result_dir

        self._logger = logging.getLogger(__name__)

        self._experiment_loops = self._cfg.getint(Experiment._CFG_SECTION, 'max_loops')

    def step(self):
        raise NotImplementedError()

    def stop(self):
        raise NotImplementedError()

    def is_running(self):
        return self._experiment_loops > 0

    def get_remaining_loops(self):
        return self._experiment_loops

    def set_remaining_loops(self, remaining_loops):
        self._experiment_loops = remaining_loops

    def finish_loop(self):
        self._experiment_loops -= 1

    def get_result_key(self):
        raise NotImplementedError()

    def get_result_key_name(self):
        raise NotImplementedError()

    def get_state(self, capture_id):
        state = {
            'experiment': self.__class__.__name__,
            'capture_id': capture_id,
            'capture_time': time.strftime('%a, %d %b %Y %H:%M:%S +0000', time.gmtime()),
            'capture_timestamp': time.time()
        }

        return state

class TemperatureExperiment(Experiment):
    _CFG_SECTION = 'temperature'
    _STATE_FILE = 'temperature.pickle'

    def __init__(self, args, cfg, result_dir):
        Experiment.__init__(self, args, cfg, result_dir)

        # Experiment parameters and hardware addresses
        self._temperature_min = self._cfg.getfloat(self._CFG_SECTION, 'temperature_min')
        self._temperature_max = self._cfg.getfloat(self._CFG_SECTION, 'temperature_max')
        self._temperature_step = self._cfg.getfloat(self._CFG_SECTION, 'temperature_step')

        self._step_time = self._cfg.getint(self._CFG_SECTION, 'step_time')

        logger_port = self._cfg.get('temperature', 'logger_port')
        logger_ambient_channel = self._cfg.getint(self._CFG_SECTION, 'logger_ambient_channel')
        logger_sensor_channel = self._cfg.getint(self._CFG_SECTION, 'logger_sensor_channel')

        supply_address = self._cfg.get(self._CFG_SECTION, 'supply_address')
        supply_bus_id = self._cfg.getint(self._CFG_SECTION, 'supply_bus_id')
        supply_limit = pid.Limit(self._cfg.getfloat(self._CFG_SECTION, 'voltage_min'), self._cfg.getfloat(self._CFG_SECTION, 'voltage_max'))

        pid_param = pid.ControllerParameters(self._cfg.getfloat(self._CFG_SECTION, 'pid_p'), self._cfg.getfloat(self._CFG_SECTION, 'pid_i'), self._cfg.getfloat(self._CFG_SECTION, 'pid_d'))
        pid_period = self._cfg.getfloat(self._CFG_SECTION, 'pid_period')

        # Reload experiment state
        try:
            with open(self._STATE_FILE, 'r') as f:
                self._temperature, self._temperature_step = pickle.load(f)
            self._logger.info("Loaded existing experiment state from file")
        except:
            self._temperature = self._temperature_min

        self._logger.info(u"Initial temperature: {}°C, step: {}°C".format(self._temperature, self._temperature_step))

        # Setup temperature regulation hardware
        logger = templogger.TemperatureLogger(logger_port)

        supply_connector = equipment.VISAConnector(supply_address, term_chars='\r', use_bus_address=True)
        supply = equipment.PowerSupply(supply_connector, supply_bus_id)

        self._logger_ambient_channel = logger_ambient_channel
        self._logger_sensor_channel = logger_sensor_channel
        self._temperature_regulator = regulator.TemperatureRegulator(logger, logger_sensor_channel, supply, pid_param, pid_period, supply_limit)

    def step(self):
        # Set temperature
        self._logger.info(u"Target temperature: {}°C".format(self._temperature))
        self._temperature_regulator.set_target(self._temperature)
        self._temperature_regulator.start()

        # Save current state
        with open(self._STATE_FILE, 'w') as f:
            pickle.dump((self._temperature, self._temperature_step), f)

        # Wait for temperature to stabilize
        resume_time = datetime.datetime.now() + datetime.timedelta(seconds=self._step_time)
        self._logger.info("Wait to {:%H:%M:%S}".format(resume_time))

        try:
            time.sleep(self._step_time)
        except (KeyboardInterrupt):
            self._logger.info("Wait interrupted")
            user_input = raw_input("Continue? ")
            if not user_input.lower() in ['y', 'yes', 'true', '1']:
                raise

        # Check that the regulator is still working
        if not self._temperature_regulator.is_running():
            self._logger.error('Temperature regulator is not running')
            raise self._temperature_regulator.get_controller_exception()

        # Update for next step
        if (self._temperature >= self._temperature_max and self._temperature_step > 0) or (self._temperature <= self._temperature_min and self._temperature_step < 0):
            self._temperature_step = -self._temperature_step

        self._temperature += self._temperature_step

    def stop(self):
        self._logger.info("Stopping temperature regulator")
        self._temperature_regulator.stop()

    def get_result_key(self, state=None):
        if not state:
            self._temperature_regulator.lock_acquire()

            key = (self._temperature_regulator.get_temperature())

            self._temperature_regulator.lock_release()
        else:
            key = (state['sensor_temperature'],)

        return key

    def get_result_key_name(self):
        return ('result_sensor_temperature',)

    def get_state(self, capture_id):
        parent_state = Experiment.get_state(self, capture_id)

        self._temperature_regulator.lock_acquire()

        state = {
            'target_temperature': self._temperature_regulator.get_target(),
            'ambient_temperature': self._temperature_regulator.get_temperature(self._logger_ambient_channel),
            'sensor_temperature': self._temperature_regulator.get_temperature(),
            'supply_voltage': self._temperature_regulator.get_voltage(),
            'supply_current': self._temperature_regulator.get_current()
        }

        self._temperature_regulator.lock_release()

        return dict(state.items() + parent_state.items())

class HumidityExperiment(Experiment):
    def get_result_key_name(self):
        return ('sensor_temperature', 'sensor_humidity')

class ExposureExperiment(Experiment):
    def get_result_key_name(self):
        return ('sensor_temperature', 'sensor_analyte')

class TimeExperiment(Experiment):
    _CFG_SECTION = 'time'

    def __init__(self, args, cfg, result_dir):
        Experiment.__init__(self, args, cfg, result_dir)

        self._step_time = self._cfg.getint(self._CFG_SECTION, 'step_time')

    def step(self):
        # Just sleep until next time increment
        try:
            time.sleep(self._step_time)
        except KeyboardInterrupt:
            self._logger.info("Wait interrupted")
            user_input = raw_input("Continue? ")
            if not user_input.lower() in ['y', 'yes', 'true', '1']:
                raise

    def stop(self):
        pass

    def get_result_key(self):
        return 0

    def get_result_key_name(self):
        return 'result_n',
<|endoftext|>