<commit_before>#
# Copyright (c) 2005 rPath, Inc.
#
# All rights reserved
#
import base64
import os
import sys
import re
import time
from urllib import quote, unquote

from mod_python import apache
from mod_python import Cookie
from mod_python.util import FieldStorage

from conary.web import fields

from mint.session import SqlSession, COOKIE_NAME
from mint import database
from mint import mint_error
from mint import mint_server
from mint import shimclient
from mint import users, userlevels

from admin import AdminHandler
from project import ProjectHandler
from site import SiteHandler
from cookie_http import ConaryHandler

from webhandler import WebHandler, normPath

class Redirect(Exception):
    def __init__(self, location):
        self.location = location

    def __str__(self):
        return "Location: %s" % self.location

#called from hooks.py if an exception was not caught
class ErrorHandler(WebHandler):
    def handle(self, context):
        self.__dict__.update(**context)
        return self.errorPage

    def errorPage(self, *args, **kwargs):
        self._write('error', error = ' An unknown error occured while handling your request. Site maintainers have been notified.')
        return apache.OK

class MintApp(WebHandler):
    project = None
    projectList = []
    userLevel = userlevels.NONMEMBER
    user = None

    def __init__(self, req, cfg, repServer = None):
        self.req = req
        self.cfg = cfg

        #If the browser can support it, give it what it wants.
        if 'application/xhtml+xml' in self.req.headers_in.get('Accept', ''):
            self.content_type = 'application/xhtml+xml'
            self.output = 'xhtml'

        self.req.content_type = self.content_type

        try:
            self.fields = dict(FieldStorage(self.req))
        except apache.SERVER_RETURN:
            # failed to parse fields; must be an incorrect POST
            # request, so fail with a better error message.
            raise apache.SERVER_RETURN, apache.HTTP_NOT_FOUND

        self.basePath = normPath(self.cfg.basePath)

        self.siteHandler = SiteHandler()
        self.projectHandler = ProjectHandler()
        self.adminHandler = AdminHandler()
        self.errorHandler = ErrorHandler()
        self.conaryHandler = ConaryHandler(req, cfg, repServer)

    def _session_start(self):
        # prepare a new session
        sessionClient = shimclient.ShimMintClient(self.cfg, (self.cfg.authUser, self.cfg.authPass))

        protocol='https'
        if self.req.subprocess_env.get('HTTPS', 'off') != 'on':
            protocol='http'
        sid = self.fields.get('sid', None)
        domain = ".".join(self.req.hostname.split(".")[1:])
        self.session = SqlSession(self.req, sessionClient,
            sid = sid,
            secret = self.cfg.cookieSecretKey,
            timeout = 86400, # XXX timeout of one day; should it be configurable?
            domain = domain,
            lock = False)
        if self.session.is_new():
            self.session['firstPage'] = "%s://%s%s" %(protocol, self.req.hostname, self.req.unparsed_uri)
            self.session['visited'] = { }
            self.session['pages'] = [ ]
        #Mark the current domain as visited
        self.session['visited'][domain] = True
        #This is just for debugging purposes
        self.session['pages'].append(self.req.hostname + self.req.unparsed_uri)

        #Now figure out if we need to redirect
        nexthop = None
        for dom in (self.cfg.siteDomainName, self.cfg.projectDomainName):
            if not self.session['visited'].get(dom, None):
                #Yeah we need to redirect
                nexthop = dom
                break
        # if we were passed a sid, specifically set a cookie
        # for the requested domain with that sid.
        if sid or nexthop:
            c = self.session.make_cookie()
            c.domain = domain
            #add it to the err_headers_out because these ALWAYS go to the browser
            self.req.err_headers_out.add('Set-Cookie', str(c))
            self.req.err_headers_out.add('Cache-Control', 'no-cache="set-cookie"')

        if nexthop:
            #Save the session
            self.session.save()
            raise Redirect("%s://%s.%s%sblank?sid=%s" % (protocol, self.cfg.hostName, nexthop, self.cfg.basePath, self.session.id()))
        else:
            if sid:
                #Clear the sid from the request by redirecting to the first page.
                self.session.save()
                raise Redirect(self.session['firstPage'])

    def _handle(self, pathInfo):
        method = self.req.method.upper()
        if method not in ('GET', 'POST'):
            return apache.HTTP_METHOD_NOT_ALLOWED

        anonToken = ('anonymous', 'anonymous')

        try:
            self._session_start()
        except Redirect, e:
            return self._redirect(e.location)

        # default to anonToken if the current session has no authToken
        self.authToken = self.session.get('authToken', anonToken)

        # open up a new client with the retrieved authToken
        self.client = shimclient.ShimMintClient(self.cfg, self.authToken)

        self.auth = self.client.checkAuth()
        if self.auth.authorized:
            self.user = self.client.getUser(self.auth.userId)
            self.projectList = self.client.getProjectsByMember(self.auth.userId)

        self.auth.setToken(self.authToken)

        try:
            method = self._getHandler(pathInfo)
        except Redirect, e:
            return self._redirect(e.location)

        d = self.fields.copy()
        d['auth'] = self.auth

        try:
            returncode = method(**d)
            self.session.save()
            return returncode
        except mint_error.MintError, e:
            self.toUrl = self.cfg.basePath
            err_name = sys.exc_info()[0].__name__
            self.req.log_error("%s: %s" % (err_name, str(e)))
            self._write("error", shortError = err_name, error = str(e))
            return apache.OK
        except fields.MissingParameterError, e:
            self._write("error", shortError = "Missing Parameter", error = str(e))
            return apache.OK
        except mint_error.PermissionDenied, e:
            self._write("error", shortError = "Permission Denied", error = str(e))
            return apache.OK

    def _getHandler(self, pathInfo):
        fullHost = self.req.hostname
        protocol='https'
        if self.req.subprocess_env.get('HTTPS', 'off') != 'on':
            protocol='http'
        self.toUrl = ("%s://%s" % (protocol, fullHost)) + self.req.unparsed_uri
        dots = fullHost.split('.')
        hostname = dots[0]

        # if it looks like we're requesting a project (hostname isn't in reserved hosts
        # and doesn't match cfg.hostName, try to request the project.
        if hostname not in mint_server.reservedHosts and hostname != self.cfg.hostName:
            try:
                project = self.client.getProjectByHostname(hostname)
            except Exception, e:
                self.req.log_error(str(e))
                raise Redirect(self.cfg.defaultRedirect)
            else:
                # coerce "external" projects to the externalSiteHost, or
                # internal projects to projectSiteHost.
                if project.external: # "external" projects are endorsed by us, so use siteHost
                    raise Redirect("%s://%s%sproject/%s/" % (protocol, self.cfg.externalSiteHost, self.cfg.basePath, hostname))
                else:
                    raise Redirect("%s://%s%sproject/%s/" % (protocol, self.cfg.projectSiteHost, self.cfg.basePath, hostname))

        self.siteHost = self.cfg.siteHost

        # eg., redirect from http://rpath.com -> <defaultRedirect>
        if self.cfg.hostName and fullHost == self.cfg.siteDomainName:
            raise Redirect(self.cfg.defaultRedirect)

        # mapping of url regexps to handlers
        urls = (
            (r'^/project/',     self.projectHandler),
            (r'^/administer',   self.adminHandler),
            (r'^/repos/',       self.conaryHandler),
            (r'^/unknownError', self.errorHandler),
            (r'^/',             self.siteHandler),
        )

        self.SITE = self.siteHost + self.basePath
        self.isOwner = self.userLevel == userlevels.OWNER or self.auth.admin

        if self.session.has_key('groupTroveId'):
            self.groupTrove = self.client.getGroupTrove(self.session['groupTroveId'])
            self.groupProject = self.client.getProject(self.groupTrove.projectId)
        else:
            self.groupTrove = None
            self.groupProject = None

        # a set of information to be passed into the next handler
        context = {
            'auth':             self.auth,
            'authToken':        self.auth.getToken(),
            'client':           self.client,
            'cfg':              self.cfg,
            'fields':           self.fields,
            'projectList':      self.projectList,
            'req':              self.req,
            'session':          self.session,
            'siteHost':         self.cfg.siteHost,
            'toUrl':            self.toUrl,
            'basePath':         self.basePath,
            'project':          None,
            'SITE':             self.SITE,
            'userLevel':        self.userLevel,
            'user':             self.user,
            'isOwner':          self.isOwner,
            'groupTrove':       self.groupTrove,
            'groupProject':     self.groupProject,
        }

        # match the requested url to the right url handler
        for match, urlHandler in urls:
            if re.match(match, pathInfo):
                urlHandler.content_type=self.content_type
                urlHandler.output = self.output
                context['cmd'] = pathInfo[len(match)-1:]

                ret = urlHandler.handle(context)
                return ret

        # fell through, nothing matched
        return self._404
<commit_msg>only set up the group trove object if the user is logged in<commit_after>#
# Copyright (c) 2005 rPath, Inc.
#
# All rights reserved
#
import base64
import os
import sys
import re
import time
from urllib import quote, unquote

from mod_python import apache
from mod_python import Cookie
from mod_python.util import FieldStorage

from conary.web import fields

from mint.session import SqlSession, COOKIE_NAME
from mint import database
from mint import mint_error
from mint import mint_server
from mint import shimclient
from mint import users, userlevels

from admin import AdminHandler
from project import ProjectHandler
from site import SiteHandler
from cookie_http import ConaryHandler

from webhandler import WebHandler, normPath

class Redirect(Exception):
    def __init__(self, location):
        self.location = location

    def __str__(self):
        return "Location: %s" % self.location

#called from hooks.py if an exception was not caught
class ErrorHandler(WebHandler):
    def handle(self, context):
        self.__dict__.update(**context)
        return self.errorPage

    def errorPage(self, *args, **kwargs):
        self._write('error', error = ' An unknown error occured while handling your request. Site maintainers have been notified.')
        return apache.OK

class MintApp(WebHandler):
    project = None
    projectList = []
    userLevel = userlevels.NONMEMBER
    user = None

    def __init__(self, req, cfg, repServer = None):
        self.req = req
        self.cfg = cfg

        #If the browser can support it, give it what it wants.
        if 'application/xhtml+xml' in self.req.headers_in.get('Accept', ''):
            self.content_type = 'application/xhtml+xml'
            self.output = 'xhtml'

        self.req.content_type = self.content_type

        try:
            self.fields = dict(FieldStorage(self.req))
        except apache.SERVER_RETURN:
            # failed to parse fields; must be an incorrect POST
            # request, so fail with a better error message.
            raise apache.SERVER_RETURN, apache.HTTP_NOT_FOUND

        self.basePath = normPath(self.cfg.basePath)

        self.siteHandler = SiteHandler()
        self.projectHandler = ProjectHandler()
        self.adminHandler = AdminHandler()
        self.errorHandler = ErrorHandler()
        self.conaryHandler = ConaryHandler(req, cfg, repServer)

    def _session_start(self):
        # prepare a new session
        sessionClient = shimclient.ShimMintClient(self.cfg, (self.cfg.authUser, self.cfg.authPass))

        protocol='https'
        if self.req.subprocess_env.get('HTTPS', 'off') != 'on':
            protocol='http'
        sid = self.fields.get('sid', None)
        domain = ".".join(self.req.hostname.split(".")[1:])
        self.session = SqlSession(self.req, sessionClient,
            sid = sid,
            secret = self.cfg.cookieSecretKey,
            timeout = 86400, # XXX timeout of one day; should it be configurable?
            domain = domain,
            lock = False)
        if self.session.is_new():
            self.session['firstPage'] = "%s://%s%s" %(protocol, self.req.hostname, self.req.unparsed_uri)
            self.session['visited'] = { }
            self.session['pages'] = [ ]
        #Mark the current domain as visited
        self.session['visited'][domain] = True
        #This is just for debugging purposes
        self.session['pages'].append(self.req.hostname + self.req.unparsed_uri)

        #Now figure out if we need to redirect
        nexthop = None
        for dom in (self.cfg.siteDomainName, self.cfg.projectDomainName):
            if not self.session['visited'].get(dom, None):
                #Yeah we need to redirect
                nexthop = dom
                break
        # if we were passed a sid, specifically set a cookie
        # for the requested domain with that sid.
        if sid or nexthop:
            c = self.session.make_cookie()
            c.domain = domain
            #add it to the err_headers_out because these ALWAYS go to the browser
            self.req.err_headers_out.add('Set-Cookie', str(c))
            self.req.err_headers_out.add('Cache-Control', 'no-cache="set-cookie"')

        if nexthop:
            #Save the session
            self.session.save()
            raise Redirect("%s://%s.%s%sblank?sid=%s" % (protocol, self.cfg.hostName, nexthop, self.cfg.basePath, self.session.id()))
        else:
            if sid:
                #Clear the sid from the request by redirecting to the first page.
                self.session.save()
                raise Redirect(self.session['firstPage'])

    def _handle(self, pathInfo):
        method = self.req.method.upper()
        if method not in ('GET', 'POST'):
            return apache.HTTP_METHOD_NOT_ALLOWED

        anonToken = ('anonymous', 'anonymous')

        try:
            self._session_start()
        except Redirect, e:
            return self._redirect(e.location)

        # default to anonToken if the current session has no authToken
        self.authToken = self.session.get('authToken', anonToken)

        # open up a new client with the retrieved authToken
        self.client = shimclient.ShimMintClient(self.cfg, self.authToken)

        self.auth = self.client.checkAuth()
        if self.auth.authorized:
            self.user = self.client.getUser(self.auth.userId)
            self.projectList = self.client.getProjectsByMember(self.auth.userId)

        self.auth.setToken(self.authToken)

        try:
            method = self._getHandler(pathInfo)
        except Redirect, e:
            return self._redirect(e.location)

        d = self.fields.copy()
        d['auth'] = self.auth

        try:
            returncode = method(**d)
            self.session.save()
            return returncode
        except mint_error.MintError, e:
            self.toUrl = self.cfg.basePath
            err_name = sys.exc_info()[0].__name__
            self.req.log_error("%s: %s" % (err_name, str(e)))
            self._write("error", shortError = err_name, error = str(e))
            return apache.OK
        except fields.MissingParameterError, e:
            self._write("error", shortError = "Missing Parameter", error = str(e))
            return apache.OK
        except mint_error.PermissionDenied, e:
            self._write("error", shortError = "Permission Denied", error = str(e))
            return apache.OK

    def _getHandler(self, pathInfo):
        fullHost = self.req.hostname
        protocol='https'
        if self.req.subprocess_env.get('HTTPS', 'off') != 'on':
            protocol='http'
        self.toUrl = ("%s://%s" % (protocol, fullHost)) + self.req.unparsed_uri
        dots = fullHost.split('.')
        hostname = dots[0]

        # if it looks like we're requesting a project (hostname isn't in reserved hosts
        # and doesn't match cfg.hostName, try to request the project.
        if hostname not in mint_server.reservedHosts and hostname != self.cfg.hostName:
            try:
                project = self.client.getProjectByHostname(hostname)
            except Exception, e:
                self.req.log_error(str(e))
                raise Redirect(self.cfg.defaultRedirect)
            else:
                # coerce "external" projects to the externalSiteHost, or
                # internal projects to projectSiteHost.
                if project.external: # "external" projects are endorsed by us, so use siteHost
                    raise Redirect("%s://%s%sproject/%s/" % (protocol, self.cfg.externalSiteHost, self.cfg.basePath, hostname))
                else:
                    raise Redirect("%s://%s%sproject/%s/" % (protocol, self.cfg.projectSiteHost, self.cfg.basePath, hostname))

        self.siteHost = self.cfg.siteHost

        # eg., redirect from http://rpath.com -> <defaultRedirect>
        if self.cfg.hostName and fullHost == self.cfg.siteDomainName:
            raise Redirect(self.cfg.defaultRedirect)

        # mapping of url regexps to handlers
        urls = (
            (r'^/project/',     self.projectHandler),
            (r'^/administer',   self.adminHandler),
            (r'^/repos/',       self.conaryHandler),
            (r'^/unknownError', self.errorHandler),
            (r'^/',             self.siteHandler),
        )

        self.SITE = self.siteHost + self.basePath
        self.isOwner = self.userLevel == userlevels.OWNER or self.auth.admin

        if self.session.has_key('groupTroveId') and self.auth.authorized:
            self.groupTrove = self.client.getGroupTrove(self.session['groupTroveId'])
            self.groupProject = self.client.getProject(self.groupTrove.projectId)
        else:
            self.groupTrove = None
            self.groupProject = None

        # a set of information to be passed into the next handler
        context = {
            'auth':             self.auth,
            'authToken':        self.auth.getToken(),
            'client':           self.client,
            'cfg':              self.cfg,
            'fields':           self.fields,
            'projectList':      self.projectList,
            'req':              self.req,
            'session':          self.session,
            'siteHost':         self.cfg.siteHost,
            'toUrl':            self.toUrl,
            'basePath':         self.basePath,
            'project':          None,
            'SITE':             self.SITE,
            'userLevel':        self.userLevel,
            'user':             self.user,
            'isOwner':          self.isOwner,
            'groupTrove':       self.groupTrove,
            'groupProject':     self.groupProject,
        }

        # match the requested url to the right url handler
        for match, urlHandler in urls:
            if re.match(match, pathInfo):
                urlHandler.content_type=self.content_type
                urlHandler.output = self.output
                context['cmd'] = pathInfo[len(match)-1:]

                ret = urlHandler.handle(context)
                return ret

        # fell through, nothing matched
        return self._404
<|endoftext|>


<commit_before>from os import path

from django.conf import settings
from django.contrib.auth.models import User
from django.core.urlresolvers import reverse
from django.db import models
from django.utils.safestring import SafeUnicode

from tardis.tardis_portal.managers import OracleSafeManager, ExperimentManager

from .license import License

import logging
logger = logging.getLogger(__name__)

class Experiment(models.Model):
    """The ``Experiment`` model inherits from :class:`django.db.models.Model`

    :attribute url: **Undocumented**
    :attribute approved: **Undocumented**
    :attribute title: the title of the experiment.
    :attribute institution_name: the name of the institution who created
       the dataset.
    :attribute start_time: **Undocumented**
    :attribute end_time: **Undocumented**
    :attribute created_time: **Undocumented**
    :attribute handle: **Undocumented**
    :attribute public: **Undocumented**
    :attribute objects: default model manager
    :attribute safe: ACL aware model manager

    """

    PUBLIC_ACCESS_NONE = 1
    PUBLIC_ACCESS_METADATA = 50
    PUBLIC_ACCESS_FULL = 100

    PUBLIC_ACCESS_CHOICES = (
        (PUBLIC_ACCESS_NONE,        'No public access (hidden)'),
        (PUBLIC_ACCESS_METADATA,    'Metadata only (no data file access)'),
        (PUBLIC_ACCESS_FULL,        'Everything'),
    )

    url = models.URLField(verify_exists=False, max_length=255,
                          null=True, blank=True)
    approved = models.BooleanField()
    title = models.CharField(max_length=400)
    institution_name = models.CharField(max_length=400,
            default=settings.DEFAULT_INSTITUTION)
    description = models.TextField(blank=True)
    start_time = models.DateTimeField(null=True, blank=True)
    end_time = models.DateTimeField(null=True, blank=True)
    created_time = models.DateTimeField(auto_now_add=True)
    update_time = models.DateTimeField(auto_now=True)
    created_by = models.ForeignKey(User)
    handle = models.TextField(null=True, blank=True)
    locked = models.BooleanField()
    public_access = \
        models.PositiveSmallIntegerField(choices=PUBLIC_ACCESS_CHOICES,
                                         null=False,
                                         default=PUBLIC_ACCESS_NONE)
    license = models.ForeignKey(License, #@ReservedAssignment
                                blank=True, null=True)
    objects = OracleSafeManager()
    safe = ExperimentManager()  # The acl-aware specific manager.

    class Meta:
        app_label = 'tardis_portal'

    def save(self, *args, **kwargs):
        super(Experiment, self).save(*args, **kwargs)
        from .hooks import publish_public_expt_rifcs
        publish_public_expt_rifcs(self)

    def getParameterSets(self, schemaType=None):
        """Return the experiment parametersets associated with this
        experiment.

        """
        from tardis.tardis_portal.models.parameters import Schema
        if schemaType == Schema.EXPERIMENT or schemaType is None:
            return self.experimentparameterset_set.filter(
                schema__type=Schema.EXPERIMENT)
        else:
            raise Schema.UnsupportedType

    def __unicode__(self):
        return self.title

    def get_or_create_directory(self):
        dirname = path.join(settings.FILE_STORE_PATH,
                            str(self.id))
        if not path.exists(dirname):
            from os import chmod, mkdir
            try:
                mkdir(dirname)
                chmod(dirname, 0770)
            except:
                dirname = None
        return dirname

    @models.permalink
    def get_absolute_url(self):
        """Return the absolute url to the current ``Experiment``"""
        return ('tardis.tardis_portal.views.view_experiment', (),
                {'experiment_id': self.id})

    @models.permalink
    def get_edit_url(self):
        """Return the absolute url to the edit view of the current
        ``Experiment``

        """
        return ('tardis.tardis_portal.views.edit_experiment', (),
                {'experiment_id': self.id})

    @models.permalink
    def get_create_token_url(self):
        """Return the absolute url to the create token view of the current
        ``Experiment``
        """
        return ('tardis.tardis_portal.views.create_token', (),
                {'experiment_id': self.id})

    def get_datafiles(self):
        from .datafile import Dataset_File
        return Dataset_File.objects.filter(dataset__experiments=self)

    def get_download_urls(self):
        from .datafile import Dataset_File
        urls = {}
        params = (('experiment_id', self.id),)
        protocols = frozenset(self.get_datafiles()\
                                  .values_list('protocol', flat=True)\
                                  .distinct())
        # Get built-in download links
        local_protocols = frozenset(('', 'tardis', 'file', 'http', 'https'))
        if any(p in protocols for p in local_protocols):
            view = 'tardis.tardis_portal.download.download_experiment'
            for comptype in ['tar', 'zip']:
                kwargs = dict(params+(('comptype', comptype),))
                urls[comptype] = reverse(view, kwargs=kwargs)

        # Get links from download providers
        for protocol in protocols - local_protocols:
            try:
                for module in settings.DOWNLOAD_PROVIDERS:
                    if module[0] == protocol:
                        view = '%s.download_experiment' % module[1]
                        urls[protocol] = reverse(view, kwargs=dict(params))
            except AttributeError:
                pass

        return urls

    def get_images(self):
        from .datafile import IMAGE_FILTER
        return self.get_datafiles().order_by('-modification_time',
                                              '-created_time') \
                                   .filter(IMAGE_FILTER)

    def get_size(self):
        from .datafile import Dataset_File
        return Dataset_File.sum_sizes(self.get_datafiles())

    @classmethod
    def public_access_implies_distribution(cls, public_access_level):
        '''
        Determines if a level of public access implies that distribution should
        be allowed, or alternately if it should not be allowed. Used to
        prevent free-distribution licences for essentially private data, and
        overly-restrictive licences for public data.
        '''
        return public_access_level > cls.PUBLIC_ACCESS_METADATA

    def get_owners(self):
        acls = ExperimentACL.objects.filter(pluginId='django_user',
                                            experiment=self,
                                            isOwner=True)
        return [acl.get_related_object() for acl in acls]

class ExperimentACL(models.Model):
    """The ExperimentACL table is the core of the `Tardis
    Authorisation framework
    <http://code.google.com/p/mytardis/wiki/AuthorisationEngineAlt>`_

    :attribute pluginId: the the name of the auth plugin being used
    :attribute entityId: a foreign key to auth plugins
    :attribute experimentId: a forign key to the
       :class:`tardis.tardis_portal.models.Experiment`
    :attribute canRead: gives the user read access
    :attribute canWrite: gives the user write access
    :attribute canDelete: gives the user delete permission
    :attribute isOwner: the experiment owner flag.
    :attribute effectiveDate: the date when access takes into effect
    :attribute expiryDate: the date when access ceases
    :attribute aclOwnershipType: system-owned or user-owned.

    System-owned ACLs will prevent users from removing or
    editing ACL entries to a particular experiment they
    own. User-owned ACLs will allow experiment owners to
    remove/add/edit ACL entries to the experiments they own.

    """

    OWNER_OWNED = 1
    SYSTEM_OWNED = 2
    __COMPARISON_CHOICES = (
        (OWNER_OWNED, 'Owner-owned'),
        (SYSTEM_OWNED, 'System-owned'),
    )

    pluginId = models.CharField(null=False, blank=False, max_length=30)
    entityId = models.CharField(null=False, blank=False, max_length=320)
    experiment = models.ForeignKey(Experiment)
    canRead = models.BooleanField(default=False)
    canWrite = models.BooleanField(default=False)
    canDelete = models.BooleanField(default=False)
    isOwner = models.BooleanField(default=False)
    effectiveDate = models.DateField(null=True, blank=True)
    expiryDate = models.DateField(null=True, blank=True)
    aclOwnershipType = models.IntegerField(
        choices=__COMPARISON_CHOICES, default=OWNER_OWNED)

    def get_related_object(self):
        """
        If possible, resolve the pluginId/entityId combination to a user or
        group object.
        """
        if self.pluginId == 'django_user':
            return User.objects.get(pk=self.entityId)
        return None

    def __unicode__(self):
        return '%i | %s' % (self.experiment.id, self.experiment.title)

    class Meta:
        app_label = 'tardis_portal'
        ordering = ['experiment__id']

class Author_Experiment(models.Model):

    experiment = models.ForeignKey(Experiment)
    author = models.CharField(max_length=255)
    order = models.PositiveIntegerField()
    url = models.URLField(
        verify_exists=True,
        max_length=2000,
        blank=True,
        help_text="URL identifier for the author")

    def save(self, *args, **kwargs):
        super(Author_Experiment, self).save(*args, **kwargs)
        try:
            from .hooks import publish_public_expt_rifcs
            publish_public_expt_rifcs(self.experiment)
        except StandardError:
            logger.exception('')

    def __unicode__(self):
        return SafeUnicode(self.author) + ' | ' \
            + SafeUnicode(self.experiment.id) + ' | ' \
            + SafeUnicode(self.order)

    class Meta:
        app_label = 'tardis_portal'
        ordering = ['order']
        unique_together = (('experiment', 'author'),)

<commit_msg>The URLField verify_exist flag is deprecated because of security issues. (And it is causing a unit test failure on our CI server due to some unexplained interaction with proxying.)<commit_after>from os import path

from django.conf import settings
from django.contrib.auth.models import User
from django.core.urlresolvers import reverse
from django.db import models
from django.utils.safestring import SafeUnicode

from tardis.tardis_portal.managers import OracleSafeManager, ExperimentManager

from .license import License

import logging
logger = logging.getLogger(__name__)

class Experiment(models.Model):
    """The ``Experiment`` model inherits from :class:`django.db.models.Model`

    :attribute url: **Undocumented**
    :attribute approved: **Undocumented**
    :attribute title: the title of the experiment.
    :attribute institution_name: the name of the institution who created
       the dataset.
    :attribute start_time: **Undocumented**
    :attribute end_time: **Undocumented**
    :attribute created_time: **Undocumented**
    :attribute handle: **Undocumented**
    :attribute public: **Undocumented**
    :attribute objects: default model manager
    :attribute safe: ACL aware model manager

    """

    PUBLIC_ACCESS_NONE = 1
    PUBLIC_ACCESS_METADATA = 50
    PUBLIC_ACCESS_FULL = 100

    PUBLIC_ACCESS_CHOICES = (
        (PUBLIC_ACCESS_NONE,        'No public access (hidden)'),
        (PUBLIC_ACCESS_METADATA,    'Metadata only (no data file access)'),
        (PUBLIC_ACCESS_FULL,        'Everything'),
    )

    url = models.URLField(verify_exists=False, max_length=255,
                          null=True, blank=True)
    approved = models.BooleanField()
    title = models.CharField(max_length=400)
    institution_name = models.CharField(max_length=400,
            default=settings.DEFAULT_INSTITUTION)
    description = models.TextField(blank=True)
    start_time = models.DateTimeField(null=True, blank=True)
    end_time = models.DateTimeField(null=True, blank=True)
    created_time = models.DateTimeField(auto_now_add=True)
    update_time = models.DateTimeField(auto_now=True)
    created_by = models.ForeignKey(User)
    handle = models.TextField(null=True, blank=True)
    locked = models.BooleanField()
    public_access = \
        models.PositiveSmallIntegerField(choices=PUBLIC_ACCESS_CHOICES,
                                         null=False,
                                         default=PUBLIC_ACCESS_NONE)
    license = models.ForeignKey(License, #@ReservedAssignment
                                blank=True, null=True)
    objects = OracleSafeManager()
    safe = ExperimentManager()  # The acl-aware specific manager.

    class Meta:
        app_label = 'tardis_portal'

    def save(self, *args, **kwargs):
        super(Experiment, self).save(*args, **kwargs)
        from .hooks import publish_public_expt_rifcs
        publish_public_expt_rifcs(self)

    def getParameterSets(self, schemaType=None):
        """Return the experiment parametersets associated with this
        experiment.

        """
        from tardis.tardis_portal.models.parameters import Schema
        if schemaType == Schema.EXPERIMENT or schemaType is None:
            return self.experimentparameterset_set.filter(
                schema__type=Schema.EXPERIMENT)
        else:
            raise Schema.UnsupportedType

    def __unicode__(self):
        return self.title

    def get_or_create_directory(self):
        dirname = path.join(settings.FILE_STORE_PATH,
                            str(self.id))
        if not path.exists(dirname):
            from os import chmod, mkdir
            try:
                mkdir(dirname)
                chmod(dirname, 0770)
            except:
                dirname = None
        return dirname

    @models.permalink
    def get_absolute_url(self):
        """Return the absolute url to the current ``Experiment``"""
        return ('tardis.tardis_portal.views.view_experiment', (),
                {'experiment_id': self.id})

    @models.permalink
    def get_edit_url(self):
        """Return the absolute url to the edit view of the current
        ``Experiment``

        """
        return ('tardis.tardis_portal.views.edit_experiment', (),
                {'experiment_id': self.id})

    @models.permalink
    def get_create_token_url(self):
        """Return the absolute url to the create token view of the current
        ``Experiment``
        """
        return ('tardis.tardis_portal.views.create_token', (),
                {'experiment_id': self.id})

    def get_datafiles(self):
        from .datafile import Dataset_File
        return Dataset_File.objects.filter(dataset__experiments=self)

    def get_download_urls(self):
        from .datafile import Dataset_File
        urls = {}
        params = (('experiment_id', self.id),)
        protocols = frozenset(self.get_datafiles()\
                                  .values_list('protocol', flat=True)\
                                  .distinct())
        # Get built-in download links
        local_protocols = frozenset(('', 'tardis', 'file', 'http', 'https'))
        if any(p in protocols for p in local_protocols):
            view = 'tardis.tardis_portal.download.download_experiment'
            for comptype in ['tar', 'zip']:
                kwargs = dict(params+(('comptype', comptype),))
                urls[comptype] = reverse(view, kwargs=kwargs)

        # Get links from download providers
        for protocol in protocols - local_protocols:
            try:
                for module in settings.DOWNLOAD_PROVIDERS:
                    if module[0] == protocol:
                        view = '%s.download_experiment' % module[1]
                        urls[protocol] = reverse(view, kwargs=dict(params))
            except AttributeError:
                pass

        return urls

    def get_images(self):
        from .datafile import IMAGE_FILTER
        return self.get_datafiles().order_by('-modification_time',
                                              '-created_time') \
                                   .filter(IMAGE_FILTER)

    def get_size(self):
        from .datafile import Dataset_File
        return Dataset_File.sum_sizes(self.get_datafiles())

    @classmethod
    def public_access_implies_distribution(cls, public_access_level):
        '''
        Determines if a level of public access implies that distribution should
        be allowed, or alternately if it should not be allowed. Used to
        prevent free-distribution licences for essentially private data, and
        overly-restrictive licences for public data.
        '''
        return public_access_level > cls.PUBLIC_ACCESS_METADATA

    def get_owners(self):
        acls = ExperimentACL.objects.filter(pluginId='django_user',
                                            experiment=self,
                                            isOwner=True)
        return [acl.get_related_object() for acl in acls]

class ExperimentACL(models.Model):
    """The ExperimentACL table is the core of the `Tardis
    Authorisation framework
    <http://code.google.com/p/mytardis/wiki/AuthorisationEngineAlt>`_

    :attribute pluginId: the the name of the auth plugin being used
    :attribute entityId: a foreign key to auth plugins
    :attribute experimentId: a forign key to the
       :class:`tardis.tardis_portal.models.Experiment`
    :attribute canRead: gives the user read access
    :attribute canWrite: gives the user write access
    :attribute canDelete: gives the user delete permission
    :attribute isOwner: the experiment owner flag.
    :attribute effectiveDate: the date when access takes into effect
    :attribute expiryDate: the date when access ceases
    :attribute aclOwnershipType: system-owned or user-owned.

    System-owned ACLs will prevent users from removing or
    editing ACL entries to a particular experiment they
    own. User-owned ACLs will allow experiment owners to
    remove/add/edit ACL entries to the experiments they own.

    """

    OWNER_OWNED = 1
    SYSTEM_OWNED = 2
    __COMPARISON_CHOICES = (
        (OWNER_OWNED, 'Owner-owned'),
        (SYSTEM_OWNED, 'System-owned'),
    )

    pluginId = models.CharField(null=False, blank=False, max_length=30)
    entityId = models.CharField(null=False, blank=False, max_length=320)
    experiment = models.ForeignKey(Experiment)
    canRead = models.BooleanField(default=False)
    canWrite = models.BooleanField(default=False)
    canDelete = models.BooleanField(default=False)
    isOwner = models.BooleanField(default=False)
    effectiveDate = models.DateField(null=True, blank=True)
    expiryDate = models.DateField(null=True, blank=True)
    aclOwnershipType = models.IntegerField(
        choices=__COMPARISON_CHOICES, default=OWNER_OWNED)

    def get_related_object(self):
        """
        If possible, resolve the pluginId/entityId combination to a user or
        group object.
        """
        if self.pluginId == 'django_user':
            return User.objects.get(pk=self.entityId)
        return None

    def __unicode__(self):
        return '%i | %s' % (self.experiment.id, self.experiment.title)

    class Meta:
        app_label = 'tardis_portal'
        ordering = ['experiment__id']

class Author_Experiment(models.Model):

    experiment = models.ForeignKey(Experiment)
    author = models.CharField(max_length=255)
    order = models.PositiveIntegerField()
    url = models.URLField(
        max_length=2000,
        blank=True,
        help_text="URL identifier for the author")

    def save(self, *args, **kwargs):
        super(Author_Experiment, self).save(*args, **kwargs)
        try:
            from .hooks import publish_public_expt_rifcs
            publish_public_expt_rifcs(self.experiment)
        except StandardError:
            logger.exception('')

    def __unicode__(self):
        return SafeUnicode(self.author) + ' | ' \
            + SafeUnicode(self.experiment.id) + ' | ' \
            + SafeUnicode(self.order)

    class Meta:
        app_label = 'tardis_portal'
        ordering = ['order']
        unique_together = (('experiment', 'author'),)

<|endoftext|>


<commit_before>from django.test.utils import override_settings

from querybuilder.logger import Logger
from querybuilder.query import Query
from querybuilder.tests.models import Uniques
from querybuilder.tests.query_tests import QueryTestCase

@override_settings(DEBUG=True)
class TestUpdate(QueryTestCase):

    def setUp(self):
        self.logger = Logger()
        self.logger.start_logging()

    def test_upsert(self):
        """
        Verifies that records get upserted correctly. Skipping this test now until travis-ci supports 9.5 addon
        """
        pass
        items = [
            Uniques(field1='1.1', field2='1.2', field3='1.3', field6='1.6', field7='1.7'),
        ]

        Query().from_table(Uniques).upsert(
            items,
            unique_fields=['field1'],
            update_fields=['field3', 'field4', 'field5']
        )

        model = Uniques.objects.get()
        self.assertEqual(model.field1, '1.1')
        self.assertEqual(model.field2, '1.2')
        self.assertEqual(model.field3, '1.3')
        self.assertEqual(model.field4, 'default_value')
        self.assertEqual(model.field5, None)
        self.assertEqual(model.field6, '1.6')
        self.assertEqual(model.field7, '1.7')

        items = [
            Uniques(
                field1='1.1',
                field2='1.2 edited',
                field3='1.3 edited',
                field4='not default',
                field5='new value',
                field6='1.6',
                field7='1.7'
            ),
        ]

        Query().from_table(Uniques).upsert(
            items,
            unique_fields=['field1'],
            update_fields=['field3', 'field4', 'field5']
        )

        # Only fields 3, 4, 5 should be updated
        model = Uniques.objects.get()
        self.assertEqual(model.field1, '1.1')
        self.assertEqual(model.field2, '1.2')
        self.assertEqual(model.field3, '1.3 edited')
        self.assertEqual(model.field4, 'not default')
        self.assertEqual(model.field5, 'new value')
        self.assertEqual(model.field6, '1.6')
        self.assertEqual(model.field7, '1.7')

        # Include a new record and an existing record
        items = [
            Uniques(field1='1.1', field2='1.2', field3='1.3', field6='1.6', field7='1.7'),
            Uniques(
                field1='2.1',
                field2='2.2',
                field3='2.3',
                field4='not default',
                field5='not null',
                field6='2.6',
                field7='2.7'
            ),
        ]

        Query().from_table(Uniques).upsert(
            items,
            unique_fields=['field1'],
            update_fields=['field3', 'field4', 'field5']
        )

        # Both records should exist and have different data
        models = list(Uniques.objects.order_by('id'))

        self.assertEqual(models[0].field1, '1.1')
        self.assertEqual(models[0].field2, '1.2')
        self.assertEqual(models[0].field3, '1.3')
        self.assertEqual(models[0].field4, 'default_value')
        self.assertEqual(models[0].field5, None)
        self.assertEqual(models[0].field6, '1.6')
        self.assertEqual(models[0].field7, '1.7')

        self.assertEqual(models[1].field1, '2.1')
        self.assertEqual(models[1].field2, '2.2')
        self.assertEqual(models[1].field3, '2.3')
        self.assertEqual(models[1].field4, 'not default')
        self.assertEqual(models[1].field5, 'not null')
        self.assertEqual(models[1].field6, '2.6')
        self.assertEqual(models[1].field7, '2.7')
<commit_msg>travis won't build<commit_after>from django.test.utils import override_settings

from querybuilder.logger import Logger
from querybuilder.query import Query
from querybuilder.tests.models import Uniques
from querybuilder.tests.query_tests import QueryTestCase

@override_settings(DEBUG=True)
class TestUpdate(QueryTestCase):

    def setUp(self):
        self.logger = Logger()
        self.logger.start_logging()

    def test_upsert(self):
        """
        Verifies that records get upserted correctly. Skipping this test now until travis-ci supports 9.5 addon.
        """
        pass
        items = [
            Uniques(field1='1.1', field2='1.2', field3='1.3', field6='1.6', field7='1.7'),
        ]

        Query().from_table(Uniques).upsert(
            items,
            unique_fields=['field1'],
            update_fields=['field3', 'field4', 'field5']
        )

        model = Uniques.objects.get()
        self.assertEqual(model.field1, '1.1')
        self.assertEqual(model.field2, '1.2')
        self.assertEqual(model.field3, '1.3')
        self.assertEqual(model.field4, 'default_value')
        self.assertEqual(model.field5, None)
        self.assertEqual(model.field6, '1.6')
        self.assertEqual(model.field7, '1.7')

        items = [
            Uniques(
                field1='1.1',
                field2='1.2 edited',
                field3='1.3 edited',
                field4='not default',
                field5='new value',
                field6='1.6',
                field7='1.7'
            ),
        ]

        Query().from_table(Uniques).upsert(
            items,
            unique_fields=['field1'],
            update_fields=['field3', 'field4', 'field5']
        )

        # Only fields 3, 4, 5 should be updated
        model = Uniques.objects.get()
        self.assertEqual(model.field1, '1.1')
        self.assertEqual(model.field2, '1.2')
        self.assertEqual(model.field3, '1.3 edited')
        self.assertEqual(model.field4, 'not default')
        self.assertEqual(model.field5, 'new value')
        self.assertEqual(model.field6, '1.6')
        self.assertEqual(model.field7, '1.7')

        # Include a new record and an existing record
        items = [
            Uniques(field1='1.1', field2='1.2', field3='1.3', field6='1.6', field7='1.7'),
            Uniques(
                field1='2.1',
                field2='2.2',
                field3='2.3',
                field4='not default',
                field5='not null',
                field6='2.6',
                field7='2.7'
            ),
        ]

        Query().from_table(Uniques).upsert(
            items,
            unique_fields=['field1'],
            update_fields=['field3', 'field4', 'field5']
        )

        # Both records should exist and have different data
        models = list(Uniques.objects.order_by('id'))

        self.assertEqual(models[0].field1, '1.1')
        self.assertEqual(models[0].field2, '1.2')
        self.assertEqual(models[0].field3, '1.3')
        self.assertEqual(models[0].field4, 'default_value')
        self.assertEqual(models[0].field5, None)
        self.assertEqual(models[0].field6, '1.6')
        self.assertEqual(models[0].field7, '1.7')

        self.assertEqual(models[1].field1, '2.1')
        self.assertEqual(models[1].field2, '2.2')
        self.assertEqual(models[1].field3, '2.3')
        self.assertEqual(models[1].field4, 'not default')
        self.assertEqual(models[1].field5, 'not null')
        self.assertEqual(models[1].field6, '2.6')
        self.assertEqual(models[1].field7, '2.7')
<|endoftext|>


<commit_before>#!/usr/bin/env python3

# Radio and server metrics, for pretty graphs at https://lainon.life/graphs/.

from datetime import datetime
from influxdb import InfluxDBClient
import json, os, psutil, time, urllib

def snapshot_icecast():
    """Return a snapshot of the icecast listener status."""

    f = urllib.request.urlopen("http://localhost:8000/status-json.xsl")
    stats = json.loads(f.read().decode("utf8"))

    snapshot = []
    for src in stats["icestats"]["source"]:
        snapshot.append({
            "channel": src["server_name"][:-6],
            "format": src["server_name"][-4:][:-1],
            "listeners": src["listeners"]
        })

    formats  = {stream["format"]  for stream in snapshot}
    channels = {stream["channel"] for stream in snapshot}

    return snapshot, formats, channels

def get_upload_download():
    """Get the current upload, in bytes, since last boot."""

    psinfo = psutil.net_io_counters(pernic=True)["eno1"]
    return psinfo[0], psinfo[1]

def get_cpu_percents():
    """Get the percentage usage of every cpu."""

    return psutil.cpu_percent(percpu=True)

def get_disk_used():
    """Get the disk usage, in bytes."""

    statinfo = os.statvfs("/")
    return statinfo.f_frsize * (statinfo.f_blocks - statinfo.f_bfree)

def get_memory_used():
    """Get the RAM and swap usage, in bytes."""

    vminfo = psutil.virtual_memory()
    swinfo = psutil.swap_memory()

    return vminfo[3], vminfo[7], vminfo[8], swinfo[1]

def get_format_listeners(snapshot, fmt):
    """Get the number of listeners on a specific format, across all channels."""

    return sum([stream["listeners"] for stream in snapshot if stream["format"] == fmt])

def get_channel_listeners(snapshot, channel):
    """Get the number of listeners on a specific channel, across all formats."""

    return sum([stream["listeners"] for stream in snapshot if stream["channel"] == channel])

def gather_metrics(now):
    """Gather metrics to send to InfluxDB."""

    snapshot, formats, channels = snapshot_icecast()
    up, down = get_upload_download()
    cpus = get_cpu_percents()
    vmused, vmbuf, vmcache, swused = get_memory_used()

    return [
        {"measurement": "network", "time": now, "fields": {
            "upload":   up,
            "download": down
        }},
        {"measurement": "disk", "time": now, "fields": {
            "used": get_disk_used(),
        }},
        {"measurement": "cpu", "time": now, "fields": {
            "core{}".format(n): percent for n, percent in enumerate(cpus)
        }},
        {"measurement": "memory", "time": now, "fields": {
            "vm_used":    vmused,
            "vm_used_no_buffers_cache": vmused - vmbuf - vmcache,
            "vm_buffers": vmbuf,
            "vm_cached":  vmcache,
            "swap_used":  swused
        }},
        {"measurement": "format_listeners", "time": now, "fields": {
            fmt: get_format_listeners(snapshot, fmt) for fmt in formats
        }},
        {"measurement": "channel_listeners", "time": now, "fields": {
            ch: get_channel_listeners(snapshot, ch) for ch in channels
        }}
    ]

if __name__ == "__main__":
    client = InfluxDBClient()

    # Ensure the database exists
    client.create_database("lainon.life")

    # We do this all in the same process to avoid the overhead of
    # launching a python interpreter every 30s, which appears to mess
    # with psutil's reporting of CPU usage.
    while True:
        now = datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%SZ")
        print("Sending report for {}".format(now))
        metrics = gather_metrics(now)
        client.write_points(metrics, database="lainon.life")
        time.sleep(30)
<commit_msg>Avoid crash when icecast gives invalid json.<commit_after>#!/usr/bin/env python3

# Radio and server metrics, for pretty graphs at https://lainon.life/graphs/.

from datetime import datetime
from influxdb import InfluxDBClient
import json, os, psutil, time, urllib

def snapshot_icecast():
    """Return a snapshot of the icecast listener status."""

    f = urllib.request.urlopen("http://localhost:8000/status-json.xsl")
    stats = json.loads(f.read().decode("utf8"))

    snapshot = []
    for src in stats["icestats"]["source"]:
        snapshot.append({
            "channel": src["server_name"][:-6],
            "format": src["server_name"][-4:][:-1],
            "listeners": src["listeners"]
        })

    formats  = {stream["format"]  for stream in snapshot}
    channels = {stream["channel"] for stream in snapshot}

    return snapshot, formats, channels

def get_upload_download():
    """Get the current upload, in bytes, since last boot."""

    psinfo = psutil.net_io_counters(pernic=True)["eno1"]
    return psinfo[0], psinfo[1]

def get_cpu_percents():
    """Get the percentage usage of every cpu."""

    return psutil.cpu_percent(percpu=True)

def get_disk_used():
    """Get the disk usage, in bytes."""

    statinfo = os.statvfs("/")
    return statinfo.f_frsize * (statinfo.f_blocks - statinfo.f_bfree)

def get_memory_used():
    """Get the RAM and swap usage, in bytes."""

    vminfo = psutil.virtual_memory()
    swinfo = psutil.swap_memory()

    return vminfo[3], vminfo[7], vminfo[8], swinfo[1]

def get_format_listeners(snapshot, fmt):
    """Get the number of listeners on a specific format, across all channels."""

    return sum([stream["listeners"] for stream in snapshot if stream["format"] == fmt])

def get_channel_listeners(snapshot, channel):
    """Get the number of listeners on a specific channel, across all formats."""

    return sum([stream["listeners"] for stream in snapshot if stream["channel"] == channel])

def gather_metrics(now):
    """Gather metrics to send to InfluxDB."""

    # Getting the icecast metrics may fail, as it sometimes produces
    # invalid json.
    try:
        snapshot, formats, channels = snapshot_icecast()
        metrics = [
            {"measurement": "format_listeners", "time": now, "fields": {
                fmt: get_format_listeners(snapshot, fmt) for fmt in formats
            }},
            {"measurement": "channel_listeners", "time": now, "fields": {
                ch: get_channel_listeners(snapshot, ch) for ch in channels
            }}
        ]
    except:
        metrics = []

    # The system metrics (shouldn't crash!)
    up, down = get_upload_download()
    cpus = get_cpu_percents()
    vmused, vmbuf, vmcache, swused = get_memory_used()

    metrics.extend([
        {"measurement": "network", "time": now, "fields": {
            "upload":   up,
            "download": down
        }},
        {"measurement": "disk", "time": now, "fields": {
            "used": get_disk_used(),
        }},
        {"measurement": "cpu", "time": now, "fields": {
            "core{}".format(n): percent for n, percent in enumerate(cpus)
        }},
        {"measurement": "memory", "time": now, "fields": {
            "vm_used":    vmused,
            "vm_used_no_buffers_cache": vmused - vmbuf - vmcache,
            "vm_buffers": vmbuf,
            "vm_cached":  vmcache,
            "swap_used":  swused
        }}
    ])

    return metrics

if __name__ == "__main__":
    client = InfluxDBClient()

    # Ensure the database exists
    client.create_database("lainon.life")

    # We do this all in the same process to avoid the overhead of
    # launching a python interpreter every 30s, which appears to mess
    # with psutil's reporting of CPU usage.
    while True:
        now = datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%SZ")
        print("Sending report for {}".format(now))
        metrics = gather_metrics(now)
        client.write_points(metrics, database="lainon.life")
        time.sleep(30)
<|endoftext|>


<commit_before>#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Wed Jul 25 15:57:03 2018

@author: neuro
"""

import matplotlib.pyplot as plt
import numpy as np
import os
import glob
from scipy import fftpack

try:
    import pyfftw

    pyfftwexists = True
    fftpack = pyfftw.interfaces.scipy_fftpack
    pyfftw.interfaces.cache.enable()
except ImportError:
    pyfftwexists = False

def filtscale(data, scalefac=1.0, reverse=False, hybrid=False, lognormalize=True, epsilon=1e-10, numorders=6):
    if not reverse:
        specvals = fftpack.fft(data)
        if lognormalize:
            themag = np.log(np.absolute(specvals) + epsilon)
            scalefac = np.max(themag)
            themag = (themag - scalefac + numorders) / numorders
            themag[np.where(themag < 0.0)] = 0.0
        else:
            scalefac = np.std(data)
            themag = np.absolute(specvals) / scalefac
        thephase = np.angle(specvals)
        thephase = thephase / (2.0 * np.pi) - 0.5
        if hybrid:
            return np.stack((thedata, themag), axis=1), scalefac
        else:
            return np.stack((themag, thephase), axis=1), scalefac
    else:
        if hybrid:
            return data[:, 0]
        else:
            thephase = (data[:, 1] + 0.5) * 2.0 * np.pi
            if lognormalize:
                themag = np.exp(data[:, 0] * numorders - numorders + scalefac)
            else:
                themag = data[:, 0] * scalefac
            specvals = themag * np.exp(1.0j * thephase)
            return  fftpack.ifft(specvals).real

def tobadpts(name):
    return name.replace('.txt', '_badpts.txt')

def tocardfmri(name):
    return name.replace('normpleth', 'cardfromfmri')

def prep(window_size,
        step=1,
        lag=0,
        excludethresh=10.0,
        usebadpts=False,
        startskip=200,
        endskip=0,
        thesuffix='sliceres',
        thedatadir='/data1/frederic/test/output',
        dofft=False,
        debug=False):

    print('entering dataload prep')
    fromfile = sorted(glob.glob(os.path.join(thedatadir, '*normpleth_' + thesuffix + '.txt')))

    # make sure all files exist
    cleanfilelist = []
    print('checking datafiles')
    for physioname in fromfile:
        if os.path.isfile(tocardfmri(physioname)):
            if usebadpts:
                if os.path.isfile(tobadpts(physioname.replace('normpleth', 'pleth'))) \
                    and os.path.isfile(tobadpts(tocardfmri(physioname))):
                    cleanfilelist.append(physioname)
                    print(cleanfilelist[-1])
            else:
                cleanfilelist.append(physioname)
                print(cleanfilelist[-1])
    if usebadpts:
        print(len(cleanfilelist), 'runs pass all 4 files present check')
    else:
        print(len(cleanfilelist), 'runs pass both files present check')

    # find out how long the files are
    tempy = np.loadtxt(cleanfilelist[0])
    tempx = np.loadtxt(cleanfilelist[0].replace('normpleth', 'cardfromfmri'))
    tclen = np.min([tempx.shape[0], tempy.shape[0]])
    print('tclen set to', tclen)

    # allocate target arrays
    print('allocating arrays')
    s = len(cleanfilelist)
    x1 = np.zeros([tclen, s])
    y1 = np.zeros([tclen, s])

    # now read the data in
    count = 0
    print('checking data')
    for i in range(s):
        print('processing ', cleanfilelist[i])
        tempy = np.loadtxt(cleanfilelist[i])
        tempx = np.loadtxt(cleanfilelist[i].replace('normpleth', 'cardfromfmri'))
        ntempx = tempx.shape[0]
        ntempy = tempy.shape[0]
        if (ntempx >= tclen) and (ntempy >= tclen):
            x1[:tclen, count] = tempx[:tclen]
            y1[:tclen, count] = tempy[:tclen]
            count += 1
    print(count, 'runs pass file length check')

    y = y1[startskip:, :count]
    x = x1[startskip:, :count]
    print('xshape, yshape:', x.shape, y.shape)

    # normalize input and output data
    print('normalizing data')
    print('y shape:', y.shape, 'count:', count)
    if debug:
        for thesubj in range(count):
            print('prenorm sub', thesubj, 'min, max mean std x, y:', thesubj,
                  np.min(x[:, thesubj]), np.max(x[:, thesubj]), np.mean(x[:, thesubj]), np.std(x[:, thesubj]),
                  np.min(y[:, thesubj]), np.max(y[:, thesubj]), np.mean(y[:, thesubj]), np.std(y[:, thesubj]))

    y -= np.mean(y, axis=0)
    thestd = np.std(y, axis=0)
    for thesubj in range(thestd.shape[0]):
        if thestd[thesubj] > 0.0:
            y[:, thesubj] /= thestd[thesubj]

    x -= np.mean(x, axis=0)
    thestd = np.std(x, axis=0)
    for thesubj in range(thestd.shape[0]):
        if thestd[thesubj] > 0.0:
            x[:, thesubj] /= thestd[thesubj]

    if debug:
        for thesubj in range(count):
            print('postnorm sub', thesubj, 'min, max mean std x, y:', thesubj,
                  np.min(x[:, thesubj]), np.max(x[:, thesubj]), np.mean(x[:, thesubj]), np.std(x[:, thesubj]),
                  np.min(y[:, thesubj]), np.max(y[:, thesubj]), np.mean(y[:, thesubj]), np.std(y[:, thesubj]))

    cleansubjs = (np.max(x, axis=0) < excludethresh) & (np.min(x, axis=0) > -excludethresh)
    x = x[:, cleansubjs]
    y = y[:, cleansubjs]

    print('after filtering, shape of x is', x.shape)

    N_pts = y.shape[0]
    N_subjs = y.shape[1]

    X = np.zeros((1, N_pts, N_subjs))
    Y = np.zeros((1, N_pts, N_subjs))

    X[0, :, :] = x
    Y[0, :, :] = y

    Xb = np.zeros((N_subjs * (N_pts - window_size - 1), window_size + lag, 1))
    print('dimensions of Xb:', Xb.shape)
    for j in range(N_subjs):
        print('sub', j, 'min, max X, Y:', j, np.min(X[0, :, j]), np.max(X[0, :, j]), np.min(Y[0, :, j]),
              np.max(Y[0, :, j]))
        for i in range((N_pts - window_size - 1)):
            Xb[j * ((N_pts - window_size - 1)) + i, :, 0] = X[0, step * i:(step * i + window_size + lag), j]

    Yb = np.zeros((N_subjs * (N_pts - window_size - 1), window_size + lag, 1))
    print('dimensions of Yb:', Yb.shape)
    for j in range(N_subjs):
        for i in range((N_pts - window_size - 1)):
            Yb[j * ((N_pts - window_size - 1)) + i, :, 0] = Y[0, step * i:(step * i + window_size + lag), j]

    if usebadpts:
        Xb_withbad = np.zeros((N_subjs * (N_pts - window_size - 1), window_size + lag, 2))
        print('dimensions of Xb_withbad:', Xb_withbad.shape)
        Xscale_withbad = np.zeros((N_subjs, N_pts - window_size - 1))
        print('dimensions of Xscale_withbad:', Xscale_withbad.shape)
        Yb_withbad = np.zeros((N_subjs * (N_pts - window_size - 1), window_size + lag, 2))
        print('dimensions of Yb_withbad:', Yb_withbad.shape)
        Yscale_withbad = np.zeros((N_subjs, N_pts - window_size - 1))
        print('dimensions of Yscale_withbad:', Yscale_withbad.shape)
        for j in range(N_subjs):
            print('transforming subject',j)
            for i in range((N_pts - window_size - 1)):
                Xb_withbad[j * ((N_pts - window_size - 1)) + i, :, :], Xscale_withbad[j, i] = \
                    filtscale(X[0, step * i:(step * i + window_size + lag), j])
                Yb_withbad[j * ((N_pts - window_size - 1)) + i, :, :], Yscale_withbad[j, i] = \
                    filtscale(Y[0, step * i:(step * i + window_size + lag), j])

    perm = np.arange(Xb.shape[0])

    if dofft:
        Xb_fourier = np.zeros((N_subjs * (N_pts - window_size - 1), window_size + lag, 2))
        print('dimensions of Xb_fourier:', Xb_fourier.shape)
        Xscale_fourier = np.zeros((N_subjs, N_pts - window_size - 1))
        print('dimensions of Xscale_fourier:', Xscale_fourier.shape)
        Yb_fourier = np.zeros((N_subjs * (N_pts - window_size - 1), window_size + lag, 2))
        print('dimensions of Yb_fourier:', Yb_fourier.shape)
        Yscale_fourier = np.zeros((N_subjs, N_pts - window_size - 1))
        print('dimensions of Yscale_fourier:', Yscale_fourier.shape)
        for j in range(N_subjs):
            print('transforming subject',j)
            for i in range((N_pts - window_size - 1)):
                Xb_fourier[j * ((N_pts - window_size - 1)) + i, :, :], Xscale_fourier[j, i] = \
                    filtscale(X[0, step * i:(step * i + window_size + lag), j])
                Yb_fourier[j * ((N_pts - window_size - 1)) + i, :, :], Yscale_fourier[j, i] = \
                    filtscale(Y[0, step * i:(step * i + window_size + lag), j])

    perm = np.arange(Xb.shape[0])
    limit = int(0.8 * Xb.shape[0])

    if dofft:
        train_x = Xb_fourier[perm[:limit], :, :]
        train_y = Yb_fourier[perm[:limit], :, :]

        val_x = Xb_fourier[perm[limit:], :, :]
        val_y = Yb_fourier[perm[limit:], :, :]
        print('train, val dims:', train_x.shape, train_y.shape, val_x.shape, val_y.shape)
        return train_x, train_y, val_x, val_y, N_subjs, tclen - startskip, Xscale_fourier, Yscale_fourier
    else:
        train_x = Xb[perm[:limit], :, :]
        train_y = Yb[perm[:limit], :, :]

        val_x = Xb[perm[limit:], :, :]
        val_y = Yb[perm[limit:], :, :]
        print('train, val dims:', train_x.shape, train_y.shape, val_x.shape, val_y.shape)
        return train_x, train_y, val_x, val_y, N_subjs, tclen - startskip
<commit_msg>Normalize with MAD, added usebadpts option<commit_after>#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Wed Jul 25 15:57:03 2018

@author: neuro
"""

import matplotlib.pyplot as plt
import numpy as np
import os
import glob
from scipy import fftpack
from statsmodels.robust.scale import mad

try:
    import pyfftw

    pyfftwexists = True
    fftpack = pyfftw.interfaces.scipy_fftpack
    pyfftw.interfaces.cache.enable()
except ImportError:
    pyfftwexists = False

def filtscale(data, scalefac=1.0, reverse=False, hybrid=False, lognormalize=True, epsilon=1e-10, numorders=6):
    if not reverse:
        specvals = fftpack.fft(data)
        if lognormalize:
            themag = np.log(np.absolute(specvals) + epsilon)
            scalefac = np.max(themag)
            themag = (themag - scalefac + numorders) / numorders
            themag[np.where(themag < 0.0)] = 0.0
        else:
            scalefac = np.std(data)
            themag = np.absolute(specvals) / scalefac
        thephase = np.angle(specvals)
        thephase = thephase / (2.0 * np.pi) - 0.5
        if hybrid:
            return np.stack((thedata, themag), axis=1), scalefac
        else:
            return np.stack((themag, thephase), axis=1), scalefac
    else:
        if hybrid:
            return data[:, 0]
        else:
            thephase = (data[:, 1] + 0.5) * 2.0 * np.pi
            if lognormalize:
                themag = np.exp(data[:, 0] * numorders - numorders + scalefac)
            else:
                themag = data[:, 0] * scalefac
            specvals = themag * np.exp(1.0j * thephase)
            return  fftpack.ifft(specvals).real

def tobadpts(name):
    return name.replace('.txt', '_badpts.txt')

def tocardfmri(name):
    return name.replace('normpleth', 'cardfromfmri')

def prep(window_size,
        step=1,
        lag=0,
        excludethresh=4.0,
        usebadpts=False,
        startskip=200,
        endskip=0,
        thesuffix='sliceres',
        thedatadir='/data1/frederic/test/output',
        dofft=False,
        debug=False):

    print('entering dataload prep')
    fromfile = sorted(glob.glob(os.path.join(thedatadir, '*normpleth_' + thesuffix + '.txt')))

    # make sure all files exist
    cleanfilelist = []
    print('checking datafiles')
    for physioname in fromfile:
        if os.path.isfile(tocardfmri(physioname)):
            if usebadpts:
                if os.path.isfile(tobadpts(physioname.replace('normpleth', 'pleth'))) \
                    and os.path.isfile(tobadpts(tocardfmri(physioname))):
                    cleanfilelist.append(physioname)
                    print(cleanfilelist[-1])
            else:
                cleanfilelist.append(physioname)
                print(cleanfilelist[-1])
    if usebadpts:
        print(len(cleanfilelist), 'runs pass all 4 files present check')
    else:
        print(len(cleanfilelist), 'runs pass both files present check')

    # find out how long the files are
    tempy = np.loadtxt(cleanfilelist[0])
    tempx = np.loadtxt(cleanfilelist[0].replace('normpleth', 'cardfromfmri'))
    tclen = np.min([tempx.shape[0], tempy.shape[0]])
    print('tclen set to', tclen)

    # allocate target arrays
    print('allocating arrays')
    s = len(cleanfilelist)
    x1 = np.zeros([tclen, s])
    y1 = np.zeros([tclen, s])
    if usebadpts:
        bad1 = np.zeros([tclen, s])

    # now read the data in
    count = 0
    print('checking data')
    for i in range(s):
        print('processing ', cleanfilelist[i])
        tempy = np.loadtxt(cleanfilelist[i])
        tempx = np.loadtxt(cleanfilelist[i].replace('normpleth', 'cardfromfmri'))
        ntempx = tempx.shape[0]
        ntempy = tempy.shape[0]
        if (ntempx >= tclen) and (ntempy >= tclen):
            x1[:tclen, count] = tempx[:tclen]
            y1[:tclen, count] = tempy[:tclen]
            if usebadpts:
                tempbad1 = np.loadtxt(tobadpts(cleanfilelist[i].replace('normpleth', 'pleth')))
                tempbad2 = np.loadtxt(tobadpts(tocardfmri(cleanfilelist[i])))
                bad1[:tclen, count] = 1.0 - (1.0 - tempbad1[:tclen]) * (1.0 - tempbad2[:tclen])
            count += 1
    print(count, 'runs pass file length check')

    count = 50

    y = y1[startskip:, :count]
    x = x1[startskip:, :count]
    if usebadpts:
        bad = bad1[startskip:, :count]
    print('xshape, yshape:', x.shape, y.shape)

    # normalize input and output data
    print('normalizing data')
    print('y shape:', y.shape, 'count:', count)
    if debug:
        for thesubj in range(count):
            print('prenorm sub', thesubj, 'min, max mean MAD x, y:', thesubj,
                  np.min(x[:, thesubj]), np.max(x[:, thesubj]), np.mean(x[:, thesubj]), mad(x[:, thesubj]),
                  np.min(y[:, thesubj]), np.max(y[:, thesubj]), np.mean(y[:, thesubj]), mad(y[:, thesubj]))

    y -= np.mean(y, axis=0)
    themad = mad(y, axis=0)
    for thesubj in range(themad.shape[0]):
        if themad[thesubj] > 0.0:
            y[:, thesubj] /= themad[thesubj]

    x -= np.mean(x, axis=0)
    themad = mad(x, axis=0)
    for thesubj in range(themad.shape[0]):
        if themad[thesubj] > 0.0:
            x[:, thesubj] /= themad[thesubj]

    if debug:
        for thesubj in range(count):
            print('postnorm sub', thesubj, 'min, max mean MAD x, y:', thesubj,
                  np.min(x[:, thesubj]), np.max(x[:, thesubj]), np.mean(x[:, thesubj]), mad(x[:, thesubj]),
                  np.min(y[:, thesubj]), np.max(y[:, thesubj]), np.mean(y[:, thesubj]), mad(y[:, thesubj]))

    cleansubjs = (np.max(x, axis=0) < excludethresh) & (np.min(x, axis=0) > -excludethresh)
    x = x[:, cleansubjs]
    y = y[:, cleansubjs]
    if usebadpts:
        bad = bad[:, cleansubjs]

    print('after filtering, shape of x is', x.shape)

    N_pts = y.shape[0]
    N_subjs = y.shape[1]

    X = np.zeros((1, N_pts, N_subjs))
    Y = np.zeros((1, N_pts, N_subjs))
    if usebadpts:
        BAD = np.zeros((1, N_pts, N_subjs))

    X[0, :, :] = x
    Y[0, :, :] = y
    if usebadpts:
        BAD[0, :, :] = bad

    Xb = np.zeros((N_subjs * (N_pts - window_size - 1), window_size + lag, 1))
    print('dimensions of Xb:', Xb.shape)
    for j in range(N_subjs):
        print('sub', j, 'min, max X, Y:', j, np.min(X[0, :, j]), np.max(X[0, :, j]), np.min(Y[0, :, j]),
              np.max(Y[0, :, j]))
        for i in range((N_pts - window_size - 1)):
            Xb[j * ((N_pts - window_size - 1)) + i, :, 0] = X[0, step * i:(step * i + window_size + lag), j]

    Yb = np.zeros((N_subjs * (N_pts - window_size - 1), window_size + lag, 1))
    print('dimensions of Yb:', Yb.shape)
    for j in range(N_subjs):
        for i in range((N_pts - window_size - 1)):
            Yb[j * ((N_pts - window_size - 1)) + i, :, 0] = Y[0, step * i:(step * i + window_size + lag), j]

    if usebadpts:
        Xb_withbad = np.zeros((N_subjs * (N_pts - window_size - 1), window_size + lag, 2))
        print('dimensions of Xb_withbad:', Xb_withbad.shape)
        for j in range(N_subjs):
            print('packing data for subject',j)
            for i in range((N_pts - window_size - 1)):
                Xb_withbad[j * ((N_pts - window_size - 1)) + i, :, 0] = \
                    X[0, step * i:(step * i + window_size + lag), j]
                Xb_withbad[j * ((N_pts - window_size - 1)) + i, :, 1] = \
                    BAD[0, step * i:(step * i + window_size + lag), j]
        Xb = Xb_withbad

    perm = np.arange(Xb.shape[0])

    if dofft:
        Xb_fourier = np.zeros((N_subjs * (N_pts - window_size - 1), window_size + lag, 2))
        print('dimensions of Xb_fourier:', Xb_fourier.shape)
        Xscale_fourier = np.zeros((N_subjs, N_pts - window_size - 1))
        print('dimensions of Xscale_fourier:', Xscale_fourier.shape)
        Yb_fourier = np.zeros((N_subjs * (N_pts - window_size - 1), window_size + lag, 2))
        print('dimensions of Yb_fourier:', Yb_fourier.shape)
        Yscale_fourier = np.zeros((N_subjs, N_pts - window_size - 1))
        print('dimensions of Yscale_fourier:', Yscale_fourier.shape)
        for j in range(N_subjs):
            print('transforming subject',j)
            for i in range((N_pts - window_size - 1)):
                Xb_fourier[j * ((N_pts - window_size - 1)) + i, :, :], Xscale_fourier[j, i] = \
                    filtscale(X[0, step * i:(step * i + window_size + lag), j])
                Yb_fourier[j * ((N_pts - window_size - 1)) + i, :, :], Yscale_fourier[j, i] = \
                    filtscale(Y[0, step * i:(step * i + window_size + lag), j])

    perm = np.arange(Xb.shape[0])
    limit = int(0.8 * Xb.shape[0])

    if dofft:
        train_x = Xb_fourier[perm[:limit], :, :]
        train_y = Yb_fourier[perm[:limit], :, :]

        val_x = Xb_fourier[perm[limit:], :, :]
        val_y = Yb_fourier[perm[limit:], :, :]
        print('train, val dims:', train_x.shape, train_y.shape, val_x.shape, val_y.shape)
        return train_x, train_y, val_x, val_y, N_subjs, tclen - startskip, Xscale_fourier, Yscale_fourier
    else:
        train_x = Xb[perm[:limit], :, :]
        train_y = Yb[perm[:limit], :, :]

        val_x = Xb[perm[limit:], :, :]
        val_y = Yb[perm[limit:], :, :]
        print('train, val dims:', train_x.shape, train_y.shape, val_x.shape, val_y.shape)
        return train_x, train_y, val_x, val_y, N_subjs, tclen - startskip
<|endoftext|>


<commit_before>
from rasa.core.brokers.broker import EventBroker

# noinspection PyAbstractClass
from rasa.utils.common import raise_warning

class EventChannel(EventBroker):
    raise_warning(
        "The `EventChannel` class is deprecated, please inherit from "
        "`EventBroker` instead. `EventChannel` will be removed "
        "in future Rasa versions.",
        DeprecationWarning,
        stacklevel=2,
    )
<commit_msg>black formatting<commit_after>from rasa.core.brokers.broker import EventBroker

# noinspection PyAbstractClass
from rasa.utils.common import raise_warning

class EventChannel(EventBroker):
    raise_warning(
        "The `EventChannel` class is deprecated, please inherit from "
        "`EventBroker` instead. `EventChannel` will be removed "
        "in future Rasa versions.",
        DeprecationWarning,
        stacklevel=2,
    )
<|endoftext|>


<commit_before>from sanic import Blueprint, response
from sanic.request import Request
from sanic.response import HTTPResponse
from twilio.twiml.voice_response import VoiceResponse, Gather
from typing import Text, Callable, Awaitable, List, Any, Dict, Optional

from rasa.shared.utils.io import raise_warning
from rasa.shared.core.events import BotUttered
from rasa.shared.exceptions import RasaException
from rasa.core.channels.channel import (
    InputChannel,
    CollectingOutputChannel,
    UserMessage,
)

class TwilioVoiceInput(InputChannel):
    """Input channel for Twilio Voice."""

    SUPPORTED_VOICES = [
        "man",
        "woman",
        "alice",
        "Polly.Mads",
        "Polly.Naja",
        "Polly.Lotte",
        "Polly.Reuben",
        "Polly.Nicole",
        "Polly.Russell",
        "Polly.Amy",
        "Polly.Brian",
        "Polly.Emma",
        "Polly.Amy-Neural",
        "Polly.Emma-Neural",
        "Polly.Brian-Neural",
        "Polly.Raveena",
        "Polly.Ivy",
        "Polly.Joanna",
        "Polly.Joey",
        "Polly.Justin",
        "Polly.Kendra",
        "Polly.Kimberly",
        "Polly.Matthew",
        "Polly.Salli",
        "Polly.Ivy-Neural",
        "Polly.Joanna-Neural",
        "Polly.Kendra-Neural",
        "Polly.Kimberly-Neural",
        "Polly.Sally-Neural",
        "Polly.Joey-Neural",
        "Polly.Justin-Neural",
        "Polly.Matthew-Neural",
        "Polly.Geraint",
        "Polly.Celine",
        "Polly.Mathieu",
        "Polly.Chantal",
        "Polly.Hans",
        "Polly.Marlene",
        "Polly.Vicki",
        "Polly.Dora",
        "Polly.Karl",
        "Polly.Carla",
        "Polly.Giorgio",
        "Polly.Mizuki",
        "Polly.Takumi",
        "Polly.Liv",
        "Polly.Jacek",
        "Polly.Jan",
        "Polly.Ewa",
        "Polly.Maja",
        "Polly.Ricardo",
        "Polly.Vitoria",
        "Polly.Camila-Neural",
        "Polly.Cristiano",
        "Polly.Ines",
        "Polly.Carmen",
        "Polly.Maxim",
        "Polly.Tatyana",
        "Polly.Conchita",
        "Polly.Enrique",
        "Polly.Miguel",
        "Polly.Penelope",
        "Polly.Lupe-Neural",
        "Polly.Astrid",
        "Polly.Filiz",
        "Polly.Gwyneth",
    ]

    @classmethod
    def name(cls) -> Text:
        """Name of your custom channel."""
        return "twilio_voice"

    @classmethod
    def from_credentials(cls, credentials: Optional[Dict[Text, Any]]) -> InputChannel:
        """Load custom configurations."""
        if not credentials:
            return cls(initial_prompt="hello", assistant_voice="woman")

        return cls(
            credentials.get("initial_prompt", "hello"),
            credentials.get("assistant_voice", "woman"),
        )

    def __init__(
        self, initial_prompt: Optional[Text], assistant_voice: Optional[Text],
    ) -> None:
        """Creates a connection to Twilio voice.

        Args:
            assistant_voice: name of the assistant voice to use.
        """
        self.initial_prompt = initial_prompt
        self.assistant_voice = assistant_voice

        if assistant_voice not in self.SUPPORTED_VOICES:
            self.raise_invalid_voice_exception()

    def raise_invalid_voice_exception(self) -> None:
        """Raises an error if an invalid voice is provided."""
        raise RasaException(
            f"{self.assistant_voice} is an invalid as an assistant voice. Please refer to the documentation for a list "
            f"of valid voices you can use for your voice assistant."
        )

    def blueprint(
        self, on_new_message: Callable[[UserMessage], Awaitable[None]]
    ) -> Blueprint:
        """Defines endpoints for Twilio voice channel."""
        twilio_voice_webhook = Blueprint("Twilio_voice_webhook", __name__)

        @twilio_voice_webhook.route("/", methods=["GET"])
        async def health(request: Request) -> HTTPResponse:
            return response.json({"status": "ok"})

        @twilio_voice_webhook.route("/webhook", methods=["POST"])
        async def receive(request: Request):
            sender_id = request.form.get("From")
            text = request.form.get("SpeechResult")
            input_channel = self.name()
            call_status = request.form.get("CallStatus")

            collector = TwilioVoiceCollectingOutputChannel()

            # Provide an initial greeting to answer the user's call.
            if (text is None) and (call_status == "ringing"):
                text = self.initial_prompt

            # determine the response.
            if text is not None:
                await on_new_message(
                    UserMessage(
                        text, collector, sender_id, input_channel=input_channel,
                    )
                )

                twilio_response = self.build_twilio_voice_response(collector.messages)
            # If the user doesn't respond resend the last message.
            else:
                # Get last user utterance from tracker.
                tracker = request.app.agent.tracker_store.retrieve(sender_id)
                last_response = next(
                    (e for e in reversed(tracker.events) if isinstance(e, BotUttered)),
                    None,
                )

                # If no previous utterance found say something generic.
                if last_response is None:
                    last_response = "I didn't get that."
                else:
                    last_response = last_response.text

                twilio_response = self.build_twilio_voice_response(
                    [{"text": last_response}]
                )
            return response.text(str(twilio_response), content_type="text/xml")

        return twilio_voice_webhook

    def build_twilio_voice_response(
        self, messages: List[Dict[Text, Any]]
    ) -> VoiceResponse:
        """Builds the Twilio Voice Response object."""
        voice_response = VoiceResponse()
        gather = Gather(
            input="speech",
            action=f"/webhooks/{self.name()}/webhook",
            actionOnEmptyResult=True,
            speechTimeout="auto",
        )

        # Add pauses between messages.
        # Add a listener to the last message to listen for user response.
        for i, message in enumerate(messages):
            if i + 1 == len(messages):
                gather.say(message["text"], voice=self.assistant_voice)
                voice_response.append(gather)
            else:
                voice_response.say(message["text"], voice=self.assistant_voice)
                voice_response.pause(length=1)

        return voice_response

class TwilioVoiceCollectingOutputChannel(CollectingOutputChannel):
    """Output channel that collects send messages in a list.

    (doesn't send them anywhere, just collects them).
    """

    @classmethod
    def name(cls) -> Text:
        """Name of the output channel."""
        return "twilio_voice"

    async def send_text_with_buttons(
        self,
        recipient_id: Text,
        text: Text,
        buttons: List[Dict[Text, Any]],
        **kwargs: Any,
    ) -> None:
        """Convert buttons into a voice representation."""
        await self._persist_message(self._message(recipient_id, text=text))

        for b in buttons:
            await self._persist_message(self._message(recipient_id, text=b["title"]))

    async def send_image_url(
        self, recipient_id: Text, image: Text, **kwargs: Any
    ) -> None:
        """For voice channel do not send images."""
        raise_warning(
            "Image removed from voice message. Only text of message is sent."
        )
<commit_msg>Update rasa/core/channels/twilio_voice.py<commit_after>from sanic import Blueprint, response
from sanic.request import Request
from sanic.response import HTTPResponse
from twilio.twiml.voice_response import VoiceResponse, Gather
from typing import Text, Callable, Awaitable, List, Any, Dict, Optional

from rasa.shared.utils.io import raise_warning
from rasa.shared.core.events import BotUttered
from rasa.shared.exceptions import RasaException
from rasa.core.channels.channel import (
    InputChannel,
    CollectingOutputChannel,
    UserMessage,
)

class TwilioVoiceInput(InputChannel):
    """Input channel for Twilio Voice."""

    SUPPORTED_VOICES = [
        "man",
        "woman",
        "alice",
        "Polly.Mads",
        "Polly.Naja",
        "Polly.Lotte",
        "Polly.Reuben",
        "Polly.Nicole",
        "Polly.Russell",
        "Polly.Amy",
        "Polly.Brian",
        "Polly.Emma",
        "Polly.Amy-Neural",
        "Polly.Emma-Neural",
        "Polly.Brian-Neural",
        "Polly.Raveena",
        "Polly.Ivy",
        "Polly.Joanna",
        "Polly.Joey",
        "Polly.Justin",
        "Polly.Kendra",
        "Polly.Kimberly",
        "Polly.Matthew",
        "Polly.Salli",
        "Polly.Ivy-Neural",
        "Polly.Joanna-Neural",
        "Polly.Kendra-Neural",
        "Polly.Kimberly-Neural",
        "Polly.Sally-Neural",
        "Polly.Joey-Neural",
        "Polly.Justin-Neural",
        "Polly.Matthew-Neural",
        "Polly.Geraint",
        "Polly.Celine",
        "Polly.Mathieu",
        "Polly.Chantal",
        "Polly.Hans",
        "Polly.Marlene",
        "Polly.Vicki",
        "Polly.Dora",
        "Polly.Karl",
        "Polly.Carla",
        "Polly.Giorgio",
        "Polly.Mizuki",
        "Polly.Takumi",
        "Polly.Liv",
        "Polly.Jacek",
        "Polly.Jan",
        "Polly.Ewa",
        "Polly.Maja",
        "Polly.Ricardo",
        "Polly.Vitoria",
        "Polly.Camila-Neural",
        "Polly.Cristiano",
        "Polly.Ines",
        "Polly.Carmen",
        "Polly.Maxim",
        "Polly.Tatyana",
        "Polly.Conchita",
        "Polly.Enrique",
        "Polly.Miguel",
        "Polly.Penelope",
        "Polly.Lupe-Neural",
        "Polly.Astrid",
        "Polly.Filiz",
        "Polly.Gwyneth",
    ]

    @classmethod
    def name(cls) -> Text:
        """Name of your custom channel."""
        return "twilio_voice"

    @classmethod
    def from_credentials(cls, credentials: Optional[Dict[Text, Any]]) -> InputChannel:
        """Load custom configurations."""
        credentials = credentials or {}

        return cls(
            credentials.get("initial_prompt", "hello"),
            credentials.get("assistant_voice", "woman"),
        )

    def __init__(
        self, initial_prompt: Optional[Text], assistant_voice: Optional[Text],
    ) -> None:
        """Creates a connection to Twilio voice.

        Args:
            assistant_voice: name of the assistant voice to use.
        """
        self.initial_prompt = initial_prompt
        self.assistant_voice = assistant_voice

        if assistant_voice not in self.SUPPORTED_VOICES:
            self.raise_invalid_voice_exception()

    def raise_invalid_voice_exception(self) -> None:
        """Raises an error if an invalid voice is provided."""
        raise RasaException(
            f"{self.assistant_voice} is an invalid as an assistant voice. Please refer to the documentation for a list "
            f"of valid voices you can use for your voice assistant."
        )

    def blueprint(
        self, on_new_message: Callable[[UserMessage], Awaitable[None]]
    ) -> Blueprint:
        """Defines endpoints for Twilio voice channel."""
        twilio_voice_webhook = Blueprint("Twilio_voice_webhook", __name__)

        @twilio_voice_webhook.route("/", methods=["GET"])
        async def health(request: Request) -> HTTPResponse:
            return response.json({"status": "ok"})

        @twilio_voice_webhook.route("/webhook", methods=["POST"])
        async def receive(request: Request):
            sender_id = request.form.get("From")
            text = request.form.get("SpeechResult")
            input_channel = self.name()
            call_status = request.form.get("CallStatus")

            collector = TwilioVoiceCollectingOutputChannel()

            # Provide an initial greeting to answer the user's call.
            if (text is None) and (call_status == "ringing"):
                text = self.initial_prompt

            # determine the response.
            if text is not None:
                await on_new_message(
                    UserMessage(
                        text, collector, sender_id, input_channel=input_channel,
                    )
                )

                twilio_response = self.build_twilio_voice_response(collector.messages)
            # If the user doesn't respond resend the last message.
            else:
                # Get last user utterance from tracker.
                tracker = request.app.agent.tracker_store.retrieve(sender_id)
                last_response = next(
                    (e for e in reversed(tracker.events) if isinstance(e, BotUttered)),
                    None,
                )

                # If no previous utterance found say something generic.
                if last_response is None:
                    last_response = "I didn't get that."
                else:
                    last_response = last_response.text

                twilio_response = self.build_twilio_voice_response(
                    [{"text": last_response}]
                )
            return response.text(str(twilio_response), content_type="text/xml")

        return twilio_voice_webhook

    def build_twilio_voice_response(
        self, messages: List[Dict[Text, Any]]
    ) -> VoiceResponse:
        """Builds the Twilio Voice Response object."""
        voice_response = VoiceResponse()
        gather = Gather(
            input="speech",
            action=f"/webhooks/{self.name()}/webhook",
            actionOnEmptyResult=True,
            speechTimeout="auto",
        )

        # Add pauses between messages.
        # Add a listener to the last message to listen for user response.
        for i, message in enumerate(messages):
            if i + 1 == len(messages):
                gather.say(message["text"], voice=self.assistant_voice)
                voice_response.append(gather)
            else:
                voice_response.say(message["text"], voice=self.assistant_voice)
                voice_response.pause(length=1)

        return voice_response

class TwilioVoiceCollectingOutputChannel(CollectingOutputChannel):
    """Output channel that collects send messages in a list.

    (doesn't send them anywhere, just collects them).
    """

    @classmethod
    def name(cls) -> Text:
        """Name of the output channel."""
        return "twilio_voice"

    async def send_text_with_buttons(
        self,
        recipient_id: Text,
        text: Text,
        buttons: List[Dict[Text, Any]],
        **kwargs: Any,
    ) -> None:
        """Convert buttons into a voice representation."""
        await self._persist_message(self._message(recipient_id, text=text))

        for b in buttons:
            await self._persist_message(self._message(recipient_id, text=b["title"]))

    async def send_image_url(
        self, recipient_id: Text, image: Text, **kwargs: Any
    ) -> None:
        """For voice channel do not send images."""
        raise_warning(
            "Image removed from voice message. Only text of message is sent."
        )
<|endoftext|>