<commit_before># Copyright (C) 2009 Umea University
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#            http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""
A plugin that allows you to use SAML2 SSO as authentication
and SAML2 attribute aggregations as metadata collector in your
WSGI application.

"""
import re
import urlparse
import urllib
import cgi
import os
import time

from paste.httpheaders import CONTENT_LENGTH
from paste.httpheaders import CONTENT_TYPE
from paste.httpheaders import LOCATION
from paste.httpexceptions import HTTPFound
from paste.httpexceptions import HTTPUnauthorized
from paste.httpexceptions import HTTPTemporaryRedirect
from paste.request import parse_dict_querystring
from paste.request import parse_formvars
from paste.request import construct_url
from paste.request import parse_querystring

from paste.response import header_value

from zope.interface import implements

from repoze.who.interfaces import IChallenger, IIdentifier, IAuthenticator
from repoze.who.interfaces import IMetadataProvider
from repoze.who.plugins.form import FormPluginBase

from saml2.client import Saml2Client
from saml2.attribute_resolver import AttributeResolver
from saml2.metadata import MetaData
from saml2.saml import NAMEID_FORMAT_TRANSIENT
from saml2.config import Config
from saml2.cache import Cache

def construct_came_from(environ):
    """ The URL that the user used when the process where interupted
    for single-sign-on processing. """

    came_from = environ.get("PATH_INFO")
    qs = environ.get("QUERY_STRING","")
    if qs:
        came_from += '?' + qs
    return came_from

# FormPluginBase defines the methods remember and forget

class SAML2Plugin(FormPluginBase):

    implements(IChallenger, IIdentifier, IAuthenticator, IMetadataProvider)

    def __init__(self, rememberer_name, saml_conf_file, virtual_organization,
                cache, path_logout, path_toskip, debug):

        self.rememberer_name = rememberer_name
        self.path_logout = path_logout
        self.path_toskip = path_toskip
        self.debug = debug

        self.conf = Config()
        self.conf.load_file(saml_conf_file)
        self.sp = self.conf["service"]["sp"]

        if virtual_organization:
            self.vo = virtual_organization
            self.vo_conf = None
#            try:
#                self.vo_conf = self.conf[
#                                "virtual_organization"][virtual_organization]
#            except KeyError:
#                self.vo = None
        else:
            self.vo = None

        try:
            self.metadata = self.conf["metadata"]
        except KeyError:
            self.metadata = None
        self.outstanding_authn = {}
        self.iam = os.uname()[1]

        if cache:
            self.cache = Cache(cache)
        else:
            self.cache = Cache()

    def _cache_session(self, session_info):
        name_id = session_info["ava"]["__userid"]
        del session_info["ava"]["__userid"]
        issuer = session_info["issuer"]
        del session_info["issuer"]
        self.cache.set(name_id, issuer, session_info,
                        session_info["not_on_or_after"])
        return name_id

    #### IChallenger ####
    def challenge(self, environ, status, app_headers, forget_headers):

        # this challenge consist in loggin out
        if environ.has_key('rwpc.logout'):
            # TODO
            pass

        logger = environ.get('repoze.who.logger','')
        # ELSE, perform a real challenge => asking for loggin
        # here by redirecting the user to a IdP.

        cl = Saml2Client(environ, self.conf)
        came_from = construct_came_from(environ)
        if self.debug:
            logger and logger.info("RelayState >> %s" % came_from)

        try:
            vo = environ["myapp.vo"]
        except KeyError:
            vo = self.vo
        logger and logger.info("VO: %s" % vo)

        # If more than one idp, I have to do wayf
        logger and logger.info("IdP URL: %s" % self.sp["idp"].values())
        if len( self.sp["idp"] ) == 1:
            idp_url = self.sp["idp"].values()[0]
        elif len( self.sp["idp"] ) == 0:
            start_response('500 Internal Server Error',
                                [('Content-Type', 'text/plain')])
            return ['Misconfiguration']
        else:
            start_response('501 Not Implemented',
                                [('Content-Type', 'text/plain')])
            return ['WAYF not implemented yet!']

        (sid, result) = cl.authenticate(self.conf["entityid"],
                                        idp_url,
                                        self.sp["url"],
                                        self.sp["name"],
                                        relay_state=came_from,
                                        log=logger,
                                        vo=vo)
        self.outstanding_authn[sid] = came_from

        if self.debug:
            logger and logger.info('sc returned: %s' % (result,))
        if isinstance(result, tuple):
            return HTTPTemporaryRedirect(headers=[result])
        else :
            # possible though normally not used
            body = "\n".join(result)
            def auth_form(environ, start_response):
                content_length = CONTENT_LENGTH.tuples(str(len(result)))
                content_type = CONTENT_TYPE.tuples('text/html')
                headers = content_length + content_type + forget_headers
                start_response('200 OK', headers)
                return [result]

            return auth_form

    #### IIdentifier ####
    def identify(self, environ):
        logger = environ.get('repoze.who.logger','')

        uri = environ.get('REQUEST_URI',construct_url(environ))
        if self.debug:
            #logger and logger.info("environ.keys(): %s" % environ.keys())
            #logger and logger.info("Environment: %s" % environ)
            logger and logger.info('identify uri: %s' % (uri,))

        query = parse_dict_querystring(environ)
        if self.debug:
            logger and logger.info('identify query: %s' % (query,))

        # path_logout for every app.
        for regex in self.path_logout:
           if re.match(regex, uri) != None:
               if self.debug :
                   logger and logger.info("LOGOUT #### ")
               # we've been asked to perform a logout

               # use all except : POST
               # trigger the challenge and tells the challenge this is a logout
               query['bhp'] = 'go'
               environ['rwpc.logout'] = \
                    self._serviceURL(environ,urllib.urlencode(query))

               return None

        # skipping, whatever it is (loggin, validating ticket etc.)
        # except for logout (see above)
        for regex in self.path_toskip:
            if re.match(regex, uri) != None:
                if self.debug :
                       logger and logger.info("########### SKIPPING")
                return None

        post_env = environ.copy()
        post_env['QUERY_STRING'] = ''

        if environ["CONTENT_LENGTH"]:
            body = environ["wsgi.input"].read(int(environ["CONTENT_LENGTH"]))
            from StringIO import StringIO
            environ['wsgi.input'] = StringIO(body)
            environ['s2repoze.body'] = body

        post = cgi.FieldStorage(
            fp=environ['wsgi.input'],
            environ=post_env,
            keep_blank_values=True
        )

        if self.debug:
            logger and logger.info('identify post: %s' % (post,))

        try:
            if not post.has_key("SAMLResponse"):
                environ["post.fieldstorage"] = post
                return {}
        except TypeError:
            environ["post.fieldstorage"] = post
            return {}

        # check for SAML2 authN
        cl = Saml2Client(environ, self.conf)
        try:
            session_info = cl.response(post,
                                            self.conf["entityid"],
                                            self.outstanding_authn,
                                            logger)
            name_id = self._cache_session(session_info)
            if self.debug:
                logger and logger.info("stored %s with key %s" % (
                                        session_info, name_id))
        except TypeError:
            return None

        if session_info["came_from"]:
            if self.debug:
                logger and logger.info(
                            "came_from << %s" % session_info["came_from"])
            try:
                path, query = session_info["came_from"].split('?')
                environ["PATH_INFO"] = path
                environ["QUERY_STRING"] = query
            except ValueError:
                environ["PATH_INFO"] = session_info["came_from"]

        identity = {}
        identity["login"] = name_id
        identity["password"] = ""
        identity['repoze.who.userid'] = name_id
        identity["user"] = session_info["ava"]
        environ["s2repoze.sessioninfo"] = session_info
        if self.debug:
            logger and logger.info("Identity: %s" % identity)
        return identity

    # IMetadataProvider
    def add_metadata(self, environ, identity):
        subject_id = identity['repoze.who.userid']

        if self.debug:
            logger = environ.get('repoze.who.logger','')
            if logger:
                logger.info(
                    "add_metadata for %s" % subject_id)
                logger.info(
                    "Known subjects: %s" % self.cache.subjects())
                try:
                    logger.info(
                        "Issuers: %s" % self.cache.entities(subject_id))
                except KeyError:
                    pass

        if "user" not in identity:
            identity["user"] = {}
        try:
            (ava, old) = self.cache.get_identity(subject_id)
            now = time.gmtime()
            if self.debug:
                logger and logger.info("Adding %s" % ava)
            identity["user"].update(ava)
        except KeyError:
            pass

        if "pysaml2_vo_expanded" not in identity:
            # is this a Virtual Organization situation
            if self.vo:
                logger and logger.info("** Do VO aggregation **")
                #try:
                    # This ought to be caseignore
                    #subject_id = identity["user"][
                    #                    self.vo_conf["common_identifier"]][0]
                #except KeyError:
                #    logger and logger.error("** No common identifier **")
                #    return
                logger and logger.info(
                    "SubjectID: %s, VO:%s" % (subject_id, self.vo))

                vo_members = [
                    member for member in self.metadata.vo_members(self.vo)\
                        if member not in self.sp["idp"].keys()]

                logger and logger.info("VO members: %s" % vo_members)
                vo_members = [m for m in vo_members \
                                if not self.cache.active(subject_id, m)]
                logger and logger.info(
                                "VO members (not cached): %s" % vo_members)

                if vo_members:
                    ar = AttributeResolver(environ, self.metadata, self.conf)

                    if self.vo_conf and "name_id_format" in self.vo_conf:
                        name_id_format = self.vo_conf["name_id_format"]
                        sp_name_qualifier=""
                    else:
                        sp_name_qualifier=self.vo
                        name_id_format = ""

                    extra = ar.extend(subject_id,
                            self.conf["entityid"],
                            vo_members,
                            name_id_format=name_id_format,
                            sp_name_qualifier=sp_name_qualifier,
                            log=logger)

                    for session_info in extra:
                        nid = self._cache_session(session_info)

                    logger.info(
                        ">Issuers: %s" % self.cache.entities(subject_id))
                    logger.info(
                        "AVA: %s" % (self.cache.get_identity(subject_id),))
                    identity["user"] = self.cache.get_identity(subject_id)[0]
                    # Only do this once
                    identity["pysaml2_vo_expanded"] = 1
                    #self.store[identity['repoze.who.userid']] = (
                    #                        not_on_or_after, identity)

# @return
# used 2 times : one to get the ticket, the other to validate it
    def _serviceURL(self,environ,qs=None):
        if qs != None:
            url = construct_url(environ, querystring=qs)
        else:
            url = construct_url(environ)
        return url

    #### IAuthenticatorPlugin ####
    def authenticate(self, environ, identity={}):
        return identity.get('login',None)

def make_plugin(rememberer_name=None, # plugin for remember
                 cache= "", # cache
                 # Which virtual organization to support
                 virtual_organization="",
                 path_logout='', # regex url to logout
                 path_toskip='',  # regex url to skip
                 saml_conf="",
                 debug=0,
                 ):

    if saml_conf is None:
        raise ValueError(
            'must include saml_conf in configuration')

    if rememberer_name is None:
        raise ValueError(
             'must include rememberer_name in configuration')
    path_logout = path_logout.lstrip().split('\n');
    path_toskip = path_toskip.lstrip().splitlines()

    plugin = SAML2Plugin(rememberer_name, saml_conf,
                virtual_organization, cache,
                path_logout, path_toskip, debug)
    return plugin

# came_from = re.sub(r'ticket=[^&]*&?', '', came_from)

<commit_msg>Removed path_logout and path_toskip for the time being, might return sometime in the future<commit_after># Copyright (C) 2009 Umea University
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#            http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""
A plugin that allows you to use SAML2 SSO as authentication
and SAML2 attribute aggregations as metadata collector in your
WSGI application.

"""
import re
import urlparse
import urllib
import cgi
import os
import time

from paste.httpheaders import CONTENT_LENGTH
from paste.httpheaders import CONTENT_TYPE
from paste.httpheaders import LOCATION
from paste.httpexceptions import HTTPFound
from paste.httpexceptions import HTTPUnauthorized
from paste.httpexceptions import HTTPTemporaryRedirect
from paste.request import parse_dict_querystring
from paste.request import parse_formvars
from paste.request import construct_url
from paste.request import parse_querystring

from paste.response import header_value

from zope.interface import implements

from repoze.who.interfaces import IChallenger, IIdentifier, IAuthenticator
from repoze.who.interfaces import IMetadataProvider
from repoze.who.plugins.form import FormPluginBase

from saml2.client import Saml2Client
from saml2.attribute_resolver import AttributeResolver
from saml2.metadata import MetaData
from saml2.saml import NAMEID_FORMAT_TRANSIENT
from saml2.config import Config
from saml2.cache import Cache

def construct_came_from(environ):
    """ The URL that the user used when the process where interupted
    for single-sign-on processing. """

    came_from = environ.get("PATH_INFO")
    qs = environ.get("QUERY_STRING","")
    if qs:
        came_from += '?' + qs
    return came_from

# FormPluginBase defines the methods remember and forget

class SAML2Plugin(FormPluginBase):

    implements(IChallenger, IIdentifier, IAuthenticator, IMetadataProvider)

    def __init__(self, rememberer_name, saml_conf_file, virtual_organization,
                cache, debug):

        self.rememberer_name = rememberer_name
        self.debug = debug

        self.conf = Config()
        self.conf.load_file(saml_conf_file)
        self.sp = self.conf["service"]["sp"]

        if virtual_organization:
            self.vo = virtual_organization
            self.vo_conf = None
#            try:
#                self.vo_conf = self.conf[
#                                "virtual_organization"][virtual_organization]
#            except KeyError:
#                self.vo = None
        else:
            self.vo = None

        try:
            self.metadata = self.conf["metadata"]
        except KeyError:
            self.metadata = None
        self.outstanding_authn = {}
        self.iam = os.uname()[1]

        if cache:
            self.cache = Cache(cache)
        else:
            self.cache = Cache()

    def _cache_session(self, session_info):
        name_id = session_info["ava"]["__userid"]
        del session_info["ava"]["__userid"]
        issuer = session_info["issuer"]
        del session_info["issuer"]
        self.cache.set(name_id, issuer, session_info,
                        session_info["not_on_or_after"])
        return name_id

    #### IChallenger ####
    def challenge(self, environ, status, app_headers, forget_headers):

        # this challenge consist in loggin out
        if environ.has_key('rwpc.logout'):
            # TODO
            pass

        logger = environ.get('repoze.who.logger','')
        # ELSE, perform a real challenge => asking for loggin
        # here by redirecting the user to a IdP.

        cl = Saml2Client(environ, self.conf)
        came_from = construct_came_from(environ)
        if self.debug:
            logger and logger.info("RelayState >> %s" % came_from)

        try:
            vo = environ["myapp.vo"]
        except KeyError:
            vo = self.vo
        logger and logger.info("VO: %s" % vo)

        # If more than one idp, I have to do wayf
        logger and logger.info("IdP URL: %s" % self.sp["idp"].values())
        if len( self.sp["idp"] ) == 1:
            idp_url = self.sp["idp"].values()[0]
        elif len( self.sp["idp"] ) == 0:
            start_response('500 Internal Server Error',
                                [('Content-Type', 'text/plain')])
            return ['Misconfiguration']
        else:
            start_response('501 Not Implemented',
                                [('Content-Type', 'text/plain')])
            return ['WAYF not implemented yet!']

        (sid, result) = cl.authenticate(self.conf["entityid"],
                                        idp_url,
                                        self.sp["url"],
                                        self.sp["name"],
                                        relay_state=came_from,
                                        log=logger,
                                        vo=vo)
        self.outstanding_authn[sid] = came_from

        if self.debug:
            logger and logger.info('sc returned: %s' % (result,))
        if isinstance(result, tuple):
            return HTTPTemporaryRedirect(headers=[result])
        else :
            # possible though normally not used
            body = "\n".join(result)
            def auth_form(environ, start_response):
                content_length = CONTENT_LENGTH.tuples(str(len(result)))
                content_type = CONTENT_TYPE.tuples('text/html')
                headers = content_length + content_type + forget_headers
                start_response('200 OK', headers)
                return [result]

            return auth_form

    #### IIdentifier ####
    def identify(self, environ):
        logger = environ.get('repoze.who.logger','')

        uri = environ.get('REQUEST_URI',construct_url(environ))
        if self.debug:
            #logger and logger.info("environ.keys(): %s" % environ.keys())
            #logger and logger.info("Environment: %s" % environ)
            logger and logger.info('identify uri: %s' % (uri,))

        query = parse_dict_querystring(environ)
        if self.debug:
            logger and logger.info('identify query: %s' % (query,))

        post_env = environ.copy()
        post_env['QUERY_STRING'] = ''

        if environ["CONTENT_LENGTH"]:
            body = environ["wsgi.input"].read(int(environ["CONTENT_LENGTH"]))
            from StringIO import StringIO
            environ['wsgi.input'] = StringIO(body)
            environ['s2repoze.body'] = body

        post = cgi.FieldStorage(
            fp=environ['wsgi.input'],
            environ=post_env,
            keep_blank_values=True
        )

        if self.debug:
            logger and logger.info('identify post: %s' % (post,))

        try:
            if not post.has_key("SAMLResponse"):
                environ["post.fieldstorage"] = post
                return {}
        except TypeError:
            environ["post.fieldstorage"] = post
            return {}

        # check for SAML2 authN
        cl = Saml2Client(environ, self.conf)
        try:
            session_info = cl.response(post,
                                            self.conf["entityid"],
                                            self.outstanding_authn,
                                            logger)
            name_id = self._cache_session(session_info)
            if self.debug:
                logger and logger.info("stored %s with key %s" % (
                                        session_info, name_id))
        except TypeError:
            return None

        if session_info["came_from"]:
            if self.debug:
                logger and logger.info(
                            "came_from << %s" % session_info["came_from"])
            try:
                path, query = session_info["came_from"].split('?')
                environ["PATH_INFO"] = path
                environ["QUERY_STRING"] = query
            except ValueError:
                environ["PATH_INFO"] = session_info["came_from"]

        identity = {}
        identity["login"] = name_id
        identity["password"] = ""
        identity['repoze.who.userid'] = name_id
        identity["user"] = session_info["ava"]
        environ["s2repoze.sessioninfo"] = session_info
        if self.debug:
            logger and logger.info("Identity: %s" % identity)
        return identity

    # IMetadataProvider
    def add_metadata(self, environ, identity):
        subject_id = identity['repoze.who.userid']

        if self.debug:
            logger = environ.get('repoze.who.logger','')
            if logger:
                logger.info(
                    "add_metadata for %s" % subject_id)
                logger.info(
                    "Known subjects: %s" % self.cache.subjects())
                try:
                    logger.info(
                        "Issuers: %s" % self.cache.entities(subject_id))
                except KeyError:
                    pass

        if "user" not in identity:
            identity["user"] = {}
        try:
            (ava, old) = self.cache.get_identity(subject_id)
            now = time.gmtime()
            if self.debug:
                logger and logger.info("Adding %s" % ava)
            identity["user"].update(ava)
        except KeyError:
            pass

        if "pysaml2_vo_expanded" not in identity:
            # is this a Virtual Organization situation
            if self.vo:
                logger and logger.info("** Do VO aggregation **")
                #try:
                    # This ought to be caseignore
                    #subject_id = identity["user"][
                    #                    self.vo_conf["common_identifier"]][0]
                #except KeyError:
                #    logger and logger.error("** No common identifier **")
                #    return
                logger and logger.info(
                    "SubjectID: %s, VO:%s" % (subject_id, self.vo))

                vo_members = [
                    member for member in self.metadata.vo_members(self.vo)\
                        if member not in self.sp["idp"].keys()]

                logger and logger.info("VO members: %s" % vo_members)
                vo_members = [m for m in vo_members \
                                if not self.cache.active(subject_id, m)]
                logger and logger.info(
                                "VO members (not cached): %s" % vo_members)

                if vo_members:
                    ar = AttributeResolver(environ, self.metadata, self.conf)

                    if self.vo_conf and "name_id_format" in self.vo_conf:
                        name_id_format = self.vo_conf["name_id_format"]
                        sp_name_qualifier=""
                    else:
                        sp_name_qualifier=self.vo
                        name_id_format = ""

                    extra = ar.extend(subject_id,
                            self.conf["entityid"],
                            vo_members,
                            name_id_format=name_id_format,
                            sp_name_qualifier=sp_name_qualifier,
                            log=logger)

                    for session_info in extra:
                        nid = self._cache_session(session_info)

                    logger.info(
                        ">Issuers: %s" % self.cache.entities(subject_id))
                    logger.info(
                        "AVA: %s" % (self.cache.get_identity(subject_id),))
                    identity["user"] = self.cache.get_identity(subject_id)[0]
                    # Only do this once
                    identity["pysaml2_vo_expanded"] = 1
                    #self.store[identity['repoze.who.userid']] = (
                    #                        not_on_or_after, identity)

# @return
# used 2 times : one to get the ticket, the other to validate it
    def _serviceURL(self,environ,qs=None):
        if qs != None:
            url = construct_url(environ, querystring=qs)
        else:
            url = construct_url(environ)
        return url

    #### IAuthenticatorPlugin ####
    def authenticate(self, environ, identity={}):
        return identity.get('login',None)

def make_plugin(rememberer_name=None, # plugin for remember
                 cache= "", # cache
                 # Which virtual organization to support
                 virtual_organization="",
                 saml_conf="",
                 debug=0,
                 ):

    if saml_conf is None:
        raise ValueError(
            'must include saml_conf in configuration')

    if rememberer_name is None:
        raise ValueError(
             'must include rememberer_name in configuration')

    plugin = SAML2Plugin(rememberer_name, saml_conf,
                virtual_organization, cache, debug)
    return plugin

# came_from = re.sub(r'ticket=[^&]*&?', '', came_from)

<|endoftext|>


<commit_before>import os, random, sys
from midi_to_statematrix import *
from data import *
import cPickle as pickle
import readxml

import signal
import time

batch_width = 10 # number of sequences in a batch
batch_len = 16*8 # length of each sequence
division_len = 16 # interval between possible start locations

def loadPieces(dirpath):

    pieces = {}

#    for fname in os.listdir(dirpath):
#        if fname[-4:] not in ('.mid','.MID'):
#            continue

#        name = fname[:-4]

#        outMatrix = midiToNoteStateMatrix(os.path.join(dirpath, fname))

#        if len(outMatrix) < batch_len:
#            continue

#        pieces[name] = outMatrix
#        print "Loaded {}".format(name)
#        sys.stdout.flush()

#    return pieces

    tmp = readxml.createStateMatrices('../musicxml')
    for key in tmp:
      pieces[key] = tmp[key][1]
    return pieces

def getPieceSegment(pieces):
    piece_output = random.choice(pieces.values())
    start = random.randrange(0,len(piece_output)-batch_len,division_len)
    # print "Range is {} {} {} -> {}".format(0,len(piece_output)-batch_len,division_len, start)

    seg_out = piece_output[start:start+batch_len]
    seg_in = noteStateMatrixToInputForm(seg_out)

    return seg_in, seg_out

def getPieceBatch(pieces):
    i,o = zip(*[getPieceSegment(pieces) for _ in range(batch_width)])
    return numpy.array(i), numpy.array(o)

def trainPiece(model,pieces,epochs,start=0):
    stopflag = [False]
    def signal_handler(signame, sf):
        stopflag[0] = True
    old_handler = signal.signal(signal.SIGINT, signal_handler)
    for i in range(start,start+epochs):
        if stopflag[0]:
            break
        error = model.update_fun(*getPieceBatch(pieces))
        if i % 100 == 0:
            print "{}: epoch {}, error={}".format(time.strftime("%c"), i, error)
            sys.stdout.flush()
        if i % 500 == 0 or (i % 100 == 0 and i < 1000):
            xIpt, xOpt = map(numpy.array, getPieceSegment(pieces))
            noteStateMatrixToMidi(numpy.concatenate((numpy.expand_dims(xOpt[0], 0), model.predict_fun(batch_len, 1, xIpt[0])), axis=0),'output/sample{}'.format(i))
            pickle.dump(model.learned_config,open('output/params{}.p'.format(i), 'wb'))
    signal.signal(signal.SIGINT, old_handler)
<commit_msg>Bugfix<commit_after>import os, random, sys
from midi_to_statematrix import *
from data import *
import cPickle as pickle
import readxml

import signal
import time

batch_width = 10 # number of sequences in a batch
batch_len = 16*8 # length of each sequence
division_len = 16 # interval between possible start locations

def loadPieces(dirpath):

    pieces = {}

#    for fname in os.listdir(dirpath):
#        if fname[-4:] not in ('.mid','.MID'):
#            continue

#        name = fname[:-4]

#        outMatrix = midiToNoteStateMatrix(os.path.join(dirpath, fname))

#        if len(outMatrix) < batch_len:
#            continue

#        pieces[name] = outMatrix
#        print "Loaded {}".format(name)
#        sys.stdout.flush()

#    return pieces

    tmp = readxml.createStateMatrices('../musicxml')
    for key in tmp:
      pieces[key] = tmp[key][1]
    return pieces

def getPieceSegment(pieces):
    piece_output = random.choice(pieces.values())
    piecelen = len(piece_output)-batch_len
    if piecelen == batch_len:
      start = 0
    else:
      start = random.randrange(0,len(piece_output)-batch_len,division_len)
    # print "Range is {} {} {} -> {}".format(0,len(piece_output)-batch_len,division_len, start)

    seg_out = piece_output[start:start+batch_len]
    seg_in = noteStateMatrixToInputForm(seg_out)

    return seg_in, seg_out

def getPieceBatch(pieces):
    i,o = zip(*[getPieceSegment(pieces) for _ in range(batch_width)])
    return numpy.array(i), numpy.array(o)

def trainPiece(model,pieces,epochs,start=0):
    stopflag = [False]
    def signal_handler(signame, sf):
        stopflag[0] = True
    old_handler = signal.signal(signal.SIGINT, signal_handler)
    for i in range(start,start+epochs):
        if stopflag[0]:
            break
        error = model.update_fun(*getPieceBatch(pieces))
        if i % 100 == 0:
            print "{}: epoch {}, error={}".format(time.strftime("%c"), i, error)
            sys.stdout.flush()
        if i % 500 == 0 or (i % 100 == 0 and i < 1000):
            xIpt, xOpt = map(numpy.array, getPieceSegment(pieces))
            noteStateMatrixToMidi(numpy.concatenate((numpy.expand_dims(xOpt[0], 0), model.predict_fun(batch_len, 1, xIpt[0])), axis=0),'output/sample{}'.format(i))
            pickle.dump(model.learned_config,open('output/params{}.p'.format(i), 'wb'))
    signal.signal(signal.SIGINT, old_handler)
<|endoftext|>


<commit_before>from datetime import datetime

from django.db import models
from django.conf import settings
from django.contrib.auth.models import User
from django.utils.translation import ugettext_lazy as _

from tagging.fields import TagField
from tagging.models import Tag

from django.contrib.auth.models import User

try:
    from notification import models as notification
    from django.db.models import signals
except ImportError:
    notification = None

try:
    markup_choices = settings.WIKI_MARKUP_CHOICES  # reuse this for now; taken from wiki
except AttributeError:
    markup_choices = (
        ('rst', _(u'reStructuredText')),
        ('txl', _(u'Textile')),
        ('mrk', _(u'Markdown')),
    )

class Post(models.Model):
    """Post model."""
    STATUS_CHOICES = (
        (1, _('Draft')),
        (2, _('Public')),
    )
    title           = models.CharField(_('title'), max_length=200)
    slug            = models.SlugField(_('slug'))
    author          = models.ForeignKey(User, related_name="added_posts")
    creator_ip      = models.IPAddressField(_("IP Address of the Post Creator"), blank=True, null=True)
    body            = models.TextField(_('body'))
    tease           = models.TextField(_('tease'), blank=True)
    status          = models.IntegerField(_('status'), choices=STATUS_CHOICES, default=2)
    allow_comments  = models.BooleanField(_('allow comments'), default=True)
    publish         = models.DateTimeField(_('publish'), default=datetime.now)
    created_at      = models.DateTimeField(_('created at'), default=datetime.now)
    updated_at      = models.DateTimeField(_('updated at'))
    markup          = models.CharField(_(u"Post Content Markup"), max_length=3,
                              choices=markup_choices,
                              null=True, blank=True)
    tags            = TagField()

    class Meta:
        verbose_name        = _('post')
        verbose_name_plural = _('posts')
        ordering            = ('-publish',)
        get_latest_by       = 'publish'

    def __unicode__(self):
        return self.title

    def get_absolute_url(self):
        return ('blog_post', None, {
            'username': self.author.username,
            'year': self.publish.year,
            'month': "%02d" % self.publish.month,
            'slug': self.slug
    })
    get_absolute_url = models.permalink(get_absolute_url)

    def save(self, force_insert=False, force_update=False):
        self.updated_at = datetime.now()
        super(Post, self).save(force_insert, force_update)

# handle notification of new comments
from threadedcomments.models import ThreadedComment
def new_comment(sender, instance, **kwargs):
    if isinstance(instance.content_object, Post):
        post = instance.content_object
        if notification:
            notification.send([post.author], "blog_post_comment", {"user": instance.user, "post": post, "comment": instance})
signals.post_save.connect(new_comment, sender=ThreadedComment)
<commit_msg>Use Django's app cache to check for presence of django-notification (in INSTALLED_APPS)<commit_after>from datetime import datetime

from django.db import models
from django.conf import settings
from django.contrib.auth.models import User
from django.utils.translation import ugettext_lazy as _
from django.core.exceptions import ImproperlyConfigured

from tagging.fields import TagField
from tagging.models import Tag

try:
    notification = models.get_app('notification')
except ImportError:
    notification = None

try:
    markup_choices = settings.WIKI_MARKUP_CHOICES  # reuse this for now; taken from wiki
except AttributeError:
    markup_choices = (
        ('rst', _(u'reStructuredText')),
        ('txl', _(u'Textile')),
        ('mrk', _(u'Markdown')),
    )

class Post(models.Model):
    """Post model."""
    STATUS_CHOICES = (
        (1, _('Draft')),
        (2, _('Public')),
    )
    title           = models.CharField(_('title'), max_length=200)
    slug            = models.SlugField(_('slug'))
    author          = models.ForeignKey(User, related_name="added_posts")
    creator_ip      = models.IPAddressField(_("IP Address of the Post Creator"), blank=True, null=True)
    body            = models.TextField(_('body'))
    tease           = models.TextField(_('tease'), blank=True)
    status          = models.IntegerField(_('status'), choices=STATUS_CHOICES, default=2)
    allow_comments  = models.BooleanField(_('allow comments'), default=True)
    publish         = models.DateTimeField(_('publish'), default=datetime.now)
    created_at      = models.DateTimeField(_('created at'), default=datetime.now)
    updated_at      = models.DateTimeField(_('updated at'))
    markup          = models.CharField(_(u"Post Content Markup"), max_length=3,
                              choices=markup_choices,
                              null=True, blank=True)
    tags            = TagField()

    class Meta:
        verbose_name        = _('post')
        verbose_name_plural = _('posts')
        ordering            = ('-publish',)
        get_latest_by       = 'publish'

    def __unicode__(self):
        return self.title

    def get_absolute_url(self):
        return ('blog_post', None, {
            'username': self.author.username,
            'year': self.publish.year,
            'month': "%02d" % self.publish.month,
            'slug': self.slug
    })
    get_absolute_url = models.permalink(get_absolute_url)

    def save(self, force_insert=False, force_update=False):
        self.updated_at = datetime.now()
        super(Post, self).save(force_insert, force_update)

# handle notification of new comments
from threadedcomments.models import ThreadedComment
def new_comment(sender, instance, **kwargs):
    if isinstance(instance.content_object, Post):
        post = instance.content_object
        if notification:
            notification.send([post.author], "blog_post_comment",
                {"user": instance.user, "post": post, "comment": instance})
models.signals.post_save.connect(new_comment, sender=ThreadedComment)
<|endoftext|>


<commit_before>import tweepy

from bottle import route, run

api = None

@route('/auth/<ckey>/<csecret>/<atoken>/<atokensecret>')
def authentication():
 consumer_key = ckey
 consumer_secret = csecret
 access_token = atoken
 access_token_secret = atokensecret

 auth = tweepy.OAuthHandler(consumer_key, consumer_secret)
 auth.set_access_token(access_token, access_token_secret)

 api = tweepy.API(auth)
 mytweets = my_tweets()
 myfirsttweet = my_first_tweet()
 print("Tweet One: {myfirsttweet}")
 print("All Tweets: {mytweets}")

def my_tweets():
 my_tweets = api.user_timeline()

 return my_tweets

def my_first_tweet():
 my_tweets = api.user_timeline()

 return my_tweets[0].text

def my_public_tweets():
 public_tweets = api.home_timeline()

 return public_tweets

def my_first_public_tweet():
 public_tweets = api.home_timeline()

 return public_tweets[0].text

run(host='localhost', port=8088, debug=True)

<commit_msg>Commit 3, client can now authenticate with their own details<commit_after>import tweepy

from bottle import route, run, template

#auth = tweepy.OAuthHandler("", "")
#auth.set_access_token("", "")
#api = tweepy.API(auth)

#Use the following url to use my (Marc's) details to login with all the keys/tokens and get back my first tweet:
#localhost:8088/auth/oj8A0mh2hMMLZBAYr9hrsyGXK/vEbApM5duqpSj3fu0oYDmuIMSDlcefk5wToHwmt5DByrqpTR3x/3088841945-9TzefMYW385iw5jsPSo7NeujLn6OigE3ZBC86hu/bve9QIWzyFUlMn4BOVZemcnZ8e3GjhwmuVViz8QhwoRse

@route('/auth/<ckey>/<csecret>/<atoken>/<atokensecret>')
def authentication(ckey, csecret, atoken, atokensecret):
 consumer_key = ckey
 consumer_secret = csecret
 access_token = atoken
 access_token_secret = atokensecret

 auth = tweepy.OAuthHandler(consumer_key, consumer_secret)
 auth.set_access_token(access_token, access_token_secret)

 api = tweepy.API(auth)
 my_tweets = api.user_timeline()
 my_first_tweet = my_tweets[0].text
 return template('My first Tweet was: {{my_first_tweet_here}}', my_first_tweet_here = my_first_tweet)

run(host='localhost', port=8088, debug=True)

<|endoftext|>


<commit_before>from Geometry import Point
from Geometry import Line

def computeHull(points):
   polygonEdges = []
   for p1 in points:
      for p2 in points:
         edge = Line(p1, p2)
         determinantSigns = []
         for p in points:
            if p not in edge:
               detSign = edge.getDeterminantSign(p)
               determinantSigns.append(detSign)
         if not allPositivesAndNegatives(determinantSigns):
            polygonEdges.append(edge)
   return polygonEdges

def allPositivesAndNegatives(array):
   return (-1 in array and 1 in array)

<commit_msg>Now the algorithm only calculate determinants for diffrenet points<commit_after>from Geometry import Point
from Geometry import Line

def computeHull(points):
   polygonEdges = []
   for p1 in points:
      for p2 in points:
         if p1 == p2:
            break
         edge = Line(p1, p2)
         determinantSigns = []
         for p in points:
            if p not in edge:
               detSign = edge.getDeterminantSign(p)
               determinantSigns.append(detSign)
         if not allPositivesAndNegatives(determinantSigns):
            polygonEdges.append(edge)
   return polygonEdges

def allPositivesAndNegatives(array):
   return (-1 in array and 1 in array)

<|endoftext|>


<commit_before>"""
A mayavi example showing 3 different isosurfaces

"""

from enthought.mayavi.scripts import mayavi2
mayavi2.standalone(globals())

# Standard library imports
from os.path import join, dirname
import gc

# Enthought library imports
import enthought.mayavi
from enthought.mayavi.modules.iso_surface import IsoSurface
from enthought.mayavi.sources.array_source import ArraySource
from enthought.mayavi.modules.outline import Outline
from enthought.mayavi.modules.vectors import Vectors

import numpy as N

def iso(img, contour, color=(1.,1.,1.), sample=None, spacing=[1,1,1],
        opacity=1):
    src = ArraySource(transpose_input_array=False, spacing=spacing)
    mayavi.add_source(src)
    if sample is not None:
        img = img[sample]
    src.scalar_data = N.nan_to_num(img[:])
    i = IsoSurface(compute_normals=True)
    mayavi.add_module(i)
    i.actor.property.color = color
    i.actor.mapper.scalar_visibility = False
    i.contour.contours[0] = contour
    i.actor.property.opacity = 0.5
    return i

if __name__ == "__main__":
    mayavi.new_scene()
    from fiac import anat, sentence, average, avg152
    iso(avg152, 230000.)
    iso(sentence, 4, color=(1,0,0))
    iso(average, 4, color=(0,0,1), opacity=0.5)
    o = Outline()
    mayavi.add_module(o)

<commit_msg>Fix mayavi example<commit_after><|endoftext|>


<commit_before>from PyQt5.QtCore import *
from PyQt5.QtGui import *
from PyQt5.QtWidgets import *
from Windows.manualAddDialog import *

from Windows.centralWindow import *
from Windows import favoriteWindow_ui
from Util import constants, scroller
from Util.enums import *

class CategoryTab():
    __slots__ = ['widget', 'horizontalLayout', 'listView']
    def __init__(self, widget = None, horizontalLayout = None, listView = None):
        self.widget = widget
        self.horizontalLayout = horizontalLayout
        self.listView = listView

class FavoriteWindow(QWidget, favoriteWindow_ui.Ui_FavoriteWindow):
    def __init__(self, centralWindow, config, dbManager, parent=None):
        super(FavoriteWindow, self).__init__(parent)
        self.setupUi(self)

        self.centralWindow = centralWindow
        self.config = config
        self.dbManager = dbManager

        self.favoritesTabModel = QStandardItemModel(4, 5, self)

        self.favoritesTabModel.setItem(0, 0, QStandardItem("Rawr"))
        self.favoritesTabModel.setItem(0, 1, QStandardItem("Really Long String of Words"))
        self.favoritesTabModel.setItem(0, 2, QStandardItem("Really Long"))
        self.favoritesTabModel.setItem(0, 3, QStandardItem("POPTARTS STRAWBERRY"))
        self.favoritesTabModel.setItem(1, 0, QStandardItem("Rawr"))
        self.favoritesTabModel.setItem(1, 1, QStandardItem("OOOOOOOOOO OOOOOOOOOO"))
        self.favoritesTabModel.setItem(1, 2, QStandardItem("OOOOOOOOOO OOOOOOOOO"))
        self.favoritesTabModel.setItem(1, 3, QStandardItem("OOOOOOOOO OOOOOOOO"))
        self.favoritesTabModel.setItem(1, 4, QStandardItem("OOOOOOOO OOOOOOO"))

        self.favoritesTableView.setModel(self.favoritesTabModel)
        scroller.setupScrolling(self.favoritesTableView)

        # Create an empty tabDict which will contain a dictionary for each tab in the window, query all categories and
        # add each tab to the window
        self.tabDict = {}
        self.categories = self.dbManager.GetCategories(True)
        for category in self.categories:
            self.addTab(category['id'], category['name'])

    def addTab(self, id, name):
        newTab = CategoryTab(QWidget(self.categoryTabWidget))
        newTab.horizontalLayout = QHBoxLayout(newTab.widget)
        newTab.horizontalLayout.setContentsMargins(0, 0, 0, 0)

        # Create list view for category and with the appropiate settings
        newTab.listView = QListView(newTab.widget)
        font = QFont()
        font.setPointSize(21)
        newTab.listView.setFont(font)
        newTab.listView.setFrameShape(QFrame.NoFrame)
        newTab.listView.setFrameShadow(QFrame.Plain)
        newTab.listView.setVerticalScrollBarPolicy(Qt.ScrollBarAlwaysOn)
        newTab.listView.setHorizontalScrollBarPolicy(Qt.ScrollBarAlwaysOff)
        newTab.listView.setEditTriggers(QAbstractItemView.NoEditTriggers)
        newTab.listView.setDragDropMode(QAbstractItemView.DragDrop)
        newTab.listView.setDefaultDropAction(Qt.ActionMask)
        newTab.listView.setAlternatingRowColors(True)
        newTab.listView.setSelectionMode(QAbstractItemView.MultiSelection)
        newTab.listView.setVerticalScrollMode(QAbstractItemView.ScrollPerPixel)
        newTab.listView.setHorizontalScrollMode(QAbstractItemView.ScrollPerPixel)
        newTab.horizontalLayout.addWidget(newTab.listView)

        # Add tab to tab widget as well as the tabDict variable
        self.categoryTabWidget.addTab(newTab.widget, name)
        self.tabDict[id] = newTab

        # Set list view to have model to display items
        self.stringList = QStringListModel(self)
        self.stringList.setStringList(['One', 'Two', 'Three', 'Four', 'Five'])

        newTab.listView.setModel(self.stringList)

    @pyqtSlot()
    def showEvent(self, event):
        self.centralWindow.primaryScanner.barcodeReceived.connect(self.primaryScanner_barcodeReceived)
        self.centralWindow.secondaryScanner.barcodeReceived.connect(self.secondaryScanner_barcodeReceived)

    @pyqtSlot()
    def hideEvent(self, event):
        self.centralWindow.primaryScanner.barcodeReceived.disconnect(self.primaryScanner_barcodeReceived)
        self.centralWindow.secondaryScanner.barcodeReceived.disconnect(self.secondaryScanner_barcodeReceived)
        print('This widget is being hidden. Handle anything necessary. Favorite Window')

    @pyqtSlot()
    def on_backBtn_clicked(self):
        self.parent().setCurrentIndex(WindowType.Main)

    @pyqtSlot()
    def on_homeBtn_clicked(self):
        self.parent().setCurrentIndex(WindowType.Main)

    @pyqtSlot()
    def on_listAddBtn_clicked(self):
        dialog = ManualAddDialog(self.config, self.dbManager, self.categories, self)
        dialog.exec()

    @pyqtSlot(str)
    def primaryScanner_barcodeReceived(self, barcode):
        print("Fav: Primary barcode scanner got: %s" % barcode)
        # TODO Send the barcode scanner information to be processed

    @pyqtSlot(str)
    def secondaryScanner_barcodeReceived(self, barcode):
        print("Fav: Secondary barcode scanner got: %s" % barcode)
        # TODO Send the barcode scanner information to be processed
<commit_msg>Sample Change. Ignore this.<commit_after>from PyQt5.QtCore import *
from PyQt5.QtGui import *
from PyQt5.QtWidgets import *
from Windows.manualAddDialog import *

from Windows.centralWindow import *
from Windows import favoriteWindow_ui
from Util import constants, scroller
from Util.enums import *

class CategoryTab():
    __slots__ = ['widget', 'horizontalLayout', 'listView']
    def __init__(self, widget = None, horizontalLayout = None, listView = None):
        self.widget = widget
        self.horizontalLayout = horizontalLayout
        self.listView = listView

class FavoriteWindow(QWidget, favoriteWindow_ui.Ui_FavoriteWindow):
    def __init__(self, centralWindow, config, dbManager, parent=None):
        super(FavoriteWindow, self).__init__(parent)
        self.setupUi(self)

        self.centralWindow = centralWindow
        self.config = config
        self.dbManager = dbManager

        self.favoritesTabModel = QStandardItemModel(4, 5, self)

        self.favoritesTabModel.setItem(0, 0, QStandardItem("Rawr"))
        self.favoritesTabModel.setItem(0, 1, QStandardItem("Really Long String of Words"))
        self.favoritesTabModel.setItem(0, 2, QStandardItem("Really Long"))
        self.favoritesTabModel.setItem(0, 3, QStandardItem("POPTARTS STRAWBERRY"))
        self.favoritesTabModel.setItem(1, 0, QStandardItem("Rawr"))
        self.favoritesTabModel.setItem(1, 1, QStandardItem("OOOOOOOOOO OOOOOOOOOO"))
        self.favoritesTabModel.setItem(1, 2, QStandardItem("OOOOOOOOOO OOOOOOOOO"))
        self.favoritesTabModel.setItem(1, 3, QStandardItem("OOOOOOOOO OOOOOOOO"))
        self.favoritesTabModel.setItem(1, 4, QStandardItem("OOOOOOOO OOOOOOO"))

        self.favoritesTableView.setModel(self.favoritesTabModel)
        scroller.setupScrolling(self.favoritesTableView)

        # Create an empty tabDict which will contain a dictionary for each tab in the window, query all categories and
        # add each tab to the window
        self.tabDict = {}
        self.categories = self.dbManager.GetCategories(True)
        for category in self.categories:
            self.addTab(category['id'], category['name'])

    def addTab(self, id, name):
        newTab = CategoryTab(QWidget(self.categoryTabWidget))
        newTab.horizontalLayout = QHBoxLayout(newTab.widget)
        newTab.horizontalLayout.setContentsMargins(0, 0, 0, 0)

        # Create list view for category and with the appropiate settings
        newTab.listView = QListView(newTab.widget)
        font = QFont()
        font.setPointSize(21)
        newTab.listView.setFont(font)
        newTab.listView.setFrameShape(QFrame.NoFrame)
        newTab.listView.setFrameShadow(QFrame.Plain)
        newTab.listView.setVerticalScrollBarPolicy(Qt.ScrollBarAlwaysOn)
        newTab.listView.setHorizontalScrollBarPolicy(Qt.ScrollBarAlwaysOff)
        newTab.listView.setEditTriggers(QAbstractItemView.NoEditTriggers)
        newTab.listView.setDragDropMode(QAbstractItemView.DragDrop)
        newTab.listView.setDefaultDropAction(Qt.ActionMask)
        newTab.listView.setAlternatingRowColors(True)
        newTab.listView.setSelectionMode(QAbstractItemView.MultiSelection)
        newTab.listView.setVerticalScrollMode(QAbstractItemView.ScrollPerPixel)
        newTab.listView.setHorizontalScrollMode(QAbstractItemView.ScrollPerPixel)
        newTab.horizontalLayout.addWidget(newTab.listView)

        # Add tab to tab widget as well as the tabDict variable
        self.categoryTabWidget.addTab(newTab.widget, name)
        self.tabDict[id] = newTab

        # Set list view to have model to display items
        self.stringList = QStringListModel(self)
        self.stringList.setStringList(['One', 'Two', 'Three', 'Four', 'Five'])

        newTab.listView.setModel(self.stringList)

    @pyqtSlot()
    def showEvent(self, event):
        self.centralWindow.primaryScanner.barcodeReceived.connect(self.primaryScanner_barcodeReceived)
        self.centralWindow.secondaryScanner.barcodeReceived.connect(self.secondaryScanner_barcodeReceived)

    @pyqtSlot()
    def hideEvent(self, event):
        self.centralWindow.primaryScanner.barcodeReceived.disconnect(self.primaryScanner_barcodeReceived)
        self.centralWindow.secondaryScanner.barcodeReceived.disconnect(self.secondaryScanner_barcodeReceived)
        print('This widget is being hidden. Handle anything necessary. Favorite Window')

    @pyqtSlot()
    def on_backBtn_clicked(self):
        self.parent().setCurrentIndex(WindowType.Main)

    @pyqtSlot()
    def on_homeBtn_clicked(self):
        self.parent().setCurrentIndex(WindowType.Main)

    @pyqtSlot()
    def on_listAddBtn_clicked(self):
        dialog = ManualAddDialog(self.config, self.dbManager, self.categories, self)

        # Run the dialog, if successfully completed, then add new item to category
        if dialog.exec():
            i = 4

        if self.categories is not dialog.categories:
            print('Categories changed!')

    @pyqtSlot(str)
    def primaryScanner_barcodeReceived(self, barcode):
        print("Fav: Primary barcode scanner got: %s" % barcode)
        # TODO Send the barcode scanner information to be processed

    @pyqtSlot(str)
    def secondaryScanner_barcodeReceived(self, barcode):
        print("Fav: Secondary barcode scanner got: %s" % barcode)
        # TODO Send the barcode scanner information to be processed
<|endoftext|>


<commit_before>"""Axis class and associated."""

# --- import --------------------------------------------------------------------------------------

import re
import numexpr
import operator
import functools

import numpy as np

from .. import exceptions as wt_exceptions
from .. import kit as wt_kit
from .. import units as wt_units

# --- define --------------------------------------------------------------------------------------

operator_to_identifier = {}
operator_to_identifier['/'] = '__d__'
operator_to_identifier['='] = '__e__'
operator_to_identifier['-'] = '__m__'
operator_to_identifier['+'] = '__p__'
operator_to_identifier['*'] = '__t__'
identifier_to_operator = {value: key for key, value in operator_to_identifier.items()}
operators = ''.join(operator_to_identifier.keys())

# --- class ---------------------------------------------------------------------------------------

class Axis(object):
    """Axis class."""

    def __init__(self, parent, expression, units=None):
        """Data axis.

        Parameters
        ----------
        parent : WrightTools.Data
            Parent data object.
        expression : string
            Axis expression.
        units : string (optional)
            Axis units. Default is None.
        """
        self.parent = parent
        self.expression = expression
        if units is None:
            self.units = self.variables[0].units
        else:
            self.units = units

    def __getitem__(self, index):
        vs = {}
        for variable in self.variables:
            arr = variable[index]
            vs[variable.natural_name] = wt_units.converter(arr, variable.units, self.units)
        return numexpr.evaluate(self.expression.split('=')[0], local_dict=vs)

    def __repr__(self):
        return '<WrightTools.Axis {0} ({1}) at {2}>'.format(self.expression, str(self.units),
                                                            id(self))

    @property
    def _leaf(self):
        out = self.expression
        if self.units is not None:
            out += ' ({0}) {1}'.format(self.units, self.shape)
        return out

    @property
    def full(self):
        arr = self[:]
        for i in range(arr.ndim):
            if arr.shape[i] == 1:
                arr = np.repeat(arr, self.parent.shape[i], axis=i)
        return arr

    @property
    def identity(self):
        """Complete identifier written to disk in data.attrs['axes']."""
        return self.natural_name + ' {%s}' % self.units

    @property
    def label(self):
        symbol = wt_units.get_symbol(self.units)
        label = r'$\mathsf{' + self.expression
        for v in self.variables:
            vl = '%s_{%s}' % (symbol, v.label)
            vl = vl.replace('_{}', '')  # label can be empty, no empty subscripts
            label = label.replace(v.natural_name, vl)
        label += '}'
        if self.units_kind:
            units_dictionary = getattr(wt_units, self.units_kind)
            label += r'\,'
            label += r'\left('
            label += units_dictionary[self.units][2]
            label += r'\right)'
        else:
            pass
        label += r'}$'
        return label

    @property
    def natural_name(self):
        name = self.expression.strip()
        for op in operators:
            name = name.replace(op, operator_to_identifier[op])
        return wt_kit.string2identifier(name)

    @property
    def ndim(self):
        """Get number of dimensions."""
        try:
            assert self._ndim is not None
        except (AssertionError, AttributeError):
            self._ndim = self.variables[0].ndim
        finally:
            return self._ndim

    @property
    def points(self):
        """Squeezed array."""
        return np.squeeze(self[:])

    @property
    def shape(self):
        """Shape."""
        return wt_kit.joint_shape(*self.variables)

    @property
    def size(self):
        """Size."""
        return functools.reduce(operator.mul, self.shape)

    @property
    def units_kind(self):
        """Units kind."""
        return wt_units.kind(self.units)

    @property
    def variables(self):
        """Variables."""
        try:
            assert self._variables is not None
        except (AssertionError, AttributeError):
            pattern = '|'.join(map(re.escape, operators))
            keys = re.split(pattern, self.expression)
            indices = []
            for key in keys:
                if key in self.parent.variable_names:
                    indices.append(self.parent.variable_names.index(key))
            self._variables = [self.parent.variables[i] for i in indices]
        finally:
            return self._variables

    def convert(self, destination_units, *, convert_variables=False):
        """Convert axis to destination_units.

        Parameters
        ----------
        destination_units : string
            Destination units.
        convert_variables : boolean (optional)
            Toggle conversion of stored arrays. Default is False.
        """
        if not wt_units.is_valid_conversion(self.units, destination_units):
            kind = wt_units.kind(self.units)
            valid = list(wt_units.dicts[kind].keys())
            raise wt_exceptions.UnitsError(valid, destination_units)
        if convert_variables:
            for v in self.variables:
                v.convert(destination_units)
        self.units = destination_units

    def max(self):
        """Axis max."""
        return np.max(self[:])

    def min(self):
        """Axis min."""
        return np.min(self[:])
<commit_msg>remove extra brace (#573)<commit_after>"""Axis class and associated."""

# --- import --------------------------------------------------------------------------------------

import re
import numexpr
import operator
import functools

import numpy as np

from .. import exceptions as wt_exceptions
from .. import kit as wt_kit
from .. import units as wt_units

# --- define --------------------------------------------------------------------------------------

operator_to_identifier = {}
operator_to_identifier['/'] = '__d__'
operator_to_identifier['='] = '__e__'
operator_to_identifier['-'] = '__m__'
operator_to_identifier['+'] = '__p__'
operator_to_identifier['*'] = '__t__'
identifier_to_operator = {value: key for key, value in operator_to_identifier.items()}
operators = ''.join(operator_to_identifier.keys())

# --- class ---------------------------------------------------------------------------------------

class Axis(object):
    """Axis class."""

    def __init__(self, parent, expression, units=None):
        """Data axis.

        Parameters
        ----------
        parent : WrightTools.Data
            Parent data object.
        expression : string
            Axis expression.
        units : string (optional)
            Axis units. Default is None.
        """
        self.parent = parent
        self.expression = expression
        if units is None:
            self.units = self.variables[0].units
        else:
            self.units = units

    def __getitem__(self, index):
        vs = {}
        for variable in self.variables:
            arr = variable[index]
            vs[variable.natural_name] = wt_units.converter(arr, variable.units, self.units)
        return numexpr.evaluate(self.expression.split('=')[0], local_dict=vs)

    def __repr__(self):
        return '<WrightTools.Axis {0} ({1}) at {2}>'.format(self.expression, str(self.units),
                                                            id(self))

    @property
    def _leaf(self):
        out = self.expression
        if self.units is not None:
            out += ' ({0}) {1}'.format(self.units, self.shape)
        return out

    @property
    def full(self):
        arr = self[:]
        for i in range(arr.ndim):
            if arr.shape[i] == 1:
                arr = np.repeat(arr, self.parent.shape[i], axis=i)
        return arr

    @property
    def identity(self):
        """Complete identifier written to disk in data.attrs['axes']."""
        return self.natural_name + ' {%s}' % self.units

    @property
    def label(self):
        symbol = wt_units.get_symbol(self.units)
        label = r'$\mathsf{' + self.expression
        for v in self.variables:
            vl = '%s_{%s}' % (symbol, v.label)
            vl = vl.replace('_{}', '')  # label can be empty, no empty subscripts
            label = label.replace(v.natural_name, vl)
        if self.units_kind:
            units_dictionary = getattr(wt_units, self.units_kind)
            label += r'\,'
            label += r'\left('
            label += units_dictionary[self.units][2]
            label += r'\right)'
        else:
            pass
        label += r'}$'
        return label

    @property
    def natural_name(self):
        name = self.expression.strip()
        for op in operators:
            name = name.replace(op, operator_to_identifier[op])
        return wt_kit.string2identifier(name)

    @property
    def ndim(self):
        """Get number of dimensions."""
        try:
            assert self._ndim is not None
        except (AssertionError, AttributeError):
            self._ndim = self.variables[0].ndim
        finally:
            return self._ndim

    @property
    def points(self):
        """Squeezed array."""
        return np.squeeze(self[:])

    @property
    def shape(self):
        """Shape."""
        return wt_kit.joint_shape(*self.variables)

    @property
    def size(self):
        """Size."""
        return functools.reduce(operator.mul, self.shape)

    @property
    def units_kind(self):
        """Units kind."""
        return wt_units.kind(self.units)

    @property
    def variables(self):
        """Variables."""
        try:
            assert self._variables is not None
        except (AssertionError, AttributeError):
            pattern = '|'.join(map(re.escape, operators))
            keys = re.split(pattern, self.expression)
            indices = []
            for key in keys:
                if key in self.parent.variable_names:
                    indices.append(self.parent.variable_names.index(key))
            self._variables = [self.parent.variables[i] for i in indices]
        finally:
            return self._variables

    def convert(self, destination_units, *, convert_variables=False):
        """Convert axis to destination_units.

        Parameters
        ----------
        destination_units : string
            Destination units.
        convert_variables : boolean (optional)
            Toggle conversion of stored arrays. Default is False.
        """
        if not wt_units.is_valid_conversion(self.units, destination_units):
            kind = wt_units.kind(self.units)
            valid = list(wt_units.dicts[kind].keys())
            raise wt_exceptions.UnitsError(valid, destination_units)
        if convert_variables:
            for v in self.variables:
                v.convert(destination_units)
        self.units = destination_units

    def max(self):
        """Axis max."""
        return np.max(self[:])

    def min(self):
        """Axis min."""
        return np.min(self[:])
<|endoftext|>


<commit_before>#!/usr/bin/env python

import sys, os, os.path, glob, math, cv2
from datetime import datetime
from optparse import OptionParser
import re

start_templates = ["lbl", "pos", "occl"]

class Sample:
    def __init__(self, l):
        self

if __name__ == "__main__":
    f = open("/home/kellan/datasets/caltech/set00/V004.txt")
    person = re.compile("^lbl=\'person\'\s+str=(\d+)\s+end=(\d+)\s+hide=0$")
    newobj = re.compile("^lbl=\'(\w+)\'\s+str=(\d+)\s+end=(\d+)\s+hide=0$")
    pos = re.compile("^pos\s=(\[[((\d+\.+\d*)|\s+|\;)]*\])$")
    nonarray = re.compile("\;\s+(?!\])|\s+(?!\])")
    lastSemicolon = re.compile("\;\s+(?=\])")
    qqq = re.compile("(?=\[)\b(?=\d*)")

    goNext = 0
    start = 0
    end = 0

    modelW = 32
    modelH = 64

    for l in f:
        qq = newobj.match(l)
        if qq is not None:
            if qq.group(1) == "person":
                goNext = 1
            else:
                goNext = 0
            print qq.group(0), qq.group(1)
        m = person.match(l)
        if m is not None:
            start = m.group(1)
            end   = m.group(2)

            print m.group(0), start, end
        else:
            m = pos.match(l)
            if m is not None:
                if not goNext:
                    continue
                strarr = re.sub(r"\s", ", ", re.sub(r"\;\s+(?=\])", "]", re.sub(r"\;\s+(?!\])", "],[", re.sub(r"(\[)(\d)", "\\1[\\2", m.group(1)))))
                list = eval(strarr)
                for idx, box in enumerate(list):
                    if (box[2] >= 32) or (box[3] >= 64):
                        x = box[0]
                        y = box[1]
                        w = box[2]
                        h = box[3]

                        ratio = w / h
                        neww = h / 2.0
                        offset = (w - neww) / 2.0
                        print "HERE is big!! ", box, ratio, offset
                        if (x + offset) > 0:
                            id = int(start) + idx
                            file = "/home/kellan/datasets/caltech/set00/V004.seq/I0%04d.jpg" % id # I00000.jpg
                            print file
                            img = cv2.imread(file)
                            cv2.rectangle(img, (int(x), int(y)), (int(x + w), int(y + h)), (0,255,0), 2)
                            cv2.imshow("sample", img)
                            cv2.waitKey(0)

<commit_msg>implement positive sample extraction<commit_after>#!/usr/bin/env python

import sys, os, os.path, glob, math, cv2
from datetime import datetime
from optparse import OptionParser
import re
import numpy as np

def extract(f):
    person = re.compile("^lbl=\'person\'\s+str=(\d+)\s+end=(\d+)\s+hide=0$")
    newobj = re.compile("^lbl=\'(\w+)\'\s+str=(\d+)\s+end=(\d+)\s+hide=0$")
    pos = re.compile("^pos\s=(\[[((\d+\.+\d*)|\s+|\;)]*\])$")
    nonarray = re.compile("\;\s+(?!\])|\s+(?!\])")
    lastSemicolon = re.compile("\;\s+(?=\])")

    goNext = 0
    start = 0
    end = 0

    modelW = 32
    modelH = 64

    for l in f:
        qq = newobj.match(l)
        if qq is not None:
            if qq.group(1) == "person":
                goNext = 1
            else:
                goNext = 0
            print qq.group(0), qq.group(1)
        m = person.match(l)
        if m is not None:
            start = m.group(1)
            end   = m.group(2)

            print m.group(0), start, end
        else:
            m = pos.match(l)
            if m is not None:
                if not goNext:
                    continue
                strarr = re.sub(r"\s", ", ", re.sub(r"\;\s+(?=\])", "]", re.sub(r"\;\s+(?!\])", "],[", re.sub(r"(\[)(\d)", "\\1[\\2", m.group(1)))))
                list = eval(strarr)
                for idx, box in enumerate(list):
                    if (box[2] >= 32) or (box[3] >= 64):
                        x = box[0]
                        y = box[1]
                        w = box[2]
                        h = box[3]

                        ratio = w / h
                        neww = h / 2.0
                        offset = (w - neww) / 2.0
                        print "HERE is big!! ", box, ratio, offset
                        if (x + offset) > 0:
                            id = int(start) + idx
                            file = "/home/kellan/datasets/caltech/set00/V004.seq/I0%04d.jpg" % id # I00000.jpg
                            print file
                            img = cv2.imread(file)
                            cv2.rectangle(img, (int(x), int(y)), (int(x + w), int(y + h)), (0,255,0), 2)
                            cv2.imshow("sample", img)
                            cv2.waitKey(50)

def showPeople(f, path, opath):
    newobj = re.compile("^lbl=\'(\w+)\'\s+str=(\d+)\s+end=(\d+)\s+hide=0$")
    pos    = re.compile("^pos\s=(\[[((\d+\.+\d*)|\s+|\;)]*\])$")
    occl   = re.compile("^occl\s*=(\[[0-1|\s]*\])$")

    goNext = 0
    start = 0
    end = 0

    person_id = -1;

    boxes = []
    occls = []

    for l in f:
        m = newobj.match(l)
        if m is not None:
            print m.group(1)
            if m.group(1) == "person":
                goNext = 1
                start = int(m.group(2))
                end   = int(m.group(3))
                person_id = person_id + 1
                print m.group(1), person_id, start, end
            else:
                goNext = 0
        else:
            m = pos.match(l)
            if m is not None:
                if not goNext:
                    continue
                strarr = re.sub(r"\s", ", ", re.sub(r"\;\s+(?=\])", "]", re.sub(r"\;\s+(?!\])", "],[", re.sub(r"(\[)(\d)", "\\1[\\2", m.group(1)))))
                boxes = eval(strarr)
            else:
                m = occl.match(l)
                if m is not None:
                    occls = eval(re.sub(r"\s+(?!\])", ",", m.group(1)))

                    if len(boxes) > 0 and len(boxes) == len(occls):
                        print len(boxes)
                        for idx, box in enumerate(boxes):
                            color = (8, 107, 255)
                            if occls[idx] == 1:
                                continue
                                # color = (255, 107, 8)
                            x = box[0]
                            y = box[1]
                            w = box[2]
                            h = box[3]
                            id = int(start) - 1 + idx
                            file = os.path.join(path, "I0%04d.jpg" % id)

                            print file

                            if (start + id) < end and w > 5 and h > 47:
                                img = cv2.imread(file)

                                fname = re.sub(r"^.*\/(set[0-1]\d)\/(V0\d\d)\.(seq)/(I\d+).jpg$", "\\1_\\2_\\4", file)#os.path.basename(file)
                                fname = os.path.join(opath, fname + "_%04d." % person_id + "png")
                                try:
                                    print "->", fname
                                    submat = img[int(y):int(y + h), int(x):int(x + w),:]
                                    cv2.imwrite(fname, submat)
                                except:
                                    print "something wrong... go next."
                                    pass
                                cv2.rectangle(img, (int(x), int(y)), (int(x + w), int(y + h)), color, 1)
                                cv2.imshow("person", img)

                                c = cv2.waitKey(10)
                                if c == 27:
                                    exit(0)

if __name__ == "__main__":
    parser = OptionParser()
    parser.add_option("-i", "--input", dest="input", metavar="DIRECTORY", type="string",
                       help="path to the Caltech dataset folder.")

    parser.add_option("-o", "--output", dest="output", metavar="DIRECTORY", type="string",
                       help="path to store data", default=".")

    (options, args) = parser.parse_args()

    if not options.input:
        parser.error("Caltech dataset folder is required.")

    opath = os.path.join(options.output, datetime.now().strftime("raw_ge48-" + "-%Y-%m-%d-%H-%M-%S"))
    os.mkdir(opath)

    # mat = cv2.imread("/home/kellan/datasets/INRIArescaled/training_set/pos/octave_-1/sample_0.png");
    # cv2.rectangle(mat, (10, 10), (42, 74), (8, 107, 255), 1)

    # cv2.imshow("person", mat)
    # cv2.waitKey(0)
    # if c == 27:
    #     exit(0)

    gl = glob.iglob( os.path.join(options.input, "set[0-1][0-9]/V0[0-9][0-9].txt"))
    for each in gl:
        path, ext = os.path.splitext(each)
        path = path + ".seq"
        print path
        showPeople(open(each), path, opath)

    # f = open("/home/kellan/datasets/caltech/set00/V004.txt")
    # extract(f)
<|endoftext|>


<commit_before>#!/usr/bin/env python
# Licensed to Cloudera, Inc. under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  Cloudera, Inc. licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import json
import re

from beeswax import models as beeswax_models
from beeswax.design import hql_query
from beeswax.models import QUERY_TYPES, HiveServerQueryHandle, QueryHistory
from beeswax.views import safe_get_design, save_design
from beeswax.server import dbms

from spark.job_server_api import get_api as get_spark_api
from desktop.lib.i18n import smart_str
from desktop.lib.rest.http_client import RestException

class Notebook():

  def __init__(self, document=None):
    self.document = None

    if document is not None:
      self.data = document.data
      self.document = document
    else:
      self.data = json.dumps({
          'name': 'My Notebook',
          'snippets': [{'type': 'scala', 'result': {}}]
      })

  def get_json(self):
    _data = self.get_data()

    return json.dumps(_data)

  def get_data(self):
    _data = json.loads(self.data)

    if self.document is not None:
      _data['id'] = self.document.id

    return _data

def get_api(user, snippet):
  if snippet['type'] in ('hive', 'impala', 'spark-sql'):
    return HS2Api(user)
  elif snippet['type'] == 'text':
    return TextApi(user)
  else:
    return SparkApi(user)

def _get_snippet_session(notebook, snippet):
  return [session for session in notebook['sessions'] if session['type'] == snippet['type']][0]

class TextApi():

  def __init__(self, user):
    self.user = user

  def create_session(self, lang):
    return {
        'type': lang,
        'id': None
    }

class HS2Api():

  def __init__(self, user):
    self.user = user

  def _get_handle(self, snippet):
    snippet['result']['handle']['secret'], snippet['result']['handle']['guid'] = HiveServerQueryHandle.get_decoded(snippet['result']['handle']['secret'], snippet['result']['handle']['guid'])
    return HiveServerQueryHandle(**snippet['result']['handle'])

  def create_session(self, lang):
    return {
        'type': lang,
        'id': None # Real one at some point
    }

  def execute(self, notebook, snippet):
    db = dbms.get(self.user)
    query = hql_query(snippet['statement'], QUERY_TYPES[0])
    handle = db.client.query(query)

#    if not handle.is_valid():
#        msg = _("Server returning invalid handle for query id %(id)d [%(query)s]...") % {'id': query_history.id, 'query': query[:40]}
#        raise QueryServerException(msg)
#    except QueryServerException, ex:
#      LOG.exception(ex)
#      # Kind of expected (hql compile/syntax error, etc.)
#      if hasattr(ex, 'handle') and ex.handle:
#        query_history.server_id, query_history.server_guid = ex.handle.id, ex.handle.id
#        query_history.log_context = ex.handle.log_context
#      query_history.save_state(QueryHistory.STATE.failed)
#      raise ex

    # All good
    server_id, server_guid  = handle.get()
    return {
        'secret': server_id,
        'guid': server_guid,
        'operation_type': handle.operation_type,
        'has_result_set': handle.has_result_set,
        'modified_row_count': handle.modified_row_count,
        'log_context': handle.log_context
    }

  def check_status(self, notebook, snippet):
    db = dbms.get(self.user)

    handle = self._get_handle(snippet)
    status =  db.get_state(handle)

    return {
        'status':
          'running' if status.index in (QueryHistory.STATE.running.index, QueryHistory.STATE.submitted.index)
          else (
             'failed' if status.index in (QueryHistory.STATE.failed.index, QueryHistory.STATE.expired.index)
              else 'available'
          )
    }

  def fetch_result(self, notebook, snippet, rows):
    db = dbms.get(self.user)

    handle = self._get_handle(snippet)
    results = db.fetch(handle, start_over=False, rows=rows)

    # no escaping...
    return {
        'data': list(results.rows()),
        'meta': [{
          'name': column.name,
          'type': column.type,
          'comment': column.comment
        } for column in results.data_table.cols()]
    }

  def fetch_result_metadata(self):
    pass

  def cancel(self, notebook, snippet):
    db = dbms.get(self.user)

    handle = self._get_handle(snippet)
    db.cancel_operation(handle)
    return {'status': 'canceled'}

  def get_log(self, snippet):
    db = dbms.get(self.user)

    handle = self._get_handle(snippet)
    return db.get_log(handle)

  def _progress(self, snippet, logs):
    if snippet['type'] == 'hive':
      match = re.search('Total jobs = (\d+)', logs, re.MULTILINE)
      total = (int(match.group(1)) if match else 1) * 2

      started = logs.count('Starting Job')
      ended = logs.count('Ended Job')

      return int((started + ended) * 100 / total)
    elif snippet['type'] == 'impala':
      match = re.search('(\d+)% Complete', logs, re.MULTILINE)
      return int(match.group(1)) if match else 0
    else: #'spark-sql'
      return 50

class SparkApi():  # Pig, DBquery, Phoenix...

  def __init__(self, user):
    self.user = user

  def create_session(self, lang='scala'):
    api = get_spark_api(self.user)
    return {
        'type': lang,
        'id': api.create_session(lang=lang)
    }

  def execute(self, notebook, snippet):
    api = get_spark_api(self.user)
    session = _get_snippet_session(notebook, snippet)

    return {'id': api.submit_statement(session['id'], snippet['statement']).split('cells/')[1]}

  def check_status(self, notebook, snippet):
    return {'status': 'ready'}

  def fetch_result(self, notebook, snippet, rows):
    api = get_spark_api(self.user)
    session = _get_snippet_session(notebook, snippet)
    cell = snippet['result']['handle']['id']

    data = api.fetch_data(session['id'], cell)

    return {
        'data': [data['output']],
        'meta': [{
          'name': column.name,
          'type': column.type,
          'comment': column.comment
        } for column in []]
    }

  def cancel(self, notebook, snippet):
    pass
<commit_msg>[spark] Switch to Impala API for Impala snippets<commit_after>#!/usr/bin/env python
# Licensed to Cloudera, Inc. under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  Cloudera, Inc. licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import json
import re

from beeswax import models as beeswax_models
from beeswax.design import hql_query
from beeswax.models import QUERY_TYPES, HiveServerQueryHandle, QueryHistory
from beeswax.views import safe_get_design, save_design
from beeswax.server import dbms
from beeswax.server.dbms import get_query_server_config

from spark.job_server_api import get_api as get_spark_api
from desktop.lib.i18n import smart_str
from desktop.lib.rest.http_client import RestException

class Notebook():

  def __init__(self, document=None):
    self.document = None

    if document is not None:
      self.data = document.data
      self.document = document
    else:
      self.data = json.dumps({
          'name': 'My Notebook',
          'snippets': [{'type': 'scala', 'result': {}}]
      })

  def get_json(self):
    _data = self.get_data()

    return json.dumps(_data)

  def get_data(self):
    _data = json.loads(self.data)

    if self.document is not None:
      _data['id'] = self.document.id

    return _data

def get_api(user, snippet):
  if snippet['type'] in ('hive', 'impala', 'spark-sql'):
    return HS2Api(user)
  elif snippet['type'] == 'text':
    return TextApi(user)
  else:
    return SparkApi(user)

def _get_snippet_session(notebook, snippet):
  return [session for session in notebook['sessions'] if session['type'] == snippet['type']][0]

class TextApi():

  def __init__(self, user):
    self.user = user

  def create_session(self, lang):
    return {
        'type': lang,
        'id': None
    }

class HS2Api():

  def __init__(self, user):
    self.user = user

  def _get_handle(self, snippet):
    snippet['result']['handle']['secret'], snippet['result']['handle']['guid'] = HiveServerQueryHandle.get_decoded(snippet['result']['handle']['secret'], snippet['result']['handle']['guid'])
    return HiveServerQueryHandle(**snippet['result']['handle'])

  def _get_db(self, snippet):
    if snippet['type'] == 'hive':
      name = 'beeswax'
    elif snippet['type'] == 'impala':
      name = 'impala'
    else:
      name = 'spark-sql'

    return dbms.get(self.user, query_server=get_query_server_config(name=name))

  def create_session(self, lang):
    return {
        'type': lang,
        'id': None # Real one at some point
    }

  def execute(self, notebook, snippet):
    db = self._get_db(snippet)
    query = hql_query(snippet['statement'], QUERY_TYPES[0])
    handle = db.client.query(query)

#    if not handle.is_valid():
#        msg = _("Server returning invalid handle for query id %(id)d [%(query)s]...") % {'id': query_history.id, 'query': query[:40]}
#        raise QueryServerException(msg)
#    except QueryServerException, ex:
#      LOG.exception(ex)
#      # Kind of expected (hql compile/syntax error, etc.)
#      if hasattr(ex, 'handle') and ex.handle:
#        query_history.server_id, query_history.server_guid = ex.handle.id, ex.handle.id
#        query_history.log_context = ex.handle.log_context
#      query_history.save_state(QueryHistory.STATE.failed)
#      raise ex

    # All good
    server_id, server_guid  = handle.get()
    return {
        'secret': server_id,
        'guid': server_guid,
        'operation_type': handle.operation_type,
        'has_result_set': handle.has_result_set,
        'modified_row_count': handle.modified_row_count,
        'log_context': handle.log_context
    }

  def check_status(self, notebook, snippet):
    db = self._get_db(snippet)

    handle = self._get_handle(snippet)
    status =  db.get_state(handle)

    return {
        'status':
          'running' if status.index in (QueryHistory.STATE.running.index, QueryHistory.STATE.submitted.index)
          else (
             'failed' if status.index in (QueryHistory.STATE.failed.index, QueryHistory.STATE.expired.index)
              else 'available'
          )
    }

  def fetch_result(self, notebook, snippet, rows):
    db = self._get_db(snippet)

    handle = self._get_handle(snippet)
    results = db.fetch(handle, start_over=False, rows=rows)

    # no escaping...
    return {
        'data': list(results.rows()),
        'meta': [{
          'name': column.name,
          'type': column.type,
          'comment': column.comment
        } for column in results.data_table.cols()]
    }

  def fetch_result_metadata(self):
    pass

  def cancel(self, notebook, snippet):
    db = self._get_db(snippet)

    handle = self._get_handle(snippet)
    db.cancel_operation(handle)
    return {'status': 'canceled'}

  def get_log(self, snippet):
    db = self._get_db(snippet)

    handle = self._get_handle(snippet)
    return db.get_log(handle)

  def _progress(self, snippet, logs):
    if snippet['type'] == 'hive':
      match = re.search('Total jobs = (\d+)', logs, re.MULTILINE)
      total = (int(match.group(1)) if match else 1) * 2

      started = logs.count('Starting Job')
      ended = logs.count('Ended Job')

      return int((started + ended) * 100 / total)
    elif snippet['type'] == 'impala':
      match = re.search('(\d+)% Complete', logs, re.MULTILINE)
      return int(match.group(1)) if match else 0
    else: #'spark-sql'
      return 50

class SparkApi():  # Pig, DBquery, Phoenix...

  def __init__(self, user):
    self.user = user

  def create_session(self, lang='scala'):
    api = get_spark_api(self.user)
    return {
        'type': lang,
        'id': api.create_session(lang=lang)
    }

  def execute(self, notebook, snippet):
    api = get_spark_api(self.user)
    session = _get_snippet_session(notebook, snippet)

    return {'id': api.submit_statement(session['id'], snippet['statement']).split('cells/')[1]}

  def check_status(self, notebook, snippet):
    return {'status': 'ready'}

  def fetch_result(self, notebook, snippet, rows):
    api = get_spark_api(self.user)
    session = _get_snippet_session(notebook, snippet)
    cell = snippet['result']['handle']['id']

    data = api.fetch_data(session['id'], cell)

    return {
        'data': [data['output']],
        'meta': [{
          'name': column.name,
          'type': column.type,
          'comment': column.comment
        } for column in []]
    }

  def cancel(self, notebook, snippet):
    pass
<|endoftext|>


<commit_before>from django.contrib import admin

from .models.observers import Observer
from .models.surveys import Survey, SurveyHour

class ObserverAdmin(admin.ModelAdmin):
    list_display = ('__str__', 'email', 'purpose',)

class SurveyHourInline(admin.StackedInline):
    model = SurveyHour
    extra = 0

class SurveyAdmin(admin.ModelAdmin):
    list_display = ('__str__', 'date', 'max_flock_size', 'observer', 'status',)
    list_filter = ('date', 'status',)
    inlines = [SurveyHourInline]
    search_fields = ('id__exact',)

class SurveyHourAdmin(admin.ModelAdmin):
    list_display = ('__str__', 'survey', 'hour', 'kea', 'activity', 'grid_tile',)
    list_filter = ('kea', 'activity',)

admin.site.register(Observer, ObserverAdmin)
admin.site.register(Survey, SurveyAdmin)
admin.site.register(SurveyHour, SurveyHourAdmin)
<commit_msg>Survey admin optimisation (#108)<commit_after>from django.contrib import admin

from .models.observers import Observer
from .models.surveys import Survey, SurveyHour

class ObserverAdmin(admin.ModelAdmin):
    list_display = ('__str__', 'email', 'purpose',)

class SurveyHourInline(admin.StackedInline):
    model = SurveyHour
    extra = 0
    list_select_related = True
    raw_id_fields = ('grid_tile',)

    def get_queryset(self, request):
        qs = super().get_queryset(request)
        return qs.all(). \
            select_related('survey', 'grid_tile'). \
            all()

class SurveyAdmin(admin.ModelAdmin):
    list_display = ('__str__', 'date', 'max_flock_size', 'observer', 'status',)
    list_filter = ('date', 'status',)
    inlines = [SurveyHourInline]
    search_fields = ('id__exact',)
    list_select_related = True

    def get_queryset(self, request):
        qs = super().get_queryset(request)
        return qs.all().select_related('observer').prefetch_related('hours')

class SurveyHourAdmin(admin.ModelAdmin):
    list_display = ('__str__', 'survey', 'hour', 'kea', 'activity', 'grid_tile',)
    list_filter = ('kea', 'activity',)
    list_select_related = True
    raw_id_fields = ('grid_tile',)

    def get_queryset(self, request):
        qs = super().get_queryset(request)
        return qs.all().select_related('survey', 'grid_tile',)

admin.site.register(Observer, ObserverAdmin)
admin.site.register(Survey, SurveyAdmin)
admin.site.register(SurveyHour, SurveyHourAdmin)
<|endoftext|>


<commit_before>#!/usr/bin/env python

# Copyright 2014 Open Connectome Project (http://openconnecto.me)
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

"""
@author: Disa Mhembere
@organization: Johns Hopkins University
@contact: disa@jhu.edu

@summary: A module to alter/update the MRdjango database as necessary
"""

'''
FileField stores files e.g. to media/documents based MEDIA_ROOT
Generally, each model maps to a single database table.
'''
from django.db import models
from django.contrib import admin
from django.contrib.auth.models import User
import os
from time import strftime, localtime

class BuildGraphModel(models.Model):
  '''
  Allows us to store data on build graph view
  '''
  project_name = models.CharField(max_length=255)
  site = models.CharField(max_length=255,)
  subject = models.CharField(max_length=255,)
  session = models.CharField(max_length=255,)
  scanId = models.CharField(max_length=255)
  location = models.TextField()
  owner = models.ForeignKey(to=User, to_field='username', null=True) # Many-to-one .Many here, other in auth_user

  def __repr__(self):
    _repr = '\n' + "project_name: " + str(self.project_name) + '\n' + \
            "site: " + str(self.site) + '\n' + \
            "subject: " + str(self.subject) + '\n' + \
            "session: " + str(self.session) + '\n' + \
            "scanId: " + str(self.scanId) + '\n' + \
            "location: " + str(self.location) + '\n' + \
            "owner: " + str(self.owner) + '\n'

    return super(BuildGraphModel, self).__repr__() + _repr

class OwnedProjects(models.Model):
  '''
  This will let us keep track of owned projects for
  integrity constraints & sharing
  '''
  project_name = models.CharField(max_length=255)
  owner = models.ForeignKey(User, 'username') # Many-to-one .Many here, other in auth_user
  is_private = models.BooleanField(null=False)
  owner_group = models.CharField(max_length=255, null=True) # Will reference other table soon
  # Really should be --> owner_groups = models.ForeignKey(to=User, to_field='groups')

class SharingTokens(models.Model):
  '''
  Class to allow you to create a project sharing token
  that allows a user to let others see a private project.
  '''
  token = models.CharField(max_length=64)
  issued_by = models.ForeignKey(User, 'username') # Many-to-one . Many here, other in auth_user
  project_name = models.ManyToManyRel(to=BuildGraphModel, related_name='project_name')
  issue_date = models.DateTimeField(auto_now_add=True)
  expire_date = models.DateField(null=True)

class GraphDownloadModel(models.Model):

  filepath = models.CharField(max_length=255, null=False, primary_key=True, unique=True)
  genus = models.CharField(max_length=128)
  region = models.CharField(max_length=128)
  # verbose name is what is rendered in the html
  project = models.CharField(max_length=255) # This is only relevant to big graphs
  numvertex = models.BigIntegerField(null=False, verbose_name="# Nodes")
  numedge = models.BigIntegerField(null=False, verbose_name="# Edges")
  graphattr = models.TextField(verbose_name="Graph Attrs") # CSVs here
  vertexattr = models.TextField(verbose_name="Node Attrs") # CSVs here
  edgeattr = models.TextField(verbose_name="Edge Attrs") #CSVs here
  sensor = models.CharField(max_length=128)
  source = models.CharField(max_length=256, verbose_name="Source host url")
  mtime = models.FloatField() # Modification Time
  url = models.URLField(max_length=2048, verbose_name="Download url")

admin.site.register(BuildGraphModel)
admin.site.register(OwnedProjects)
admin.site.register(SharingTokens)
<commit_msg>No-ops<commit_after>#!/usr/bin/env python

# Copyright 2014 Open Connectome Project (http://openconnecto.me)
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

"""
@author: Disa Mhembere
@organization: Johns Hopkins University
@contact: disa@jhu.edu

@summary: A module to alter/update the MRdjango database as necessary
"""

'''
FileField stores files e.g. to media/documents based MEDIA_ROOT
Generally, each model maps to a single database table.
'''
from django.db import models
from django.contrib import admin
from django.contrib.auth.models import User
import os

class BuildGraphModel(models.Model):
  '''
  Allows us to store data on build graph view
  '''
  project_name = models.CharField(max_length=255)
  site = models.CharField(max_length=255,)
  subject = models.CharField(max_length=255,)
  session = models.CharField(max_length=255,)
  scanId = models.CharField(max_length=255)
  location = models.TextField()
  owner = models.ForeignKey(to=User, to_field='username', null=True) # Many-to-one .Many here, other in auth_user

  def __repr__(self):
    _repr = '\n' + "project_name: " + str(self.project_name) + '\n' + \
            "site: " + str(self.site) + '\n' + \
            "subject: " + str(self.subject) + '\n' + \
            "session: " + str(self.session) + '\n' + \
            "scanId: " + str(self.scanId) + '\n' + \
            "location: " + str(self.location) + '\n' + \
            "owner: " + str(self.owner) + '\n'

    return super(BuildGraphModel, self).__repr__() + _repr

class OwnedProjects(models.Model):
  '''
  This will let us keep track of owned projects for
  integrity constraints & sharing
  '''
  project_name = models.CharField(max_length=255)
  owner = models.ForeignKey(User, 'username') # Many-to-one .Many here, other in auth_user
  is_private = models.BooleanField(null=False)
  owner_group = models.CharField(max_length=255, null=True) # Will reference other table soon
  # Really should be --> owner_groups = models.ForeignKey(to=User, to_field='groups')

class SharingTokens(models.Model):
  '''
  Class to allow you to create a project sharing token
  that allows a user to let others see a private project.
  '''
  token = models.CharField(max_length=64)
  issued_by = models.ForeignKey(User, 'username') # Many-to-one . Many here, other in auth_user
  project_name = models.ManyToManyRel(to=BuildGraphModel, related_name='project_name')
  issue_date = models.DateTimeField(auto_now_add=True)
  expire_date = models.DateField(null=True)

class GraphDownloadModel(models.Model):

  filepath = models.CharField(max_length=255, null=False, primary_key=True, unique=True)
  genus = models.CharField(max_length=128)
  region = models.CharField(max_length=128)
  # verbose name is what is rendered in the html
  project = models.CharField(max_length=255) # This is only relevant to big graphs
  numvertex = models.BigIntegerField(null=False, verbose_name="# Nodes")
  numedge = models.BigIntegerField(null=False, verbose_name="# Edges")
  graphattr = models.TextField(verbose_name="Graph Attrs") # CSVs here
  vertexattr = models.TextField(verbose_name="Node Attrs") # CSVs here
  edgeattr = models.TextField(verbose_name="Edge Attrs") #CSVs here
  sensor = models.CharField(max_length=128)
  source = models.CharField(max_length=256, verbose_name="Source host url")
  mtime = models.FloatField() # Modification Time
  url = models.URLField(max_length=2048, verbose_name="Download url")

admin.site.register(BuildGraphModel)
admin.site.register(OwnedProjects)
admin.site.register(SharingTokens)
<|endoftext|>