<commit_before>import MarkovDecisionProcess as MDP
import numpy as np
from numpy.random import randn
from scipy.linalg import solve, cholesky, block_diag, eigh, eig
from bovy_mcmc.elliptical_slice import elliptical_slice as eslice

def eigMin(M):
    return eigh(M,eigvals_only=True,eigvals=(0,0))[0]

class Controller:
    """
    Base class for a controller.
    It is by default an open-loop controller, i.e. it returns input 0
    at each step.
    """
    def __init__(self,Horizon=0,NumInputs = 1,label=''):
        self.NumInputs = NumInputs
        self.Horizon = Horizon
        self.label = label

    def action(self,x,k):
        u = np.zeros(self.NumInputs)
        return u

class openLoopPolicy(Controller):
    def __init__(self,U,*args,**kwargs):
        Controller.__init__(self,*args,**kwargs)
        self.U = U

    def action(self,x,k):
        return self.U[k]

class flatOpenLoopPolicy(Controller):
    """
    Open loop policy from flat array of inputs
    """
    def __init__(self,U=0,NumInputs=1,*args,**kwargs):
        Controller.__init__(self,*args,**kwargs)
        self.U=U
        self.NumInputs = NumInputs
    def action(self,x,k):
        u = self.U[self.NumInputs*k : self.NumInputs * (k+1)]
        return u

class staticGain(Controller):
    def __init__(self,gain=0,*args,**kwargs):
        self.gain = gain
        Controller.__init__(self,*args,**kwargs)

    def action(self,x,k):
        u = np.dot(self.gain,x)
        return u

class varyingGainAndFeedforward(Controller):
    def __init__(self,gain,*args,**kwargs):
        self.Gain = gain
        Controller.__init__(self,*args,**kwargs)

    def action(self,x,k):
        vec = np.hstack((1,x))
        u = np.dot(self.Gain[k],vec)
        return u

class staticFunction(Controller):
    def __init__(self,func,*args,**kwargs):
        self.func = func
        Controller.__init__(self,*args,**kwargs)

    def action(self,x,k):
        u = self.func(x)
        return u

def schurComplement(M,p):
    A = M[:-p,:-p]
    B = M[:-p,-p:]
    C = M[-p:,-p:]
    S = A - np.dot(B,solve(C,B.T,sym_pos=True))
    return S

def gainMatrix(M,p):
    B = M[:-p,-p:]
    C = M[-p:,-p:]
    return -solve(C,B.T,sym_pos=True)

class linearQuadraticRegulator(Controller):
    def __init__(self,SYS,*args,**kwargs):
        Controller.__init__(self,*args,**kwargs)
        self.computeGains(SYS)

    def computeGains(self,sys):
        n = sys.NumStates
        p = sys.NumInputs

        self.RiccatiSolution = np.zeros((self.Horizon,n+1,n+1))
        self.Gain = np.zeros((self.Horizon,p,n+1))

        if sys.timeInvariant:
            costMat = sys.costMatrix
            dynMat = sys.dynamicsMatrix
            bigDynMat = np.vstack((np.hstack((1,np.zeros(n+p))),
                                   dynMat))
        else:
            costMat = sys.costMatrix[-1]
        self.RiccatiSolution[-1] = schurComplement(costMat,p)
        self.RiccatiSolution[-1] = .5*(self.RiccatiSolution[-1]+\
                                       self.RiccatiSolution[-1].T)
        self.Gain[-1] = gainMatrix(costMat,p)

        for k in range(self.Horizon-1)[::-1]:
            if not sys.timeInvariant:
                costMat = sys.costMatrix[k]
                dynMat = sys.dynamicsMatrix[k]
                bigDynMat = np.vstack((np.hstack((1,np.zeros(n+p))),
                                       dynMat))

            A = dynMat[:,1:n+1]
            v = eig(A)[0]
            # print 'max eigenvalue: %g' % np.abs(v).max()
            Ric = self.RiccatiSolution[k+1]
            OlDyn = bigDynMat[:,:-p]
            RicBound = costMat[:-p,:-p] + np.dot(OlDyn.T,np.dot(Ric,OlDyn))
            UpdateMat = np.dot(bigDynMat.T,np.dot(Ric,bigDynMat))
            M = costMat + UpdateMat
            M = .5*(M+M.T)

            newRic = schurComplement(M,p)
            self.RiccatiSolution[k] = .5*(newRic+newRic.T)
            self.Gain[k] = gainMatrix(M,p)

        # print np.diag(self.RiccatiSolution[0][:-p,:-p])
        self.Gain = self.Gain
    def action(self,x,k):
        curVec = np.hstack((1,x))
        return np.dot(self.Gain[k],curVec)

class modelPredictiveControl(Controller):
    def __init__(self,SYS,predictiveHorizon,*args,**kwargs):
        Controller.__init__(self,*args,**kwargs)
        self.SYS = SYS
        self.previousAction = np.zeros(SYS.NumInputs)
        self.predictiveHorizon = predictiveHorizon
    def action(self,x,k):
        # Currently only supporting time invariant systems
        # It will probably be weird if a time-varying system is used.
        # This will need to be updated for consistency with nonlinear systems
        predictiveController = approximateLQR(self.SYS,
                                              x,
                                              self.previousAction,
                                              k,
                                              Horizon=self.predictiveHorizon)

        curVec = np.hstack((1,x))
        gain = predictiveController.Gain[0]
        u = np.dot(gain,curVec)
        self.previousAction = u
        return u

class iterativeLQR(varyingGainAndFeedforward):
    def __init__(self,SYS,initialPolicy = None,Horizon=1,
                 stoppingTolerance=1e-3,*args,**kwargs):
        self.Horizon = Horizon
        if initialPolicy is None:
            gain = np.zeros((SYS.NumInputs,SYS.NumStates))
            initialPolicy = staticGain(gain=gain,Horizon=Horizon)
        else:
            self.Horizon = initialPolicy.Horizon

        X,U,initCost = SYS.simulatePolicy(initialPolicy)

        gainShape = (self.Horizon, SYS.NumInputs, SYS.NumStates+1)

        bestCost = initCost
        bestGain = np.zeros(gainShape)
        for k in range(self.Horizon):
            bestGain[k,:,0] = U[k]

        costChange = np.inf

        # Cost regularization Parameters
        alpha = 1
        n = SYS.NumStates
        p = SYS.NumInputs

        testController = varyingGainAndFeedforward(gain=bestGain,
                                                   Horizon=self.Horizon,
                                                   *args,**kwargs)

        run = 0
        while np.abs(costChange)>stoppingTolerance:
            run += 1
            if alpha > 1e12:
                print 'regularization parameter too large'
                break

            correctionSys = MDP.buildCorrectionSystem(SYS,X,U)

            for k in range(self.Horizon):
                dynMat = correctionSys.dynamicsMatrix[k]
                A = dynMat[:,1:n+1]
                v = eig(A)[0]
                # print 'max eigenvalue: %g' % np.abs(v).max(0)
                curCost = correctionSys.costMatrix[k]
                M = curCost[1:,1:]

                nv = M.shape[0]

                minEig = eigMin(M)
                if minEig < 0:
                    beta = -1.1 * minEig
                    M += beta * np.eye(nv)

                M[-p:,-p:] += alpha * np.eye(p)
                curCost[1:,1:] = M
                correctionSys.costMatrix[k] = curCost

            correctionCtrl = linearQuadraticRegulator(SYS=correctionSys,
                                                      Horizon=self.Horizon,
                                                      *args,**kwargs)

            testController.Gain = np.zeros(gainShape)
            #print 'largest correction gain: %g' % np.abs(correctionCtrl.Gain).max()
            for k in range(self.Horizon):
                testController.Gain[k,:,0] = correctionCtrl.Gain[k,:,0] + \
                                             U[k] - \
                                             np.dot(correctionCtrl.Gain[k,:,1:],X[k])
                testController.Gain[k,:,1:] = correctionCtrl.Gain[k,:,1:]

            try:
                newX,newU,newCost = SYS.simulatePolicy(testController)
                costChange = newCost-bestCost
                print 'iLQR cost: %g, costChange %g' % (newCost,costChange)
                if newCost < bestCost:
                    X = newX
                    U = newU
                    bestCost = newCost
                    bestGain = testController.Gain
                    alpha = 1
                else:
                    alpha *= 2
            except (ValueError,np.linalg.LinAlgError):
                alpha *= 2
                print 'numerical problem'
                # print 'increasing regularization parameter to %g' % alpha

        varyingGainAndFeedforward.__init__(self,
                                           gain=bestGain,
                                           Horizon=self.Horizon,
                                           *args,**kwargs)

class approximateLQR(linearQuadraticRegulator):
    def __init__(self,SYS,x,u,k=0,*args,**kwargs):
        dynMat,costMat = MDP.convexApproximationMatrices(SYS,x,u,k)

        approxSYS = MDP.LinearQuadraticSystem(dynMat,costMat,x0=x)
        linearQuadraticRegulator.__init__(self,SYS=approxSYS,*args,**kwargs)

class samplingControl(flatOpenLoopPolicy):
    def __init__(self,
                 SYS,
                 KLWeight=1,
                 burnIn=0,
                 ExplorationCovariance=1.,
                 initialPolicy = None,
                 *args, **kwargs):

        flatOpenLoopPolicy.__init__(self,NumInputs=SYS.NumInputs,
                                    *args,**kwargs)

        self.SYS = SYS
        self.KLWeight = KLWeight

        if isinstance(ExplorationCovariance,np.ndarray):
            cholSig = cholesky(ExplorationCovariance,lower=True)
            NoiseGain = np.kron(np.eye(self.Horizon),cholSig)
        else:
            NoiseGain = np.sqrt(ExplorationCovariance) * np.eye(self.Horizon)

        lenW = NoiseGain.shape[0]

        if initialPolicy is None:
            U = np.zeros(lenW)
        else:
            U = SYS.simulatePolicy(initialPolicy)[1].flatten()

        logLik = self.loglikelihood(U)
        bestCost = np.inf
        cost = np.inf

        eps = 1e-3

        samp = 0
        while samp < burnIn:
            samp += 1
            W = np.dot(NoiseGain,randn(lenW))
            U,logLik = eslice(U,W,self.loglikelihood,cur_lnpdf=logLik)
            newCost = -logLik * self.KLWeight
            if newCost < bestCost:
                bestCost = newCost
                bestU = U

            costChange = newCost - cost
            cost = newCost

            if burnIn < np.inf:
                if samp % 10 == 0:
                    print 'run %d of %d, Cost: %g, Best Cost: %g' % \
                        (samp,burnIn,cost,bestCost)
            else:
                if costChange>=0:
                    break

                if samp % 10 == 0:
                    print 'run %d, Cost: %g, Cost Change %g' % \
                        (samp,cost,costChange)

        self.U = bestU

    def loglikelihood(self,U):
        self.U = U
        cost = self.SYS.simulatePolicy(self)[2]
        return -cost / self.KLWeight
<commit_msg>made a flat affine policy<commit_after>import MarkovDecisionProcess as MDP
import numpy as np
from numpy.random import randn
from scipy.linalg import solve, cholesky, block_diag, eigh, eig
from bovy_mcmc.elliptical_slice import elliptical_slice as eslice

def eigMin(M):
    return eigh(M,eigvals_only=True,eigvals=(0,0))[0]

class Controller:
    """
    Base class for a controller.
    It is by default an open-loop controller, i.e. it returns input 0
    at each step.
    """
    def __init__(self,Horizon=0,NumInputs = 1,label=''):
        self.NumInputs = NumInputs
        self.Horizon = Horizon
        self.label = label

    def action(self,x,k):
        u = np.zeros(self.NumInputs)
        return u

class openLoopPolicy(Controller):
    def __init__(self,U,*args,**kwargs):
        Controller.__init__(self,*args,**kwargs)
        self.U = U

    def action(self,x,k):
        return self.U[k]

class flatOpenLoopPolicy(Controller):
    """
    Open loop policy from flat array of inputs
    """
    def __init__(self,U=0,NumInputs=1,*args,**kwargs):
        Controller.__init__(self,*args,**kwargs)
        self.U=U
        self.NumInputs = NumInputs
    def action(self,x,k):
        u = self.U[self.NumInputs*k : self.NumInputs * (k+1)]
        return u

class staticGain(Controller):
    def __init__(self,gain=0,*args,**kwargs):
        self.gain = gain
        Controller.__init__(self,*args,**kwargs)

    def action(self,x,k):
        u = np.dot(self.gain,x)
        return u

class varyingAffine(Controller):
    def __init__(self,gain,*args,**kwargs):
        self.Gain = gain
        Controller.__init__(self,*args,**kwargs)

    def action(self,x,k):
        vec = np.hstack((1,x))
        u = np.dot(self.Gain[k],vec)
        return u

class flatVaryingAffine(varyingAffine):
    def __init__(self,flatGain,NumInputs=1,Horizon=1,*args,**kwargs):
        NumStates = -1 + len(Gain) / (Horizon*NumInputs)
        gain = np.reshape(flatGain,(Horizon,NumInputs,NumStates+1))
        varyingAffine.__init__(self,gain=gain,
                               Horizon=Horizon,NumInputs=NumInputs,
                               *args,**kwargs)

class staticFunction(Controller):
    def __init__(self,func,*args,**kwargs):
        self.func = func
        Controller.__init__(self,*args,**kwargs)

    def action(self,x,k):
        u = self.func(x)
        return u

def schurComplement(M,p):
    A = M[:-p,:-p]
    B = M[:-p,-p:]
    C = M[-p:,-p:]
    S = A - np.dot(B,solve(C,B.T,sym_pos=True))
    return S

def gainMatrix(M,p):
    B = M[:-p,-p:]
    C = M[-p:,-p:]
    return -solve(C,B.T,sym_pos=True)

class linearQuadraticRegulator(Controller):
    def __init__(self,SYS,*args,**kwargs):
        Controller.__init__(self,*args,**kwargs)
        self.computeGains(SYS)

    def computeGains(self,sys):
        n = sys.NumStates
        p = sys.NumInputs

        self.RiccatiSolution = np.zeros((self.Horizon,n+1,n+1))
        self.Gain = np.zeros((self.Horizon,p,n+1))

        if sys.timeInvariant:
            costMat = sys.costMatrix
            dynMat = sys.dynamicsMatrix
            bigDynMat = np.vstack((np.hstack((1,np.zeros(n+p))),
                                   dynMat))
        else:
            costMat = sys.costMatrix[-1]
        self.RiccatiSolution[-1] = schurComplement(costMat,p)
        self.RiccatiSolution[-1] = .5*(self.RiccatiSolution[-1]+\
                                       self.RiccatiSolution[-1].T)
        self.Gain[-1] = gainMatrix(costMat,p)

        for k in range(self.Horizon-1)[::-1]:
            if not sys.timeInvariant:
                costMat = sys.costMatrix[k]
                dynMat = sys.dynamicsMatrix[k]
                bigDynMat = np.vstack((np.hstack((1,np.zeros(n+p))),
                                       dynMat))

            A = dynMat[:,1:n+1]
            v = eig(A)[0]
            # print 'max eigenvalue: %g' % np.abs(v).max()
            Ric = self.RiccatiSolution[k+1]
            OlDyn = bigDynMat[:,:-p]
            RicBound = costMat[:-p,:-p] + np.dot(OlDyn.T,np.dot(Ric,OlDyn))
            UpdateMat = np.dot(bigDynMat.T,np.dot(Ric,bigDynMat))
            M = costMat + UpdateMat
            M = .5*(M+M.T)

            newRic = schurComplement(M,p)
            self.RiccatiSolution[k] = .5*(newRic+newRic.T)
            self.Gain[k] = gainMatrix(M,p)

        # print np.diag(self.RiccatiSolution[0][:-p,:-p])
        self.Gain = self.Gain
    def action(self,x,k):
        curVec = np.hstack((1,x))
        return np.dot(self.Gain[k],curVec)

class modelPredictiveControl(Controller):
    def __init__(self,SYS,predictiveHorizon,*args,**kwargs):
        Controller.__init__(self,*args,**kwargs)
        self.SYS = SYS
        self.previousAction = np.zeros(SYS.NumInputs)
        self.predictiveHorizon = predictiveHorizon
    def action(self,x,k):
        # Currently only supporting time invariant systems
        # It will probably be weird if a time-varying system is used.
        # This will need to be updated for consistency with nonlinear systems
        predictiveController = approximateLQR(self.SYS,
                                              x,
                                              self.previousAction,
                                              k,
                                              Horizon=self.predictiveHorizon)

        curVec = np.hstack((1,x))
        gain = predictiveController.Gain[0]
        u = np.dot(gain,curVec)
        self.previousAction = u
        return u

class iterativeLQR(varyingAffine):
    def __init__(self,SYS,initialPolicy = None,Horizon=1,
                 stoppingTolerance=1e-3,*args,**kwargs):
        self.Horizon = Horizon
        if initialPolicy is None:
            gain = np.zeros((SYS.NumInputs,SYS.NumStates))
            initialPolicy = staticGain(gain=gain,Horizon=Horizon)
        else:
            self.Horizon = initialPolicy.Horizon

        X,U,initCost = SYS.simulatePolicy(initialPolicy)

        gainShape = (self.Horizon, SYS.NumInputs, SYS.NumStates+1)

        bestCost = initCost
        bestGain = np.zeros(gainShape)
        for k in range(self.Horizon):
            bestGain[k,:,0] = U[k]

        costChange = np.inf

        # Cost regularization Parameters
        alpha = 1
        n = SYS.NumStates
        p = SYS.NumInputs

        testController = varyingAffine(gain=bestGain,
                                                   Horizon=self.Horizon,
                                                   *args,**kwargs)

        run = 0
        while np.abs(costChange)>stoppingTolerance:
            run += 1
            if alpha > 1e12:
                print 'regularization parameter too large'
                break

            correctionSys = MDP.buildCorrectionSystem(SYS,X,U)

            for k in range(self.Horizon):
                dynMat = correctionSys.dynamicsMatrix[k]
                A = dynMat[:,1:n+1]
                v = eig(A)[0]
                # print 'max eigenvalue: %g' % np.abs(v).max(0)
                curCost = correctionSys.costMatrix[k]
                M = curCost[1:,1:]

                nv = M.shape[0]

                minEig = eigMin(M)
                if minEig < 0:
                    beta = -1.1 * minEig
                    M += beta * np.eye(nv)

                M[-p:,-p:] += alpha * np.eye(p)
                curCost[1:,1:] = M
                correctionSys.costMatrix[k] = curCost

            correctionCtrl = linearQuadraticRegulator(SYS=correctionSys,
                                                      Horizon=self.Horizon,
                                                      *args,**kwargs)

            testController.Gain = np.zeros(gainShape)
            #print 'largest correction gain: %g' % np.abs(correctionCtrl.Gain).max()
            for k in range(self.Horizon):
                testController.Gain[k,:,0] = correctionCtrl.Gain[k,:,0] + \
                                             U[k] - \
                                             np.dot(correctionCtrl.Gain[k,:,1:],X[k])
                testController.Gain[k,:,1:] = correctionCtrl.Gain[k,:,1:]

            try:
                newX,newU,newCost = SYS.simulatePolicy(testController)
                costChange = newCost-bestCost
                print 'iLQR cost: %g, costChange %g' % (newCost,costChange)
                if newCost < bestCost:
                    X = newX
                    U = newU
                    bestCost = newCost
                    bestGain = testController.Gain
                    alpha = 1
                else:
                    alpha *= 2
            except (ValueError,np.linalg.LinAlgError):
                alpha *= 2
                print 'numerical problem'
                # print 'increasing regularization parameter to %g' % alpha

        varyingAffine.__init__(self,
                                           gain=bestGain,
                                           Horizon=self.Horizon,
                                           *args,**kwargs)

class approximateLQR(linearQuadraticRegulator):
    def __init__(self,SYS,x,u,k=0,*args,**kwargs):
        dynMat,costMat = MDP.convexApproximationMatrices(SYS,x,u,k)

        approxSYS = MDP.LinearQuadraticSystem(dynMat,costMat,x0=x)
        linearQuadraticRegulator.__init__(self,SYS=approxSYS,*args,**kwargs)

class samplingControl(flatOpenLoopPolicy):
    def __init__(self,
                 SYS,
                 KLWeight=1,
                 burnIn=0,
                 ExplorationCovariance=1.,
                 initialPolicy = None,
                 *args, **kwargs):

        flatOpenLoopPolicy.__init__(self,NumInputs=SYS.NumInputs,
                                    *args,**kwargs)

        self.SYS = SYS
        self.KLWeight = KLWeight

        if isinstance(ExplorationCovariance,np.ndarray):
            cholSig = cholesky(ExplorationCovariance,lower=True)
            NoiseGain = np.kron(np.eye(self.Horizon),cholSig)
        else:
            NoiseGain = np.sqrt(ExplorationCovariance) * np.eye(self.Horizon)

        lenW = NoiseGain.shape[0]

        if initialPolicy is None:
            U = np.zeros(lenW)
        else:
            U = SYS.simulatePolicy(initialPolicy)[1].flatten()

        logLik = self.loglikelihood(U)
        bestCost = np.inf
        cost = np.inf

        eps = 1e-3

        samp = 0
        while samp < burnIn:
            samp += 1
            W = np.dot(NoiseGain,randn(lenW))
            U,logLik = eslice(U,W,self.loglikelihood,cur_lnpdf=logLik)
            newCost = -logLik * self.KLWeight
            if newCost < bestCost:
                bestCost = newCost
                bestU = U

            costChange = newCost - cost
            cost = newCost

            if burnIn < np.inf:
                if samp % 10 == 0:
                    print 'run %d of %d, Cost: %g, Best Cost: %g' % \
                        (samp,burnIn,cost,bestCost)
            else:
                if costChange>=0:
                    break

                if samp % 10 == 0:
                    print 'run %d, Cost: %g, Cost Change %g' % \
                        (samp,cost,costChange)

        self.U = bestU

    def loglikelihood(self,U):
        self.U = U
        cost = self.SYS.simulatePolicy(self)[2]
        return -cost / self.KLWeight
<|endoftext|>


<commit_before>import base64
from flask import (
    current_app,
    jsonify
)

from app import celery
from app.comms.email import get_email_html, send_smtp_email
from app.dao import dao_update_record
from app.dao.emails_dao import dao_create_email, dao_get_email_by_magazine_id
from app.dao.magazines_dao import dao_get_magazine_by_id
from app.dao.users_dao import dao_get_users
from app.models import Email, Magazine, MAGAZINE, READY
from app.utils.pdf import extract_topics
from app.utils.storage import Storage

@celery.task()
def upload_magazine(magazine_id, pdf_data):
    current_app.logger.info('Upload magazine pdf: {}'.format(magazine_id))

    try:
        magazine = dao_get_magazine_by_id(magazine_id)

        storage = Storage(current_app.config['STORAGE'])

        storage.upload_blob_from_base64string(
            magazine.filename,
            magazine.filename,
            base64.b64decode(pdf_data),
            content_type='application/pdf'
        )

        topics = extract_topics(base64.b64decode(pdf_data))

        dao_update_record(Magazine, magazine_id, topics=topics)

        email = dao_get_email_by_magazine_id(magazine_id)

        if not email:
            email = Email(
                magazine_id=magazine.id,
                email_state=READY,
                email_type=MAGAZINE
            )
            dao_create_email(email)

        emails_to = [user.email for user in dao_get_users()]

        subject = 'Please review {}'.format(magazine.title)

        # send email to admin users and ask them to log in in order to approve the email
        review_part = '
Please review this email: {}/emails/{}
'.format(
            current_app.config['FRONTEND_ADMIN_URL'], str(email.id))
        magazine_html = get_email_html(MAGAZINE, magazine_id=magazine.id)
        response = send_smtp_email(emails_to, subject, review_part + magazine_html)

        if response != 200:
            current_app.logger.error('Error sending review email {}, for {}'.format(email.id, magazine.id))
    except Exception as e:
        current_app.logger.error('Task error uploading magazine: {}'.format(str(e)))
<commit_msg>Update topic extraction<commit_after>import base64
from flask import (
    current_app,
    jsonify
)

from app import celery
from app.comms.email import get_email_html, send_smtp_email
from app.dao import dao_update_record
from app.dao.emails_dao import dao_create_email, dao_get_email_by_magazine_id
from app.dao.magazines_dao import dao_get_magazine_by_id
from app.dao.users_dao import dao_get_users
from app.models import Email, Magazine, MAGAZINE, READY
from app.utils.pdf import extract_topics
from app.utils.storage import Storage

@celery.task()
def upload_magazine(magazine_id, pdf_data):
    current_app.logger.info('Upload magazine pdf: {}'.format(magazine_id))

    try:
        magazine = dao_get_magazine_by_id(magazine_id)

        storage = Storage(current_app.config['STORAGE'])
        decoded_data = base64.b64decode(pdf_data)
        storage.upload_blob_from_base64string(
            magazine.filename,
            magazine.filename,
            decoded_data,
            content_type='application/pdf'
        )

        try:
            topics = extract_topics(base64.b64decode(decoded_data))
        except Exception as e:
            topics = []
            current_app.logger.error("Error extracting topics: %r", e)

        dao_update_record(Magazine, magazine_id, topics=topics)

        email = dao_get_email_by_magazine_id(magazine_id)

        if not email:
            email = Email(
                magazine_id=magazine.id,
                email_state=READY,
                email_type=MAGAZINE
            )
            dao_create_email(email)

        emails_to = [user.email for user in dao_get_users()]

        subject = 'Please review {}'.format(magazine.title)

        # send email to admin users and ask them to log in in order to approve the email
        review_part = '
Please review this email: {}/emails/{}
'.format(
            current_app.config['FRONTEND_ADMIN_URL'], str(email.id))
        magazine_html = get_email_html(MAGAZINE, magazine_id=magazine.id)
        response = send_smtp_email(emails_to, subject, review_part + magazine_html)

        if response != 200:
            current_app.logger.error('Error sending review email {}, for {}'.format(email.id, magazine.id))
    except Exception as e:
        current_app.logger.error('Task error uploading magazine: {}'.format(str(e)))
<|endoftext|>


<commit_before># Copyright 2020 Kaggle Inc
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import os
import json
import jsonschema
import sys
import traceback
from copy import deepcopy
from io import StringIO
from pathlib import Path
from threading import Thread
from .errors import DeadlineExceeded, InvalidArgument, NotFound

# Path Utilities.
root_path = Path(__file__).parent.resolve()
envs_path = Path.joinpath(root_path, "envs")

# Primative Utilities.
def get(o, classinfo=None, default=None, path=[], is_callable=None, fallback=None):
    if o == None and default != None:
        o = default
    if has(o, classinfo, default, path, is_callable):
        cur = o
        for p in path:
            cur = cur[p]
        return cur
    else:
        if default != None:
            return default
        return fallback

def has(o, classinfo=None, default=None, path=[], is_callable=None):
    try:
        cur = o
        for p in path:
            cur = cur[p]
        if classinfo != None and not isinstance(cur, classinfo):
            raise "Not a match"
        if is_callable == True and not callable(cur):
            raise "Not callable"
        if is_callable == False and callable(cur):
            raise "Is callable"
        return True
    except:
        if default != None and o != None and len(path) > 0:
            cur = o
            for p in path[:-1]:
                if not has(cur, dict, path=[p]):
                    cur[p] = {}
                cur = cur[p]
            cur[path[-1]] = default
        return False

def call(o, default=None, path=[], args=[], kwargs={}):
    o = get(o, default=False, path=path, is_callable=True)
    if o != False:
        return o(*args, **kwargs)
    else:
        return default

class Struct(dict):
    def __init__(self, **entries):
        entries = {k: v for k, v in entries.items() if k != "items"}
        dict.__init__(self, entries)
        self.__dict__.update(entries)

    def __setattr__(self, attr, value):
        self.__dict__[attr] = value
        self[attr] = value

def timeout(fn, *args, **kwargs):
    seconds = get(kwargs, int, 60, path=["seconds"])
    rtn = [DeadlineExceeded(f"Timed out after {seconds} seconds.")]

    def target():
        try:
            rtn[0] = fn(*args)
        except Exception as e:
            rtn[0] = e

    t = Thread(target=target)
    t.daemon = True
    try:
        t.start()
        t.join(seconds)
    except Exception as e:
        raise e
    if isinstance(rtn[0], BaseException):
        raise rtn[0]
    return rtn[0]

# Added benifit of cloning lists and dicts.
def structify(o):
    if isinstance(o, list):
        return [structify(o[i]) for i in range(len(o))]
    elif isinstance(o, dict):
        return Struct(**{k: structify(v) for k, v in o.items()})
    return o

# File Utilities.
def read_file(path, fallback=None):
    try:
        with open(path, "r", encoding="utf-8") as file:
            return file.read()
    except:
        if fallback != None:
            return fallback
        raise NotFound(f"{path} not found")

def get_exec(raw, fallback=None):
    orig_out = sys.stdout
    buffer = StringIO()
    sys.stdout = buffer
    try:
        code_object = compile(raw, "<string>", "exec")
        env = {}
        exec(code_object, env)
        sys.stdout = orig_out
        print(buffer.getvalue())
        return env
    except Exception as e:
        sys.stdout = orig_out
        print(buffer.getvalue())
        if fallback != None:
            return fallback
        raise InvalidArgument("Invalid raw Python: " + str(e))

def get_last_callable(raw, fallback=None):
    try:
        local = get_exec(raw)
        callables = [v for v in local.values() if callable(v)]
        if len(callables) > 0:
            return callables[-1]
        raise "Nope"
    except:
        if fallback != None:
            return fallback
        raise InvalidArgument("No callable found")

def get_file_json(path, fallback=None):
    try:
        with open(path, "r") as json_file:
            return json.load(json_file)
    except:
        if fallback != None:
            return fallback
        raise InvalidArgument(f"{path} does not contain valid JSON")

# Schema Utilities.
schemas = structify(get_file_json(Path.joinpath(root_path, "schemas.json")))

def default_schema(schema, data):
    default = get(schema, path=["default"])
    if default == None and data == None:
        return

    if get(schema, path=["type"]) == "object":
        default = deepcopy(get(default, dict, {}))
        if data == None or not has(data, dict):
            obj = default
        else:
            obj = data
            for k, v in default.items():
                if not k in obj:
                    obj[k] = v
        properties = get(schema, dict, {}, ["properties"])
        for key, prop_schema in properties.items():
            new_value = default_schema(prop_schema, get(obj, path=[key]))
            if new_value != None:
                obj[key] = new_value
        return obj

    if get(schema, path=["type"]) == "array":
        default = deepcopy(get(default, list, []))
        arr = get(data, list, default)
        item_schema = get(schema, dict, {}, ["items"])
        for index, value in enumerate(arr):
            if value == None and len(default) > index:
                new_value = default[index]
            else:
                new_value = default_schema(item_schema, value)
            if new_value != None:
                arr[index] = new_value
        return arr

    return data if data != None else default

def process_schema(schema, data, use_default=True):
    error = None
    if use_default == True:
        data = default_schema(schema, deepcopy(data))
    try:
        jsonschema.validate(data, schema)
    except Exception as err:
        traceback.print_exc()
        error = str(err)
    return error, data

# Player utilities
def get_player(window_kaggle, renderer):
    key = "/*window.kaggle*/"
    value = f"""
window.kaggle = {json.dumps(window_kaggle, indent=2)};\n\n
window.kaggle.renderer = {renderer.strip()};\n\n
    """
    return read_file(Path.joinpath(root_path, "static", "player.html")).replace(
        key, value
    )
<commit_msg>Remove timeout util and add isUrl check.<commit_after># Copyright 2020 Kaggle Inc
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import json
import jsonschema
import sys
import traceback
from copy import deepcopy
from io import StringIO
from pathlib import Path
from urllib.parse import urlparse
from .errors import InvalidArgument, NotFound

# Path Utilities.
root_path = Path(__file__).parent.resolve()
envs_path = Path.joinpath(root_path, "envs")

# Primative Utilities.
def get(o, classinfo=None, default=None, path=[], is_callable=None, fallback=None):
    if o == None and default != None:
        o = default
    if has(o, classinfo, default, path, is_callable):
        cur = o
        for p in path:
            cur = cur[p]
        return cur
    else:
        if default != None:
            return default
        return fallback

def has(o, classinfo=None, default=None, path=[], is_callable=None):
    try:
        cur = o
        for p in path:
            cur = cur[p]
        if classinfo != None and not isinstance(cur, classinfo):
            raise "Not a match"
        if is_callable == True and not callable(cur):
            raise "Not callable"
        if is_callable == False and callable(cur):
            raise "Is callable"
        return True
    except:
        if default != None and o != None and len(path) > 0:
            cur = o
            for p in path[:-1]:
                if not has(cur, dict, path=[p]):
                    cur[p] = {}
                cur = cur[p]
            cur[path[-1]] = default
        return False

def call(o, default=None, path=[], args=[], kwargs={}):
    o = get(o, default=False, path=path, is_callable=True)
    if o != False:
        return o(*args, **kwargs)
    else:
        return default

class Struct(dict):
    def __init__(self, **entries):
        entries = {k: v for k, v in entries.items() if k != "items"}
        dict.__init__(self, entries)
        self.__dict__.update(entries)

    def __setattr__(self, attr, value):
        self.__dict__[attr] = value
        self[attr] = value

# Added benifit of cloning lists and dicts.
def structify(o):
    if isinstance(o, list):
        return [structify(o[i]) for i in range(len(o))]
    elif isinstance(o, dict):
        return Struct(**{k: structify(v) for k, v in o.items()})
    return o

# File Utilities.
def read_file(path, fallback=None):
    try:
        with open(path, "r", encoding="utf-8") as file:
            return file.read()
    except:
        if fallback != None:
            return fallback
        raise NotFound(f"{path} not found")

def get_exec(raw, fallback=None):
    orig_out = sys.stdout
    buffer = StringIO()
    sys.stdout = buffer
    try:
        code_object = compile(raw, "<string>", "exec")
        env = {}
        exec(code_object, env)
        sys.stdout = orig_out
        print(buffer.getvalue())
        return env
    except Exception as e:
        sys.stdout = orig_out
        print(buffer.getvalue())
        if fallback != None:
            return fallback
        raise InvalidArgument("Invalid raw Python: " + str(e))

def get_last_callable(raw, fallback=None):
    try:
        local = get_exec(raw)
        callables = [v for v in local.values() if callable(v)]
        if len(callables) > 0:
            return callables[-1]
        raise "Nope"
    except:
        if fallback != None:
            return fallback
        raise InvalidArgument("No callable found")

def get_file_json(path, fallback=None):
    try:
        with open(path, "r") as json_file:
            return json.load(json_file)
    except:
        if fallback != None:
            return fallback
        raise InvalidArgument(f"{path} does not contain valid JSON")

# Schema Utilities.
schemas = structify(get_file_json(Path.joinpath(root_path, "schemas.json")))

def default_schema(schema, data):
    default = get(schema, path=["default"])
    if default == None and data == None:
        return

    if get(schema, path=["type"]) == "object":
        default = deepcopy(get(default, dict, {}))
        if data == None or not has(data, dict):
            obj = default
        else:
            obj = data
            for k, v in default.items():
                if not k in obj:
                    obj[k] = v
        properties = get(schema, dict, {}, ["properties"])
        for key, prop_schema in properties.items():
            new_value = default_schema(prop_schema, get(obj, path=[key]))
            if new_value != None:
                obj[key] = new_value
        return obj

    if get(schema, path=["type"]) == "array":
        default = deepcopy(get(default, list, []))
        arr = get(data, list, default)
        item_schema = get(schema, dict, {}, ["items"])
        for index, value in enumerate(arr):
            if value == None and len(default) > index:
                new_value = default[index]
            else:
                new_value = default_schema(item_schema, value)
            if new_value != None:
                arr[index] = new_value
        return arr

    return data if data != None else default

def process_schema(schema, data, use_default=True):
    error = None
    if use_default == True:
        data = default_schema(schema, deepcopy(data))
    try:
        jsonschema.validate(data, schema)
    except Exception as err:
        traceback.print_exc()
        error = str(err)
    return error, data

def is_url(url):
    try:
        result = urlparse(url)
        return all([result.scheme, result.netloc])
    except ValueError:
        return False

# Player utilities
def get_player(window_kaggle, renderer):
    key = "/*window.kaggle*/"
    value = f"""
window.kaggle = {json.dumps(window_kaggle, indent=2)};\n\n
window.kaggle.renderer = {renderer.strip()};\n\n
    """
    return read_file(Path.joinpath(root_path, "static", "player.html")).replace(
        key, value
    )
<|endoftext|>


<commit_before>import json
import logging
import os
import os.path as osp
import re

from .read_machinery import DATA_ROOT_DIR

log = logging.getLogger(__name__)

class RunFilesMap:
    """Cached data about HDF5 files in a run directory

    Stores the train IDs and source names in each file, along with some
    metadata to check that the cache is still valid. The cached information
    can be stored in:

    - (run dir)/karabo_data_map.json
    - (proposal dir)/scratch/.karabo_data_maps/raw_r0032.json
    """
    cache_file = None

    def __init__(self, directory):
        self.directory = osp.abspath(directory)
        self.files_data = {}

        self.candidate_paths = [osp.join(directory, 'karabo_data_map.json')]
        m = re.match(
            r'(%s/\w+/\w+/\w+)/(raw|proc)/(r\d+)/?$' % DATA_ROOT_DIR, directory
        )
        if m:
            prop_dir, raw_proc, run_nr = m.groups()
            fname = '%s_%s.json' % (raw_proc, run_nr)
            self.candidate_paths.append(
                osp.join(prop_dir, 'scratch', '.karabo_data_maps', fname)
            )

        self.load()

    def load(self):
        """Load the cached data

        This skips over invalid cache entries(based on the file's size & mtime).
        """
        loaded_data = []

        for path in self.candidate_paths:
            if osp.isfile(path):
                with open(path) as f:
                    loaded_data = json.load(f)

                self.cache_file = path
                log.debug("Loaded cached files map from %s", path)
                break

        for info in loaded_data:
            filename = info['filename']
            st = os.stat(osp.join(self.directory, filename))
            if (st.st_mtime == info['mtime']) and (st.st_size == info['size']):
                self.files_data[filename] = info

    def get(self, path):
        """Get cache entry for a file path

        Returns a dict or None
        """
        dirname, fname = osp.split(osp.abspath(path))
        if (dirname == self.directory) and (fname in self.files_data):
            return self.files_data[fname]

        return None

    def save(self, files):
        """Save the cache if needed

        This skips writing the cache out if all the data files already have
        valid cache entries. It also silences permission errors from writing
        the cache file.
        """
        need_save = False

        for file_access in files:
            dirname, fname = osp.split(osp.abspath(file_access.filename))
            if (dirname == self.directory) and (fname not in self.files_data):
                log.debug("Will save cached data for %s", fname)
                need_save = True

                st = os.stat(file_access.filename)
                self.files_data[fname] = {
                    'filename': fname,
                    'mtime': st.st_mtime,
                    'size': st.st_size,
                    'train_ids': [int(t) for t in file_access.train_ids],
                    'control_sources': sorted(file_access.control_sources),
                    'instrument_sources': sorted(file_access.instrument_sources),
                }

        if need_save:
            save_data = [info for (_, info) in sorted(self.files_data.items())]
            for path in self.candidate_paths:
                try:
                    os.makedirs(osp.dirname(path), exist_ok=True)
                    f = open(path, 'w')
                except PermissionError:
                    continue

                with f:
                    json.dump(save_data, f, indent=2)
<commit_msg>Write cache file atomically to avoid interleaved writes<commit_after>import json
import logging
import os
import os.path as osp
import re
from tempfile import mkstemp

from .read_machinery import DATA_ROOT_DIR

log = logging.getLogger(__name__)

def atomic_dump(obj, path, **kwargs):
    """Write JSON to a file atomically

    This aims to avoid garbled files from multiple processes writing the same
    cache. It doesn't try to protect against e.g. sudden power failures, as
    forcing the OS to flush changes to disk may hurt performance.
    """
    dirname, basename = osp.split(path)
    fd, tmp_filename = mkstemp(dir=dirname, prefix=basename)
    try:
        with open(fd, 'w') as f:
            json.dump(obj, f, **kwargs)
    except:
        os.unlink(tmp_filename)
        raise

    os.replace(tmp_filename, path)

class RunFilesMap:
    """Cached data about HDF5 files in a run directory

    Stores the train IDs and source names in each file, along with some
    metadata to check that the cache is still valid. The cached information
    can be stored in:

    - (run dir)/karabo_data_map.json
    - (proposal dir)/scratch/.karabo_data_maps/raw_r0032.json
    """
    cache_file = None

    def __init__(self, directory):
        self.directory = osp.abspath(directory)
        self.files_data = {}

        self.candidate_paths = [osp.join(directory, 'karabo_data_map.json')]
        m = re.match(
            r'(%s/\w+/\w+/\w+)/(raw|proc)/(r\d+)/?$' % DATA_ROOT_DIR, directory
        )
        if m:
            prop_dir, raw_proc, run_nr = m.groups()
            fname = '%s_%s.json' % (raw_proc, run_nr)
            self.candidate_paths.append(
                osp.join(prop_dir, 'scratch', '.karabo_data_maps', fname)
            )

        self.load()

    def load(self):
        """Load the cached data

        This skips over invalid cache entries(based on the file's size & mtime).
        """
        loaded_data = []

        for path in self.candidate_paths:
            if osp.isfile(path):
                with open(path) as f:
                    loaded_data = json.load(f)

                self.cache_file = path
                log.debug("Loaded cached files map from %s", path)
                break

        for info in loaded_data:
            filename = info['filename']
            st = os.stat(osp.join(self.directory, filename))
            if (st.st_mtime == info['mtime']) and (st.st_size == info['size']):
                self.files_data[filename] = info

    def get(self, path):
        """Get cache entry for a file path

        Returns a dict or None
        """
        dirname, fname = osp.split(osp.abspath(path))
        if (dirname == self.directory) and (fname in self.files_data):
            return self.files_data[fname]

        return None

    def save(self, files):
        """Save the cache if needed

        This skips writing the cache out if all the data files already have
        valid cache entries. It also silences permission errors from writing
        the cache file.
        """
        need_save = False

        for file_access in files:
            dirname, fname = osp.split(osp.abspath(file_access.filename))
            if (dirname == self.directory) and (fname not in self.files_data):
                log.debug("Will save cached data for %s", fname)
                need_save = True

                st = os.stat(file_access.filename)
                self.files_data[fname] = {
                    'filename': fname,
                    'mtime': st.st_mtime,
                    'size': st.st_size,
                    'train_ids': [int(t) for t in file_access.train_ids],
                    'control_sources': sorted(file_access.control_sources),
                    'instrument_sources': sorted(file_access.instrument_sources),
                }

        if need_save:
            save_data = [info for (_, info) in sorted(self.files_data.items())]
            for path in self.candidate_paths:
                try:
                    os.makedirs(osp.dirname(path), exist_ok=True)
                    atomic_dump(save_data, path, indent=2)
                except PermissionError:
                    continue
<|endoftext|>


<commit_before># Standard library
import sys
import importlib
import contextlib

# Pyblish libraries
import pyblish.api

# Host libraries
import hou

# Local libraries
from . import lib
from ..lib import logger
from avalon import api, schema

self = sys.modules[__name__]
self._has_been_setup = False
self._parent = None
self._events = dict()

main_container = "/obj/AVALON_CONTAINERS"
IS_HEADLESS = not hasattr(hou, "ui")

def install(config):
    """Setup integration
    Register plug-ins and integrate into the host

    """

    _register_callbacks()

    pyblish.api.register_host("houdini")
    pyblish.api.register_host("hython")
    pyblish.api.register_host("hpython")

    self._has_been_setup = True

    config = find_host_config(config)
    if hasattr(config, "install"):
        config.install()

def uninstall(config):
    """Uninstall Houdini-specific functionality of avalon-core.

    This function is called automatically on calling `api.uninstall()`.

    Args:
        config: configuration module

    """

    config = find_host_config(config)
    if hasattr(config, "uninstall"):
        config.uninstall()

    pyblish.api.deregister_host("hython")
    pyblish.api.deregister_host("hpython")
    pyblish.api.deregister_host("houdini")

def find_host_config(config):
    config_name = config.__name__
    try:
        config = importlib.import_module(config_name + ".houdini")
    except ImportError as exc:
        if str(exc) != "No module name {}".format(config_name + ".houdini"):
            raise
        config = None

    return config

def reload_pipeline(*args):
    """Attempt to reload pipeline at run-time.

    CAUTION: This is primarily for development and debugging purposes.

    """

    import importlib

    api.uninstall()

    for module in ("avalon.io",
                   "avalon.lib",
                   "avalon.pipeline",

                   "avalon.houdini.pipeline",
                   "avalon.houdini.lib",
                   "avalon.tools.loader.app",
                   "avalon.tools.creator.app",
                   "avalon.tools.manager.app",

                   # NOTE(marcus): These have circular depenendencies
                   #               that is preventing reloadability
                   # "avalon.tools.cbloader.delegates",
                   # "avalon.tools.cbloader.model",
                   # "avalon.tools.cbloader.widgets",
                   # "avalon.tools.cbloader.app",
                   # "avalon.tools.cbsceneinventory.model",
                   # "avalon.tools.cbsceneinventory.proxy",
                   # "avalon.tools.cbsceneinventory.app",
                   # "avalon.tools.projectmanager.dialogs",
                   # "avalon.tools.projectmanager.lib",
                   # "avalon.tools.projectmanager.model",
                   # "avalon.tools.projectmanager.style",
                   # "avalon.tools.projectmanager.widget",
                   # "avalon.tools.projectmanager.app",

                   "avalon.api",
                   "avalon.tools",
                   "avalon.houdini"):
        module = importlib.import_module(module)
        reload(module)

    self._parent = {hou.ui.mainQtWindow().objectName(): hou.ui.mainQtWindow()}

    import avalon.houdini
    api.install(avalon.houdini)

def _discover_gui():
    """Return the most desirable of the currently registered GUIs"""

    # Prefer last registered
    guis = reversed(pyblish.api.registered_guis())

    for gui in guis:
        try:
            gui = __import__(gui).show
        except (ImportError, AttributeError):
            continue
        else:
            return gui

def teardown():
    """Remove integration"""
    if not self._has_been_setup:
        return

    self._has_been_setup = False
    print("pyblish: Integration torn down successfully")

@contextlib.contextmanager
def maintained_selection():
    """Maintain selection during context
    Example:
        >>> with maintained_selection():
        ...     # Modify selection
        ...     node.setSelected(on=False, clear_all_selected=True)
        >>> # Selection restored
    """

    previous_selection = hou.selectedNodes()
    try:
        yield
    finally:
        if previous_selection:
            for node in previous_selection:
                node.setSelected(on=True)
        else:
            for node in previous_selection:
                node.setSelected(on=False)

def containerise(name,
                 namespace,
                 nodes,
                 context,
                 loader=None,
                 suffix=""):
    """Bundle `nodes` into an assembly and imprint it with metadata

    Containerisation enables a tracking of version, author and origin
    for loaded assets.

    In Houdini it is not possible to next goemetry nodes in geometry nodes
    directly. To counter this we place a Object Network node calles ROOT
    in the HOUDINI_CONTAINERS node.

    Arguments:
        name (str): Name of resulting assembly
        namespace (str): Namespace under which to host container
        nodes (list): Long names of nodes to containerise
        context (dict): Asset information
        loader (str, optional): Name of loader used to produce this container.
        suffix (str, optional): Suffix of container, defaults to `_CON`.

    Returns:
        container (str): Name of container assembly

    """

    # Check if the AVALON_CONTAINERS exists
    geo = hou.node(main_container)
    if geo is None:
        obj_network = hou.node("/obj")
        geo = obj_network.createNode("geo", node_name="AVALON_CONTAINERS")
        # Delete file node
        geo.node("file1").destroy()

    # Check if root Object Network exists
    container_network = hou.node("{}/ROOT".format(main_container))
    if container_network is None:
        container_network = geo.createNode("objnet", node_name="ROOT")

    container = hou.node("/obj/{}".format(name))
    data = {
        "schema": "avalon-core:container-2.0",
        "id": "pyblish.avalon.container",
        "name": name,
        "namespace": namespace,
        "loader": str(loader),
        "representation": str(context["representation"]["_id"]),
    }

    lib.imprint(container, data)

    # "Parent" the container under the container network
    hou.moveNodesTo([container], container_network)

    # Get the container and set good position
    container_network.node(name).moveToGoodPosition()

    return container

def parse_container(container, validate=True):
    """Return the container node's full container data.

    Args:
        container (str): A container node name.

    Returns:
        dict: The container schema data for this container node.

    """
    data = lib.read(container)

    # Backwards compatibility pre-schemas for containers
    data["schema"] = data.get("schema", "avalon-core:container-1.0")

    # Append transient data
    data["objectName"] = container.path()
    data["node"] = container

    if validate:
        schema.validate(data)

    return data

def ls():
    containers = []
    for identifier in ("pyblish.avalon.container",
                       "pyblish.mindbender.container"):
        containers += lib.lsattr("id", identifier)

    for container in sorted(containers):
        data = parse_container(container)
        yield data

class Creator(api.Creator):
    """Creator plugin to create instances in Houdini

    To support the wide range of node types for render output (Alembic, VDB,
    Mantra) the Creator needs a node type to create the correct instance

    By default, if none is given, is `geometry`. An example of accepted node
    types: geometry, alembic, ifd (mantra)

    Please check the Houdini documentation for more node types.

    Tip: to find the exact node type to create press the `i` left of the node
    when hovering over a node. The information is visible under the name of
    the node.

    """

    def __init__(self, *args, **kwargs):
        api.Creator.__init__(self, *args, **kwargs)
        self.nodes = list()

    def process(self):
        """This is the base functionality to create instances in Houdini

        The selected nodes are stored in self to be used in an override method.
        This is currently necessary in order to support the multiple output
        types in Houdini which can only be rendered through their own node.

        Default node type if none is given is `geometry`

        It also makes it easier to apply custom settings per instance type

        Example of override method for Alembic:

            def process(self):
                instance =  super(CreateEpicNode, self, process()
                # Set paramaters for Alembic node
                instance.setParms(
                    {"sop_path": "$HIP/%s.abc" % self.nodes[0]}
                )

        Returns:
            hou.Node

        """

        if (self.options or {}).get("useSelection"):
            self.nodes = hou.selectedNodes()

        # Get the node type and remove it from the data, not needed
        node_type = self.data.pop("node_type", None)
        if node_type is None:
            node_type = "geometry"

        # Get out node
        out = hou.node("out")
        instance = out.createNode(node_type, node_name=self.name)
        instance.moveToGoodPosition()

        lib.imprint(instance, self.data)

        return instance

def _on_scene_open(*args):
    api.emit("open", *[])

def _on_scene_new(*args):
    api.emit("new", *[])

def _on_scene_save(*args):
    api.emit("save", *[])

def on_houdini_initialize():

    main_window = hou.qt.mainWindow()
    self._parent = {main_window.objectName(): main_window}

def _register_callbacks():

    for handler, event in self._events.copy().items():
        if event is None:
            continue

        try:
            hou.hipFile.removeEventCallback(event)
        except RuntimeError as e:
            logger.info(e)

    self._events[_on_scene_save] = hou.hipFile.addEventCallback(_on_scene_save)

    self._events[_on_scene_new] = hou.hipFile.addEventCallback(_on_scene_new)

    self._events[_on_scene_open] = hou.hipFile.addEventCallback(_on_scene_open)
<commit_msg>improved callbacks<commit_after># Standard library
import sys
import importlib
import contextlib

# Pyblish libraries
import pyblish.api

# Host libraries
import hou

# Local libraries
from . import lib
from ..lib import logger
from avalon import api, schema

self = sys.modules[__name__]
self._has_been_setup = False
self._parent = None
self._events = dict()

main_container = "/obj/AVALON_CONTAINERS"
IS_HEADLESS = not hasattr(hou, "ui")

def install(config):
    """Setup integration
    Register plug-ins and integrate into the host

    """

    print("Registering callbacks")
    _register_callbacks()

    pyblish.api.register_host("houdini")
    pyblish.api.register_host("hython")
    pyblish.api.register_host("hpython")

    self._has_been_setup = True

    config = find_host_config(config)
    if hasattr(config, "install"):
        config.install()

def uninstall(config):
    """Uninstall Houdini-specific functionality of avalon-core.

    This function is called automatically on calling `api.uninstall()`.

    Args:
        config: configuration module

    """

    config = find_host_config(config)
    if hasattr(config, "uninstall"):
        config.uninstall()

    pyblish.api.deregister_host("hython")
    pyblish.api.deregister_host("hpython")
    pyblish.api.deregister_host("houdini")

def find_host_config(config):
    config_name = config.__name__
    try:
        config = importlib.import_module(config_name + ".houdini")
    except ImportError as exc:
        if str(exc) != "No module name {}".format(config_name + ".houdini"):
            raise
        config = None

    return config

def reload_pipeline(*args):
    """Attempt to reload pipeline at run-time.

    CAUTION: This is primarily for development and debugging purposes.

    """

    import importlib

    api.uninstall()

    for module in ("avalon.io",
                   "avalon.lib",
                   "avalon.pipeline",

                   "avalon.houdini.pipeline",
                   "avalon.houdini.lib",
                   "avalon.tools.loader.app",
                   "avalon.tools.creator.app",
                   "avalon.tools.manager.app",

                   # NOTE(marcus): These have circular depenendencies
                   #               that is preventing reloadability
                   # "avalon.tools.cbloader.delegates",
                   # "avalon.tools.cbloader.model",
                   # "avalon.tools.cbloader.widgets",
                   # "avalon.tools.cbloader.app",
                   # "avalon.tools.cbsceneinventory.model",
                   # "avalon.tools.cbsceneinventory.proxy",
                   # "avalon.tools.cbsceneinventory.app",
                   # "avalon.tools.projectmanager.dialogs",
                   # "avalon.tools.projectmanager.lib",
                   # "avalon.tools.projectmanager.model",
                   # "avalon.tools.projectmanager.style",
                   # "avalon.tools.projectmanager.widget",
                   # "avalon.tools.projectmanager.app",

                   "avalon.api",
                   "avalon.tools",
                   "avalon.houdini"):
        module = importlib.import_module(module)
        reload(module)

    self._parent = {hou.ui.mainQtWindow().objectName(): hou.ui.mainQtWindow()}

    import avalon.houdini
    api.install(avalon.houdini)

def _discover_gui():
    """Return the most desirable of the currently registered GUIs"""

    # Prefer last registered
    guis = reversed(pyblish.api.registered_guis())

    for gui in guis:
        try:
            gui = __import__(gui).show
        except (ImportError, AttributeError):
            continue
        else:
            return gui

def teardown():
    """Remove integration"""
    if not self._has_been_setup:
        return

    self._has_been_setup = False
    print("pyblish: Integration torn down successfully")

@contextlib.contextmanager
def maintained_selection():
    """Maintain selection during context
    Example:
        >>> with maintained_selection():
        ...     # Modify selection
        ...     node.setSelected(on=False, clear_all_selected=True)
        >>> # Selection restored
    """

    previous_selection = hou.selectedNodes()
    try:
        yield
    finally:
        if previous_selection:
            for node in previous_selection:
                node.setSelected(on=True)
        else:
            for node in previous_selection:
                node.setSelected(on=False)

def containerise(name,
                 namespace,
                 nodes,
                 context,
                 loader=None,
                 suffix=""):
    """Bundle `nodes` into an assembly and imprint it with metadata

    Containerisation enables a tracking of version, author and origin
    for loaded assets.

    In Houdini it is not possible to next goemetry nodes in geometry nodes
    directly. To counter this we place a Object Network node calles ROOT
    in the HOUDINI_CONTAINERS node.

    Arguments:
        name (str): Name of resulting assembly
        namespace (str): Namespace under which to host container
        nodes (list): Long names of nodes to containerise
        context (dict): Asset information
        loader (str, optional): Name of loader used to produce this container.
        suffix (str, optional): Suffix of container, defaults to `_CON`.

    Returns:
        container (str): Name of container assembly

    """

    # Check if the AVALON_CONTAINERS exists
    geo = hou.node(main_container)
    if geo is None:
        obj_network = hou.node("/obj")
        geo = obj_network.createNode("geo", node_name="AVALON_CONTAINERS")
        # Delete file node
        geo.node("file1").destroy()

    # Check if root Object Network exists
    container_network = hou.node("{}/ROOT".format(main_container))
    if container_network is None:
        container_network = geo.createNode("objnet", node_name="ROOT")

    container = hou.node("/obj/{}".format(name))
    data = {
        "schema": "avalon-core:container-2.0",
        "id": "pyblish.avalon.container",
        "name": name,
        "namespace": namespace,
        "loader": str(loader),
        "representation": str(context["representation"]["_id"]),
    }

    lib.imprint(container, data)

    # "Parent" the container under the container network
    hou.moveNodesTo([container], container_network)

    # Get the container and set good position
    container_network.node(name).moveToGoodPosition()

    return container

def parse_container(container, validate=True):
    """Return the container node's full container data.

    Args:
        container (str): A container node name.
        validate (bool): If True, validate schema and data

    Returns:
        dict: The container schema data for this container node.

    """
    data = lib.read(container)

    # Backwards compatibility pre-schemas for containers
    data["schema"] = data.get("schema", "avalon-core:container-1.0")

    # Append transient data
    data["objectName"] = container.path()
    data["node"] = container

    if validate:
        schema.validate(data)

    return data

def ls():
    containers = []
    for identifier in ("pyblish.avalon.container",
                       "pyblish.mindbender.container"):
        containers += lib.lsattr("id", identifier)

    for container in sorted(containers):
        data = parse_container(container)
        yield data

class Creator(api.Creator):
    """Creator plugin to create instances in Houdini

    To support the wide range of node types for render output (Alembic, VDB,
    Mantra) the Creator needs a node type to create the correct instance

    By default, if none is given, is `geometry`. An example of accepted node
    types: geometry, alembic, ifd (mantra)

    Please check the Houdini documentation for more node types.

    Tip: to find the exact node type to create press the `i` left of the node
    when hovering over a node. The information is visible under the name of
    the node.

    """

    def __init__(self, *args, **kwargs):
        api.Creator.__init__(self, *args, **kwargs)
        self.nodes = list()

    def process(self):
        """This is the base functionality to create instances in Houdini

        The selected nodes are stored in self to be used in an override method.
        This is currently necessary in order to support the multiple output
        types in Houdini which can only be rendered through their own node.

        Default node type if none is given is `geometry`

        It also makes it easier to apply custom settings per instance type

        Example of override method for Alembic:

            def process(self):
                instance =  super(CreateEpicNode, self, process()
                # Set paramaters for Alembic node
                instance.setParms(
                    {"sop_path": "$HIP/%s.abc" % self.nodes[0]}
                )

        Returns:
            hou.Node

        """

        if (self.options or {}).get("useSelection"):
            self.nodes = hou.selectedNodes()

        # Get the node type and remove it from the data, not needed
        node_type = self.data.pop("node_type", None)
        if node_type is None:
            node_type = "geometry"

        # Get out node
        out = hou.node("out")
        instance = out.createNode(node_type, node_name=self.name)
        instance.moveToGoodPosition()

        lib.imprint(instance, self.data)

        return instance

def on_file_event_callback(event):
    if event == hou.hipFileEventType.AfterLoad:
        api.emit("open", [event])
    elif event == hou.hipFileEventType.AfterSave:
        api.emit("save", [event])
    elif event == hou.hipFileEventType.BeforeSave:
        api.emit("before_save", [event])
    elif event == hou.hipFileEventType.AfterClear:
        api.emit("new", [event])

def on_houdini_initialize():

    main_window = hou.qt.mainWindow()
    self._parent = {main_window.objectName(): main_window}

def _register_callbacks():

    for handler, event in self._events.copy().items():
        if event is None:
            continue

        try:
            hou.hipFile.removeEventCallback(event)
        except RuntimeError as e:
            logger.info(e)

    self._events[on_file_event_callback] = hou.hipFile.addEventCallback(
        on_file_event_callback
    )
<|endoftext|>


<commit_before>#
# ------------------------------------------------------------
# Copyright (c) All rights reserved
# SiLab, Institute of Physics, University of Bonn
# ------------------------------------------------------------
#

import os

from basil.TL.TransferLayer import TransferLayer
from SiLibUSB import GetUSBBoards, SiUSBDevice

class SiUsb (TransferLayer):
    '''SiLab USB device
    '''
    BASE_ADDRESS_I2C = 0x00000
    HIGH_ADDRESS_I2C = BASE_ADDRESS_I2C + 256

    BASE_ADDRESS_EXTERNAL = 0x10000
    HIGH_ADDRESS_EXTERNAL = 0x10000 + 0x10000

    BASE_ADDRESS_BLOCK = 0x0001000000000000
    HIGH_ADDRESS_BLOCK = 0xffffffffffffffff

    _sidev = None

    def __init__(self, conf):
        super(SiUsb, self).__init__(conf)

    def init(self, **kwargs):
        self._init.setdefault('board_id', None)
        self._init.setdefault('avoid_download', False)
        if self._init['board_id'] and int(self._init['board_id']) >= 0:
            self._sidev = SiUSBDevice.from_board_id(self._init['board_id'])
        else:
            # search for any available device
            devices = GetUSBBoards()
            if not devices:
                raise IOError('Can\'t find USB board. Connect or reset USB board!')
            else:
                print 'Found following USB boards: {}'.format(', '.join(('%s with ID %s (FW %s)' % (device.board_name, filter(type(device.board_id).isdigit, device.board_id), filter(type(device.fw_version).isdigit, device.fw_version))) for device in devices))
                if len(devices) > 1:
                    raise ValueError('Please specify ID of USB board')
                self._sidev = devices[0]
        if 'bit_file' in self._init.keys():
            if 'avoid_download' in self._init.keys() and self._init['avoid_download'] is True and self._sidev.XilinxAlreadyLoaded():
                print "FPGA already programmed, skipping download"
            else:
                if os.path.exists(self._init['bit_file']):
                    bit_file = self._init['bit_file']
                elif os.path.exists(os.path.join(os.path.dirname(self.parent.conf_path), self._init['bit_file'])):
                    bit_file = os.path.join(os.path.dirname(self.parent.conf_path), self._init['bit_file'])
                else:
                    raise ValueError('No such bit file: %s' % self._init['bit_file'])
                print "Programming FPGA: %s ... %s" % (self._init['bit_file'], 'SUCCESS' if self._sidev.DownloadXilinx(bit_file) else 'FAILED')
        else:
            if not self._sidev.XilinxAlreadyLoaded():
                raise ValueError('FPGA not initialized, bit_file not specified')
            else:
                print "Programming FPGA: bit_file not specified"

    def write(self, addr, data):
        if(addr >= self.BASE_ADDRESS_I2C and addr < self.HIGH_ADDRESS_I2C):
            self._sidev.WriteI2C(addr - self.BASE_ADDRESS_I2C, data)
        elif(addr >= self.BASE_ADDRESS_EXTERNAL and addr < self.HIGH_ADDRESS_EXTERNAL):
            self._sidev.WriteExternal(addr - self.BASE_ADDRESS_EXTERNAL, data)
        elif(addr >= self.BASE_ADDRESS_BLOCK and addr < self.HIGH_ADDRESS_BLOCK):
            self._sidev.FastBlockWrite(data)

    def read(self, addr, size):
        if(addr >= self.BASE_ADDRESS_I2C and addr < self.HIGH_ADDRESS_I2C):
            return self._sidev.ReadI2C(addr - self.BASE_ADDRESS_I2C, size)
        elif(addr >= self.BASE_ADDRESS_EXTERNAL and addr < self.HIGH_ADDRESS_EXTERNAL):
            data = self._sidev.ReadExternal(addr - self.BASE_ADDRESS_EXTERNAL, size)
            return data
        elif(addr >= self.BASE_ADDRESS_BLOCK and addr < self.HIGH_ADDRESS_BLOCK):
            return self._sidev.FastBlockRead(size)

    def get_configuration(self):
        conf = dict(self._init)
        conf['board_id'] = self._sidev.board_id
        return conf

    def close(self):
        self._sidev.dispose()
<commit_msg>ENH: use logging<commit_after>#
# ------------------------------------------------------------
# Copyright (c) All rights reserved
# SiLab, Institute of Physics, University of Bonn
# ------------------------------------------------------------
#

import logging
import os

from basil.TL.TransferLayer import TransferLayer
from SiLibUSB import GetUSBBoards, SiUSBDevice

class SiUsb (TransferLayer):
    '''SiLab USB device
    '''
    BASE_ADDRESS_I2C = 0x00000
    HIGH_ADDRESS_I2C = BASE_ADDRESS_I2C + 256

    BASE_ADDRESS_EXTERNAL = 0x10000
    HIGH_ADDRESS_EXTERNAL = 0x10000 + 0x10000

    BASE_ADDRESS_BLOCK = 0x0001000000000000
    HIGH_ADDRESS_BLOCK = 0xffffffffffffffff

    _sidev = None

    def __init__(self, conf):
        super(SiUsb, self).__init__(conf)

    def init(self, **kwargs):
        self._init.setdefault('board_id', None)
        self._init.setdefault('avoid_download', False)
        if self._init['board_id'] and int(self._init['board_id']) >= 0:
            self._sidev = SiUSBDevice.from_board_id(self._init['board_id'])
        else:
            # search for any available device
            devices = GetUSBBoards()
            if not devices:
                raise IOError('Can\'t find USB board. Connect or reset USB board!')
            else:
                logging.info('Found following USB boards: {}'.format(', '.join(('%s with ID %s (FW %s)' % (device.board_name, filter(type(device.board_id).isdigit, device.board_id), filter(type(device.fw_version).isdigit, device.fw_version))) for device in devices)))
                if len(devices) > 1:
                    raise ValueError('Please specify ID of USB board')
                self._sidev = devices[0]
        if 'bit_file' in self._init.keys():
            if 'avoid_download' in self._init.keys() and self._init['avoid_download'] is True and self._sidev.XilinxAlreadyLoaded():
                logging.info("FPGA already programmed, skipping download")
            else:
                if os.path.exists(self._init['bit_file']):
                    bit_file = self._init['bit_file']
                elif os.path.exists(os.path.join(os.path.dirname(self.parent.conf_path), self._init['bit_file'])):
                    bit_file = os.path.join(os.path.dirname(self.parent.conf_path), self._init['bit_file'])
                else:
                    raise ValueError('No such bit file: %s' % self._init['bit_file'])
                logging.info("Programming FPGA: %s..." % (self._init['bit_file']))
                status = self._sidev.DownloadXilinx(bit_file)
                logging.log(logging.INFO if status else logging.ERROR, 'Success!' if status else 'Failed!')
        else:
            if not self._sidev.XilinxAlreadyLoaded():
                raise ValueError('FPGA not initialized, bit_file not specified')
            else:
                logging.info("Programming FPGA: bit_file not specified")

    def write(self, addr, data):
        if(addr >= self.BASE_ADDRESS_I2C and addr < self.HIGH_ADDRESS_I2C):
            self._sidev.WriteI2C(addr - self.BASE_ADDRESS_I2C, data)
        elif(addr >= self.BASE_ADDRESS_EXTERNAL and addr < self.HIGH_ADDRESS_EXTERNAL):
            self._sidev.WriteExternal(addr - self.BASE_ADDRESS_EXTERNAL, data)
        elif(addr >= self.BASE_ADDRESS_BLOCK and addr < self.HIGH_ADDRESS_BLOCK):
            self._sidev.FastBlockWrite(data)

    def read(self, addr, size):
        if(addr >= self.BASE_ADDRESS_I2C and addr < self.HIGH_ADDRESS_I2C):
            return self._sidev.ReadI2C(addr - self.BASE_ADDRESS_I2C, size)
        elif(addr >= self.BASE_ADDRESS_EXTERNAL and addr < self.HIGH_ADDRESS_EXTERNAL):
            data = self._sidev.ReadExternal(addr - self.BASE_ADDRESS_EXTERNAL, size)
            return data
        elif(addr >= self.BASE_ADDRESS_BLOCK and addr < self.HIGH_ADDRESS_BLOCK):
            return self._sidev.FastBlockRead(size)

    def get_configuration(self):
        conf = dict(self._init)
        conf['board_id'] = self._sidev.board_id
        return conf

    def close(self):
        self._sidev.dispose()
<|endoftext|>


<commit_before>import json
from django.http import HttpResponse
from houdini_stats.models import *
from datetime import datetime, timedelta
from django.contrib.gis.geoip import GeoIP
import re

#-------------------------------------------------------------------------------

def text_http_response(content, status=200):
    """
    Translate a response into HTML text.
    """
    # FIXME (LM): Why doesn't Django set the Content-Length header?
    response = HttpResponse(content, status=status)
    response["Content-Length"] = str(len(response.content))
    return response

#-------------------------------------------------------------------------------

class ApiError(Exception):
    """
    Parent class for all api exceptions.  Requires an HTTP status
    code, an error message template, and optionally some formatting
    arguments for that template.
    """
    def __init__(self, status_code, msg_template, **kwargs):
        Exception.__init__(self, msg_template % kwargs)
        self.status_code = status_code

#-------------------------------------------------------------------------------

class ServerError(ApiError):
    """
    Internal error.
    """
    def __init__(self, msg_template, **kwargs):
        ApiError.__init__(self, 500, msg_template, **kwargs)

#---------------------------from datetime import datetime, timedelta-------------------------------------------------

def json_http_response(content, status=200):
    """
    Translate a response JSON and return.
    """
    return text_http_response(json.dumps(content), status=status)

#-------------------------------------------------------------------------------

def parse_byte_size_string(string):
    """
    Attempts to guess the string format based on default symbols
    set and return the corresponding bytes as an integer.
    When unable to recognize the format ValueError is raised.

      >>> parse_byte_size_string('1 KB')
      1024
      >>> parse_byte_size_string('2.2 GB')
      2362232012
    """

    if not string:
        return
    # Find out the numerical part.
    initial_string = string
    num_string = ""
    while string and string[0:1].isdigit() or string[0:1] == '.':
        num_string += string[0]
        string = string[1:]

    num = float(num_string)

    # Look for the suffix.
    suffix = string.strip() or "B"
    suffix_set = ('B', 'KB', 'MB', 'GB', 'TB', 'PB', 'EB', 'ZB', 'YB')

    prefix = {suffix_set[0]: 1}
    for i, string in enumerate(suffix_set[1:]):
        prefix[string] = 1 << (i+1)*10

    return int(num * prefix[suffix])

#-------------------------------------------------------------------------------

def str_to_datetime(str_date):
    """
    Converts a string into a datetime object
    """
    return datetime.strptime(str_date.strip(), "%a %b  %d %H:%M:%S %Y")

#-------------------------------------------------------------------------------

def date_range_to_seconds(datetime1, datetime2):
    """
    Computes the number of seconds between two datetime
    """
    return (datetime2 - datetime1).total_seconds()

#-------------------------------------------------------------------------------

def get_time(secs):
    """
    This function receives a number of seconds and return how many min, hours,
    days those seconds represent.
    """
    return {
        "seconds": int(secs),
        "minutes": round(int(secs) / 60.0),
        "hours": round(int(secs) / (60.0 * 60.0)),
        "days": round(int(secs) / (60.0 * 60.0 * 24.0)),
    }

#-------------------------------------------------------------------------------

def get_percent(part, whole):
    """
    Get which percentage is a from b, and round it to 2 decimal numbers.
    """
    return round(100 * float(part)/float(whole)  if whole !=0 else 0.0, 2)

#-------------------------------------------------------------------------------

def get_or_save_machine_config(user_info, ip_address):
    """
    Get or save if not already in db the machine config

    User Info:{ 'config_hash': '7ef9c42fe4d3748dc9aad755e02852d8',
                'houdini_build_version': '146',
                'houdini_major_version': '13',
                'houdini_minor_version': '0',
                'application_name': 'houdini',
                'operating_system': 'linux-x86_64-gcc4.7',
                'system_memory': '23.55 GB',
                'license_category': 'Commercial',
                'number_of_processors': '12',
                'graphics_card': 'Quadro 600/PCIe/SSE2',
                'graphics_card_version': '4.2.0 NVIDIA 304.88'
                'mac_address_hash'      : '05e8458a3e60776298ece4af002dcef7',
                ""
              }
    """

    # 1. Get or save Machine by hardware_id
    hardware_id = user_info.get('mac_address_hash','')

    try:
        machine = Machine.objects.get(hardware_id=hardware_id)

    except Machine.DoesNotExist:
        machine = Machine(hardware_id=hardware_id)
        machine.save()

    # 2. Get or save Machine Config
    config_hash = user_info['config_hash']

    try:
        machine_config = MachineConfig.objects.get(machine = machine,
                                                config_hash__exact=config_hash)
    except MachineConfig.DoesNotExist:

        sys_memory = user_info.get('system_memory', "0")
        product = user_info.get('application_name',"") + " " + user_info.get('license_category',"")

        # Create new machine config
        machine_config = MachineConfig(config_hash = config_hash,
             ip_address = ip_address,
             machine = machine,
             creation_date = datetime.now(),
             houdini_major_version = user_info.get('houdini_major_version',0),
             houdini_minor_version = user_info.get('houdini_minor_version',0),
             houdini_build_number = user_info.get('houdini_build_number',0),
             product = user_info.get('application_name',"").title(),
             is_apprentice = user_info.get('license_category',"") == 'Apprentice',
             graphics_card = user_info.get('graphics_card',''),
             graphics_card_version = user_info.get('graphics_card_version',''),
             operating_system = user_info.get('operating_system', ""),
             system_memory = parse_byte_size_string(sys_memory),
             number_of_processors = user_info.get('number_of_processors',0),
             raw_user_info = str(user_info)
        )
        # Save the asset version status
        machine_config.save()

    return machine_config

#-------------------------------------------------------------------------------

def save_uptime(machine_config, num_seconds):
    """
    Create Uptime record and save it in DB.
    """
    uptime = Uptime(machine_config = machine_config,
                    date = datetime.now(),
                    number_of_seconds = num_seconds)
    uptime.save()

#-------------------------------------------------------------------------------

def save_tools_usage(machine_config, counts_dict):
    """
    Create HoudiniToolUsage object and save it in DB.

    Schema: tools|location|tool_name
    - location can be "shelf", "viewer/Object", "viewer/Sop",
      "network/Object", "network/Sop", etc.
    - tool_name can be "sop_box", or "SideFX::spaceship" or
      blank if it's a custom tool
    - the tool name can be followed by "(orbolt)" (if it's an orbolt tool) or
      "(custom_tool)" if it's a nameless custom tool.
    """

    is_asset = False
    is_custom = False

    for key, count in counts_dict.iteritems():

        for mode, name in HoudiniToolUsage.TOOL_CREATION_MODES:
            prefix = "tools/"+ name
            if key.startswith(prefix):

                # Find "|" to get tool creation mode
                pipe_pos = key.index("|")
                tool_creation_location = key[len(prefix)+1: pipe_pos]
                tool_name = key[pipe_pos +1:]

                # Verify if tool type is a custom_tool
                if "(custom_tool)" in tool_name:
                    tool_name = re.sub('[\(\)]', "", tool_name)
                    is_custom = True

                # Verify if tool type is an Orbolt asset
                elif "(orbolt)" in tool_name:
                    tool_name = tool_name.replace("(orbolt)","")
                    is_asset = True

                tools_usage = HoudiniToolUsage(machine_config = machine_config,
                                date = datetime.now(), tool_name = tool_name,
                                tool_creation_location = tool_creation_location,
                                tool_creation_mode= mode, count = count,
                                is_builtin = (not is_asset and not is_custom),
                                is_asset = is_asset)
                tools_usage.save()
                break

#-------------------------------------------------------------------------------

def save_crash(machine_config, crash_log):
    """
    Create a HoudiniCrash object and save it in DB..

    crash_log: {
                 u'log_data': u'Caught signal 11\\n\\nAP_Interface::
                              createCrashLog(UTsignalHandlerArg....'
                 u'log_type': u'crash',
                 u'logged_date_and_time': u'2013-08-20 16:25:43'
    }
    """
    crash = HoudiniCrash(machine_config = machine_config,
                  date = crash_log.get('logged_date_and_time',
                                       datetime.now()),
                  stack_trace = crash_log.get('log_data',""),
                  type = crash_log.get('log_type',"")
                  )
    crash.save()

#-------------------------------------------------------------------------------

def get_ip_address(request):
    """
    Get the ip address from the machine doing the request.
    """
    return request.META.get("REMOTE_ADDR", "0.0.0.0")
    <commit_msg>Simplifying the format of the crash data that's sent.<commit_after>import json
from django.http import HttpResponse
from houdini_stats.models import *
from datetime import datetime, timedelta
from django.contrib.gis.geoip import GeoIP
import re

#-------------------------------------------------------------------------------

def text_http_response(content, status=200):
    """
    Translate a response into HTML text.
    """
    # FIXME (LM): Why doesn't Django set the Content-Length header?
    response = HttpResponse(content, status=status)
    response["Content-Length"] = str(len(response.content))
    return response

#-------------------------------------------------------------------------------

class ApiError(Exception):
    """
    Parent class for all api exceptions.  Requires an HTTP status
    code, an error message template, and optionally some formatting
    arguments for that template.
    """
    def __init__(self, status_code, msg_template, **kwargs):
        Exception.__init__(self, msg_template % kwargs)
        self.status_code = status_code

#-------------------------------------------------------------------------------

class ServerError(ApiError):
    """
    Internal error.
    """
    def __init__(self, msg_template, **kwargs):
        ApiError.__init__(self, 500, msg_template, **kwargs)

#---------------------------from datetime import datetime, timedelta-------------------------------------------------

def json_http_response(content, status=200):
    """
    Translate a response JSON and return.
    """
    return text_http_response(json.dumps(content), status=status)

#-------------------------------------------------------------------------------

def parse_byte_size_string(string):
    """
    Attempts to guess the string format based on default symbols
    set and return the corresponding bytes as an integer.
    When unable to recognize the format ValueError is raised.

      >>> parse_byte_size_string('1 KB')
      1024
      >>> parse_byte_size_string('2.2 GB')
      2362232012
    """

    if not string:
        return
    # Find out the numerical part.
    initial_string = string
    num_string = ""
    while string and string[0:1].isdigit() or string[0:1] == '.':
        num_string += string[0]
        string = string[1:]

    num = float(num_string)

    # Look for the suffix.
    suffix = string.strip() or "B"
    suffix_set = ('B', 'KB', 'MB', 'GB', 'TB', 'PB', 'EB', 'ZB', 'YB')

    prefix = {suffix_set[0]: 1}
    for i, string in enumerate(suffix_set[1:]):
        prefix[string] = 1 << (i+1)*10

    return int(num * prefix[suffix])

#-------------------------------------------------------------------------------

def str_to_datetime(str_date):
    """
    Converts a string into a datetime object
    """
    return datetime.strptime(str_date.strip(), "%a %b  %d %H:%M:%S %Y")

#-------------------------------------------------------------------------------

def date_range_to_seconds(datetime1, datetime2):
    """
    Computes the number of seconds between two datetime
    """
    return (datetime2 - datetime1).total_seconds()

#-------------------------------------------------------------------------------

def get_time(secs):
    """
    This function receives a number of seconds and return how many min, hours,
    days those seconds represent.
    """
    return {
        "seconds": int(secs),
        "minutes": round(int(secs) / 60.0),
        "hours": round(int(secs) / (60.0 * 60.0)),
        "days": round(int(secs) / (60.0 * 60.0 * 24.0)),
    }

#-------------------------------------------------------------------------------

def get_percent(part, whole):
    """
    Get which percentage is a from b, and round it to 2 decimal numbers.
    """
    return round(100 * float(part)/float(whole)  if whole !=0 else 0.0, 2)

#-------------------------------------------------------------------------------

def get_or_save_machine_config(user_info, ip_address):
    """
    Get or save if not already in db the machine config

    User Info:{ 'config_hash': '7ef9c42fe4d3748dc9aad755e02852d8',
                'houdini_build_version': '146',
                'houdini_major_version': '13',
                'houdini_minor_version': '0',
                'application_name': 'houdini',
                'operating_system': 'linux-x86_64-gcc4.7',
                'system_memory': '23.55 GB',
                'license_category': 'Commercial',
                'number_of_processors': '12',
                'graphics_card': 'Quadro 600/PCIe/SSE2',
                'graphics_card_version': '4.2.0 NVIDIA 304.88'
                'mac_address_hash'      : '05e8458a3e60776298ece4af002dcef7',
                ""
              }
    """

    # 1. Get or save Machine by hardware_id
    hardware_id = user_info.get('mac_address_hash','')

    try:
        machine = Machine.objects.get(hardware_id=hardware_id)

    except Machine.DoesNotExist:
        machine = Machine(hardware_id=hardware_id)
        machine.save()

    # 2. Get or save Machine Config
    config_hash = user_info['config_hash']

    try:
        machine_config = MachineConfig.objects.get(machine = machine,
                                                config_hash__exact=config_hash)
    except MachineConfig.DoesNotExist:

        sys_memory = user_info.get('system_memory', "0")
        product = user_info.get('application_name',"") + " " + user_info.get('license_category',"")

        # Create new machine config
        machine_config = MachineConfig(config_hash = config_hash,
             ip_address = ip_address,
             machine = machine,
             creation_date = datetime.now(),
             houdini_major_version = user_info.get('houdini_major_version',0),
             houdini_minor_version = user_info.get('houdini_minor_version',0),
             houdini_build_number = user_info.get('houdini_build_number',0),
             product = user_info.get('application_name',"").title(),
             is_apprentice = user_info.get('license_category',"") == 'Apprentice',
             graphics_card = user_info.get('graphics_card',''),
             graphics_card_version = user_info.get('graphics_card_version',''),
             operating_system = user_info.get('operating_system', ""),
             system_memory = parse_byte_size_string(sys_memory),
             number_of_processors = user_info.get('number_of_processors',0),
             raw_user_info = str(user_info)
        )
        # Save the asset version status
        machine_config.save()

    return machine_config

#-------------------------------------------------------------------------------

def save_uptime(machine_config, num_seconds):
    """
    Create Uptime record and save it in DB.
    """
    uptime = Uptime(machine_config = machine_config,
                    date = datetime.now(),
                    number_of_seconds = num_seconds)
    uptime.save()

#-------------------------------------------------------------------------------

def save_tools_usage(machine_config, counts_dict):
    """
    Create HoudiniToolUsage object and save it in DB.

    Schema: tools|location|tool_name
    - location can be "shelf", "viewer/Object", "viewer/Sop",
      "network/Object", "network/Sop", etc.
    - tool_name can be "sop_box", or "SideFX::spaceship" or
      blank if it's a custom tool
    - the tool name can be followed by "(orbolt)" (if it's an orbolt tool) or
      "(custom_tool)" if it's a nameless custom tool.
    """

    is_asset = False
    is_custom = False

    for key, count in counts_dict.iteritems():

        for mode, name in HoudiniToolUsage.TOOL_CREATION_MODES:
            prefix = "tools/"+ name
            if key.startswith(prefix):

                # Find "|" to get tool creation mode
                pipe_pos = key.index("|")
                tool_creation_location = key[len(prefix)+1: pipe_pos]
                tool_name = key[pipe_pos +1:]

                # Verify if tool type is a custom_tool
                if "(custom_tool)" in tool_name:
                    tool_name = re.sub('[\(\)]', "", tool_name)
                    is_custom = True

                # Verify if tool type is an Orbolt asset
                elif "(orbolt)" in tool_name:
                    tool_name = tool_name.replace("(orbolt)","")
                    is_asset = True

                tools_usage = HoudiniToolUsage(machine_config = machine_config,
                                date = datetime.now(), tool_name = tool_name,
                                tool_creation_location = tool_creation_location,
                                tool_creation_mode= mode, count = count,
                                is_builtin = (not is_asset and not is_custom),
                                is_asset = is_asset)
                tools_usage.save()
                break

#-------------------------------------------------------------------------------

def save_crash(machine_config, crash_log):
    """
    Create a HoudiniCrash object and save it in DB..

    crash_log: {
                 u'traceback': u'Caught signal 11\\n\\nAP_Interface::
                              createCrashLog(UTsignalHandlerArg....'
    }
    """
    crash = HoudiniCrash(
        machine_config=machine_config,
        date=datetime.now(),
        stack_trace=crash_log['traceback'],
        type="crash",
    )
    crash.save()

#-------------------------------------------------------------------------------

def get_ip_address(request):
    """
    Get the ip address from the machine doing the request.
    """
    return request.META.get("REMOTE_ADDR", "0.0.0.0")

<|endoftext|>