<commit_before>#!/usr/bin/env python
# -*- coding: UTF-8 -*-
'''
Quantile Regression and diagnostics

http://pastebin.com/egLfUkms
translated from matlab by Christian Prinoth
see pystatsmodel mailing list

'''
import numpy as np
import scipy.stats as stats
import statsmodels.api as sm
from scipy.linalg import inv
from statsmodels.tools.tools import chain_dot as dot

#function [beta tstats VCboot itrat PseudoR2 betaboot]=quantilereg(y,x,p)
def quantilereg(y,x,p):
    '''
     estimates quantile regression based on weighted least squares.
     constant term is added to x matrix
    __________________________________________________________________________

     Inputs:
      y: Dependent variable
      x: matrix of independent variables
      p: quantile

     Outputs:
      beta:     estimated Coefficients.
      tstats:   T- students of the coefficients.
      VCboot:   Variance-Covariance of coefficients by bootstrapping method.
      itrat:    number of iterations for convergence to roots.
      PseudoR2: in quatile regression another definition of R2 is used namely PseudoR2.
      betaboot: estimated coefficients by bootstrapping method.

     This code can be used for quantile regression estimation as whole,and LAD
     regression as special case of it, when one sets p=0.5.

     Copyright(c) Shapour Mohammadi, University of Tehran, 2008
     shmohammadi@gmail.com

     Translated to python with permission from original author by Christian Prinoth (christian at prinoth dot name)

    Ref:
     1-Birkes, D. and Y. Dodge(1993). Alternative Methods of Regression, John Wiley and Sons.
     2-Green,W. H. (2008). Econometric Analysis. Sixth Edition. International Student Edition.
     3-LeSage, J. P.(1999),Applied Econometrics Using MATLAB,

     Keywords: Least Absolute Deviation(LAD) Regression, Quantile Regression,
     Regression, Robust Estimation.
    __________________________________________________________________________
    '''

    tstats=0
    VCboot=0
    itrat=0
    PseudoR2=0
    betaboot=0

    ry=len(y)
    rx, cx=x.shape
    x=np.c_[np.ones(rx),x]
    cx=cx+1
    #______________Finding first estimates by solving the system_______________
    # Some lines of this section is based on a code written by
    # James P. Lesage in Applied Econometrics Using MATLAB(1999).PP. 73-4.
    itrat=0
    xstar=x
    diff=1
    beta=np.ones(cx)
    z=np.zeros((rx,cx))
    while itrat<1000 and diff>1e-6:
        itrat+=1
        beta0=beta
        beta=dot(inv(dot(xstar.T,x)),xstar.T,y)
        resid=y-dot(x,beta)
        resid[np.abs(resid)<.000001]=.000001
        resid[resid<0]=p*resid[resid<0]
        resid[resid>0]=(1-p)*resid[resid>0]
        resid=np.abs(resid)
        for i in range(cx):
            z[:,i] = x[:,i]/resid

        xstar=z
        beta1=beta
        diff=np.max(np.abs(beta1-beta0))

    return beta

    #_______estimating variances based on Green 2008(quantile regression)______

    e=y-dot(x,beta)
    iqre=np.percentile(e,0.75)-np.percentile(e,0.25)
    if p==0.5:
        h=0.9*np.std(e)/(ry**0.2)
    else:
        h=0.9*np.min(np.std(e),iqre/1.34)/(ry**0.2)
    u=(e/h)
    fhat0=(1/(ry*h))*(sum(exp(-u)/((1+exp(-u))**2)))
    D=np.zeros((ry,ry))
    DIAGON=np.diag(D)
    DIAGON[e>0]=(p/fhat0)**2
    DIAGON[e<=0]=((1-p)/fhat0)**2
    D=np.diag(DIAGON)
    VCQ=np.dot(inv(dot(x.T,x)),dot(x.T,D,x),inv(np.dot(x.T,x)))

    #____________________Standarad errores and t-stats_________________________

    tstats=beta/np.sqrt(np.diag(VCQ))
    stderrors=np.sqrt(np.diag(VCQ))
    PValues=2*(1-stats.t.cdf(np.abs(tstats),ry-cx))

    #______________________________ Quasi R square_____________________________

    ef=y-dot(x,beta)
    ef[ef<0]=(1-p)*ef[ef<0]
    ef[ef>0]=p*ef[ef>0]
    ef=np.abs(ef)

    ered=y-np.percentile(y,p)
    ered[ered<0]=(1-p)*ered[ered<0]
    ered[ered>0]=p*ered[ered>0]
    ered=np.abs(ered)

    PseudoR2=1-np.sum(ef)/np.sum(ered)

    #__________________Bootstrap standard deviation (Green 2008)_______________

    betaboot=np.zeros((cx,cx))
    for ii in range(100):
        bootm, estar=bootstrp(1,np.mean,e)
        #
        ystar=dot(x,beta)+e[estar]
        #
        itratstar=0
        xstarstar=x
        diffstar=1
        betastar=np.ones(cx)
        while itratstar<1000 and diffstar>1e-6:
            itratstar=itratstar+1
            betastar0=betastar
            betastar=dot(inv(dot(xstarstar.T,x)),xstarstar.T,ystar)
            #
            residstar=ystar-dot(x,betastar)
            residstar[np.abs(residstar)<.000001]=.000001
            residstar[residstar<0]=p*residstar[residstar<0]
            residstar[residstar>0]=(1-p)*residstar[residstar>0]
            residstar=np.abs(residstar)
            zstar=np.zeros((rx,cx))
            for i in range(cx):
                zstar[:,i] = x[:,i]/residstar
            xstarstar=zstar
            beta1star=betastar
            diffstar=np.max(np.abs(beta1star-betastar0))
        #
        betaboot=[betaboot + dot((betastar-beta),(betastar-beta).T)]
    VCboot=(1/100)*betaboot
    #
    tstatsboot=beta/diag(VCboot)**0.5
    stderrorsboot=diag(VCboot)**0.5
    PValuesboot=2*(1-stats.t.cdf(np.abs(tstatsboot),ry-cx))

    #_______________________________Display Results____________________________

    print
    print(' Results of Quantile Regression')
    print('_'*70)
    print("%10s %10s %10s %10s %10s %10s %10s" % ['Coef.', 'SE.Ker', 't.Ker', 'P.Ker', 'SE.Boot', 't.Boot', 'P.Boot'])
    print('_'*70)
    print("%10f %10f %10f %10f %10f %10f %10f" % [ beta,stderrors,tstats,PValues,stderrorsboot,tstatsboot,PValuesboot])
    print('_'*70)
    print('Pseudo R2: %10f' % PseudoR2 )
    print('_'*70)
<commit_msg>STY: add whitespace, fix some undefined objects in unused code (pylint)<commit_after>#!/usr/bin/env python
# -*- coding: UTF-8 -*-
'''
Quantile Regression and diagnostics

http://pastebin.com/egLfUkms
translated from matlab by Christian Prinoth
see pystatsmodel mailing list

'''
import numpy as np
import scipy.stats as stats
import statsmodels.api as sm
from scipy.linalg import inv
from statsmodels.tools.tools import chain_dot as dot

#function [beta tstats VCboot itrat PseudoR2 betaboot]=quantilereg(y,x,p)
def quantilereg(y,x,p):
    '''
     estimates quantile regression based on weighted least squares.
     constant term is added to x matrix
    __________________________________________________________________________

     Inputs:
      y: Dependent variable
      x: matrix of independent variables
      p: quantile

     Outputs:
      beta:     estimated Coefficients.
      tstats:   T- students of the coefficients.
      VCboot:   Variance-Covariance of coefficients by bootstrapping method.
      itrat:    number of iterations for convergence to roots.
      PseudoR2: in quatile regression another definition of R2 is used namely PseudoR2.
      betaboot: estimated coefficients by bootstrapping method.

     This code can be used for quantile regression estimation as whole,and LAD
     regression as special case of it, when one sets p=0.5.

     Copyright(c) Shapour Mohammadi, University of Tehran, 2008
     shmohammadi@gmail.com

     Translated to python with permission from original author by Christian Prinoth (christian at prinoth dot name)

    Ref:
     1-Birkes, D. and Y. Dodge(1993). Alternative Methods of Regression, John Wiley and Sons.
     2-Green,W. H. (2008). Econometric Analysis. Sixth Edition. International Student Edition.
     3-LeSage, J. P.(1999),Applied Econometrics Using MATLAB,

     Keywords: Least Absolute Deviation(LAD) Regression, Quantile Regression,
     Regression, Robust Estimation.
    __________________________________________________________________________
    '''

    tstats = 0
    VCboot = 0
    itrat = 0
    PseudoR2 = 0
    betaboot = 0

    ry = len(y)
    rx, cx = x.shape
    x = np.c_[np.ones(rx), x]
    cx = cx + 1
    #______________Finding first estimates by solving the system_______________
    # Some lines of this section is based on a code written by
    # James P. Lesage in Applied Econometrics Using MATLAB(1999).PP. 73-4.
    itrat = 0
    xstar = x
    diff = 1
    beta = np.ones(cx)
    z = np.zeros((rx, cx))
    while itrat < 1000 and diff > 1e-6:
        itrat += 1
        beta0 = beta
        beta = dot(inv(dot(xstar.T, x)), xstar.T, y)
        resid = y - dot(x, beta)
        resid[np.abs(resid) < .000001] = .000001
        resid[resid < 0] = p * resid[resid < 0]
        resid[resid > 0] = (1 - p) * resid[resid > 0]
        resid = np.abs(resid)

        for i in range(cx):
            z[:,i] = x[:,i] / resid

        xstar = z
        beta1 = beta
        diff = np.max(np.abs(beta1 - beta0))

    return beta

    #_______estimating variances based on Green 2008(quantile regression)______

    e = y - dot(x, beta)
    iqre = np.percentile(e, 0.75) - np.percentile(e, 0.25)
    if p == 0.5:
        h = 0.9 * np.std(e) / (ry**0.2)
    else:
        h = 0.9 * np.min(np.std(e), iqre / 1.34) / (ry**0.2)

    u = e / h
    fhat0 = (1. / (ry * h)) * (np.sum(np.exp(-u) / ((1 + np.exp(-u))**2)))
    D = np.zeros((ry, ry))
    DIAGON = np.diag(D)
    DIAGON[e > 0] = (p / fhat0)**2
    DIAGON[e <= 0] = ((1-p) / fhat0)**2
    D = np.diag(DIAGON)
    VCQ = np.dot(inv(dot(x.T, x)), dot(x.T, D, x), inv(np.dot(x.T, x)))
    #BUG:dot with 3 arguments

    #____________________Standarad errores and t-stats_________________________

    tstats = beta / np.sqrt(np.diag(VCQ))
    stderrors = np.sqrt(np.diag(VCQ))
    PValues = 2 * (1 - stats.t.cdf(np.abs(tstats), ry - cx))

    #______________________________ Quasi R square_____________________________

    ef = y - dot(x, beta)
    ef[ef < 0] = (1 - p) * ef[ef < 0]
    ef[ef > 0] = p * ef[ef > 0]
    ef = np.abs(ef)

    ered = y-np.percentile(y,p)
    ered[ered < 0] = (1-p) * ered[ered < 0]
    ered[ered > 0] = p * ered[ered > 0]
    ered = np.abs(ered)

    PseudoR2 = 1 - np.sum(ef) / np.sum(ered)

    #__________________Bootstrap standard deviation (Green 2008)_______________

    betaboot = np.zeros((cx, cx))
    for ii in range(100):
        bootm, estar = bootstrp(1, np.mean, e)
        #BUG: bootstrp undefined, matlab function ?
        #
        ystar = dot(x, beta) + e[estar]
        #
        itratstar = 0
        xstarstar = x
        diffstar = 1
        betastar = np.ones(cx)
        while itratstar < 1000 and diffstar > 1e-6:
            itratstar = itratstar + 1
            betastar0 = betastar
            betastar = dot(inv(dot(xstarstar.T, x)), xstarstar.T, ystar)
            #
            residstar = ystar - dot(x, betastar)
            residstar[np.abs(residstar) < .000001] = .000001
            residstar[residstar < 0] = p * residstar[residstar < 0]
            residstar[residstar > 0] = (1 - p) * residstar[residstar > 0]
            residstar = np.abs(residstar)
            zstar = np.zeros((rx, cx))

            for i in range(cx):
                zstar[:,i] = x[:,i] / residstar

            xstarstar = zstar
            beta1star = betastar
            diffstar = np.max(np.abs(beta1star - betastar0))
        #
        betaboot = [betaboot + dot((betastar - beta), (betastar - beta).T)]

    VCboot = betaboot / 100.
    #
    tstatsboot = beta / np.diag(VCboot)**0.5
    stderrorsboot = np.diag(VCboot)**0.5
    PValuesboot = 2 * (1 - stats.t.cdf(np.abs(tstatsboot), ry - cx))

    #_______________________________Display Results____________________________

    print
    print(' Results of Quantile Regression')
    print('_'*70)
    print("%10s %10s %10s %10s %10s %10s %10s" % ('Coef.', 'SE.Ker', 't.Ker',
                                                  'P.Ker', 'SE.Boot', 't.Boot',
                                                  'P.Boot'))
    print('_'*70)
    print("%10f %10f %10f %10f %10f %10f %10f" % (beta, stderrors, tstats,
                                                  PValues, stderrorsboot,
                                                  tstatsboot, PValuesboot))
    print('_'*70)
    print('Pseudo R2: %10f' % PseudoR2 )
    print('_'*70)
<|endoftext|>


<commit_before>#!/usr/bin/env python3

class TextWrapper:
    """
    A simple wrapper that creates a class with a render method
    for just text
    This allows the Element classes to render either Element objects or
    plain text
    """
    def __init__(self, text):
        self.text = text

    def render(self, file_out, current_ind=""):
        file_out.write(current_ind + self.text)

class Element:

    tag = 'html'
    indent = "    "

    def __init__(self, content=None, **kwargs):
        self.attributes = kwargs
        self.content = []
        if content:
            self.append(content)

    def append(self, content):
        if hasattr(content, 'render'):
            self.content.append(content)
        else:
            self.content.append(TextWrapper(str(content)))

    def render(self, out_file, ind=''):
        open_tag, close_tag = self.make_tags()
        out_file.write(ind + open_tag + '\n')
        for stuff in self.content:
            stuff.render(out_file, ind + self.indent)
            out_file.write('\n')
        out_file.write(ind + close_tag)

    def make_tags(self):
        """
        create the tags
        -- in a separate method so different subclass's render methods can use it
        """
        attrs = " ".join(['{}="{}"'.format(key, val) for key, val in self.attributes.items()])
        if attrs.strip():
            open_tag = "<{} {}>".format(self.tag, attrs.strip())
        else:
            open_tag = "<{}>".format(self.tag)
        close_tag = "</{}>".format(self.tag)

        return open_tag, close_tag

class OneLineTag(Element):
    # note: by re-writting the render
    def render(self, out_file, ind=""):
        # there is some repition here -- maybe factor that out?
        open_tag, close_tag = self.make_tags()
        out_file.write(ind + open_tag)
        for stuff in self.content:
            stuff.render(out_file)
        out_file.write(close_tag)

class Html(Element):
    tag = 'html'

class Body(Element):
    tag = 'body'

class P(Element):
    tag = 'p'

class Head(Element):
    tag = 'head'

class Title(OneLineTag):
    tag = 'title'

class SelfClosingTag(Element):
    """
    base class for tags that have no content
    """
    def render(self, out_file, ind=""):
        # there is some repition here -- maybe factor that out?
        open_tag, _ = self.make_tags()
        # make it a self cloding tag by adding the /
        out_file.write(ind + open_tag.replace(">", " />"))

class Hr(SelfClosingTag):
    tag = 'hr'

class Br(SelfClosingTag):
    tag = 'br'

class A(OneLineTag):
    """
    anchor element
    """
    tag = "a"

    def __init__(self, link, content, **kwargs):
        kwargs['href'] = link
        super().__init__(content, **kwargs)
        # this could also be direct:
        # Element.__init__(self, content, **kwargs)

class Ul(Element):
    """
    unordered list
    """
    tag = "ul"

class Li(Element):
    """
    list element
    """
    tag = "li"

class H(OneLineTag):
    """
    section head
    """
    tag = "H"

    def __init__(self, level, content=None, **kwargs):
        self.tag = "h" + str(int(level))
        super().__init__(content, **kwargs)

class Meta(SelfClosingTag):
    """
    metadata tag
    """
    tag = "meta"

# print(help(Body))
<commit_msg>Fixing conflict<commit_after>#!/usr/bin/env python3

class Element:
    tag = 'html'
    def __init__(self, content=None):
        self.content = []
        if content:
            self.content.append(content)

    def append(self, content):
        self.content.append(content)

    def render(self, out_file, ind=''):
        out_file.write("<{}>\n".format(self.tag))
        for stuff in self.content:
            out_file.write(stuff+"\n")
        out_file.write("</{}>\n".format(self.tag))

class Html(Element):
    tag = 'html'

class Body(Element):
    tag = 'body'

class P(Element):
    tag = 'p'

# class Element:

#     tag = 'aaa'

#     def __init__(self, content=None):
#         self.content = []
#         if content:
#             self.content.append(content)

#     def append(self, content):
#         self.content.append(content)

#     def render(self, out_file, ind=''):
#         if self.tag == 'html':
#             out_file.write('\n')
#         for stuff in self.content:
#             out_file.write(stuff + '\n')

#         # if self.tag == 'body':
#         #     out_file.write('\n')
#         # for stuff in self.content:
#         #     out_file.write(stuff + '\n')
#         # if self.tag == 'body':
#         #     out_file.write('\n')

#         if self.tag == 'html':
#             out_file.write('\n')
<|endoftext|>


<commit_before>#
# Copyright (c) 2005-2006 rPath, Inc.
#
# This program is distributed under the terms of the Common Public License,
# version 1.0. A copy of this license should have been distributed with this
# source file in a file called LICENSE. If it is not present, the license
# is always available at http://www.opensource.org/licenses/cpl.php.
#
# This program is distributed in the hope that it will be useful, but
# without any waranty; without even the implied warranty of merchantability
# or fitness for a particular purpose. See the Common Public License for
# full details.
#

import os
import tempfile
import cPickle

from conary import dbstore
from conary.local import schema, sqldb, versiontable
from conary.dbstore import idtable

CACHE_SCHEMA_VERSION = 17

class NullCacheSet:
    def getEntry(self, item, recurse, withFiles, withFileContents,
                 excludeAutoSource):
        return None

    def addEntry(self, item, recurse, withFiles, withFileContents,
                 excludeAutoSource, returnVal, size):
        (fd, path) = tempfile.mkstemp(dir = self.tmpDir,
                                      suffix = '.ccs-out')
        os.close(fd)
        return None, path

    def invalidateEntry(self, name, version, flavor):
        pass

    def __init__(self, tmpDir):
        self.tmpDir = tmpDir

class CacheSet:
    filePattern = "%s/cache-%s.ccs-out"

    def getEntry(self, item, recurse, withFiles, withFileContents,
                 excludeAutoSource):
        (name, (oldVersion, oldFlavor), (newVersion, newFlavor), absolute) = \
            item

        oldVersionId = 0
        oldFlavorId = 0
        newFlavorId = 0

        if oldVersion:
            oldVersionId = self.versions.get(oldVersion, None)
            if oldVersionId is None:
                return None

        if oldFlavor:
            oldFlavorId = self.flavors.get(oldFlavor, None)
            if oldFlavorId is None:
                return None

        if newFlavor:
            newFlavorId = self.flavors.get(newFlavor, None)
            if newFlavorId is None:
                return None

        newVersionId = self.versions.get(newVersion, None)
        if newVersionId is None:
            return None

        cu = self.db.cursor()
        cu.execute("""
            SELECT row, returnValue, size FROM CacheContents WHERE
                troveName=? AND
                oldFlavorId=? AND oldVersionId=? AND
                newFlavorId=? AND newVersionId=? AND
                absolute=? AND recurse=? AND withFiles=?
                AND withFileContents=? AND excludeAutoSource=?
            """, (name, oldFlavorId, oldVersionId, newFlavorId,
                  newVersionId, absolute, recurse, withFiles, withFileContents,
                  excludeAutoSource))

        # since we begin and commit a transaction inside the loop
        # over the returned rows, we must use fetchall() here so that we
        # release our read lock.
        for (row, returnVal, size) in cu.fetchall():
            path = self.filePattern % (self.tmpDir, row)
            # if we have no size or we can't access the file, it's
            # bad entry.  delete it.
            if not size or not os.access(path, os.R_OK):
                cu.execute("DELETE FROM CacheContents WHERE row=?", row)
                self.db.commit()
                continue
            return (path, cPickle.loads(returnVal), size)

        return None

    def addEntry(self, item, recurse, withFiles, withFileContents,
                 excludeAutoSource, returnVal, size):
        (name, (oldVersion, oldFlavor), (newVersion, newFlavor), absolute) = \
            item

        oldVersionId = 0
        oldFlavorId = 0
        newFlavorId = 0

        # start a transaction now to avoid race conditions when getting
        # or adding IDs for versions and flavors
        cu = self.db.transaction()

        try:
            if oldVersion:
                oldVersionId = self.versions.get(oldVersion, None)
                if oldVersionId is None:
                    oldVersionId = self.versions.addId(oldVersion)

            if oldFlavor:
                oldFlavorId = self.flavors.get(oldFlavor, None)
                if oldFlavorId is None:
                    oldFlavorId = self.flavors.addId(oldFlavor)

            if newFlavor:
                newFlavorId = self.flavors.get(newFlavor, None)
                if newFlavorId is None:
                    newFlavorId = self.flavors.addId(newFlavor)

            newVersionId = self.versions.get(newVersion, None)
            if newVersionId is None:
                newVersionId = self.versions.addId(newVersion)

            cu.execute("""
            INSERT INTO CacheContents
            (row, troveName, size,
            oldFlavorId, oldVersionId, newFlavorId, newVersionId,
            absolute, recurse, withFiles, withFileContents,
            excludeAutoSource, returnValue )
            VALUES (NULL,?,?,   ?,?,?,?,   ?,?,?,?,   ?,?)""",
                       (name, size,
                       oldFlavorId, oldVersionId, newFlavorId, newVersionId,
                       absolute, recurse, withFiles, withFileContents,
                       excludeAutoSource, cPickle.dumps(returnVal,
                                                        protocol = -1)))

            row = cu.lastrowid
            path = self.filePattern % (self.tmpDir, row)

            self.db.commit()
        except:
            # something went wrong.  make sure that we roll
            # back any pending change
            self.db.rollback()
            raise

        return (row, path)

    def invalidateEntry(self, name, version, flavor):
        """
        invalidates (and deletes) any cached changeset that matches
        the given name, version, flavor.
        """
        flavorId = self.flavors.get(flavor, None)
        versionId = self.versions.get(version, None)

        if flavorId is None or versionId is None:
            # this should not happen, but we'll handle it anyway
            return

        # start a transaction to retain a consistent state
        cu = self.db.transaction()
        cu.execute("""
        SELECT row, returnValue, size
        FROM CacheContents
        WHERE troveName=? AND newFlavorId=? AND newVersionId=?
        """, (name, flavorId, versionId))

        # delete all matching entries from the db and the file system
        for (row, returnVal, size) in cu.fetchall():
            cu.execute("DELETE FROM CacheContents WHERE row=?", row)
            path = self.filePattern % (self.tmpDir, row)
            if os.path.exists(path):
                try:
                    os.unlink(path)
                except OSError:
                    pass
        self.db.commit()

    def __cleanCache(self):
        cu = self.db.cursor()
        cu.execute("SELECT row from CacheContents")
        for (row,) in cu:
            fn = self.filePattern % (self.tmpDir, row)
            if os.path.exists(fn):
                try:
                    os.unlink(fn)
                except OSError:
                    pass

    def createSchema(self, dbInfo, schemaVersion):
        self.db = dbstore.connect(dbInfo[1], driver = dbInfo[0])
        cu = self.db.cursor()
        if "CacheContents" in self.db.tables:
            if self.db.version != CACHE_SCHEMA_VERSION:
                self.__cleanCache()
                for t in self.db.tables:
                    cu.execute("DROP TABLE %s" % (t,))
                self.db.setVersion(CACHE_SCHEMA_VERSION)

        if "CacheContents" not in self.db.tables:
            cu.execute("""
            CREATE TABLE CacheContents(
               row              %(PRIMARYKEY)s,
               troveName        VARCHAR(254),
               oldFlavorId      INTEGER,
               oldVersionId     INTEGER,
               newFlavorId      INTEGER,
               newVersionId     INTEGER,
               absolute         BOOLEAN,
               recurse          BOOLEAN,
               withFiles        BOOLEAN,
               withFileContents BOOLEAN,
               excludeAutoSource BOOLEAN,
               returnValue      BINARY,
               size             INTEGER
            )""" % self.db.keywords)
            cu.execute("""
            CREATE INDEX CacheContentsIdx ON
                CacheContents(troveName)
            """)
            schema.createFlavors(self.db)

            cu.execute("CREATE TABLE Versions(
                versionId       %(PRIMARYKEY)s,
                version         VARCHAR(767)
            )""" % self.db.keywords)

            self.db.commit()

    def __init__(self, cacheDB, tmpDir):
	self.tmpDir = tmpDir
        self.createSchema(cacheDB, CACHE_SCHEMA_VERSION)
        self.flavors = sqldb.Flavors(self.db)
        self.versions = versiontable.VersionTable(self.db)

<commit_msg>use correct quoting<commit_after>#
# Copyright (c) 2005-2006 rPath, Inc.
#
# This program is distributed under the terms of the Common Public License,
# version 1.0. A copy of this license should have been distributed with this
# source file in a file called LICENSE. If it is not present, the license
# is always available at http://www.opensource.org/licenses/cpl.php.
#
# This program is distributed in the hope that it will be useful, but
# without any waranty; without even the implied warranty of merchantability
# or fitness for a particular purpose. See the Common Public License for
# full details.
#

import os
import tempfile
import cPickle

from conary import dbstore
from conary.local import schema, sqldb, versiontable
from conary.dbstore import idtable

CACHE_SCHEMA_VERSION = 17

class NullCacheSet:
    def getEntry(self, item, recurse, withFiles, withFileContents,
                 excludeAutoSource):
        return None

    def addEntry(self, item, recurse, withFiles, withFileContents,
                 excludeAutoSource, returnVal, size):
        (fd, path) = tempfile.mkstemp(dir = self.tmpDir,
                                      suffix = '.ccs-out')
        os.close(fd)
        return None, path

    def invalidateEntry(self, name, version, flavor):
        pass

    def __init__(self, tmpDir):
        self.tmpDir = tmpDir

class CacheSet:
    filePattern = "%s/cache-%s.ccs-out"

    def getEntry(self, item, recurse, withFiles, withFileContents,
                 excludeAutoSource):
        (name, (oldVersion, oldFlavor), (newVersion, newFlavor), absolute) = \
            item

        oldVersionId = 0
        oldFlavorId = 0
        newFlavorId = 0

        if oldVersion:
            oldVersionId = self.versions.get(oldVersion, None)
            if oldVersionId is None:
                return None

        if oldFlavor:
            oldFlavorId = self.flavors.get(oldFlavor, None)
            if oldFlavorId is None:
                return None

        if newFlavor:
            newFlavorId = self.flavors.get(newFlavor, None)
            if newFlavorId is None:
                return None

        newVersionId = self.versions.get(newVersion, None)
        if newVersionId is None:
            return None

        cu = self.db.cursor()
        cu.execute("""
            SELECT row, returnValue, size FROM CacheContents WHERE
                troveName=? AND
                oldFlavorId=? AND oldVersionId=? AND
                newFlavorId=? AND newVersionId=? AND
                absolute=? AND recurse=? AND withFiles=?
                AND withFileContents=? AND excludeAutoSource=?
            """, (name, oldFlavorId, oldVersionId, newFlavorId,
                  newVersionId, absolute, recurse, withFiles, withFileContents,
                  excludeAutoSource))

        # since we begin and commit a transaction inside the loop
        # over the returned rows, we must use fetchall() here so that we
        # release our read lock.
        for (row, returnVal, size) in cu.fetchall():
            path = self.filePattern % (self.tmpDir, row)
            # if we have no size or we can't access the file, it's
            # bad entry.  delete it.
            if not size or not os.access(path, os.R_OK):
                cu.execute("DELETE FROM CacheContents WHERE row=?", row)
                self.db.commit()
                continue
            return (path, cPickle.loads(returnVal), size)

        return None

    def addEntry(self, item, recurse, withFiles, withFileContents,
                 excludeAutoSource, returnVal, size):
        (name, (oldVersion, oldFlavor), (newVersion, newFlavor), absolute) = \
            item

        oldVersionId = 0
        oldFlavorId = 0
        newFlavorId = 0

        # start a transaction now to avoid race conditions when getting
        # or adding IDs for versions and flavors
        cu = self.db.transaction()

        try:
            if oldVersion:
                oldVersionId = self.versions.get(oldVersion, None)
                if oldVersionId is None:
                    oldVersionId = self.versions.addId(oldVersion)

            if oldFlavor:
                oldFlavorId = self.flavors.get(oldFlavor, None)
                if oldFlavorId is None:
                    oldFlavorId = self.flavors.addId(oldFlavor)

            if newFlavor:
                newFlavorId = self.flavors.get(newFlavor, None)
                if newFlavorId is None:
                    newFlavorId = self.flavors.addId(newFlavor)

            newVersionId = self.versions.get(newVersion, None)
            if newVersionId is None:
                newVersionId = self.versions.addId(newVersion)

            cu.execute("""
            INSERT INTO CacheContents
            (row, troveName, size,
            oldFlavorId, oldVersionId, newFlavorId, newVersionId,
            absolute, recurse, withFiles, withFileContents,
            excludeAutoSource, returnValue )
            VALUES (NULL,?,?,   ?,?,?,?,   ?,?,?,?,   ?,?)""",
                       (name, size,
                       oldFlavorId, oldVersionId, newFlavorId, newVersionId,
                       absolute, recurse, withFiles, withFileContents,
                       excludeAutoSource, cPickle.dumps(returnVal,
                                                        protocol = -1)))

            row = cu.lastrowid
            path = self.filePattern % (self.tmpDir, row)

            self.db.commit()
        except:
            # something went wrong.  make sure that we roll
            # back any pending change
            self.db.rollback()
            raise

        return (row, path)

    def invalidateEntry(self, name, version, flavor):
        """
        invalidates (and deletes) any cached changeset that matches
        the given name, version, flavor.
        """
        flavorId = self.flavors.get(flavor, None)
        versionId = self.versions.get(version, None)

        if flavorId is None or versionId is None:
            # this should not happen, but we'll handle it anyway
            return

        # start a transaction to retain a consistent state
        cu = self.db.transaction()
        cu.execute("""
        SELECT row, returnValue, size
        FROM CacheContents
        WHERE troveName=? AND newFlavorId=? AND newVersionId=?
        """, (name, flavorId, versionId))

        # delete all matching entries from the db and the file system
        for (row, returnVal, size) in cu.fetchall():
            cu.execute("DELETE FROM CacheContents WHERE row=?", row)
            path = self.filePattern % (self.tmpDir, row)
            if os.path.exists(path):
                try:
                    os.unlink(path)
                except OSError:
                    pass
        self.db.commit()

    def __cleanCache(self):
        cu = self.db.cursor()
        cu.execute("SELECT row from CacheContents")
        for (row,) in cu:
            fn = self.filePattern % (self.tmpDir, row)
            if os.path.exists(fn):
                try:
                    os.unlink(fn)
                except OSError:
                    pass

    def createSchema(self, dbInfo, schemaVersion):
        self.db = dbstore.connect(dbInfo[1], driver = dbInfo[0])
        cu = self.db.cursor()
        if "CacheContents" in self.db.tables:
            if self.db.version != CACHE_SCHEMA_VERSION:
                self.__cleanCache()
                for t in self.db.tables:
                    cu.execute("DROP TABLE %s" % (t,))
                self.db.setVersion(CACHE_SCHEMA_VERSION)

        if "CacheContents" not in self.db.tables:
            cu.execute("""
            CREATE TABLE CacheContents(
               row              %(PRIMARYKEY)s,
               troveName        VARCHAR(254),
               oldFlavorId      INTEGER,
               oldVersionId     INTEGER,
               newFlavorId      INTEGER,
               newVersionId     INTEGER,
               absolute         BOOLEAN,
               recurse          BOOLEAN,
               withFiles        BOOLEAN,
               withFileContents BOOLEAN,
               excludeAutoSource BOOLEAN,
               returnValue      BINARY,
               size             INTEGER
            )""" % self.db.keywords)
            cu.execute("""
            CREATE INDEX CacheContentsIdx ON
                CacheContents(troveName)
            """)
            schema.createFlavors(self.db)

            cu.execute("""
            CREATE TABLE Versions(
                versionId       %(PRIMARYKEY)s,
                version         VARCHAR(767)
            )""" % self.db.keywords)

            self.db.commit()

    def __init__(self, cacheDB, tmpDir):
	self.tmpDir = tmpDir
        self.createSchema(cacheDB, CACHE_SCHEMA_VERSION)
        self.flavors = sqldb.Flavors(self.db)
        self.versions = versiontable.VersionTable(self.db)

<|endoftext|>


<commit_before>#!/usr/bin/env python

# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
#
# See LICENSE for more details.
#
# Copyright (c) 2016 ScyllaDB

from sdcm.tester import ClusterTester

class BuildClusterTest(ClusterTester):
    """
    Build a Scylla cluster with the appropriate parameters.
    """

    default_params = {'timeout': 650000}

    def test_build(self):
        """
        Build a Scylla cluster with params defined in data_dir/scylla.yaml
        """
        self.log.info('DB cluster is: %s', self.db_cluster)
        for node in self.db_cluster.nodes:
            self.log.info(node.remoter.ssh_debug_cmd())
        self.log.info('Loader Nodes: %s', self.loaders)
        for node in self.loaders.nodes:
            self.log.info(node.remoter.ssh_debug_cmd())
        self.log.info('Monitor Nodes: %s', self.monitors)
        for node in self.monitors.nodes:
            self.log.info(node.remoter.ssh_debug_cmd())
        if self.monitors.nodes:
            self.log.info('Prometheus Web UI: http://%s:9090',
                          self.monitors.nodes[0].public_ip_address)
            self.log.info('Grafana Web UI: http://%s:3000',
                          self.monitors.nodes[0].public_ip_address)

    def test_use_public_dns_names(self):
        """
        Build a Scylla cluster with params defined in data_dir/scylla.yaml
        Stop cluster.
        Replace IPs for seeds, listen_address & broadcast_rpc_address on public dns names
        The cluster works properly.
        """
        if self.params.get('cluster_backend') != 'aws':
            self.log.error("Test supports only AWS instances")
            return

        self.log.info('DB cluster is: %s', self.db_cluster)
        for node in self.db_cluster.nodes:
            node.stop_scylla(verify_up=False, verify_down=True)

        addresses = {}
        seeds = []
        for node in self.db_cluster.nodes:
            if hasattr(node._instance, 'public_dns_name'):  # pylint: disable=protected-access
                addresses[node.private_ip_address] = node._instance.public_dns_name  # pylint: disable=protected-access
                if node.is_seed:
                    seeds.append(node.private_ip_address)
            else:
                self.log.error("Node instance doesn't have public dns name. "
                               "Please check AMI!")

        for node in self.db_cluster.nodes:
            node.remoter.run('sudo sed -i.bak s/{0}/{1}/g /etc/scylla/scylla.yaml'.format(node.private_ip_address,
                                                                                addresses.get(
                                                                                node.private_ip_address,
                                                                                "")))
            for seed in seeds:
                node.remoter.run(
                    'sudo sed -i.bak2 s/{0}/{1}/g /etc/scylla/scylla.yaml'.format(seed, addresses.get(seed, "")))

        for node in self.db_cluster.nodes:
            if node.is_seed:
                node.start_scylla(verify_down=False, verify_up=True)
        for node in self.db_cluster.nodes:
            if not node.is_seed:
                node.start_scylla(verify_down=False, verify_up=True)
        base_cmd_w = self.params.get('stress_cmd')

        # run a workload
        cs_thread_pool = self.run_stress_thread(stress_cmd=base_cmd_w)
        self.verify_stress_thread(cs_thread_pool=cs_thread_pool)
<commit_msg>fix(build_cluster_test): Use remote_scylla_yaml to edit the yaml file<commit_after>#!/usr/bin/env python

# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
#
# See LICENSE for more details.
#
# Copyright (c) 2016 ScyllaDB

from sdcm.tester import ClusterTester

class BuildClusterTest(ClusterTester):
    """
    Build a Scylla cluster with the appropriate parameters.
    """

    default_params = {'timeout': 650000}

    def test_build(self):
        """
        Build a Scylla cluster with params defined in data_dir/scylla.yaml
        """
        self.log.info('DB cluster is: %s', self.db_cluster)
        for node in self.db_cluster.nodes:
            self.log.info(node.remoter.ssh_debug_cmd())
        self.log.info('Loader Nodes: %s', self.loaders)
        for node in self.loaders.nodes:
            self.log.info(node.remoter.ssh_debug_cmd())
        self.log.info('Monitor Nodes: %s', self.monitors)
        for node in self.monitors.nodes:
            self.log.info(node.remoter.ssh_debug_cmd())
        if self.monitors.nodes:
            self.log.info('Prometheus Web UI: http://%s:9090',
                          self.monitors.nodes[0].public_ip_address)
            self.log.info('Grafana Web UI: http://%s:3000',
                          self.monitors.nodes[0].public_ip_address)

    def test_use_public_dns_names(self):
        """
        Build a Scylla cluster with params defined in data_dir/scylla.yaml
        Stop cluster.
        Replace IPs for seeds, listen_address & broadcast_rpc_address on public dns names
        The cluster works properly.
        """
        if self.params.get('cluster_backend') != 'aws':
            self.log.error("Test supports only AWS instances")
            return

        self.log.info('DB cluster is: %s', self.db_cluster)
        for node in self.db_cluster.nodes:
            node.stop_scylla(verify_up=False, verify_down=True)

        addresses = {}
        seeds = []
        for node in self.db_cluster.nodes:
            if hasattr(node._instance, 'public_dns_name'):  # pylint: disable=protected-access
                addresses[node.private_ip_address] = node._instance.public_dns_name  # pylint: disable=protected-access
                if node.is_seed:
                    seeds.append(node.private_ip_address)
            else:
                self.log.error("Node instance doesn't have public dns name. "
                               "Please check AMI!")

        for node in self.db_cluster.nodes:
            with node.remote_scylla_yaml() as scylla_yml:
                scylla_yml.listen_address = addresses[node.private_ip_address]
                scylla_yml.rpc_address = addresses[node.private_ip_address]
                current_seed_provider = scylla_yml.seed_provider
                seeds_string = ",".join([addresses.get(seed) for seed in seeds])
                current_seed_provider[0].parameters = [{"seeds": seeds_string}]
                scylla_yml.seed_provider = current_seed_provider

        for node in self.db_cluster.nodes:
            if node.is_seed:
                node.start_scylla(verify_down=False, verify_up=True)
        for node in self.db_cluster.nodes:
            if not node.is_seed:
                node.start_scylla(verify_down=False, verify_up=True)
        base_cmd_w = self.params.get('stress_cmd')

        # run a workload
        cs_thread_pool = self.run_stress_thread(stress_cmd=base_cmd_w)
        self.verify_stress_thread(cs_thread_pool=cs_thread_pool)
<|endoftext|>


<commit_before>#!/usr/bin/python

# Copyright (c) 2011 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""Main builder code for Chromium OS.

Used by Chromium OS buildbot configuration for all Chromium OS builds including
full and pre-flight-queue builds.
"""

import optparse
import os
import pprint
import sys

if __name__ == '__main__':
  import constants
  sys.path.append(constants.SOURCE_ROOT)

from chromite.buildbot import cbuildbot_config
from chromite.buildbot import cbuildbot_stages as stages
from chromite.lib import cros_build_lib as cros_lib

def _GetConfig(config_name, options):
  """Gets the configuration for the build"""
  if not cbuildbot_config.config.has_key(config_name):
    print 'Non-existent configuration specified.'
    print 'Please specify one of:'
    config_names = cbuildbot_config.config.keys()
    config_names.sort()
    for name in config_names:
      print '  %s' % name
    sys.exit(1)

  result = cbuildbot_config.config[config_name]

  # Use the config specific url, if not given on command line.
  if options.url:
    result['git_url'] = options.url

  return result

def _GetChromiteTrackingBranch():
  """Returns the current Chromite tracking_branch.

  If Chromite is on a detached HEAD, we assume it's the manifest branch.
  """
  cwd = os.path.dirname(os.path.realpath(__file__))
  current_branch = cros_lib.GetCurrentBranch(cwd)
  if current_branch:
    (_, tracking_branch) = cros_lib.GetPushBranch(current_branch, cwd)
  else:
    tracking_branch = cros_lib.GetManifestDefaultBranch(cwd)

  return tracking_branch

def RunBuildStages(bot_id, options, build_config):
  """Run the requested build stages."""

  completed_stages_file = os.path.join(options.buildroot, '.completed_stages')

  if options.resume and os.path.exists(completed_stages_file):
    with open(completed_stages_file, 'r') as load_file:
      stages.Results.RestoreCompletedStages(load_file)

  # TODO, Remove here and in config after bug chromium-os:14649 is fixed.
  if build_config['chromeos_official']: os.environ['CHROMEOS_OFFICIAL'] = '1'

  # Determine the stages to use for syncing and completion.
  sync_stage = stages.SyncStage
  completion_stage = None
  if build_config['manifest_version']:
    if build_config['build_type'] == 'binary':
      sync_stage = stages.LGKMVersionedSyncStage
      completion_stage = stages.LGKMVersionedSyncCompletionStage
    else:
      sync_stage = stages.ManifestVersionedSyncStage
      completion_stage = stages.ManifestVersionedSyncCompletionStage

  tracking_branch = _GetChromiteTrackingBranch()
  stages.BuilderStage.SetTrackingBranch(tracking_branch)

  build_success = False
  build_and_test_success = False

  try:
    if options.sync:
      sync_stage(bot_id, options, build_config).Run()

    if not options.clobber:
      # If we are doing an incremental checkout, make sure we are running on a
      # buildroot checked out to same branch as chromite.  Use
      # <build_root>/src/scripts as the spot check.
      manifest_branch = cros_lib.GetManifestDefaultBranch(options.buildroot)
      if manifest_branch != tracking_branch:
        cros_lib.Die('Chromite is not on same branch as buildroot checkout\n' +
                     'Chromite is on branch %s.\n' % tracking_branch +
                     'Buildroot checked out to %s\n' % manifest_branch)

    if options.build:
      stages.BuildBoardStage(bot_id, options, build_config).Run()
      if build_config['build_type'] == 'chroot':
        stages.Results.Report(sys.stdout)
        return stages.Results.Success()

    if options.uprev:
      stages.UprevStage(bot_id, options, build_config).Run()

    if options.build:
      stages.BuildTargetStage(bot_id, options, build_config).Run()

    build_success = True

    if options.tests:
      stages.TestStage(bot_id, options, build_config).Run()

    if options.remote_test_status:
      stages.RemoteTestStatusStage(bot_id, options, build_config).Run()

    build_and_test_success = True
  except stages.BuildException:
    # We skipped out of this build block early, all we need to do.
    pass

  if options.sync:
    if (completion_stage and
        stages.ManifestVersionedSyncStage.manifest_manager):
      completion_stage(bot_id, options, build_config,
                       success=build_and_test_success).Run()

    if build_config['master'] and build_and_test_success:
      try:
        stages.PushChangesStage(bot_id, options, build_config).Run()
      except stages.BuildException:
        pass

  if build_success and options.archive:
    try:
      stages.ArchiveStage(bot_id, options, build_config).Run()
    except stages.BuildException:
      pass

  if os.path.exists(options.buildroot):
    with open(completed_stages_file, 'w+') as save_file:
      stages.Results.SaveCompletedStages(save_file)

  stages.Results.Report(sys.stdout)
  return stages.Results.Success()

def main():
  # Parse options
  usage = "usage: %prog [options] cbuildbot_config"
  parser = optparse.OptionParser(usage=usage)
  parser.add_option('--buildbot', action='store_false', dest='debug',
                    help='This is running on a buildbot')
  parser.add_option('-r', '--buildroot',
                    help='root directory where build occurs', default=".")
  parser.add_option('-n', '--buildnumber',
                    help='build number', type='int', default=0)
  parser.add_option('--chrome_rev', default=None, type='string',
                    dest='chrome_rev',
                    help=('Chrome_rev of type [tot|latest_release|'
                          'sticky_release]'))
  parser.add_option('--clobber', action='store_true', dest='clobber',
                    default=False,
                    help='Clobbers an old checkout before syncing')
  parser.add_option('--debug', action='store_true', dest='debug',
                    default=True,
                    help='Override some options to run as a developer.')
  parser.add_option('--dump_config', action='store_true', dest='dump_config',
                    default=False,
                    help='Dump out build config options, and exit.')
  parser.add_option('--noarchive', action='store_false', dest='archive',
                    default=True,
                    help="Don't run archive stage.")
  parser.add_option('--nobuild', action='store_false', dest='build',
                    default=True,
                    help="Don't actually build (for cbuildbot dev")
  parser.add_option('--noprebuilts', action='store_false', dest='prebuilts',
                    default=True,
                    help="Don't upload prebuilts.")
  parser.add_option('--nosync', action='store_false', dest='sync',
                    default=True,
                    help="Don't sync before building.")
  parser.add_option('--notests', action='store_false', dest='tests',
                    default=True,
                    help='Override values from buildconfig and run no tests.')
  parser.add_option('--remoteteststatus', dest='remote_test_status',
                    default=None, help='List of remote jobs to check status')
  parser.add_option('--resume', action='store_true',
                    default=False,
                    help='Skip stages already successfully completed.')
  parser.add_option('-f', '--revisionfile',
                    help='file where new revisions are stored')
  parser.add_option('-t', '--tracking-branch', dest='tracking_branch_old',
                    default='cros/master', help='Run the buildbot on a branch')
  parser.add_option('--nouprev', action='store_false', dest='uprev',
                    default=True,
                    help='Override values from buildconfig and never uprev.')
  parser.add_option('-u', '--url', dest='url',
                    default=None,
                    help='Override the GIT repo URL from the build config.')

  (options, args) = parser.parse_args()

  if len(args) >= 1:
    bot_id = args[-1]
    build_config = _GetConfig(bot_id, options)
  else:
    parser.error('Invalid usage.  Use -h to see usage.')

  if options.dump_config:
    # This works, but option ordering is bad...
    print 'Configuration %s:' % bot_id
    pp = pprint.PrettyPrinter(indent=2)
    pp.pprint(build_config)
    sys.exit(0)

  if not RunBuildStages(bot_id, options, build_config):
    sys.exit(1)

if __name__ == '__main__':
    main()
<commit_msg>Add a little whitespace before the final report to seperate final stage end, and the report.<commit_after>#!/usr/bin/python

# Copyright (c) 2011 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""Main builder code for Chromium OS.

Used by Chromium OS buildbot configuration for all Chromium OS builds including
full and pre-flight-queue builds.
"""

import optparse
import os
import pprint
import sys

if __name__ == '__main__':
  import constants
  sys.path.append(constants.SOURCE_ROOT)

from chromite.buildbot import cbuildbot_config
from chromite.buildbot import cbuildbot_stages as stages
from chromite.lib import cros_build_lib as cros_lib

def _GetConfig(config_name, options):
  """Gets the configuration for the build"""
  if not cbuildbot_config.config.has_key(config_name):
    print 'Non-existent configuration specified.'
    print 'Please specify one of:'
    config_names = cbuildbot_config.config.keys()
    config_names.sort()
    for name in config_names:
      print '  %s' % name
    sys.exit(1)

  result = cbuildbot_config.config[config_name]

  # Use the config specific url, if not given on command line.
  if options.url:
    result['git_url'] = options.url

  return result

def _GetChromiteTrackingBranch():
  """Returns the current Chromite tracking_branch.

  If Chromite is on a detached HEAD, we assume it's the manifest branch.
  """
  cwd = os.path.dirname(os.path.realpath(__file__))
  current_branch = cros_lib.GetCurrentBranch(cwd)
  if current_branch:
    (_, tracking_branch) = cros_lib.GetPushBranch(current_branch, cwd)
  else:
    tracking_branch = cros_lib.GetManifestDefaultBranch(cwd)

  return tracking_branch

def RunBuildStages(bot_id, options, build_config):
  """Run the requested build stages."""

  completed_stages_file = os.path.join(options.buildroot, '.completed_stages')

  if options.resume and os.path.exists(completed_stages_file):
    with open(completed_stages_file, 'r') as load_file:
      stages.Results.RestoreCompletedStages(load_file)

  # TODO, Remove here and in config after bug chromium-os:14649 is fixed.
  if build_config['chromeos_official']: os.environ['CHROMEOS_OFFICIAL'] = '1'

  # Determine the stages to use for syncing and completion.
  sync_stage = stages.SyncStage
  completion_stage = None
  if build_config['manifest_version']:
    if build_config['build_type'] == 'binary':
      sync_stage = stages.LGKMVersionedSyncStage
      completion_stage = stages.LGKMVersionedSyncCompletionStage
    else:
      sync_stage = stages.ManifestVersionedSyncStage
      completion_stage = stages.ManifestVersionedSyncCompletionStage

  tracking_branch = _GetChromiteTrackingBranch()
  stages.BuilderStage.SetTrackingBranch(tracking_branch)

  build_success = False
  build_and_test_success = False

  try:
    if options.sync:
      sync_stage(bot_id, options, build_config).Run()

    if not options.clobber:
      # If we are doing an incremental checkout, make sure we are running on a
      # buildroot checked out to same branch as chromite.  Use
      # <build_root>/src/scripts as the spot check.
      manifest_branch = cros_lib.GetManifestDefaultBranch(options.buildroot)
      if manifest_branch != tracking_branch:
        cros_lib.Die('Chromite is not on same branch as buildroot checkout\n' +
                     'Chromite is on branch %s.\n' % tracking_branch +
                     'Buildroot checked out to %s\n' % manifest_branch)

    if options.build:
      stages.BuildBoardStage(bot_id, options, build_config).Run()
      if build_config['build_type'] == 'chroot':
        stages.Results.Report(sys.stdout)
        return stages.Results.Success()

    if options.uprev:
      stages.UprevStage(bot_id, options, build_config).Run()

    if options.build:
      stages.BuildTargetStage(bot_id, options, build_config).Run()

    build_success = True

    if options.tests:
      stages.TestStage(bot_id, options, build_config).Run()

    if options.remote_test_status:
      stages.RemoteTestStatusStage(bot_id, options, build_config).Run()

    build_and_test_success = True
  except stages.BuildException:
    # We skipped out of this build block early, all we need to do.
    pass

  if options.sync:
    if (completion_stage and
        stages.ManifestVersionedSyncStage.manifest_manager):
      completion_stage(bot_id, options, build_config,
                       success=build_and_test_success).Run()

    if build_config['master'] and build_and_test_success:
      try:
        stages.PushChangesStage(bot_id, options, build_config).Run()
      except stages.BuildException:
        pass

  if build_success and options.archive:
    try:
      stages.ArchiveStage(bot_id, options, build_config).Run()
    except stages.BuildException:
      pass

  if os.path.exists(options.buildroot):
    with open(completed_stages_file, 'w+') as save_file:
      stages.Results.SaveCompletedStages(save_file)

  print '\n\n\n'
  stages.Results.Report(sys.stdout)
  return stages.Results.Success()

def main():
  # Parse options
  usage = "usage: %prog [options] cbuildbot_config"
  parser = optparse.OptionParser(usage=usage)
  parser.add_option('--buildbot', action='store_false', dest='debug',
                    help='This is running on a buildbot')
  parser.add_option('-r', '--buildroot',
                    help='root directory where build occurs', default=".")
  parser.add_option('-n', '--buildnumber',
                    help='build number', type='int', default=0)
  parser.add_option('--chrome_rev', default=None, type='string',
                    dest='chrome_rev',
                    help=('Chrome_rev of type [tot|latest_release|'
                          'sticky_release]'))
  parser.add_option('--clobber', action='store_true', dest='clobber',
                    default=False,
                    help='Clobbers an old checkout before syncing')
  parser.add_option('--debug', action='store_true', dest='debug',
                    default=True,
                    help='Override some options to run as a developer.')
  parser.add_option('--dump_config', action='store_true', dest='dump_config',
                    default=False,
                    help='Dump out build config options, and exit.')
  parser.add_option('--noarchive', action='store_false', dest='archive',
                    default=True,
                    help="Don't run archive stage.")
  parser.add_option('--nobuild', action='store_false', dest='build',
                    default=True,
                    help="Don't actually build (for cbuildbot dev")
  parser.add_option('--noprebuilts', action='store_false', dest='prebuilts',
                    default=True,
                    help="Don't upload prebuilts.")
  parser.add_option('--nosync', action='store_false', dest='sync',
                    default=True,
                    help="Don't sync before building.")
  parser.add_option('--notests', action='store_false', dest='tests',
                    default=True,
                    help='Override values from buildconfig and run no tests.')
  parser.add_option('--remoteteststatus', dest='remote_test_status',
                    default=None, help='List of remote jobs to check status')
  parser.add_option('--resume', action='store_true',
                    default=False,
                    help='Skip stages already successfully completed.')
  parser.add_option('-f', '--revisionfile',
                    help='file where new revisions are stored')
  parser.add_option('-t', '--tracking-branch', dest='tracking_branch_old',
                    default='cros/master', help='Run the buildbot on a branch')
  parser.add_option('--nouprev', action='store_false', dest='uprev',
                    default=True,
                    help='Override values from buildconfig and never uprev.')
  parser.add_option('-u', '--url', dest='url',
                    default=None,
                    help='Override the GIT repo URL from the build config.')

  (options, args) = parser.parse_args()

  if len(args) >= 1:
    bot_id = args[-1]
    build_config = _GetConfig(bot_id, options)
  else:
    parser.error('Invalid usage.  Use -h to see usage.')

  if options.dump_config:
    # This works, but option ordering is bad...
    print 'Configuration %s:' % bot_id
    pp = pprint.PrettyPrinter(indent=2)
    pp.pprint(build_config)
    sys.exit(0)

  if not RunBuildStages(bot_id, options, build_config):
    sys.exit(1)

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before># Copyright (c) 2012 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""This module contains constants used by cbuildbot and related code."""

import os

SOURCE_ROOT = os.path.dirname(os.path.abspath(__file__))
SOURCE_ROOT = os.path.realpath(os.path.join(SOURCE_ROOT, '..', '..'))
CROSUTILS_DIR = os.path.join(SOURCE_ROOT, 'src/scripts')
CHROMITE_BIN_SUBDIR = 'chromite/bin'
CHROMITE_BIN_DIR = os.path.join(SOURCE_ROOT, CHROMITE_BIN_SUBDIR)
PATH_TO_CBUILDBOT = os.path.join(CHROMITE_BIN_SUBDIR, 'cbuildbot')
DEFAULT_CHROOT_DIR = 'chroot'

GERRIT_PORT = '29418'
GERRIT_INT_PORT = '29419'

GERRIT_HOST = 'gerrit.chromium.org'
GERRIT_INT_HOST = 'gerrit-int.chromium.org'
GIT_HOST = 'git.chromium.org'

GERRIT_SSH_URL = 'ssh://%s:%s' % (GERRIT_HOST, GERRIT_PORT)
GERRIT_INT_SSH_URL = 'ssh://%s:%s' % (GERRIT_INT_HOST, GERRIT_INT_PORT)
GERRIT_HTTP_URL = 'https://%s' % GERRIT_HOST
GIT_HTTP_URL = 'https://%s/git' % GIT_HOST

REPO_PROJECT = 'external/repo'
REPO_URL = '%s/%s' % (GIT_HTTP_URL, REPO_PROJECT)

CHROMITE_PROJECT = 'chromiumos/chromite'
CHROMITE_URL = '%s/%s' % (GIT_HTTP_URL, CHROMITE_PROJECT)

MANIFEST_PROJECT = 'chromiumos/manifest'
MANIFEST_INT_PROJECT = 'chromeos/manifest-internal'

MANIFEST_URL = '%s/%s' % (GIT_HTTP_URL, MANIFEST_PROJECT)
MANIFEST_INT_URL = '%s/%s' % (GERRIT_INT_SSH_URL, MANIFEST_INT_PROJECT)

DEFAULT_MANIFEST = 'default.xml'

# CrOS remotes specified in the manifests.
EXTERNAL_REMOTE = 'cros'
INTERNAL_REMOTE = 'cros-internal'
CROS_REMOTES = {
    EXTERNAL_REMOTE : GERRIT_SSH_URL,
    INTERNAL_REMOTE : GERRIT_INT_SSH_URL
}

# TODO(sosa): Move to manifest-versions-external once its created
MANIFEST_VERSIONS_SUFFIX = '/chromiumos/manifest-versions'
MANIFEST_VERSIONS_INT_SUFFIX = '/chromeos/manifest-versions'

PATCH_BRANCH = 'patch_branch'
STABLE_EBUILD_BRANCH = 'stabilizing_branch'
MERGE_BRANCH = 'merge_branch'

# These branches are deleted at the beginning of every buildbot run.
CREATED_BRANCHES = [
    PATCH_BRANCH,
    STABLE_EBUILD_BRANCH,
    MERGE_BRANCH
]

# Constants for uprevving Chrome

# Builds and validates _alpha ebuilds.  These builds sync to the latest
# revsion of the Chromium src tree and build with that checkout.
CHROME_REV_TOT = 'tot'

# Builds and validates chrome at a given revision through cbuildbot
# --chrome_version
CHROME_REV_SPEC = 'spec'

# Builds and validates the latest Chromium release as defined by
# ~/trunk/releases in the Chrome src tree.  These ebuilds are suffixed with rc.
CHROME_REV_LATEST = 'latest_release'

# Builds and validates the latest Chromium release for a specific Chromium
# branch that we want to watch.  These ebuilds are suffixed with rc.
CHROME_REV_STICKY = 'stable_release'

# Builds and validates Chromium for a pre-populated directory.
# Also uses _alpha, since portage doesn't have anything lower.
CHROME_REV_LOCAL = 'local'
VALID_CHROME_REVISIONS = [CHROME_REV_TOT, CHROME_REV_LATEST,
                          CHROME_REV_STICKY, CHROME_REV_LOCAL, CHROME_REV_SPEC]

# Build types supported.

# TODO(sosa): Deprecate PFQ type.
# Incremental builds that are built using binary packages when available.
# These builds have less validation than other build types.
INCREMENTAL_TYPE = 'binary'

# These builds serve as PFQ builders.  This is being deprecated.
PFQ_TYPE = 'pfq'

# TODO(sosa): Deprecate CQ type.
# Commit Queue type that is similar to PFQ_TYPE but uses Commit Queue sync
# logic.
COMMIT_QUEUE_TYPE = 'commit-queue'

# Hybrid Commit and PFQ type.  Ultimate protection.
PALADIN_TYPE = 'paladin'

# Chrome PFQ type.  Incremental build type that builds and validates new
# versions of Chrome.  Only valid if set with CHROME_REV.  See
# VALID_CHROME_REVISIONS for more information.
CHROME_PFQ_TYPE = 'chrome'

# Builds from source and non-incremental.  This builds fully wipe their
# chroot before the start of every build and no not use a BINHOST.
BUILD_FROM_SOURCE_TYPE = 'full'

# Full but with versioned logic.
CANARY_TYPE = 'canary'

# Special build type for Chroot builders.  These builds focus on building
# toolchains and validate that they work.
CHROOT_BUILDER_TYPE = 'chroot'
CHROOT_BUILDER_BOARD = 'amd64-host'

# Build that refreshes the online Portage package status spreadsheet.
REFRESH_PACKAGES_TYPE = 'refresh_packages'

# Define pool of machines for Hardware tests.
HWTEST_DEFAULT_NUM = 4
HWTEST_TRYBOT_NUM = 1
HWTEST_MACH_POOL = 'bvt'
HWTEST_PALADIN_POOL = 'cq-bvt'
HWTEST_CHROME_PFQ_POOL = 'chromepfq'
HWTEST_TRYBOT_POOL = 'try-bot'

# Defines VM Test types.
SMOKE_SUITE_TEST_TYPE = 'smoke_suite'
SIMPLE_AU_TEST_TYPE = 'pfq_suite'
FULL_AU_TEST_TYPE = 'full_suite'

VALID_AU_TEST_TYPES = [SMOKE_SUITE_TEST_TYPE, SIMPLE_AU_TEST_TYPE,
                       FULL_AU_TEST_TYPE]

VERSION_FILE = os.path.join('src/third_party/chromiumos-overlay',
                            'chromeos/config/chromeos_version.sh')

BOTH_OVERLAYS = 'both'
PUBLIC_OVERLAYS = 'public'
PRIVATE_OVERLAYS = 'private'
VALID_OVERLAYS = [BOTH_OVERLAYS, PUBLIC_OVERLAYS, PRIVATE_OVERLAYS, None]

# Common default logging settings for use with the logging module.
LOGGER_FMT = '%(asctime)s: %(levelname)s: %(message)s'
LOGGER_DATE_FMT = '%H:%M:%S'

# Used by remote patch serialization/deserialzation.
INTERNAL_PATCH_TAG = 'i'
EXTERNAL_PATCH_TAG = 'e'
PATCH_TAGS = (INTERNAL_PATCH_TAG, EXTERNAL_PATCH_TAG)

# Default gerrit query used to find changes for CQ.
DEFAULT_CQ_READY_QUERY = ('status:open AND CodeReview=+2 AND Verified=+1 '
                          'AND CommitReady=+1 '
                          'AND NOT ( CodeReview=-2 OR Verified=-1 )')

# Some files need permissions set for several distinct groups. A google storage
# acl (xml) file will be necessary in those cases. Make available well known
# locations and standardize.
KNOWN_ACL_FILES = {'slave': os.path.expanduser('~/slave_archive_acl')}
<commit_msg>Increase the number of devices running HWTest from 4 to 6.<commit_after># Copyright (c) 2012 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""This module contains constants used by cbuildbot and related code."""

import os

SOURCE_ROOT = os.path.dirname(os.path.abspath(__file__))
SOURCE_ROOT = os.path.realpath(os.path.join(SOURCE_ROOT, '..', '..'))
CROSUTILS_DIR = os.path.join(SOURCE_ROOT, 'src/scripts')
CHROMITE_BIN_SUBDIR = 'chromite/bin'
CHROMITE_BIN_DIR = os.path.join(SOURCE_ROOT, CHROMITE_BIN_SUBDIR)
PATH_TO_CBUILDBOT = os.path.join(CHROMITE_BIN_SUBDIR, 'cbuildbot')
DEFAULT_CHROOT_DIR = 'chroot'

GERRIT_PORT = '29418'
GERRIT_INT_PORT = '29419'

GERRIT_HOST = 'gerrit.chromium.org'
GERRIT_INT_HOST = 'gerrit-int.chromium.org'
GIT_HOST = 'git.chromium.org'

GERRIT_SSH_URL = 'ssh://%s:%s' % (GERRIT_HOST, GERRIT_PORT)
GERRIT_INT_SSH_URL = 'ssh://%s:%s' % (GERRIT_INT_HOST, GERRIT_INT_PORT)
GERRIT_HTTP_URL = 'https://%s' % GERRIT_HOST
GIT_HTTP_URL = 'https://%s/git' % GIT_HOST

REPO_PROJECT = 'external/repo'
REPO_URL = '%s/%s' % (GIT_HTTP_URL, REPO_PROJECT)

CHROMITE_PROJECT = 'chromiumos/chromite'
CHROMITE_URL = '%s/%s' % (GIT_HTTP_URL, CHROMITE_PROJECT)

MANIFEST_PROJECT = 'chromiumos/manifest'
MANIFEST_INT_PROJECT = 'chromeos/manifest-internal'

MANIFEST_URL = '%s/%s' % (GIT_HTTP_URL, MANIFEST_PROJECT)
MANIFEST_INT_URL = '%s/%s' % (GERRIT_INT_SSH_URL, MANIFEST_INT_PROJECT)

DEFAULT_MANIFEST = 'default.xml'

# CrOS remotes specified in the manifests.
EXTERNAL_REMOTE = 'cros'
INTERNAL_REMOTE = 'cros-internal'
CROS_REMOTES = {
    EXTERNAL_REMOTE : GERRIT_SSH_URL,
    INTERNAL_REMOTE : GERRIT_INT_SSH_URL
}

# TODO(sosa): Move to manifest-versions-external once its created
MANIFEST_VERSIONS_SUFFIX = '/chromiumos/manifest-versions'
MANIFEST_VERSIONS_INT_SUFFIX = '/chromeos/manifest-versions'

PATCH_BRANCH = 'patch_branch'
STABLE_EBUILD_BRANCH = 'stabilizing_branch'
MERGE_BRANCH = 'merge_branch'

# These branches are deleted at the beginning of every buildbot run.
CREATED_BRANCHES = [
    PATCH_BRANCH,
    STABLE_EBUILD_BRANCH,
    MERGE_BRANCH
]

# Constants for uprevving Chrome

# Builds and validates _alpha ebuilds.  These builds sync to the latest
# revsion of the Chromium src tree and build with that checkout.
CHROME_REV_TOT = 'tot'

# Builds and validates chrome at a given revision through cbuildbot
# --chrome_version
CHROME_REV_SPEC = 'spec'

# Builds and validates the latest Chromium release as defined by
# ~/trunk/releases in the Chrome src tree.  These ebuilds are suffixed with rc.
CHROME_REV_LATEST = 'latest_release'

# Builds and validates the latest Chromium release for a specific Chromium
# branch that we want to watch.  These ebuilds are suffixed with rc.
CHROME_REV_STICKY = 'stable_release'

# Builds and validates Chromium for a pre-populated directory.
# Also uses _alpha, since portage doesn't have anything lower.
CHROME_REV_LOCAL = 'local'
VALID_CHROME_REVISIONS = [CHROME_REV_TOT, CHROME_REV_LATEST,
                          CHROME_REV_STICKY, CHROME_REV_LOCAL, CHROME_REV_SPEC]

# Build types supported.

# TODO(sosa): Deprecate PFQ type.
# Incremental builds that are built using binary packages when available.
# These builds have less validation than other build types.
INCREMENTAL_TYPE = 'binary'

# These builds serve as PFQ builders.  This is being deprecated.
PFQ_TYPE = 'pfq'

# TODO(sosa): Deprecate CQ type.
# Commit Queue type that is similar to PFQ_TYPE but uses Commit Queue sync
# logic.
COMMIT_QUEUE_TYPE = 'commit-queue'

# Hybrid Commit and PFQ type.  Ultimate protection.
PALADIN_TYPE = 'paladin'

# Chrome PFQ type.  Incremental build type that builds and validates new
# versions of Chrome.  Only valid if set with CHROME_REV.  See
# VALID_CHROME_REVISIONS for more information.
CHROME_PFQ_TYPE = 'chrome'

# Builds from source and non-incremental.  This builds fully wipe their
# chroot before the start of every build and no not use a BINHOST.
BUILD_FROM_SOURCE_TYPE = 'full'

# Full but with versioned logic.
CANARY_TYPE = 'canary'

# Special build type for Chroot builders.  These builds focus on building
# toolchains and validate that they work.
CHROOT_BUILDER_TYPE = 'chroot'
CHROOT_BUILDER_BOARD = 'amd64-host'

# Build that refreshes the online Portage package status spreadsheet.
REFRESH_PACKAGES_TYPE = 'refresh_packages'

# Define pool of machines for Hardware tests.
HWTEST_DEFAULT_NUM = 6
HWTEST_TRYBOT_NUM = 1
HWTEST_MACH_POOL = 'bvt'
HWTEST_PALADIN_POOL = 'cq-bvt'
HWTEST_CHROME_PFQ_POOL = 'chromepfq'
HWTEST_TRYBOT_POOL = 'try-bot'

# Defines VM Test types.
SMOKE_SUITE_TEST_TYPE = 'smoke_suite'
SIMPLE_AU_TEST_TYPE = 'pfq_suite'
FULL_AU_TEST_TYPE = 'full_suite'

VALID_AU_TEST_TYPES = [SMOKE_SUITE_TEST_TYPE, SIMPLE_AU_TEST_TYPE,
                       FULL_AU_TEST_TYPE]

VERSION_FILE = os.path.join('src/third_party/chromiumos-overlay',
                            'chromeos/config/chromeos_version.sh')

BOTH_OVERLAYS = 'both'
PUBLIC_OVERLAYS = 'public'
PRIVATE_OVERLAYS = 'private'
VALID_OVERLAYS = [BOTH_OVERLAYS, PUBLIC_OVERLAYS, PRIVATE_OVERLAYS, None]

# Common default logging settings for use with the logging module.
LOGGER_FMT = '%(asctime)s: %(levelname)s: %(message)s'
LOGGER_DATE_FMT = '%H:%M:%S'

# Used by remote patch serialization/deserialzation.
INTERNAL_PATCH_TAG = 'i'
EXTERNAL_PATCH_TAG = 'e'
PATCH_TAGS = (INTERNAL_PATCH_TAG, EXTERNAL_PATCH_TAG)

# Default gerrit query used to find changes for CQ.
DEFAULT_CQ_READY_QUERY = ('status:open AND CodeReview=+2 AND Verified=+1 '
                          'AND CommitReady=+1 '
                          'AND NOT ( CodeReview=-2 OR Verified=-1 )')

# Some files need permissions set for several distinct groups. A google storage
# acl (xml) file will be necessary in those cases. Make available well known
# locations and standardize.
KNOWN_ACL_FILES = {'slave': os.path.expanduser('~/slave_archive_acl')}
<|endoftext|>


<commit_before># Copyright 2014 PerfKitBenchmarker Authors. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Runs plain vanilla bonnie++."""

import logging

from perfkitbenchmarker import configs
from perfkitbenchmarker import regex_util
from perfkitbenchmarker import sample

BENCHMARK_NAME = 'bonnie++'
BENCHMARK_CONFIG = """
bonnie++:
  description: >
      Runs Bonnie++. Running this benchmark inside
      a container is currently not supported,
      since Docker tries to run it as root, which
      is not recommended.
  vm_groups:
    default:
      vm_spec: *default_single_core
      disk_spec: *default_500_gb
"""

LATENCY_REGEX = r'([0-9]*\.?[0-9]+)(\w+)'
# Bonnie++ result fields mapping, see man bon_csv2txt for details.
BONNIE_RESULTS_MAPPING = {
    'format_version': 0,
    'bonnie_version': 1,
    'name': 2,
    'concurrency': 3,
    'seed': 4,
    'file_size': 5,
    'chunk_size': 6,
    'putc': 7,
    'putc_cpu': 8,
    'put_block': 9,
    'put_block_cpu': 10,
    'rewrite': 11,
    'rewrite_cpu': 12,
    'getc': 13,
    'getc_cpu': 14,
    'get_block': 15,
    'get_block_cpu': 16,
    'seeks': 17,
    'seeks_cpu': 18,
    'num_files': 19,
    'max_size': 20,
    'min_size': 21,
    'num_dirs': 22,
    'file_chunk_size': 23,
    'seq_create': 24,
    'seq_create_cpu': 25,
    'seq_stat': 26,
    'seq_stat_cpu': 27,
    'seq_del': 28,
    'seq_del_cpu': 29,
    'ran_create': 30,
    'ran_create_cpu': 31,
    'ran_stat': 32,
    'ran_stat_cpu': 33,
    'ran_del': 34,
    'ran_del_cpu': 35,
    'putc_latency': 36,
    'put_block_latency': 37,
    'rewrite_latency': 38,
    'getc_latency': 39,
    'get_block_latency': 40,
    'seeks_latency': 41,
    'seq_create_latency': 42,
    'seq_stat_latency': 43,
    'seq_del_latency': 44,
    'ran_create_latency': 45,
    'ran_stat_latency': 46,
    'ran_del_latency': 47}

def GetConfig(user_config):
  return configs.LoadConfig(BENCHMARK_CONFIG, user_config, BENCHMARK_NAME)

def Prepare(benchmark_spec):
  """Install Bonnie++ on the target vm.

  Args:
    benchmark_spec: The benchmark specification. Contains all data that is
        required to run the benchmark.

  """
  vms = benchmark_spec.vms
  vm = vms[0]
  logging.info('Bonnie++ prepare on %s', vm)
  vm.Install('bonnieplusplus')

def IsValueValid(value):
  """Validate the value.

  An invalid value is either an empty string or a string of multiple '+'.

  Args:
    value: string. The value in raw result.

  Returns:
    A boolean indicates if the value is valid or not.
  """
  if value == '' or '+' in value:
    return False
  return True

def IsCpuField(field):
  """Check if the field is cpu percentage.

  Args:
    field: string. The name of the field.

  Returns:
    A boolean indicates if the field contains keyword 'cpu'.
  """
  return 'cpu' in field

def IsLatencyField(field):
  """Check if the field is latency.

  Args:
    field: string. The name of the field.

  Returns:
    A boolean indicates if the field contains keyword 'latency'.
  """
  return 'latency' in field

def ParseLatencyResult(result):
  """Parse latency result into value and unit.

  Args:
    result: string. Latency value in string format, contains value and unit.
            eg. 200ms

  Returns:
    A tuple of value (float) and unit (string).
  """
  match = regex_util.ExtractAllMatches(LATENCY_REGEX, result)[0]
  return float(match[0]), match[1]

def UpdateMetadata(metadata, key, value):
  """Check if the value is valid, update metadata with the key, value pair.

  Args:
    metadata: dict. A dictionary of sample metadata.
    key: string. Key that will be added into metadata dictionary.
    value: Value that of the key.
  """
  if IsValueValid(value):
    metadata[key] = value

def CreateSamples(results, start_index, end_index, metadata,
                  field_index_mapping):
  """Create samples with data in results from start_index to end_index.

  Args:
    results: A list of string representing bonnie++ results.
    start_index: integer. The start index in results list of the samples.
    end_index: integer. The end index in results list of the samples.
    metadata: dict. A dictionary of metadata added into samples.
    field_index_mapping: dict. A dictionary maps field index to field names.

  Returns:
    A list of sample.Sample instances.
  """
  samples = []
  for field_index in range(start_index, end_index):
    field_name = field_index_mapping[field_index]
    value = results[field_index]
    if not IsValueValid(value):
      continue
    if IsCpuField(field_name):
      unit = '%s'
    elif IsLatencyField(field_name):
      value, unit = ParseLatencyResult(value)
    else:
      unit = 'K/sec'
    samples.append(sample.Sample(field_name, float(value), unit, metadata))
  return samples

def ParseCSVResults(results):
  """Parse csv format bonnie++ results.

  Sample Results:
    1.96,1.96,perfkit-7b22f510-0,1,1421800799,7423M,,,,72853,15,47358,5,,,
    156821,7,537.7,10,100,,,,,49223,58,+++++,+++,54405,53,2898,97,+++++,+++,
    59089,60,,512ms,670ms,,44660us,200ms,3747us,1759us,1643us,33518us,192us,
    839us

  Args:
    results: string. Bonnie++ results.

  Returns:
    A list of samples in the form of 3 or 4 tuples. The tuples contain
        the sample metric (string), value (float), and unit (string).
        If a 4th element is included, it is a dictionary of sample
        metadata.
  """
  field_index_mapping = {}
  for field, value in BONNIE_RESULTS_MAPPING.iteritems():
    field_index_mapping[value] = field
  results = results.split(',')
  assert len(results) == len(BONNIE_RESULTS_MAPPING)
  samples = []
  metadata = {}
  for field_index in range(BONNIE_RESULTS_MAPPING['format_version'],
                           BONNIE_RESULTS_MAPPING['chunk_size'] + 1):
    UpdateMetadata(metadata, field_index_mapping[field_index],
                   results[field_index])

  for field_index in range(BONNIE_RESULTS_MAPPING['num_files'],
                           BONNIE_RESULTS_MAPPING['file_chunk_size'] + 1):
    UpdateMetadata(metadata, field_index_mapping[field_index],
                   results[field_index])
  samples.extend(CreateSamples(results,
                               BONNIE_RESULTS_MAPPING['putc'],
                               BONNIE_RESULTS_MAPPING['num_files'],
                               metadata, field_index_mapping))
  samples.extend(CreateSamples(results,
                               BONNIE_RESULTS_MAPPING['seq_create'],
                               BONNIE_RESULTS_MAPPING['ran_del_latency'] + 1,
                               metadata, field_index_mapping))
  return samples

def Run(benchmark_spec):
  """Run Bonnie++ on the target vm.

  Args:
    benchmark_spec: The benchmark specification. Contains all data that is
        required to run the benchmark.

  Returns:
    A list of samples in the form of 3 or 4 tuples. The tuples contain
        the sample metric (string), value (float), and unit (string).
        If a 4th element is included, it is a dictionary of sample
        metadata.
  """
  vms = benchmark_spec.vms
  vm = vms[0]
  logging.info('Bonnie++ running on %s', vm)
  bonnie_command = ('/usr/sbin/bonnie++ -q -d %s -s %d -n 100 -f' %
                    (vm.GetScratchDir(),
                     2 * vm.total_memory_kb / 1024))
  logging.info('Bonnie++ Results:')
  out, _ = vm.RemoteCommand(bonnie_command, should_log=True)
  return ParseCSVResults(out.strip())

def Cleanup(benchmark_spec):
  """Cleanup Bonnie++ on the target vm (by uninstalling).

  Args:
    benchmark_spec: The benchmark specification. Contains all data that is
        required to run the benchmark.
  """
  pass
<commit_msg>Fix for "Cannot run bonnie++ on Azure #1217" <commit_after># Copyright 2014 PerfKitBenchmarker Authors. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Runs plain vanilla bonnie++."""

import logging

from perfkitbenchmarker import configs
from perfkitbenchmarker import regex_util
from perfkitbenchmarker import sample

BENCHMARK_NAME = 'bonnieplusplus'
BENCHMARK_CONFIG = """
bonnieplusplus:
  description: >
      Runs Bonnie++. Running this benchmark inside
      a container is currently not supported,
      since Docker tries to run it as root, which
      is not recommended.
  vm_groups:
    default:
      vm_spec: *default_single_core
      disk_spec: *default_500_gb
"""

LATENCY_REGEX = r'([0-9]*\.?[0-9]+)(\w+)'
# Bonnie++ result fields mapping, see man bon_csv2txt for details.
BONNIE_RESULTS_MAPPING = {
    'format_version': 0,
    'bonnie_version': 1,
    'name': 2,
    'concurrency': 3,
    'seed': 4,
    'file_size': 5,
    'chunk_size': 6,
    'putc': 7,
    'putc_cpu': 8,
    'put_block': 9,
    'put_block_cpu': 10,
    'rewrite': 11,
    'rewrite_cpu': 12,
    'getc': 13,
    'getc_cpu': 14,
    'get_block': 15,
    'get_block_cpu': 16,
    'seeks': 17,
    'seeks_cpu': 18,
    'num_files': 19,
    'max_size': 20,
    'min_size': 21,
    'num_dirs': 22,
    'file_chunk_size': 23,
    'seq_create': 24,
    'seq_create_cpu': 25,
    'seq_stat': 26,
    'seq_stat_cpu': 27,
    'seq_del': 28,
    'seq_del_cpu': 29,
    'ran_create': 30,
    'ran_create_cpu': 31,
    'ran_stat': 32,
    'ran_stat_cpu': 33,
    'ran_del': 34,
    'ran_del_cpu': 35,
    'putc_latency': 36,
    'put_block_latency': 37,
    'rewrite_latency': 38,
    'getc_latency': 39,
    'get_block_latency': 40,
    'seeks_latency': 41,
    'seq_create_latency': 42,
    'seq_stat_latency': 43,
    'seq_del_latency': 44,
    'ran_create_latency': 45,
    'ran_stat_latency': 46,
    'ran_del_latency': 47}

def GetConfig(user_config):
  return configs.LoadConfig(BENCHMARK_CONFIG, user_config, BENCHMARK_NAME)

def Prepare(benchmark_spec):
  """Install Bonnie++ on the target vm.

  Args:
    benchmark_spec: The benchmark specification. Contains all data that is
        required to run the benchmark.

  """
  vms = benchmark_spec.vms
  vm = vms[0]
  logging.info('Bonnie++ prepare on %s', vm)
  vm.Install('bonnieplusplus')

def IsValueValid(value):
  """Validate the value.

  An invalid value is either an empty string or a string of multiple '+'.

  Args:
    value: string. The value in raw result.

  Returns:
    A boolean indicates if the value is valid or not.
  """
  if value == '' or '+' in value:
    return False
  return True

def IsCpuField(field):
  """Check if the field is cpu percentage.

  Args:
    field: string. The name of the field.

  Returns:
    A boolean indicates if the field contains keyword 'cpu'.
  """
  return 'cpu' in field

def IsLatencyField(field):
  """Check if the field is latency.

  Args:
    field: string. The name of the field.

  Returns:
    A boolean indicates if the field contains keyword 'latency'.
  """
  return 'latency' in field

def ParseLatencyResult(result):
  """Parse latency result into value and unit.

  Args:
    result: string. Latency value in string format, contains value and unit.
            eg. 200ms

  Returns:
    A tuple of value (float) and unit (string).
  """
  match = regex_util.ExtractAllMatches(LATENCY_REGEX, result)[0]
  return float(match[0]), match[1]

def UpdateMetadata(metadata, key, value):
  """Check if the value is valid, update metadata with the key, value pair.

  Args:
    metadata: dict. A dictionary of sample metadata.
    key: string. Key that will be added into metadata dictionary.
    value: Value that of the key.
  """
  if IsValueValid(value):
    metadata[key] = value

def CreateSamples(results, start_index, end_index, metadata,
                  field_index_mapping):
  """Create samples with data in results from start_index to end_index.

  Args:
    results: A list of string representing bonnie++ results.
    start_index: integer. The start index in results list of the samples.
    end_index: integer. The end index in results list of the samples.
    metadata: dict. A dictionary of metadata added into samples.
    field_index_mapping: dict. A dictionary maps field index to field names.

  Returns:
    A list of sample.Sample instances.
  """
  samples = []
  for field_index in range(start_index, end_index):
    field_name = field_index_mapping[field_index]
    value = results[field_index]
    if not IsValueValid(value):
      continue
    if IsCpuField(field_name):
      unit = '%s'
    elif IsLatencyField(field_name):
      value, unit = ParseLatencyResult(value)
    else:
      unit = 'K/sec'
    samples.append(sample.Sample(field_name, float(value), unit, metadata))
  return samples

def ParseCSVResults(results):
  """Parse csv format bonnie++ results.

  Sample Results:
    1.96,1.96,perfkit-7b22f510-0,1,1421800799,7423M,,,,72853,15,47358,5,,,
    156821,7,537.7,10,100,,,,,49223,58,+++++,+++,54405,53,2898,97,+++++,+++,
    59089,60,,512ms,670ms,,44660us,200ms,3747us,1759us,1643us,33518us,192us,
    839us

  Args:
    results: string. Bonnie++ results.

  Returns:
    A list of samples in the form of 3 or 4 tuples. The tuples contain
        the sample metric (string), value (float), and unit (string).
        If a 4th element is included, it is a dictionary of sample
        metadata.
  """
  field_index_mapping = {}
  for field, value in BONNIE_RESULTS_MAPPING.iteritems():
    field_index_mapping[value] = field
  results = results.split(',')
  assert len(results) == len(BONNIE_RESULTS_MAPPING)
  samples = []
  metadata = {}
  for field_index in range(BONNIE_RESULTS_MAPPING['format_version'],
                           BONNIE_RESULTS_MAPPING['chunk_size'] + 1):
    UpdateMetadata(metadata, field_index_mapping[field_index],
                   results[field_index])

  for field_index in range(BONNIE_RESULTS_MAPPING['num_files'],
                           BONNIE_RESULTS_MAPPING['file_chunk_size'] + 1):
    UpdateMetadata(metadata, field_index_mapping[field_index],
                   results[field_index])
  samples.extend(CreateSamples(results,
                               BONNIE_RESULTS_MAPPING['putc'],
                               BONNIE_RESULTS_MAPPING['num_files'],
                               metadata, field_index_mapping))
  samples.extend(CreateSamples(results,
                               BONNIE_RESULTS_MAPPING['seq_create'],
                               BONNIE_RESULTS_MAPPING['ran_del_latency'] + 1,
                               metadata, field_index_mapping))
  return samples

def Run(benchmark_spec):
  """Run Bonnie++ on the target vm.

  Args:
    benchmark_spec: The benchmark specification. Contains all data that is
        required to run the benchmark.

  Returns:
    A list of samples in the form of 3 or 4 tuples. The tuples contain
        the sample metric (string), value (float), and unit (string).
        If a 4th element is included, it is a dictionary of sample
        metadata.
  """
  vms = benchmark_spec.vms
  vm = vms[0]
  logging.info('Bonnie++ running on %s', vm)
  bonnie_command = ('/usr/sbin/bonnie++ -q -d %s -s %d -n 100 -f' %
                    (vm.GetScratchDir(),
                     2 * vm.total_memory_kb / 1024))
  logging.info('Bonnie++ Results:')
  out, _ = vm.RemoteCommand(bonnie_command, should_log=True)
  return ParseCSVResults(out.strip())

def Cleanup(benchmark_spec):
  """Cleanup Bonnie++ on the target vm (by uninstalling).

  Args:
    benchmark_spec: The benchmark specification. Contains all data that is
        required to run the benchmark.
  """
  pass
<|endoftext|>


<commit_before>from django.db.models import signals

from haystack.indexes import *
from haystack import site

from cab.models import Snippet

class SnippetIndex(SearchIndex):
    text = CharField(document=True, use_template=True)
    author = CharField()
    title = CharField(model_attr='title')
    tags = CharField()
    tag_list = MultiValueField()
    language = CharField()
    pub_date = DateTimeField(model_attr='pub_date')
    django_version = FloatField(model_attr='django_version')
    bookmark_count = IntegerField(model_attr='bookmark_count')
    rating_score = IntegerField(model_attr='rating_score')
    url = CharField(indexed=False)

    def _setup_delete(self, model):
        # copied more or less from `haystack.indexes.RealTimeSearchIndex`
        signals.post_delete.connect(
            self.remove_object,
            sender=model,
            dispatch_uid='cab.snippet_index.remove_object',
        )

    def prepare_author(self, obj):
        return obj.author.username

    def prepare_language(self, obj):
        return obj.language.name

    def prepare_tags(self, obj):
        return ' '.join([tag.name for tag in obj.tags.all()])

    def prepare_tag_list(self, obj):
        return [tag.name for tag in obj.tags.all()]

    def prepare_url(self, obj):
        return obj.get_absolute_url()

    def get_updated_field(self):
        return 'updated_date'

site.register(Snippet, SnippetIndex)
<commit_msg>Revert "Hm, back again."<commit_after>from django.db.models import signals

from haystack.indexes import *
from haystack import site

from cab.models import Snippet

class SnippetIndex(RealTimeSearchIndex):
    text = CharField(document=True, use_template=True)
    author = CharField()
    title = CharField(model_attr='title')
    tags = CharField()
    tag_list = MultiValueField()
    language = CharField()
    pub_date = DateTimeField(model_attr='pub_date')
    django_version = FloatField(model_attr='django_version')
    bookmark_count = IntegerField(model_attr='bookmark_count')
    rating_score = IntegerField(model_attr='rating_score')
    url = CharField(indexed=False)

    def _setup_delete(self, model):
        # copied more or less from `haystack.indexes.RealTimeSearchIndex`
        signals.post_delete.connect(
            self.remove_object,
            sender=model,
            dispatch_uid='cab.snippet_index.remove_object',
        )

    def prepare_author(self, obj):
        return obj.author.username

    def prepare_language(self, obj):
        return obj.language.name

    def prepare_tags(self, obj):
        return ' '.join([tag.name for tag in obj.tags.all()])

    def prepare_tag_list(self, obj):
        return [tag.name for tag in obj.tags.all()]

    def prepare_url(self, obj):
        return obj.get_absolute_url()

    def get_updated_field(self):
        return 'updated_date'

site.register(Snippet, SnippetIndex)
<|endoftext|>


<commit_before># coding: utf8
"""
    cairocffi
    ~~~~~~~~~

    CFFI-based cairo bindings for Python. See README for details.

    :copyright: Copyright 2013 by Simon Sapin
    :license: BSD, see LICENSE for details.

"""

import sys
from cffi import FFI

from . import constants
from .compat import FileNotFoundError

VERSION = '0.4.3'
# pycairo compat:
version = VERSION
version_info = version.split('.')

def dlopen(ffi, *names):
    """Try various names for the same library, for different platforms."""
    for name in names:
        try:
            return ffi.dlopen(name)
        except OSError:
            pass
    # Re-raise the exception.
    return ffi.dlopen(names[0])  # pragma: no cover

ffi = FFI()
ffi.cdef(constants._CAIRO_HEADERS)
cairo = dlopen(ffi, 'libcairo.so.2', 'libcairo-2.dll', 'cairo', 'libcairo-2')

class CairoError(Exception):
    """Raised when cairo returns an error status."""
    def __init__(self, message, status):
        super(CairoError, self).__init__(message)
        self.status = status

Error = CairoError  # pycairo compat

STATUS_TO_EXCEPTION = {
    constants.STATUS_NO_MEMORY: MemoryError,
    constants.STATUS_READ_ERROR: IOError,
    constants.STATUS_WRITE_ERROR: IOError,
    constants.STATUS_TEMP_FILE_ERROR: IOError,
    constants.STATUS_FILE_NOT_FOUND: FileNotFoundError,
}

def _check_status(status):
    """Take a cairo status code and raise an exception if/as appropriate."""
    if status != constants.STATUS_SUCCESS:
        exception = STATUS_TO_EXCEPTION.get(status, CairoError)
        status_name = ffi.string(ffi.cast("cairo_status_t", status))
        message = 'cairo returned %s: %s' % (
            status_name, ffi.string(cairo.cairo_status_to_string(status)))
        raise exception(message, status)

def cairo_version():
    """Return the cairo version number as a single integer,
    such as 11208 for ``1.12.8``.
    Major, minor and micro versions are "worth" 10000, 100 and 1 respectively.

    Can be useful as a guard for method not available in older cairo versions::

        if cairo_version() >= 11000:
            surface.set_mime_data('image/jpeg', jpeg_bytes)

    """
    return cairo.cairo_version()

def cairo_version_string():
    """Return the cairo version number as a string, such as ``1.12.8``."""
    return ffi.string(cairo.cairo_version_string()).decode('ascii')

def install_as_pycairo():
    """Install cairocffi so that ``import cairo`` imports it.

    cairoffi’s API is compatible with pycairo as much as possible.

    """
    sys.modules['cairo'] = sys.modules[__name__]

# Implementation is in submodules, but public API is all here.

from .surfaces import (Surface, ImageSurface, PDFSurface, PSSurface,
                       SVGSurface, RecordingSurface)
from .patterns import (Pattern, SolidPattern, SurfacePattern,
                       Gradient, LinearGradient, RadialGradient)
from .fonts import FontFace, ToyFontFace, ScaledFont, FontOptions
from .context import Context
from .matrix import Matrix

from .constants import *
<commit_msg>Better (?) pycairo compat<commit_after># coding: utf8
"""
    cairocffi
    ~~~~~~~~~

    CFFI-based cairo bindings for Python. See README for details.

    :copyright: Copyright 2013 by Simon Sapin
    :license: BSD, see LICENSE for details.

"""

import sys
from cffi import FFI

from . import constants
from .compat import FileNotFoundError

VERSION = '0.4.3'
# pycairo compat:
version = '1.10.0'
version_info = (1, 10, 0)

def dlopen(ffi, *names):
    """Try various names for the same library, for different platforms."""
    for name in names:
        try:
            return ffi.dlopen(name)
        except OSError:
            pass
    # Re-raise the exception.
    return ffi.dlopen(names[0])  # pragma: no cover

ffi = FFI()
ffi.cdef(constants._CAIRO_HEADERS)
cairo = dlopen(ffi, 'libcairo.so.2', 'libcairo-2.dll', 'cairo', 'libcairo-2')

class CairoError(Exception):
    """Raised when cairo returns an error status."""
    def __init__(self, message, status):
        super(CairoError, self).__init__(message)
        self.status = status

Error = CairoError  # pycairo compat

STATUS_TO_EXCEPTION = {
    constants.STATUS_NO_MEMORY: MemoryError,
    constants.STATUS_READ_ERROR: IOError,
    constants.STATUS_WRITE_ERROR: IOError,
    constants.STATUS_TEMP_FILE_ERROR: IOError,
    constants.STATUS_FILE_NOT_FOUND: FileNotFoundError,
}

def _check_status(status):
    """Take a cairo status code and raise an exception if/as appropriate."""
    if status != constants.STATUS_SUCCESS:
        exception = STATUS_TO_EXCEPTION.get(status, CairoError)
        status_name = ffi.string(ffi.cast("cairo_status_t", status))
        message = 'cairo returned %s: %s' % (
            status_name, ffi.string(cairo.cairo_status_to_string(status)))
        raise exception(message, status)

def cairo_version():
    """Return the cairo version number as a single integer,
    such as 11208 for ``1.12.8``.
    Major, minor and micro versions are "worth" 10000, 100 and 1 respectively.

    Can be useful as a guard for method not available in older cairo versions::

        if cairo_version() >= 11000:
            surface.set_mime_data('image/jpeg', jpeg_bytes)

    """
    return cairo.cairo_version()

def cairo_version_string():
    """Return the cairo version number as a string, such as ``1.12.8``."""
    return ffi.string(cairo.cairo_version_string()).decode('ascii')

def install_as_pycairo():
    """Install cairocffi so that ``import cairo`` imports it.

    cairoffi’s API is compatible with pycairo as much as possible.

    """
    sys.modules['cairo'] = sys.modules[__name__]

# Implementation is in submodules, but public API is all here.

from .surfaces import (Surface, ImageSurface, PDFSurface, PSSurface,
                       SVGSurface, RecordingSurface)
from .patterns import (Pattern, SolidPattern, SurfacePattern,
                       Gradient, LinearGradient, RadialGradient)
from .fonts import FontFace, ToyFontFace, ScaledFont, FontOptions
from .context import Context
from .matrix import Matrix

from .constants import *
<|endoftext|>