<commit_before>import socket, string, types
from pysnmp.entity import engine, config
from pysnmp.entity.rfc3413 import cmdgen, mibvar
from pysnmp.carrier.asynsock.dgram import udp
from pysnmp.smi import view
from pysnmp import error
from pyasn1.type import univ

# Auth protocol
usmHMACMD5AuthProtocol = config.usmHMACMD5AuthProtocol
usmHMACSHAAuthProtocol = config.usmHMACSHAAuthProtocol
usmNoAuthProtocol = config.usmNoAuthProtocol

# Privacy protocol
usmDESPrivProtocol = config.usmDESPrivProtocol
usmNoPrivProtocol = config.usmNoPrivProtocol

class CommunityData:
    mpModel=1 # Default is SMIv2
    securityModel=mpModel+1
    securityLevel='noAuthNoPriv'
    def __init__(self, securityName, communityName, mpModel=None):
        self.securityName = securityName
        self.communityName = communityName
        if mpModel is not None:
            self.mpModel = mpModel
            self.securityModel = mpModel + 1

class UsmUserData:
    authKey = privKey = ''
    securityLevel='noAuthNoPriv'
    securityModel=3
    mpModel=2
    def __init__(self, securityName,
                 authKey='', privKey='',
                 authProtocol=usmNoAuthProtocol,
                 privProtocol=usmNoPrivProtocol):
        self.securityName = securityName
        if authKey:
            self.authKey = authKey
            if authProtocol == usmNoAuthProtocol:
                self.authProtocol = usmHMACMD5AuthProtocol
            else:
                self.authProtocol = authProtocol
            if self.securityLevel != 'authPriv':
                self.securityLevel = 'authNoPriv'
        else:
            self.authProtocol = usmNoAuthProtocol
            self.privProtocol = usmNoPrivProtocol
        if privKey:
            self.privKey = privKey
            if self.authProtocol == usmNoAuthProtocol:
                raise error.PySnmpError('Privacy implies authenticity')
            self.securityLevel = 'authPriv'
            if privProtocol == usmNoPrivProtocol:
                self.privProtocol = usmDESPrivProtocol
            else:
                self.privProtocol = privProtocol
        else:
            self.privProtocol = usmNoPrivProtocol

class UdpTransportTarget:
    transportDomain = udp.domainName
    transport = udp.UdpSocketTransport().openClientMode()
    retries = timeout = None  # XXX
    def __init__(self, transportAddr):
        self.transportAddr = (
            socket.gethostbyname(transportAddr[0]), transportAddr[1]
            )

class AsynCommandGenerator:
    _null = univ.Null('')
    def __init__(self, snmpEngine=None):
        if snmpEngine is None:
            self.snmpEngine = engine.SnmpEngine()
        else:
            self.snmpEngine = snmpEngine
        self.mibViewController = view.MibViewController(
            self.snmpEngine.msgAndPduDsp.mibInstrumController.mibBuilder
            )
        self.__knownAuths = {}
        self.__knownTransports = {}
        self.__knownTransportAddrs = {}

    def _configure(self, authData, transportTarget, tagList=''):
        paramsName = '%s-params' % (authData.securityName,)
        if not self.__knownAuths.has_key(authData):
            if isinstance(authData, CommunityData):
                config.addV1System(
                    self.snmpEngine,
                    authData.securityName,
                    authData.communityName
                    )
                config.addTargetParams(
                    self.snmpEngine, paramsName,
                    authData.securityName, authData.securityLevel,
                    authData.mpModel
                    )
            elif isinstance(authData, UsmUserData):
                config.addV3User(
                    self.snmpEngine,
                    authData.securityName,
                    authData.authProtocol, authData.authKey,
                    authData.privProtocol, authData.privKey
                    )
                config.addTargetParams(
                    self.snmpEngine, paramsName,
                    authData.securityName, authData.securityLevel
                    )
            else:
                raise error.PySnmpError('Unsupported SNMP version')
            self.__knownAuths[authData] = 1

        if not self.__knownTransports.has_key(transportTarget.transport):
            config.addSocketTransport(
                self.snmpEngine,
                transportTarget.transportDomain,
                transportTarget.transport
                )
            self.__knownTransports[transportTarget.transport] = 1

        addrName = str(transportTarget.transportAddr)
        if not self.__knownTransportAddrs.has_key(addrName):
            config.addTargetAddr(
                self.snmpEngine, addrName,
                transportTarget.transportDomain,
                transportTarget.transportAddr,
                paramsName,
                tagList=tagList
                )
            self.__knownTransportAddrs[addrName] = 1

        return addrName, paramsName

    # Async SNMP apps

    def asyncGetCmd(
        self, authData, transportTarget, varNames, (cbFun, cbCtx)
        ):
        addrName, paramsName = self._configure(
            authData, transportTarget
            )
        varBinds = []
        for varName in varNames:
            name, oid = mibvar.mibNameToOid(
                self.mibViewController, varName
                )
            varBinds.append((name + oid, self._null))
        return cmdgen.GetCommandGenerator().sendReq(
            self.snmpEngine, addrName, varBinds, cbFun, cbCtx
            )

    def asyncSetCmd(
        self, authData, transportTarget, varBinds, (cbFun, cbCtx)
        ):
        addrName, paramsName = self._configure(
            authData, transportTarget
            )
        __varBinds = []
        for varName, varVal in varBinds:
            name, oid = mibvar.mibNameToOid(
                self.mibViewController, varName
                )
            __varBinds.append((name + oid, varVal))
        return cmdgen.SetCommandGenerator().sendReq(
            self.snmpEngine, addrName, __varBinds, cbFun, cbCtx
            )

    def asyncNextCmd(
        self, authData, transportTarget, varNames, (cbFun, cbCtx)
        ):
        addrName, paramsName = self._configure(
            authData, transportTarget
            )
        varBinds = []
        for varName in varNames:
            name, oid = mibvar.mibNameToOid(
                self.mibViewController, varName
                )
            varBinds.append((name + oid, self._null))
        return cmdgen.NextCommandGenerator().sendReq(
            self.snmpEngine, addrName, varBinds, cbFun, cbCtx
            )

    def asyncBulkCmd(
        self, authData, transportTarget, nonRepeaters, maxRepetitions,
        varNames, (cbFun, cbCtx)
        ):
        addrName, paramsName = self._configure(
            authData, transportTarget
            )
        varBinds = []
        for varName in varNames:
            name, oid = mibvar.mibNameToOid(
                self.mibViewController, varName
                )
            varBinds.append((name + oid, self._null))
        return cmdgen.BulkCommandGenerator().sendReq(
            self.snmpEngine, addrName, nonRepeaters, maxRepetitions,
            varBinds, cbFun, cbCtx
            )

class CommandGenerator(AsynCommandGenerator):
    def getCmd(self, authData, transportTarget, *varNames):
        def __cbFun(
            sendRequestHandle, errorIndication, errorStatus, errorIndex,
            varBinds, appReturn
            ):
            appReturn['errorIndication'] = errorIndication
            appReturn['errorStatus'] = errorStatus
            appReturn['errorIndex'] = errorIndex
            appReturn['varBinds'] = varBinds

        appReturn = {}
        self.asyncGetCmd(
            authData, transportTarget, varNames, (__cbFun, appReturn)
            )
        self.snmpEngine.transportDispatcher.runDispatcher()
        return (
            appReturn['errorIndication'],
            appReturn['errorStatus'],
            appReturn['errorIndex'],
            appReturn['varBinds']
            )

    def setCmd(self, authData, transportTarget, *varBinds):
        def __cbFun(
            sendRequestHandle, errorIndication, errorStatus, errorIndex,
            varBinds, appReturn
            ):
            appReturn['errorIndication'] = errorIndication
            appReturn['errorStatus'] = errorStatus
            appReturn['errorIndex'] = errorIndex
            appReturn['varBinds'] = varBinds

        appReturn = {}
        self.asyncSetCmd(
            authData, transportTarget, varBinds, (__cbFun, appReturn)
            )
        self.snmpEngine.transportDispatcher.runDispatcher()
        return (
            appReturn['errorIndication'],
            appReturn['errorStatus'],
            appReturn['errorIndex'],
            appReturn['varBinds']
            )

    def nextCmd(self, authData, transportTarget, *varNames):
        def __cbFun(
            sendRequestHandle, errorIndication, errorStatus, errorIndex,
            varBindTable, (varBindHead, varBindTotalTable, appReturn)
            ):
            if errorIndication or errorStatus:
                appReturn['errorIndication'] = errorIndication
                appReturn['errorStatus'] = errorStatus
                appReturn['errorIndex'] = errorIndex
                appReturn['varBindTable'] = varBindTable
                return
            else:
                varBindTableRow = varBindTable[-1]
                for idx in range(len(varBindTableRow)):
                    name, val = varBindTableRow[idx]
                    if head[idx].isPrefixOf(name):  # XXX causes extra rows
                        break
                else:
                    appReturn['errorIndication'] = errorIndication
                    appReturn['errorStatus'] = errorStatus
                    appReturn['errorIndex'] = errorIndex
                    appReturn['varBindTable'] = varBindTotalTable
                    return
                varBindTotalTable.extend(varBindTable)

            return 1 # continue table retrieval

        head = map(lambda (x,y): univ.ObjectIdentifier(x+y), map(lambda x,self=self: mibvar.mibNameToOid(self.mibViewController, x), varNames))

        appReturn = {}
        self.asyncNextCmd(
            authData, transportTarget, varNames, (__cbFun, (head,[],appReturn))
            )

        self.snmpEngine.transportDispatcher.runDispatcher()

        return (
            appReturn['errorIndication'],
            appReturn['errorStatus'],
            appReturn['errorIndex'],
            appReturn['varBindTable']
            )

    def bulkCmd(self, authData, transportTarget,
                nonRepeaters, maxRepetitions, *varNames):
        def __cbFun(
            sendRequestHandle, errorIndication, errorStatus, errorIndex,
            varBindTable, (varBindHead, varBindTotalTable, appReturn)
            ):
            if errorIndication or errorStatus:
                appReturn['errorIndication'] = errorIndication
                appReturn['errorStatus'] = errorStatus
                appReturn['errorIndex'] = errorIndex
                appReturn['varBindTable'] = varBindTable
                return
            else:
                varBindTotalTable.extend(varBindTable) # XXX out of table
                                                       # rows possible
                varBindTableRow = varBindTable[-1]
                for idx in range(len(varBindTableRow)):
                    name, val = varBindTableRow[idx]
                    if head[idx].isPrefixOf(name):
                        break
                else:
                    appReturn['errorIndication'] = errorIndication
                    appReturn['errorStatus'] = errorStatus
                    appReturn['errorIndex'] = errorIndex
                    appReturn['varBindTable'] = varBindTotalTable
                    return

            return 1 # continue table retrieval

        head = map(lambda (x,y): univ.ObjectIdentifier(x+y), map(lambda x,self=self: mibvar.mibNameToOid(self.mibViewController, x), varNames))

        appReturn = {}

        self.asyncBulkCmd(
            authData, transportTarget, nonRepeaters, maxRepetitions,
            varNames, (__cbFun, (head, [], appReturn))
            )

        self.snmpEngine.transportDispatcher.runDispatcher()

        return (
            appReturn['errorIndication'],
            appReturn['errorStatus'],
            appReturn['errorIndex'],
            appReturn['varBindTable']
            )
<commit_msg>* python1.5 compatibity fix related to names scoping * check if value == None in varBinds what indicates EOM<commit_after>import socket, string, types
from pysnmp.entity import engine, config
from pysnmp.entity.rfc3413 import cmdgen, mibvar
from pysnmp.carrier.asynsock.dgram import udp
from pysnmp.smi import view
from pysnmp import error
from pyasn1.type import univ

# Auth protocol
usmHMACMD5AuthProtocol = config.usmHMACMD5AuthProtocol
usmHMACSHAAuthProtocol = config.usmHMACSHAAuthProtocol
usmNoAuthProtocol = config.usmNoAuthProtocol

# Privacy protocol
usmDESPrivProtocol = config.usmDESPrivProtocol
usmNoPrivProtocol = config.usmNoPrivProtocol

class CommunityData:
    mpModel=1 # Default is SMIv2
    securityModel=mpModel+1
    securityLevel='noAuthNoPriv'
    def __init__(self, securityName, communityName, mpModel=None):
        self.securityName = securityName
        self.communityName = communityName
        if mpModel is not None:
            self.mpModel = mpModel
            self.securityModel = mpModel + 1

class UsmUserData:
    authKey = privKey = ''
    securityLevel='noAuthNoPriv'
    securityModel=3
    mpModel=2
    def __init__(self, securityName,
                 authKey='', privKey='',
                 authProtocol=usmNoAuthProtocol,
                 privProtocol=usmNoPrivProtocol):
        self.securityName = securityName
        if authKey:
            self.authKey = authKey
            if authProtocol == usmNoAuthProtocol:
                self.authProtocol = usmHMACMD5AuthProtocol
            else:
                self.authProtocol = authProtocol
            if self.securityLevel != 'authPriv':
                self.securityLevel = 'authNoPriv'
        else:
            self.authProtocol = usmNoAuthProtocol
            self.privProtocol = usmNoPrivProtocol
        if privKey:
            self.privKey = privKey
            if self.authProtocol == usmNoAuthProtocol:
                raise error.PySnmpError('Privacy implies authenticity')
            self.securityLevel = 'authPriv'
            if privProtocol == usmNoPrivProtocol:
                self.privProtocol = usmDESPrivProtocol
            else:
                self.privProtocol = privProtocol
        else:
            self.privProtocol = usmNoPrivProtocol

class UdpTransportTarget:
    transportDomain = udp.domainName
    transport = udp.UdpSocketTransport().openClientMode()
    retries = timeout = None  # XXX
    def __init__(self, transportAddr):
        self.transportAddr = (
            socket.gethostbyname(transportAddr[0]), transportAddr[1]
            )

class AsynCommandGenerator:
    _null = univ.Null('')
    def __init__(self, snmpEngine=None):
        if snmpEngine is None:
            self.snmpEngine = engine.SnmpEngine()
        else:
            self.snmpEngine = snmpEngine
        self.mibViewController = view.MibViewController(
            self.snmpEngine.msgAndPduDsp.mibInstrumController.mibBuilder
            )
        self.__knownAuths = {}
        self.__knownTransports = {}
        self.__knownTransportAddrs = {}

    def _configure(self, authData, transportTarget, tagList=''):
        paramsName = '%s-params' % (authData.securityName,)
        if not self.__knownAuths.has_key(authData):
            if isinstance(authData, CommunityData):
                config.addV1System(
                    self.snmpEngine,
                    authData.securityName,
                    authData.communityName
                    )
                config.addTargetParams(
                    self.snmpEngine, paramsName,
                    authData.securityName, authData.securityLevel,
                    authData.mpModel
                    )
            elif isinstance(authData, UsmUserData):
                config.addV3User(
                    self.snmpEngine,
                    authData.securityName,
                    authData.authProtocol, authData.authKey,
                    authData.privProtocol, authData.privKey
                    )
                config.addTargetParams(
                    self.snmpEngine, paramsName,
                    authData.securityName, authData.securityLevel
                    )
            else:
                raise error.PySnmpError('Unsupported SNMP version')
            self.__knownAuths[authData] = 1

        if not self.__knownTransports.has_key(transportTarget.transport):
            config.addSocketTransport(
                self.snmpEngine,
                transportTarget.transportDomain,
                transportTarget.transport
                )
            self.__knownTransports[transportTarget.transport] = 1

        addrName = str(transportTarget.transportAddr)
        if not self.__knownTransportAddrs.has_key(addrName):
            config.addTargetAddr(
                self.snmpEngine, addrName,
                transportTarget.transportDomain,
                transportTarget.transportAddr,
                paramsName,
                tagList=tagList
                )
            self.__knownTransportAddrs[addrName] = 1

        return addrName, paramsName

    # Async SNMP apps

    def asyncGetCmd(
        self, authData, transportTarget, varNames, (cbFun, cbCtx)
        ):
        addrName, paramsName = self._configure(
            authData, transportTarget
            )
        varBinds = []
        for varName in varNames:
            name, oid = mibvar.mibNameToOid(
                self.mibViewController, varName
                )
            varBinds.append((name + oid, self._null))
        return cmdgen.GetCommandGenerator().sendReq(
            self.snmpEngine, addrName, varBinds, cbFun, cbCtx
            )

    def asyncSetCmd(
        self, authData, transportTarget, varBinds, (cbFun, cbCtx)
        ):
        addrName, paramsName = self._configure(
            authData, transportTarget
            )
        __varBinds = []
        for varName, varVal in varBinds:
            name, oid = mibvar.mibNameToOid(
                self.mibViewController, varName
                )
            __varBinds.append((name + oid, varVal))
        return cmdgen.SetCommandGenerator().sendReq(
            self.snmpEngine, addrName, __varBinds, cbFun, cbCtx
            )

    def asyncNextCmd(
        self, authData, transportTarget, varNames, (cbFun, cbCtx)
        ):
        addrName, paramsName = self._configure(
            authData, transportTarget
            )
        varBinds = []
        for varName in varNames:
            name, oid = mibvar.mibNameToOid(
                self.mibViewController, varName
                )
            varBinds.append((name + oid, self._null))
        return cmdgen.NextCommandGenerator().sendReq(
            self.snmpEngine, addrName, varBinds, cbFun, cbCtx
            )

    def asyncBulkCmd(
        self, authData, transportTarget, nonRepeaters, maxRepetitions,
        varNames, (cbFun, cbCtx)
        ):
        addrName, paramsName = self._configure(
            authData, transportTarget
            )
        varBinds = []
        for varName in varNames:
            name, oid = mibvar.mibNameToOid(
                self.mibViewController, varName
                )
            varBinds.append((name + oid, self._null))
        return cmdgen.BulkCommandGenerator().sendReq(
            self.snmpEngine, addrName, nonRepeaters, maxRepetitions,
            varBinds, cbFun, cbCtx
            )

class CommandGenerator(AsynCommandGenerator):
    def getCmd(self, authData, transportTarget, *varNames):
        def __cbFun(
            sendRequestHandle, errorIndication, errorStatus, errorIndex,
            varBinds, appReturn
            ):
            appReturn['errorIndication'] = errorIndication
            appReturn['errorStatus'] = errorStatus
            appReturn['errorIndex'] = errorIndex
            appReturn['varBinds'] = varBinds

        appReturn = {}
        self.asyncGetCmd(
            authData, transportTarget, varNames, (__cbFun, appReturn)
            )
        self.snmpEngine.transportDispatcher.runDispatcher()
        return (
            appReturn['errorIndication'],
            appReturn['errorStatus'],
            appReturn['errorIndex'],
            appReturn['varBinds']
            )

    def setCmd(self, authData, transportTarget, *varBinds):
        def __cbFun(
            sendRequestHandle, errorIndication, errorStatus, errorIndex,
            varBinds, appReturn
            ):
            appReturn['errorIndication'] = errorIndication
            appReturn['errorStatus'] = errorStatus
            appReturn['errorIndex'] = errorIndex
            appReturn['varBinds'] = varBinds

        appReturn = {}
        self.asyncSetCmd(
            authData, transportTarget, varBinds, (__cbFun, appReturn)
            )
        self.snmpEngine.transportDispatcher.runDispatcher()
        return (
            appReturn['errorIndication'],
            appReturn['errorStatus'],
            appReturn['errorIndex'],
            appReturn['varBinds']
            )

    def nextCmd(self, authData, transportTarget, *varNames):
        def __cbFun(
            sendRequestHandle, errorIndication, errorStatus, errorIndex,
            varBindTable, (varBindHead, varBindTotalTable, appReturn)
            ):
            if errorIndication or errorStatus:
                appReturn['errorIndication'] = errorIndication
                appReturn['errorStatus'] = errorStatus
                appReturn['errorIndex'] = errorIndex
                appReturn['varBindTable'] = varBindTable
                return
            else:
                varBindTableRow = varBindTable[-1]
                for idx in range(len(varBindTableRow)):
                    name, val = varBindTableRow[idx]
                    # XXX extra rows
                    if val is not None and varBindHead[idx].isPrefixOf(name):
                        break
                else:
                    appReturn['errorIndication'] = errorIndication
                    appReturn['errorStatus'] = errorStatus
                    appReturn['errorIndex'] = errorIndex
                    appReturn['varBindTable'] = varBindTotalTable
                    return
                varBindTotalTable.extend(varBindTable)

            return 1 # continue table retrieval

        varBindHead = map(lambda (x,y): univ.ObjectIdentifier(x+y), map(lambda x,self=self: mibvar.mibNameToOid(self.mibViewController, x), varNames))

        appReturn = {}
        self.asyncNextCmd(
            authData, transportTarget, varNames,
            (__cbFun, (varBindHead,[],appReturn))
            )

        self.snmpEngine.transportDispatcher.runDispatcher()

        return (
            appReturn['errorIndication'],
            appReturn['errorStatus'],
            appReturn['errorIndex'],
            appReturn['varBindTable']
            )

    def bulkCmd(self, authData, transportTarget,
                nonRepeaters, maxRepetitions, *varNames):
        def __cbFun(
            sendRequestHandle, errorIndication, errorStatus, errorIndex,
            varBindTable, (varBindHead, varBindTotalTable, appReturn)
            ):
            if errorIndication or errorStatus:
                appReturn['errorIndication'] = errorIndication
                appReturn['errorStatus'] = errorStatus
                appReturn['errorIndex'] = errorIndex
                appReturn['varBindTable'] = varBindTable
                return
            else:
                varBindTotalTable.extend(varBindTable) # XXX out of table
                                                       # rows possible
                varBindTableRow = varBindTable[-1]
                for idx in range(len(varBindTableRow)):
                    name, val = varBindTableRow[idx]
                    if val is not None and varBindHead[idx].isPrefixOf(name):
                        break
                else:
                    appReturn['errorIndication'] = errorIndication
                    appReturn['errorStatus'] = errorStatus
                    appReturn['errorIndex'] = errorIndex
                    appReturn['varBindTable'] = varBindTotalTable
                    return

            return 1 # continue table retrieval

        varBindHead = map(lambda (x,y): univ.ObjectIdentifier(x+y), map(lambda x,self=self: mibvar.mibNameToOid(self.mibViewController, x), varNames))

        appReturn = {}

        self.asyncBulkCmd(
            authData, transportTarget, nonRepeaters, maxRepetitions,
            varNames, (__cbFun, (varBindHead, [], appReturn))
            )

        self.snmpEngine.transportDispatcher.runDispatcher()

        return (
            appReturn['errorIndication'],
            appReturn['errorStatus'],
            appReturn['errorIndex'],
            appReturn['varBindTable']
            )
<|endoftext|>


<commit_before>#!/usr/bin/env python
# Copyright (c) 2011 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""Retrieve passing and failing WebKit revision numbers from canaries.

From each canary,
- the last WebKit revision number for which all the tests have passed,
- the last WebKit revision number for which the tests were run, and
- the names of failing layout tests
are retrieved and printed.
"""

import json
import optparse
import re
import sys
import urllib2

_WEBKIT_REVISION_IN_DEPS_RE = re.compile(r'"webkit_revision"\s*:\s*"(\d+)"')
_DEPS_FILE_URL = "http://src.chromium.org/viewvc/chrome/trunk/src/DEPS"
_DEFAULT_BUILDERS = [
  "Webkit Win",
  "Webkit Vista",
  "Webkit Win7",
  "Webkit Win (dbg)(1)",
  "Webkit Win (dbg)(2)",
  "Webkit Mac10.5",
  "Webkit Mac10.6",
  "Webkit Mac10.5 (dbg)(1)",
  "Webkit Mac10.5 (dbg)(2)",
  "Webkit Mac10.6 (dbg)",
  "Webkit Linux",
  "Webkit Linux 32",
  "Webkit Linux (dbg)(1)",
  "Webkit Linux (dbg)(2)",
]
_DEFAULT_MAX_BUILDS = 10
_TEST_PREFIX = "&tests="
_TEST_SUFFIX = '">'
_WEBKIT_TESTS = "webkit_tests"

def _OpenUrl(url):
  """Opens a URL.

  Returns:
      A file-like object in case of success, an empty list otherwise.
  """
  try:
    return urllib2.urlopen(url)
  except urllib2.URLError, url_error:
    message = ""
    # Surprisingly, urllib2.URLError has different attributes based on the
    # kinds of errors -- "code" for HTTP-level errors, "reason" for others.
    if hasattr(url_error, "code"):
      message = "Status code: %d" % url_error.code
    if hasattr(url_error, "reason"):
      message = url_error.reason
    print >>sys.stderr, "Failed to open %s: %s" % (url, message)
  return []

def _WebkitRevisionInDeps():
  """Returns the WebKit revision specified in DEPS file.

  Returns:
      Revision number as int. -1 in case of error.
  """
  for line in _OpenUrl(_DEPS_FILE_URL):
    match = _WEBKIT_REVISION_IN_DEPS_RE.search(line)
    if match:
      return int(match.group(1))
  return -1

class _BuildResult(object):
  """Build result for a builder.

  Holds builder name, the last passing revision, the last run revision, and
  a list of names of failing tests. Revision nubmer 0 is used to represent
  that the revision doesn't exist.
  """
  def __init__(self, builder, last_passing_revision, last_run_revision,
               failing_tests):
    """Constructs build results."""
    self.builder = builder
    self.last_passing_revision = last_passing_revision
    self.last_run_revision = last_run_revision
    self.failing_tests = failing_tests

def _BuilderUrlFor(builder, max_builds):
  """Constructs the URL for a builder to retrieve the last results."""
  url = ("http://build.chromium.org/p/chromium.webkit/json/builders/%s/builds" %
         urllib2.quote(builder))
  if max_builds == -1:
    return url + "/_all?as_text=1"
  return (url + "?as_text=1&" +
          '&'.join(["select=%d" % -i for i in range(1, 1 + max_builds)]))

def _ExtractFailingTests(build):
  """Extracts failing test names from a build result entry JSON object."""
  failing_tests = []
  for step in build["steps"]:
    if step["name"] == _WEBKIT_TESTS:
      for text in step["text"]:
        prefix = text.find(_TEST_PREFIX)
        suffix = text.find(_TEST_SUFFIX)
        if prefix != -1 and suffix != -1:
          failing_tests += sorted(
              text[prefix + len(_TEST_PREFIX): suffix].split(","))
    elif "results" in step:
      # Existence of "results" entry seems to mean failure.
      failing_tests.append(" ".join(step["text"]))
  return failing_tests

def _RetrieveBuildResult(builder, max_builds, oldest_revision_to_check):
  """Retrieves build results for a builder.

  Checks the last passing revision, the last run revision, and failing tests
  for the last builds of a builder.

  Args:
      builder: Builder name.
      max_builds: Maximum number of builds to check.
      oldest_revision_to_check: Oldest WebKit revision to check.

  Returns:
      _BuildResult instance.
  """
  last_run_revision = 0
  failing_tests = []
  succeeded = False
  builds_json = _OpenUrl(_BuilderUrlFor(builder, max_builds))
  if not builds_json:
    return _BuildResult(builder, 0, 0, failing_tests)
  builds = [(int(value["number"]), value) for unused_key, value
            in json.loads(''.join(builds_json)).items()
            if value.has_key("number")]
  builds.sort()
  builds.reverse()
  for unused_key, build in builds:
    if not build.has_key("text"):
      continue
    if len(build["text"]) < 2:
      continue
    if not build.has_key("sourceStamp"):
      continue
    if build["text"][1] == "successful":
      succeeded = True
    elif not failing_tests:
      failing_tests = _ExtractFailingTests(build)
    revision = 0
    if build["sourceStamp"]["branch"] == "trunk":
      revision = int(build["sourceStamp"]["changes"][-1]["revision"])
    if revision and not last_run_revision:
      last_run_revision = revision
    if revision and revision < oldest_revision_to_check:
      break
    if not succeeded or not revision:
      continue
    return _BuildResult(builder, revision, last_run_revision, failing_tests)
  return _BuildResult(builder, 0, last_run_revision, failing_tests)

def _PrintPassingRevisions(results, unused_verbose):
  """Prints passing revisions and the range of such revisions.

  Args:
      results: A list of build results.
  """
  print "**** Passing revisions *****"
  min_passing_revision = sys.maxint
  max_passing_revision = 0
  for result in results:
    if result.last_passing_revision:
      min_passing_revision = min(min_passing_revision,
                                 result.last_passing_revision)
      max_passing_revision = max(max_passing_revision,
                                 result.last_passing_revision)
      print 'The last passing run was at r%d on "%s"' % (
          result.last_passing_revision, result.builder)
    else:
      print 'No passing runs on "%s"' % result.builder
  if max_passing_revision:
    print "Passing revision range: r%d - r%d" % (
          min_passing_revision, max_passing_revision)

def _PrintFailingRevisions(results, verbose):
  """Prints failing revisions and the failing tests.

  Args:
      results: A list of build results.
  """
  failing_test_to_builders = {}
  print "**** Failing revisions *****"
  for result in results:
    if result.last_run_revision and result.failing_tests:
      print ('The last run was at r%d on "%s" and the following %d tests'
             ' failed' % (result.last_run_revision, result.builder,
                          len(result.failing_tests)))
      for test in result.failing_tests:
        print "  " + test
        failing_test_to_builders.setdefault(test, set()).add(result.builder)
  if verbose:
    _PrintFailingTestsForBuilderSubsets(failing_test_to_builders)

class _FailingTestsForBuilderSubset(object):
  def __init__(self, subset_size):
    self._subset_size = subset_size
    self._tests = []

  def SubsetSize(self):
    return self._subset_size

  def Tests(self):
    return self._tests

def _PrintFailingTestsForBuilderSubsets(failing_test_to_builders):
  """Prints failing test for builder subsets.

  Prints failing tests for each subset of builders, in descending order of the
  set size.
  """
  print "**** Failing tests ****"
  builders_to_tests = {}
  for test in failing_test_to_builders:
    builders = sorted(failing_test_to_builders[test])
    subset_name = ", ".join(builders)
    tests = builders_to_tests.setdefault(
        subset_name, _FailingTestsForBuilderSubset(len(builders))).Tests()
    tests.append(test)
  # Sort subsets in descending order of size and then name.
  builder_subsets = [(builders_to_tests[subset_name].SubsetSize(), subset_name)
                     for subset_name in builders_to_tests]
  for subset_size, subset_name in reversed(sorted(builder_subsets)):
    print "** Tests failing for %d builders: %s **" % (subset_size,
                                                       subset_name)
    for test in sorted(builders_to_tests[subset_name].Tests()):
      print test

def _ParseOptions():
  """Parses command-line options."""
  parser = optparse.OptionParser(usage="%prog [options] [builders]")
  parser.add_option("-m", "--max_builds", type="int",
                    default=-1,
                    help="Maximum number of builds to check for each builder."
                         " Defaults to all builds for which record is"
                         " available. Checking is ended either when the maximum"
                         " number is reached, the remaining builds are older"
                         " than the DEPS WebKit revision, or a passing"
                         " revision is found.")
  parser.add_option("-v", "--verbose", action="store_true", default=False,
                    dest="verbose")
  return parser.parse_args()

def _Main():
  """The main function."""
  options, builders = _ParseOptions()
  if not builders:
    builders = _DEFAULT_BUILDERS
  oldest_revision_to_check = _WebkitRevisionInDeps()
  if options.max_builds == -1 and oldest_revision_to_check == -1:
    options.max_builds = _DEFAULT_MAX_BUILDS
  if options.max_builds != -1:
    print "Maxium number of builds to check: %d" % options.max_builds
  if oldest_revision_to_check != -1:
    print "Oldest revision to check: %d" % oldest_revision_to_check
  sys.stdout.flush()
  results = []
  for builder in builders:
    print '"%s"' % builder
    sys.stdout.flush()
    results.append(_RetrieveBuildResult(
        builder, options.max_builds, oldest_revision_to_check))
  _PrintFailingRevisions(results, options.verbose)
  _PrintPassingRevisions(results, options.verbose)

if __name__ == "__main__":
  _Main()
<commit_msg>Pick up Mac bot renames.<commit_after>#!/usr/bin/env python
# Copyright (c) 2011 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""Retrieve passing and failing WebKit revision numbers from canaries.

From each canary,
- the last WebKit revision number for which all the tests have passed,
- the last WebKit revision number for which the tests were run, and
- the names of failing layout tests
are retrieved and printed.
"""

import json
import optparse
import re
import sys
import urllib2

_WEBKIT_REVISION_IN_DEPS_RE = re.compile(r'"webkit_revision"\s*:\s*"(\d+)"')
_DEPS_FILE_URL = "http://src.chromium.org/viewvc/chrome/trunk/src/DEPS"
_DEFAULT_BUILDERS = [
  "Webkit Win",
  "Webkit Vista",
  "Webkit Win7",
  "Webkit Win (dbg)(1)",
  "Webkit Win (dbg)(2)",
  "Webkit Mac10.5 (CG)",
  "Webkit Mac10.6 (CG)",
  "Webkit Mac10.5 (CG)(dbg)(1)",
  "Webkit Mac10.5 (CG)(dbg)(2)",
  "Webkit Mac10.6 (CG)(dbg)",
  "Webkit Linux",
  "Webkit Linux 32",
  "Webkit Linux (dbg)(1)",
  "Webkit Linux (dbg)(2)",
]
_DEFAULT_MAX_BUILDS = 10
_TEST_PREFIX = "&tests="
_TEST_SUFFIX = '">'
_WEBKIT_TESTS = "webkit_tests"

def _OpenUrl(url):
  """Opens a URL.

  Returns:
      A file-like object in case of success, an empty list otherwise.
  """
  try:
    return urllib2.urlopen(url)
  except urllib2.URLError, url_error:
    message = ""
    # Surprisingly, urllib2.URLError has different attributes based on the
    # kinds of errors -- "code" for HTTP-level errors, "reason" for others.
    if hasattr(url_error, "code"):
      message = "Status code: %d" % url_error.code
    if hasattr(url_error, "reason"):
      message = url_error.reason
    print >>sys.stderr, "Failed to open %s: %s" % (url, message)
  return []

def _WebkitRevisionInDeps():
  """Returns the WebKit revision specified in DEPS file.

  Returns:
      Revision number as int. -1 in case of error.
  """
  for line in _OpenUrl(_DEPS_FILE_URL):
    match = _WEBKIT_REVISION_IN_DEPS_RE.search(line)
    if match:
      return int(match.group(1))
  return -1

class _BuildResult(object):
  """Build result for a builder.

  Holds builder name, the last passing revision, the last run revision, and
  a list of names of failing tests. Revision nubmer 0 is used to represent
  that the revision doesn't exist.
  """
  def __init__(self, builder, last_passing_revision, last_run_revision,
               failing_tests):
    """Constructs build results."""
    self.builder = builder
    self.last_passing_revision = last_passing_revision
    self.last_run_revision = last_run_revision
    self.failing_tests = failing_tests

def _BuilderUrlFor(builder, max_builds):
  """Constructs the URL for a builder to retrieve the last results."""
  url = ("http://build.chromium.org/p/chromium.webkit/json/builders/%s/builds" %
         urllib2.quote(builder))
  if max_builds == -1:
    return url + "/_all?as_text=1"
  return (url + "?as_text=1&" +
          '&'.join(["select=%d" % -i for i in range(1, 1 + max_builds)]))

def _ExtractFailingTests(build):
  """Extracts failing test names from a build result entry JSON object."""
  failing_tests = []
  for step in build["steps"]:
    if step["name"] == _WEBKIT_TESTS:
      for text in step["text"]:
        prefix = text.find(_TEST_PREFIX)
        suffix = text.find(_TEST_SUFFIX)
        if prefix != -1 and suffix != -1:
          failing_tests += sorted(
              text[prefix + len(_TEST_PREFIX): suffix].split(","))
    elif "results" in step:
      # Existence of "results" entry seems to mean failure.
      failing_tests.append(" ".join(step["text"]))
  return failing_tests

def _RetrieveBuildResult(builder, max_builds, oldest_revision_to_check):
  """Retrieves build results for a builder.

  Checks the last passing revision, the last run revision, and failing tests
  for the last builds of a builder.

  Args:
      builder: Builder name.
      max_builds: Maximum number of builds to check.
      oldest_revision_to_check: Oldest WebKit revision to check.

  Returns:
      _BuildResult instance.
  """
  last_run_revision = 0
  failing_tests = []
  succeeded = False
  builds_json = _OpenUrl(_BuilderUrlFor(builder, max_builds))
  if not builds_json:
    return _BuildResult(builder, 0, 0, failing_tests)
  builds = [(int(value["number"]), value) for unused_key, value
            in json.loads(''.join(builds_json)).items()
            if value.has_key("number")]
  builds.sort()
  builds.reverse()
  for unused_key, build in builds:
    if not build.has_key("text"):
      continue
    if len(build["text"]) < 2:
      continue
    if not build.has_key("sourceStamp"):
      continue
    if build["text"][1] == "successful":
      succeeded = True
    elif not failing_tests:
      failing_tests = _ExtractFailingTests(build)
    revision = 0
    if build["sourceStamp"]["branch"] == "trunk":
      revision = int(build["sourceStamp"]["changes"][-1]["revision"])
    if revision and not last_run_revision:
      last_run_revision = revision
    if revision and revision < oldest_revision_to_check:
      break
    if not succeeded or not revision:
      continue
    return _BuildResult(builder, revision, last_run_revision, failing_tests)
  return _BuildResult(builder, 0, last_run_revision, failing_tests)

def _PrintPassingRevisions(results, unused_verbose):
  """Prints passing revisions and the range of such revisions.

  Args:
      results: A list of build results.
  """
  print "**** Passing revisions *****"
  min_passing_revision = sys.maxint
  max_passing_revision = 0
  for result in results:
    if result.last_passing_revision:
      min_passing_revision = min(min_passing_revision,
                                 result.last_passing_revision)
      max_passing_revision = max(max_passing_revision,
                                 result.last_passing_revision)
      print 'The last passing run was at r%d on "%s"' % (
          result.last_passing_revision, result.builder)
    else:
      print 'No passing runs on "%s"' % result.builder
  if max_passing_revision:
    print "Passing revision range: r%d - r%d" % (
          min_passing_revision, max_passing_revision)

def _PrintFailingRevisions(results, verbose):
  """Prints failing revisions and the failing tests.

  Args:
      results: A list of build results.
  """
  failing_test_to_builders = {}
  print "**** Failing revisions *****"
  for result in results:
    if result.last_run_revision and result.failing_tests:
      print ('The last run was at r%d on "%s" and the following %d tests'
             ' failed' % (result.last_run_revision, result.builder,
                          len(result.failing_tests)))
      for test in result.failing_tests:
        print "  " + test
        failing_test_to_builders.setdefault(test, set()).add(result.builder)
  if verbose:
    _PrintFailingTestsForBuilderSubsets(failing_test_to_builders)

class _FailingTestsForBuilderSubset(object):
  def __init__(self, subset_size):
    self._subset_size = subset_size
    self._tests = []

  def SubsetSize(self):
    return self._subset_size

  def Tests(self):
    return self._tests

def _PrintFailingTestsForBuilderSubsets(failing_test_to_builders):
  """Prints failing test for builder subsets.

  Prints failing tests for each subset of builders, in descending order of the
  set size.
  """
  print "**** Failing tests ****"
  builders_to_tests = {}
  for test in failing_test_to_builders:
    builders = sorted(failing_test_to_builders[test])
    subset_name = ", ".join(builders)
    tests = builders_to_tests.setdefault(
        subset_name, _FailingTestsForBuilderSubset(len(builders))).Tests()
    tests.append(test)
  # Sort subsets in descending order of size and then name.
  builder_subsets = [(builders_to_tests[subset_name].SubsetSize(), subset_name)
                     for subset_name in builders_to_tests]
  for subset_size, subset_name in reversed(sorted(builder_subsets)):
    print "** Tests failing for %d builders: %s **" % (subset_size,
                                                       subset_name)
    for test in sorted(builders_to_tests[subset_name].Tests()):
      print test

def _ParseOptions():
  """Parses command-line options."""
  parser = optparse.OptionParser(usage="%prog [options] [builders]")
  parser.add_option("-m", "--max_builds", type="int",
                    default=-1,
                    help="Maximum number of builds to check for each builder."
                         " Defaults to all builds for which record is"
                         " available. Checking is ended either when the maximum"
                         " number is reached, the remaining builds are older"
                         " than the DEPS WebKit revision, or a passing"
                         " revision is found.")
  parser.add_option("-v", "--verbose", action="store_true", default=False,
                    dest="verbose")
  return parser.parse_args()

def _Main():
  """The main function."""
  options, builders = _ParseOptions()
  if not builders:
    builders = _DEFAULT_BUILDERS
  oldest_revision_to_check = _WebkitRevisionInDeps()
  if options.max_builds == -1 and oldest_revision_to_check == -1:
    options.max_builds = _DEFAULT_MAX_BUILDS
  if options.max_builds != -1:
    print "Maxium number of builds to check: %d" % options.max_builds
  if oldest_revision_to_check != -1:
    print "Oldest revision to check: %d" % oldest_revision_to_check
  sys.stdout.flush()
  results = []
  for builder in builders:
    print '"%s"' % builder
    sys.stdout.flush()
    results.append(_RetrieveBuildResult(
        builder, options.max_builds, oldest_revision_to_check))
  _PrintFailingRevisions(results, options.verbose)
  _PrintPassingRevisions(results, options.verbose)

if __name__ == "__main__":
  _Main()
<|endoftext|>


<commit_before>"""Define attributes on ORM-mapped classes that have 'hybrid' behavior.

'hybrid' means the attribute has distinct behaviors defined at the
class level and at the instance level.

Consider a table `interval` as below::

    from sqlalchemy import MetaData, Table, Column, Integer
    from sqlalchemy.orm import mapper, create_session

    engine = create_engine('sqlite://')
    metadata = MetaData()

    interval_table = Table('interval', metadata,
        Column('id', Integer, primary_key=True),
        Column('start', Integer, nullable=False),
        Column('end', Integer, nullable=False))
    metadata.create_all(engine)

We can define higher level functions on mapped classes that produce SQL
expressions at the class level, and Python expression evaluation at the
instance level.  Below, each function decorated with :func:`hybrid.method`
or :func:`hybrid.property` may receive ``self`` as an instance of the class,
or as the class itself::

    # A base class for intervals

    from sqlalchemy.orm import hybrid

    class Interval(object):
        def __init__(self, start, end):
            self.start = start
            self.end = end

        @hybrid.property
        def length(self):
            return self.end - self.start

        @hybrid.method
        def contains(self,point):
            return (self.start <= point) & (point < self.end)

        @hybrid.method
        def intersects(self, other):
            return (self.start < other.end) & (self.end > other.start)

    mapper(Interval1, interval_table1)

    session = sessionmaker(engine)()

    session.add_all(
        [Interval1(1,4), Interval1(3,15), Interval1(11,16)]
    )
    intervals =

    for interval in intervals:
        session.add(interval)
        session.add(Interval2(interval.start, interval.length))

    session.commit()

    ### TODO ADD EXAMPLES HERE AND STUFF THIS ISN'T FINISHED ###

"""
from sqlalchemy import util
from sqlalchemy.orm import attributes, interfaces

class method(object):
    def __init__(self, func, expr=None):
        self.func = func
        self.expr = expr or func

    def __get__(self, instance, owner):
        if instance is None:
            return new.instancemethod(self.expr, owner, owner.__class__)
        else:
            return new.instancemethod(self.func, instance, owner)

    def expression(self, expr):
        self.expr = expr
        return self

class property_(object):
    def __init__(self, fget, fset=None, fdel=None, expr=None):
        self.fget = fget
        self.fset = fset
        self.fdel = fdel
        self.expr = expr or fget
        util.update_wrapper(self, fget)

    def __get__(self, instance, owner):
        if instance is None:
            return self.expr(owner)
        else:
            return self.fget(instance)

    def __set__(self, instance, value):
        self.fset(instance, value)

    def __delete__(self, instance):
        self.fdel(instance)

    def setter(self, fset):
        self.fset = fset
        return self

    def deleter(self, fdel):
        self.fdel = fdel
        return self

    def expression(self, expr):
        self.expr = expr
        return self

    def comparator(self, comparator):
        proxy_attr = attributes.\
                        create_proxied_attribute(self)
        def expr(owner):
            return proxy_attr(self.__name__, self, comparator(owner))
        self.expr = expr
        return self

class Comparator(interfaces.PropComparator):
    def __init__(self, expression):
        self.expression = expression

    def __clause_element__(self):
        expr = self.expression
        while hasattr(expr, '__clause_element__'):
            expr = expr.__clause_element__()
        return expr

    def adapted(self, adapter):
        # interesting....
        return self

def hybrid_and(self):
    if isinstance(self, type):
        return expression.and_
    else:
        return operator.and_
<commit_msg>this doesnt actually work<commit_after>"""Define attributes on ORM-mapped classes that have 'hybrid' behavior.

'hybrid' means the attribute has distinct behaviors defined at the
class level and at the instance level.

Consider a table `interval` as below::

    from sqlalchemy import MetaData, Table, Column, Integer
    from sqlalchemy.orm import mapper, create_session

    engine = create_engine('sqlite://')
    metadata = MetaData()

    interval_table = Table('interval', metadata,
        Column('id', Integer, primary_key=True),
        Column('start', Integer, nullable=False),
        Column('end', Integer, nullable=False))
    metadata.create_all(engine)

We can define higher level functions on mapped classes that produce SQL
expressions at the class level, and Python expression evaluation at the
instance level.  Below, each function decorated with :func:`hybrid.method`
or :func:`hybrid.property` may receive ``self`` as an instance of the class,
or as the class itself::

    # A base class for intervals

    from sqlalchemy.orm import hybrid

    class Interval(object):
        def __init__(self, start, end):
            self.start = start
            self.end = end

        @hybrid.property
        def length(self):
            return self.end - self.start

        @hybrid.method
        def contains(self,point):
            return (self.start <= point) & (point < self.end)

        @hybrid.method
        def intersects(self, other):
            return (self.start < other.end) & (self.end > other.start)

    mapper(Interval1, interval_table1)

    session = sessionmaker(engine)()

    session.add_all(
        [Interval1(1,4), Interval1(3,15), Interval1(11,16)]
    )
    intervals =

    for interval in intervals:
        session.add(interval)
        session.add(Interval2(interval.start, interval.length))

    session.commit()

    ### TODO ADD EXAMPLES HERE AND STUFF THIS ISN'T FINISHED ###

"""
from sqlalchemy import util
from sqlalchemy.orm import attributes, interfaces

class method(object):
    def __init__(self, func, expr=None):
        self.func = func
        self.expr = expr or func

    def __get__(self, instance, owner):
        if instance is None:
            return new.instancemethod(self.expr, owner, owner.__class__)
        else:
            return new.instancemethod(self.func, instance, owner)

    def expression(self, expr):
        self.expr = expr
        return self

class property_(object):
    def __init__(self, fget, fset=None, fdel=None, expr=None):
        self.fget = fget
        self.fset = fset
        self.fdel = fdel
        self.expr = expr or fget
        util.update_wrapper(self, fget)

    def __get__(self, instance, owner):
        if instance is None:
            return self.expr(owner)
        else:
            return self.fget(instance)

    def __set__(self, instance, value):
        self.fset(instance, value)

    def __delete__(self, instance):
        self.fdel(instance)

    def setter(self, fset):
        self.fset = fset
        return self

    def deleter(self, fdel):
        self.fdel = fdel
        return self

    def expression(self, expr):
        self.expr = expr
        return self

    def comparator(self, comparator):
        proxy_attr = attributes.\
                        create_proxied_attribute(self)
        def expr(owner):
            return proxy_attr(self.__name__, self, comparator(owner))
        self.expr = expr
        return self

class Comparator(interfaces.PropComparator):
    def __init__(self, expression):
        self.expression = expression

    def __clause_element__(self):
        expr = self.expression
        while hasattr(expr, '__clause_element__'):
            expr = expr.__clause_element__()
        return expr

    def adapted(self, adapter):
        # interesting....
        return self

<|endoftext|>


<commit_before>import os
import base64
from django.db.models import F, Q
from xos.config import Config
from observer.openstacksyncstep import OpenStackSyncStep
from core.models import Controller
from core.models.network import *
from util.logger import observer_logger as logger

class SyncNetworkSlivers(OpenStackSyncStep):
    requested_interval = 0 # 3600
    provides=[NetworkSliver]
    observes=NetworkSliver

    #     The way it works is to enumerate the all of the ports that quantum
    #     has, and then work backward from each port's network-id to determine
    #     which Network is associated from the port.

    def call(self, **args):
        logger.info("sync'ing network slivers")

        networkSlivers = NetworkSliver.objects.all()
        networkSlivers_by_id = {}
        networkSlivers_by_port = {}
        for networkSliver in networkSlivers:
            networkSlivers_by_id[networkSliver.id] = networkSliver
            networkSlivers_by_port[networkSliver.port_id] = networkSliver

        networks = Network.objects.all()
        networks_by_id = {}
        for network in networks:
            for nd in network.controllernetworks.all():
                networks_by_id[nd.net_id] = network

        #logger.info("networks_by_id = ")
        #for (network_id, network) in networks_by_id.items():
        #    logger.info("   %s: %s" % (network_id, network.name))

        slivers = Sliver.objects.all()
        slivers_by_instance_uuid = {}
        for sliver in slivers:
            slivers_by_instance_uuid[sliver.instance_uuid] = sliver

        # Get all ports in all controllers

        ports_by_id = {}
        templates_by_id = {}
        for controller in Controller.objects.all():
            if not controller.admin_tenant:
                logger.info("controller %s has no admin_tenant" % controller)
                continue
            try:
                driver = self.driver.admin_driver(controller = controller,tenant='admin')
                ports = driver.shell.quantum.list_ports()["ports"]
            except:
                logger.log_exc("failed to get ports from controller %s" % controller)
                continue

            for port in ports:
                ports_by_id[port["id"]] = port

            # public-nat and public-dedicated networks don't have a net-id anywhere
            # in the data model, so build up a list of which ids map to which network
            # templates.
            try:
                neutron_networks = driver.shell.quantum.list_networks()["networks"]
            except:
                print "failed to get networks from controller %s" % controller
                continue
            for network in neutron_networks:
                for template in NetworkTemplate.objects.all():
                    if template.shared_network_name == network["name"]:
                        templates_by_id[network["id"]] = template

        for port in ports_by_id.values():
            #logger.info("port %s" % str(port))
            if port["id"] in networkSlivers_by_port:
                # we already have it
                #logger.info("already accounted for port %s" % port["id"])
                continue

            if port["device_owner"] != "compute:nova":
                # we only want the ports that connect to instances
                #logger.info("port %s is not a compute port, it is a %s" % (port["id"], port["device_owner"]))
                continue

            sliver = slivers_by_instance_uuid.get(port['device_id'], None)
            if not sliver:
                logger.info("no sliver for port %s device_id %s" % (port["id"], port['device_id']))
                continue

            network = networks_by_id.get(port['network_id'], None)
            if not network:
                # maybe it's public-nat or public-dedicated. Search the templates for
                # the id, then see if the sliver's slice has some network that uses
                # that template
                template = templates_by_id.get(port['network_id'], None)
                if template and sliver.slice:
                    for candidate_network in sliver.slice.networks.all():
                         if candidate_network.template == template:
                             network=candidate_network
            if not network:
                logger.info("no network for port %s network %s" % (port["id"], port["network_id"]))

                # we know it's associated with a sliver, but we don't know
                # which network it is part of.

                continue

            if network.template.shared_network_name:
                # If it's a shared network template, then more than one network
                # object maps to the quantum network. We have to do a whole bunch
                # of extra work to find the right one.
                networks = network.template.network_set.all()
                network = None
                for candidate_network in networks:
                    if (candidate_network.owner == sliver.slice):
                        print "found network", candidate_network
                        network = candidate_network

                if not network:
                    logger.info("failed to find the correct network for a shared template for port %s network %s" % (port["id"], port["network_id"]))
                    continue

            if not port["fixed_ips"]:
                logger.info("port %s has no fixed_ips" % port["id"])
                continue

            ip=port["fixed_ips"][0]["ip_address"]
            logger.info("creating NetworkSliver (%s, %s, %s, %s)" % (str(network), str(sliver), ip, str(port["id"])))

            ns = NetworkSliver(network=network,
                               sliver=sliver,
                               ip=ip,
                               port_id=port["id"])

            try:
                ns.save()
            except:
                logger.log_exc("failed to save networksliver %s" % str(ns))
                continue

        # For networkSlivers that were created by the user, find that ones
        # that don't have neutron ports, and create them.
        for networkSliver in NetworkSliver.objects.filter(port_id__isnull=True, sliver__isnull=False):
            #logger.info("working on networksliver %s" % networkSliver)
            controller = sliver.node.site_deployment.controller
            if controller:
                cn=networkSliver.network.controllernetworks.filter(controller=controller)
                if not cn:
                    logger.log_exc("no controllernetwork for %s" % networkSliver)
                    continue
                cn=cn[0]
                try:
                    driver = self.driver.admin_driver(controller = controller,tenant='admin')
                    #networkSliver.port = driver.shell.quantum.create_port(network_id = cn.net_id)
                    networkSliver.port_id = driver.shell.quantum.create_port({"port": {"network_id": cn.net_id}})["port"]["id"]
                except:
                    logger.log_exc("failed to create neutron port for %s" % networkSliver)
                    continue
                networkSliver.save()

        # Now, handle port forwarding
        # We get the list of NetworkSlivers again, since we might have just
        # added a few. Then, for each one of them we find it's quantum port and
        # make sure quantum's nat:forward_ports argument is the same.

        for networkSliver in NetworkSliver.objects.all():
            try:
                nat_list = networkSliver.network.nat_list
            except (TypeError, ValueError), e:
                logger.info("Failed to decode nat_list: %s" % str(e))
                continue

            if not networkSliver.port_id:
                continue

            neutron_port = ports_by_id.get(networkSliver.port_id, None)
            if not neutron_port:
                continue

            neutron_nat_list = neutron_port.get("nat:forward_ports", None)
            if not neutron_nat_list:
                # make sure that None and the empty set are treated identically
                neutron_nat_list = []

            if (neutron_nat_list != nat_list):
                logger.info("Setting nat:forward_ports for port %s network %s sliver %s to %s" % (str(networkSliver.port_id), str(networkSliver.network.id), str(networkSliver.sliver), str(nat_list)))
                try:
                    driver = self.driver.admin_driver(controller=networkSliver.sliver.node.site_deployment.controller,tenant='admin')
                    driver.shell.quantum.update_port(networkSliver.port_id, {"port": {"nat:forward_ports": nat_list}})
                except:
                    logger.log_exc("failed to update port with nat_list %s" % str(nat_list))
                    continue
            else:
                #logger.info("port %s network %s sliver %s nat %s is already set" % (str(networkSliver.port_id), str(networkSliver.network.id), str(networkSliver.sliver), str(nat_list)))
                pass

    def delete_record(self, network_sliver):
        # Nothing to do, this is an OpenCloud object
        pass

<commit_msg>store the ip address too<commit_after>import os
import base64
from django.db.models import F, Q
from xos.config import Config
from observer.openstacksyncstep import OpenStackSyncStep
from core.models import Controller
from core.models.network import *
from util.logger import observer_logger as logger

class SyncNetworkSlivers(OpenStackSyncStep):
    requested_interval = 0 # 3600
    provides=[NetworkSliver]
    observes=NetworkSliver

    #     The way it works is to enumerate the all of the ports that quantum
    #     has, and then work backward from each port's network-id to determine
    #     which Network is associated from the port.

    def call(self, **args):
        logger.info("sync'ing network slivers")

        networkSlivers = NetworkSliver.objects.all()
        networkSlivers_by_id = {}
        networkSlivers_by_port = {}
        for networkSliver in networkSlivers:
            networkSlivers_by_id[networkSliver.id] = networkSliver
            networkSlivers_by_port[networkSliver.port_id] = networkSliver

        networks = Network.objects.all()
        networks_by_id = {}
        for network in networks:
            for nd in network.controllernetworks.all():
                networks_by_id[nd.net_id] = network

        #logger.info("networks_by_id = ")
        #for (network_id, network) in networks_by_id.items():
        #    logger.info("   %s: %s" % (network_id, network.name))

        slivers = Sliver.objects.all()
        slivers_by_instance_uuid = {}
        for sliver in slivers:
            slivers_by_instance_uuid[sliver.instance_uuid] = sliver

        # Get all ports in all controllers

        ports_by_id = {}
        templates_by_id = {}
        for controller in Controller.objects.all():
            if not controller.admin_tenant:
                logger.info("controller %s has no admin_tenant" % controller)
                continue
            try:
                driver = self.driver.admin_driver(controller = controller,tenant='admin')
                ports = driver.shell.quantum.list_ports()["ports"]
            except:
                logger.log_exc("failed to get ports from controller %s" % controller)
                continue

            for port in ports:
                ports_by_id[port["id"]] = port

            # public-nat and public-dedicated networks don't have a net-id anywhere
            # in the data model, so build up a list of which ids map to which network
            # templates.
            try:
                neutron_networks = driver.shell.quantum.list_networks()["networks"]
            except:
                print "failed to get networks from controller %s" % controller
                continue
            for network in neutron_networks:
                for template in NetworkTemplate.objects.all():
                    if template.shared_network_name == network["name"]:
                        templates_by_id[network["id"]] = template

        for port in ports_by_id.values():
            #logger.info("port %s" % str(port))
            if port["id"] in networkSlivers_by_port:
                # we already have it
                #logger.info("already accounted for port %s" % port["id"])
                continue

            if port["device_owner"] != "compute:nova":
                # we only want the ports that connect to instances
                #logger.info("port %s is not a compute port, it is a %s" % (port["id"], port["device_owner"]))
                continue

            sliver = slivers_by_instance_uuid.get(port['device_id'], None)
            if not sliver:
                logger.info("no sliver for port %s device_id %s" % (port["id"], port['device_id']))
                continue

            network = networks_by_id.get(port['network_id'], None)
            if not network:
                # maybe it's public-nat or public-dedicated. Search the templates for
                # the id, then see if the sliver's slice has some network that uses
                # that template
                template = templates_by_id.get(port['network_id'], None)
                if template and sliver.slice:
                    for candidate_network in sliver.slice.networks.all():
                         if candidate_network.template == template:
                             network=candidate_network
            if not network:
                logger.info("no network for port %s network %s" % (port["id"], port["network_id"]))

                # we know it's associated with a sliver, but we don't know
                # which network it is part of.

                continue

            if network.template.shared_network_name:
                # If it's a shared network template, then more than one network
                # object maps to the quantum network. We have to do a whole bunch
                # of extra work to find the right one.
                networks = network.template.network_set.all()
                network = None
                for candidate_network in networks:
                    if (candidate_network.owner == sliver.slice):
                        print "found network", candidate_network
                        network = candidate_network

                if not network:
                    logger.info("failed to find the correct network for a shared template for port %s network %s" % (port["id"], port["network_id"]))
                    continue

            if not port["fixed_ips"]:
                logger.info("port %s has no fixed_ips" % port["id"])
                continue

            ip=port["fixed_ips"][0]["ip_address"]
            logger.info("creating NetworkSliver (%s, %s, %s, %s)" % (str(network), str(sliver), ip, str(port["id"])))

            ns = NetworkSliver(network=network,
                               sliver=sliver,
                               ip=ip,
                               port_id=port["id"])

            try:
                ns.save()
            except:
                logger.log_exc("failed to save networksliver %s" % str(ns))
                continue

        # For networkSlivers that were created by the user, find that ones
        # that don't have neutron ports, and create them.
        for networkSliver in NetworkSliver.objects.filter(port_id__isnull=True, sliver__isnull=False):
            #logger.info("working on networksliver %s" % networkSliver)
            controller = sliver.node.site_deployment.controller
            if controller:
                cn=networkSliver.network.controllernetworks.filter(controller=controller)
                if not cn:
                    logger.log_exc("no controllernetwork for %s" % networkSliver)
                    continue
                cn=cn[0]
                try:
                    driver = self.driver.admin_driver(controller = controller,tenant='admin')
                    #networkSliver.port = driver.shell.quantum.create_port(network_id = cn.net_id)
                    port = driver.shell.quantum.create_port({"port": {"network_id": cn.net_id}})["port"]
                    networkSliver.port_id = port["id"]
                    if port["fixed_ips"]:
                        networkSliver.ip = port["fixed_ips"][0]["ip_address"]
                except:
                    logger.log_exc("failed to create neutron port for %s" % networkSliver)
                    continue
                networkSliver.save()

        # Now, handle port forwarding
        # We get the list of NetworkSlivers again, since we might have just
        # added a few. Then, for each one of them we find it's quantum port and
        # make sure quantum's nat:forward_ports argument is the same.

        for networkSliver in NetworkSliver.objects.all():
            try:
                nat_list = networkSliver.network.nat_list
            except (TypeError, ValueError), e:
                logger.info("Failed to decode nat_list: %s" % str(e))
                continue

            if not networkSliver.port_id:
                continue

            neutron_port = ports_by_id.get(networkSliver.port_id, None)
            if not neutron_port:
                continue

            neutron_nat_list = neutron_port.get("nat:forward_ports", None)
            if not neutron_nat_list:
                # make sure that None and the empty set are treated identically
                neutron_nat_list = []

            if (neutron_nat_list != nat_list):
                logger.info("Setting nat:forward_ports for port %s network %s sliver %s to %s" % (str(networkSliver.port_id), str(networkSliver.network.id), str(networkSliver.sliver), str(nat_list)))
                try:
                    driver = self.driver.admin_driver(controller=networkSliver.sliver.node.site_deployment.controller,tenant='admin')
                    driver.shell.quantum.update_port(networkSliver.port_id, {"port": {"nat:forward_ports": nat_list}})
                except:
                    logger.log_exc("failed to update port with nat_list %s" % str(nat_list))
                    continue
            else:
                #logger.info("port %s network %s sliver %s nat %s is already set" % (str(networkSliver.port_id), str(networkSliver.network.id), str(networkSliver.sliver), str(nat_list)))
                pass

    def delete_record(self, network_sliver):
        # Nothing to do, this is an OpenCloud object
        pass

<|endoftext|>


<commit_before>import numpy as N
import numpy.linalg as L
import enthought.traits as traits
from model import Model
import utils

class RegressionOutput(traits.HasTraits):

    """
    A generic output for regression. Key feature is that it has
    an \'extract\' method which is called on an instance of
    RegressionModelResults.
    """

    Tmax = traits.Float(100.)
    Tmin = traits.Float(-100.)
    Fmax = traits.Float(100.)

    def __init__(self, iterator, **keywords):
        self.iterator = iter(iterator)
        traits.HasTraits(**keywords)

    def __iter__(self):
        return self

    def extract(self, results):
        return 0.

class RegressionModelResults(traits.HasTraits):
    """
    A container for results from fitting a regression model.
    Key attributes
    RegressionModelResults.
    """

    def __init__(self):
        return

    def t(self, column=None):
        if not hasattr(self, '_sd'):
            self.sd()
        if column is None:
            _t = N.zeros(_beta.shape, N.Float)
            for i in range(self.beta.shape[0]):
                _t[i] = _beta[i] / (self._sd * self.sqrt(self.normalized_cov_beta[i,i]))
        else:
            i = column
            _t = _beta[i] / (self._sd * self.sqrt(self.normalized_cov_beta[i,i]))
        return _t

    def sd(self):
        if not hasattr(self, 'resid'):
            raise ValueError, 'need residuals to estimate standard deviation'
        self._sd = N.sqrt(self.scale)

    def norm_resid(self):
        if not hasattr(self, 'resid'):
            raise ValueError, 'need normalized residuals to estimate standard deviation'
        if not hasattr(self, '_sd'):
            self.sd()
        test = N.greater(_sd, 0)
        sdd = utils.recipr(self._sd) / N.sqrt(self.df)
        norm_resid = self.resid * N.multiply.outer(N.ones(Y.shape[0]), sdd)
        return norm_resid

    def predict(self, design):
        return N.dot(design, self.beta)

    def Rsq(self):
        """
        Return the R^2 value for each row of the response Y.
        """
        self.Ssq = N.std(Y)**2
        Rsq = self.scale / self.Ssq
        return Rsq

    def cov_beta(self, matrix=None, column=None, scale=None, other=None):
        """
        Returns the variance/covariance matrix of a linear contrast
        of the estimates of beta, multiplied by scale which
        will usually be an estimate of sigma^2. The covariance of
        interest is either specified as a (set of) column(s) or a matrix.
        """
        if scale is None:
            scale = 1.

        if column is not None:
            return self.normalized_cov_beta[column, column] * scale

        elif matrix is not None:
            if other is None:
                other = matrix
            tmp = N.dot(matrix, N.dot(self.normalized_cov_beta, N.transpose(other)))
            return tmp * scale

    def Tcontrast(self, matrix, t=True, sd=True, scale=None):
        """
        Compute a Tcontrast for a row vector matrix. To get the t-statistic
        for a single column, use self.t(column=column).
        """

        if not hasattr(self, '_sd'):
            self.sd()
        results = ContrastResults()
        results.effect = N.dot(matrix, self.beta)
        if sd:
            results.sd = N.sqrt(self.cov_beta(matrix=matrix)) * self._sd
        if t:
            results.t = results.effect * utils.recipr(results.sd)
        return results

    def Fcontrast(self, matrix, eff=True, t=True, sd=True, scale=None, invcov=None):
        """
        Compute an Fcontrast for a contrast matrix. To get the t-statistic
        for a single column, use self.t(column=column).

        Here, matrix M is assumed to be non-singular. More precisely,

        M pX pX' M'

        is assumed invertible. Here, pX is the generalized inverse of the
        design matrix of the model. There can be problems in non-OLS models where
        the rank of the covariance of the noise is not full.

        See the statistics.contrast module to see how to specify contrasts.
        In particular, the matrices from these contrasts will always be non-singular
        in the sense above.

        """

        results = ContrastResults()
        cbeta = N.dot(matrix, self.beta)

        q = matrix.shape[0]
        if invcov is None:
            invcov = L.inv(self.cov_beta(matrix=matrix, scale=1.0))
        results.F = N.add.reduce(N.dot(invcov, cbeta) * cbeta, 0) / (q * self.scale)
        return results

class ContrastResults:
    """
    Results from looking at a particular contrast of coefficients in
    a regression model. The class does nothing, it is a container
    for the results from T and F contrasts.
    """
    pass

class OLSModel(Model):

    """
    A simple ordinary least squares model.
    """

    design = traits.Any()

    def __init__(self, **keywords):
        Model.__init__(self, **keywords)
        try:
            self.setup()
        except:
            pass

    def _design_changed(self):
        self.wdesign = self.design

    def _wdesign_changed(self):
        self.setup()

    def setup(self):
        self.calc_beta = L.pinv(self.wdesign)
        self.normalized_cov_beta = N.dot(self.calc_beta, N.transpose(self.calc_beta))
        self._df_resid = int(N.around(self.wdesign.shape[0] - N.add.reduce(N.diagonal(N.dot(self.calc_beta, self.wdesign)))))

    def initialize(self, design, **keywords):
        self.design = design

    def df_resid(self, **keywords):
        return self._df_resid

    def fit(self, Y, **keywords):

        if not hasattr(self, 'calc_beta'):
            self.setup()

        if hasattr(self, 'whiten'):
            Z = self.whiten(Y)
        else:
            Z = Y

        lfit = RegressionModelResults()

        lfit.beta = N.dot(self.calc_beta, Z)
        lfit.normalized_cov_beta = self.normalized_cov_beta
        lfit.df_resid = self.df_resid
        lfit.resid = Z - N.dot(self.wdesign, lfit.beta)

        lfit.scale = N.add.reduce(lfit.resid**2) / lfit.df_resid()

        lfit.Z = Z # just in case

        return lfit

class ARModel(OLSModel):
    """
    A regression model with an AR(1) covariance structure.

    Eventually, this will be AR(p) -- all that is needed is to
    determine the self.whiten method from AR(p) parameters.
    """

    rho = traits.Float() # for now just do AR(1) -- not hard to do AR(p)
                         # the only the needs to be done is the whiten method
                         # for this -- we should use the cholesky trick of
                         # a pxp matrix

    def _design_changed(self):
        self.wdesign = self.whiten(self.design)

    def _rho_changed(self):
        self._design_changed()

    def whiten(self, X):
        factor = 1. / N.sqrt(1 - self.rho**2)
        return N.concatenate([[X[0]], (X[1:] - self.rho * X[0:-1]) * factor])

def contrastfromcols(T, D, pinv=None, warn=True):
    """
    From an n x p design matrix D and a matrix T, tries
    to determine a p x q contrast matrix C which
    determines a contrast of full rank, i.e. the
    n x q matrix

    dot(transpose(C), pinv(D))

    is full rank.

    T must satisfy either T.shape[0] == n or T.shape[1] == p.

    Note that this always produces a meaningful contrast, not always
    with the intended properties because q is always non-zero unless
    T is identically 0. That is, it produces a contrast that spans
    the column space of T (after projection onto the column space of D).

    """

    n, p = D.shape

    if T.shape[0] != n and T.shape[1] != p:
        raise ValueError, 'shape of T and D mismatched'

    if pinv is None:
        pinv = L.pinv(D)

    if T.shape[0] == n:
        C = N.transpose(N.dot(pinv, T))
    else:
        C = T

    Tp = N.dot(D, N.transpose(C))

    if utils.rank(Tp) != Tp.shape[1]:
        Tp = utils.fullrank(Tp)
        C = N.transpose(N.dot(pinv, Tp))

    return N.squeeze(C)

def isestimable(C, D, pinv=None, warn=True):
    """
    From an q x p contrast matrix C and an n x p design matrix T, tries
    if the contrast C is estimable by looking at the rank of [D,C] and
    verifying it is the same as the rank of D.
    """

    print C.shape, D.shape

    new = N.concatenate([C, D])
    print utils.rank(new), utils.rank(D)
    if utils.rank(new) != utils.rank(D):
        return False
    return True
<commit_msg>moving fmri.regression to image.regression<commit_after>import numpy as N
import numpy.linalg as L
import enthought.traits as traits
from model import Model
import utils

class RegressionOutput(traits.HasTraits):

    """
    A generic output for regression. Key feature is that it has
    an \'extract\' method which is called on an instance of
    RegressionModelResults.
    """

    Tmax = traits.Float(100.)
    Tmin = traits.Float(-100.)
    Fmax = traits.Float(100.)

    def __init__(self, iterator, **keywords):
        self.iterator = iter(iterator)
        traits.HasTraits(**keywords)

    def __iter__(self):
        return self

    def extract(self, results):
        return 0.

class RegressionModelResults(traits.HasTraits):
    """
    A container for results from fitting a regression model.
    Key attributes
    RegressionModelResults.
    """

    def __init__(self):
        return

    def t(self, column=None):
        if not hasattr(self, '_sd'):
            self.sd()
        if column is None:
            _t = N.zeros(_beta.shape, N.Float)
            for i in range(self.beta.shape[0]):
                _t[i] = _beta[i] / (self._sd * self.sqrt(self.normalized_cov_beta[i,i]))
        else:
            i = column
            _t = _beta[i] / (self._sd * self.sqrt(self.normalized_cov_beta[i,i]))
        return _t

    def sd(self):
        if not hasattr(self, 'resid'):
            raise ValueError, 'need residuals to estimate standard deviation'
        self._sd = N.sqrt(self.scale)

    def norm_resid(self):
        if not hasattr(self, 'resid'):
            raise ValueError, 'need normalized residuals to estimate standard deviation'
        if not hasattr(self, '_sd'):
            self.sd()
        test = N.greater(_sd, 0)
        sdd = utils.recipr(self._sd) / N.sqrt(self.df)
        norm_resid = self.resid * N.multiply.outer(N.ones(Y.shape[0]), sdd)
        return norm_resid

    def predict(self, design):
        return N.dot(design, self.beta)

    def Rsq(self):
        """
        Return the R^2 value for each row of the response Y.
        """
        self.Ssq = N.std(Y)**2
        Rsq = self.scale / self.Ssq
        return Rsq

    def cov_beta(self, matrix=None, column=None, scale=None, other=None):
        """
        Returns the variance/covariance matrix of a linear contrast
        of the estimates of beta, multiplied by scale which
        will usually be an estimate of sigma^2. The covariance of
        interest is either specified as a (set of) column(s) or a matrix.
        """
        if scale is None:
            scale = 1.

        if column is not None:
            return self.normalized_cov_beta[column, column] * scale

        elif matrix is not None:
            if other is None:
                other = matrix
            tmp = N.dot(matrix, N.dot(self.normalized_cov_beta, N.transpose(other)))
            return tmp * scale

    def Tcontrast(self, matrix, t=True, sd=True, scale=None):
        """
        Compute a Tcontrast for a row vector matrix. To get the t-statistic
        for a single column, use self.t(column=column).
        """

        if not hasattr(self, '_sd'):
            self.sd()
        results = ContrastResults()
        results.effect = N.dot(matrix, self.beta)
        if sd:
            results.sd = N.sqrt(self.cov_beta(matrix=matrix)) * self._sd
        if t:
            results.t = results.effect * utils.recipr(results.sd)
        return results

    def Fcontrast(self, matrix, eff=True, t=True, sd=True, scale=None, invcov=None):
        """
        Compute an Fcontrast for a contrast matrix. To get the t-statistic
        for a single column, use self.t(column=column).

        Here, matrix M is assumed to be non-singular. More precisely,

        M pX pX' M'

        is assumed invertible. Here, pX is the generalized inverse of the
        design matrix of the model. There can be problems in non-OLS models where
        the rank of the covariance of the noise is not full.

        See the statistics.contrast module to see how to specify contrasts.
        In particular, the matrices from these contrasts will always be non-singular
        in the sense above.

        """

        results = ContrastResults()
        cbeta = N.dot(matrix, self.beta)

        q = matrix.shape[0]
        if invcov is None:
            invcov = L.inv(self.cov_beta(matrix=matrix, scale=1.0))
        results.F = N.add.reduce(N.dot(invcov, cbeta) * cbeta, 0) / (q * self.scale)
        return results

class ContrastResults:
    """
    Results from looking at a particular contrast of coefficients in
    a regression model. The class does nothing, it is a container
    for the results from T and F contrasts.
    """
    pass

class OLSModel(Model):

    """
    A simple ordinary least squares model.
    """

    design = traits.Any()

    def __init__(self, **keywords):
        Model.__init__(self, **keywords)
        try:
            self.setup()
        except:
            pass

    def _design_changed(self):
        self.wdesign = self.design

    def _wdesign_changed(self):
        self.setup()

    def setup(self):
        self.calc_beta = L.pinv(self.wdesign)
        self.normalized_cov_beta = N.dot(self.calc_beta, N.transpose(self.calc_beta))
        self._df_resid = int(N.around(self.wdesign.shape[0] - N.add.reduce(N.diagonal(N.dot(self.calc_beta, self.wdesign)))))

    def initialize(self, design, **keywords):
        self.design = design

    def df_resid(self, **keywords):
        return self._df_resid

    def fit(self, Y, **keywords):

        if not hasattr(self, 'calc_beta'):
            self.setup()

        if hasattr(self, 'whiten'):
            Z = self.whiten(Y)
        else:
            Z = Y

        lfit = RegressionModelResults()

        lfit.beta = N.dot(self.calc_beta, Z)
        lfit.normalized_cov_beta = self.normalized_cov_beta
        lfit.df_resid = self.df_resid
        lfit.resid = Z - N.dot(self.wdesign, lfit.beta)

        lfit.scale = N.add.reduce(lfit.resid**2) / lfit.df_resid()

        lfit.Z = Z # just in case

        return lfit

class ARModel(OLSModel):
    """
    A regression model with an AR(1) covariance structure.

    Eventually, this will be AR(p) -- all that is needed is to
    determine the self.whiten method from AR(p) parameters.
    """

    rho = traits.Float() # for now just do AR(1) -- not hard to do AR(p)
                         # the only the needs to be done is the whiten method
                         # for this -- we should use the cholesky trick of
                         # a pxp matrix

    def _design_changed(self):
        self.wdesign = self.whiten(self.design)

    def _rho_changed(self):
        self._design_changed()

    def whiten(self, X):
        factor = 1. / N.sqrt(1 - self.rho**2)
        return N.concatenate([[X[0]], (X[1:] - self.rho * X[0:-1]) * factor])

class WLSModel(ARModel):
    """

    A regression model with diagonal but non-identity covariance
    structure. The weights are proportional to the inverse of the

    """

    weights = traits.Any()

    def _weights_changed(self):
        self._design_changed()

    def whiten(self, X):
        if X.ndim == 1:
            return X / N.sqrt(self.weights)
        elif X.ndim == 2:
            c = N.sqrt(self.weights)
            v = N.zeros(X.shape, N.Float)
            for i in range(X.shape[1]):
                v[:,i] = X[:,i] / c
            return v

def contrastfromcols(T, D, pinv=None, warn=True):
    """
    From an n x p design matrix D and a matrix T, tries
    to determine a p x q contrast matrix C which
    determines a contrast of full rank, i.e. the
    n x q matrix

    dot(transpose(C), pinv(D))

    is full rank.

    T must satisfy either T.shape[0] == n or T.shape[1] == p.

    Note that this always produces a meaningful contrast, not always
    with the intended properties because q is always non-zero unless
    T is identically 0. That is, it produces a contrast that spans
    the column space of T (after projection onto the column space of D).

    """

    n, p = D.shape

    if T.shape[0] != n and T.shape[1] != p:
        raise ValueError, 'shape of T and D mismatched'

    if pinv is None:
        pinv = L.pinv(D)

    if T.shape[0] == n:
        C = N.transpose(N.dot(pinv, T))
    else:
        C = T

    Tp = N.dot(D, N.transpose(C))

    if utils.rank(Tp) != Tp.shape[1]:
        Tp = utils.fullrank(Tp)
        C = N.transpose(N.dot(pinv, Tp))

    return N.squeeze(C)

def isestimable(C, D, pinv=None, warn=True):
    """
    From an q x p contrast matrix C and an n x p design matrix T, tries
    if the contrast C is estimable by looking at the rank of [D,C] and
    verifying it is the same as the rank of D.
    """

    print C.shape, D.shape

    new = N.concatenate([C, D])
    print utils.rank(new), utils.rank(D)
    if utils.rank(new) != utils.rank(D):
        return False
    return True
<|endoftext|>