<commit_before># ===================================================================================================
#                           _  __     _ _
#                          | |/ /__ _| | |_ _  _ _ _ __ _
#                          | ' </ _` | |  _| || | '_/ _` |
#                          |_|\_\__,_|_|\__|\_,_|_| \__,_|
#
# This file is part of the Kaltura Collaborative Media Suite which allows users
# to do with audio, video, and animation what Wiki platfroms allow them to do with
# text.
#
# Copyright (C) 2006-2011  Kaltura Inc.
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http:#www.gnu.org/licenses/>.
#
# @ignore
# ===================================================================================================
from KalturaCoreClient import *
from KalturaClientBase import *
from xml.parsers.expat import ExpatError
from xml.dom import minidom
from threading import Timer
import hashlib
import random
import base64
import socket
import urllib
import time
import sys
import os

from poster.streaminghttp import register_openers
from poster.encode import multipart_encode
import urllib2

# Register the streaming http handlers with urllib2
register_openers()

pluginsFolder = os.path.normpath(os.path.join(os.path.dirname(__file__), 'KalturaPlugins'))
if not pluginsFolder in sys.path:
    sys.path.append(pluginsFolder)

class MultiRequestSubResult:
    def __init__(self, value):
        self.value = value
    def __str__(self):
        return '{%s}' % self.value
    def __repr__(self):
        return '{%s}' % self.value
    def __getattr__(self, name):
        if name.startswith('__') or name.endswith('__'):
            raise AttributeError
        return MultiRequestSubResult('%s:%s' % (self.value, name))
    def __getitem__(self, key):
        return MultiRequestSubResult('%s:%s' % (self.value, key))

class PluginServicesProxy:
    def addService(self, serviceName, serviceClass):
        setattr(self, serviceName, serviceClass)

class KalturaClient:
    METHOD_POST = 0
    METHOD_GET = 1

    def __init__(self, config):
        self.apiVersion = API_VERSION
        self.config = None
        self.ks = NotImplemented
        self.shouldLog = False
        self.multiRequest = False
        self.callsQueue = []

        self.method = KalturaClient.METHOD_POST
        self.requestHeaders = {}

        self.config = config
        logger = self.config.getLogger()
        if (logger):
            self.shouldLog = True

        self.loadPlugins()

    def loadPlugins(self):
        if not os.path.isdir(pluginsFolder):
            return

        pluginList = ['KalturaCoreClient']
        for fileName in os.listdir(pluginsFolder):
            (pluginClass, fileExt) = os.path.splitext(fileName)
            if fileExt.lower() != '.py':
                continue
            pluginList.append(pluginClass)

        for pluginClass in pluginList:
            self.loadPlugin(pluginClass)

    def loadPlugin(self, pluginClass):
        pluginModule = __import__(pluginClass)
        if not pluginClass in dir(pluginModule):
            return

        pluginClassType = getattr(pluginModule, pluginClass)

        plugin = pluginClassType.get()
        if not isinstance(plugin, IKalturaClientPlugin):
            return

        self.registerPluginServices(plugin)
        self.registerPluginObjects(plugin)

    def registerPluginServices(self, plugin):
        pluginName = plugin.getName()
        if pluginName != '':
            pluginProxy = PluginServicesProxy()
            setattr(self, pluginName, pluginProxy)

        for (serviceName, serviceFactory) in plugin.getServices().items():
            serviceClass = serviceFactory(self)
            if pluginName == '':
                self.addCoreService(serviceName, serviceClass)
            else:
                pluginProxy.addService(serviceName, serviceClass)

    def registerPluginObjects(self, plugin):
        KalturaEnumsFactory.registerEnums(plugin.getEnums())
        KalturaObjectFactory.registerObjects(plugin.getTypes())

    def addCoreService(self, serviceName, serviceClass):
        setattr(self, serviceName, serviceClass)

    def getServeUrl(self):
        if len(self.callsQueue) != 1:
            return None

        (url, params, _) = self.getRequestParams()

        # reset state
        self.callsQueue = []
        self.multiRequest = False

        result = '%s&%s' % (url, urllib.urlencode(params.get()))
        self.log("Returned url [%s]" % result)
        return result

    def queueServiceActionCall(self, service, action, params = KalturaParams(), files = KalturaFiles()):
        # in start session partner id is optional (default -1). if partner id was not set, use the one in the config
        if not params.get().has_key("partnerId") or params.get()["partnerId"] == -1:
            if self.config.partnerId != None:
                params.put("partnerId", self.config.partnerId)
        params.addStringIfDefined("ks", self.ks)
        call = KalturaServiceActionCall(service, action, params, files)
        self.callsQueue.append(call)

    def getRequestParams(self):
        params = KalturaParams()
        files = KalturaFiles()
        params.put("apiVersion", self.apiVersion)
        params.put("format", self.config.format)
        params.put("clientTag", self.config.clientTag)
        url = self.config.serviceUrl + "/api_v3/index.php?service="
        if self.multiRequest:
            url += "multirequest"
            i = 1
            for call in self.callsQueue:
                callParams = call.getParamsForMultiRequest(i)
                callFiles = call.getFilesForMultiRequest(i)
                params.update(callParams)
                files.update(callFiles)
                i += 1
        else:
            call = self.callsQueue[0]
            url += call.service + "&action=" + call.action
            params.update(call.params)
            files.update(call.files)

        signature = params.signature()
        params.put("kalsig", signature)

        if self.method == KalturaClient.METHOD_GET and len(files.get()) == 0:
            url += '&' + urllib.urlencode(params.get())
            params = None

        self.log("request url: [%s]" % url)

        return (url, params, files)

    @staticmethod
    def closeHandle(fh):
        fh.close()

    @staticmethod
    def openRequestUrl(url, params, files, requestHeaders):
        if len(files.get()) == 0:
            data = None
            if params != None:
                data = urllib.urlencode(params.get())
            request = urllib2.Request(url, data, requestHeaders)
        else:
            fullParams = params
            fullParams.update(files)
            datagen, headers = multipart_encode(fullParams.get())
            headers.update(requestHeaders)
            request = urllib2.Request(url, datagen, headers)

        try:
            f = urllib2.urlopen(request)
        except Exception, e:
            raise KalturaClientException(e, KalturaClientException.ERROR_CONNECTION_FAILED)
        return f

    @staticmethod
    def readHttpResponse(f, requestTimeout):
        if requestTimeout != None:
            readTimer = Timer(requestTimeout, KalturaClient.closeHandle, [f])
            readTimer.start()
        try:
            try:
                data = f.read()
            except AttributeError, e:      # socket was closed while reading
                raise KalturaClientException(e, KalturaClientException.ERROR_READ_TIMEOUT)
            except Exception, e:
                raise KalturaClientException(e, KalturaClientException.ERROR_READ_FAILED)
        finally:
            if requestTimeout != None:
                readTimer.cancel()
        return data

    # Send http request
    def doHttpRequest(self, url, params = KalturaParams(), files = KalturaFiles()):
        if len(files.get()) == 0:
            requestTimeout = self.config.requestTimeout
        else:
            requestTimeout = None

        if requestTimeout != None:
            origSocketTimeout = socket.getdefaulttimeout()
            socket.setdefaulttimeout(requestTimeout)
        try:
            f = self.openRequestUrl(url, params, files, self.requestHeaders)
            data = self.readHttpResponse(f, requestTimeout)
            self.responseHeaders = f.info().headers
        finally:
            if requestTimeout != None:
                socket.setdefaulttimeout(origSocketTimeout)
        return data

    def parsePostResult(self, postResult):
        if len(postResult) > 1024:
            self.log("result (xml): %s bytes" % len(postResult))
        else:
            self.log("result (xml): %s" % postResult)

        try:
            resultXml = minidom.parseString(postResult)
        except ExpatError, e:
            raise KalturaClientException(e, KalturaClientException.ERROR_INVALID_XML)

        resultNode = getChildNodeByXPath(resultXml, 'xml/result')
        if resultNode == None:
            raise KalturaClientException('Could not find result node in response xml', KalturaClientException.ERROR_RESULT_NOT_FOUND)

        execTime = getChildNodeByXPath(resultXml, 'xml/executionTime')
        if execTime != None:
            self.executionTime = getXmlNodeFloat(execTime)

        self.throwExceptionIfError(resultNode)

        return resultNode

    # Call all API services that are in queue
    def doQueue(self):
        self.responseHeaders = None
        self.executionTime = None
        if len(self.callsQueue) == 0:
            self.multiRequest = False
            return None

        if self.config.format != KALTURA_SERVICE_FORMAT_XML:
            raise KalturaClientException("unsupported format: %s" % (postResult), KalturaClientException.ERROR_FORMAT_NOT_SUPPORTED)

        startTime = time.time()

        # get request params
        (url, params, files) = self.getRequestParams()

        # reset state
        self.callsQueue = []
        self.multiRequest = False

        # issue the request
        postResult = self.doHttpRequest(url, params, files)

        # parse the result
        resultNode = self.parsePostResult(postResult)

        endTime = time.time()
        self.log("execution time for [%s]: [%s]" % (url, endTime - startTime))

        return resultNode

    def getKs(self):
        return self.ks

    def setKs(self, ks):
        self.ks = ks

    def getConfig(self):
        return self.config

    def setConfig(self, config):
        self.config = config
        logger = self.config.getLogger()
        if isinstance(logger, IKalturaLogger):
            self.shouldLog = True

    def getExceptionIfError(self, resultNode):
        errorNode = getChildNodeByXPath(resultNode, 'error')
        if errorNode == None:
            return None
        messageNode = getChildNodeByXPath(errorNode, 'message')
        codeNode = getChildNodeByXPath(errorNode, 'code')
        if messageNode == None or codeNode == None:
            return None
        return KalturaException(getXmlNodeText(messageNode), getXmlNodeText(codeNode))

    # Validate the result xml node and raise exception if its an error
    def throwExceptionIfError(self, resultNode):
        exceptionObj = self.getExceptionIfError(resultNode)
        if exceptionObj == None:
            return
        raise exceptionObj

    def startMultiRequest(self):
        self.multiRequest = True

    def doMultiRequest(self):
        resultXml = self.doQueue()
        if resultXml == None:
            return []
        result = []
        for childNode in resultXml.childNodes:
            exceptionObj = self.getExceptionIfError(childNode)
            if exceptionObj != None:
                result.append(exceptionObj)
            elif getChildNodeByXPath(childNode, 'objectType') != None:
                result.append(KalturaObjectFactory.create(childNode, KalturaObjectBase))
            elif getChildNodeByXPath(childNode, 'item/objectType') != None:
                result.append(KalturaObjectFactory.createArray(childNode, KalturaObjectBase))
            else:
                result.append(getXmlNodeText(childNode))
        return result

    def isMultiRequest(self):
        return self.multiRequest

    def getMultiRequestResult(self):
        return MultiRequestSubResult('%s:result' % len(self.callsQueue))

    def log(self, msg):
        if self.shouldLog:
            self.config.getLogger().log(msg)

    @staticmethod
    def generateSession(adminSecretForSigning, userId, type, partnerId, expiry = 86400, privileges = ''):
        rand = random.randint(0, 0x10000)
        expiry = int(time.time()) + expiry
        fields = [partnerId, partnerId, expiry, type, rand, userId, privileges]
        fields = map(lambda x: str(x), fields)
        info = ';'.join(fields)
        signature = KalturaClient.hash(adminSecretForSigning, info)
        decodedKS = signature + "|" + info
        KS = base64.b64encode(decodedKS)
        return KS

    @staticmethod
    def hash(salt, msg):
        m = hashlib.sha1()
        m.update(salt)
        m.update(msg)
        return m.digest().encode('hex')

class KalturaServiceActionCall:
    def __init__(self, service, action, params = KalturaParams(), files = KalturaFiles()):
        self.service = service
        self.action = action
        self.params = params
        self.files = files

    # Return the parameters for a multi request
    def getParamsForMultiRequest(self, multiRequestIndex):
        multiRequestParams = KalturaParams()
        multiRequestParams.put("%s:service" % multiRequestIndex, self.service)
        multiRequestParams.put("%s:action" % multiRequestIndex, self.action)
        for (key, val) in self.params.get().items():
            multiRequestParams.put("%s:%s" % (multiRequestIndex, key), val)
        return multiRequestParams

    def getFilesForMultiRequest(self, multiRequestIndex):
        multiRequestParams = KalturaFiles()
        for (key, val) in self.files.get().items():
            multiRequestParams.put("%s:%s" % (multiRequestIndex, key), val)
        return multiRequestParams
<commit_msg>qnd: python client library - added support for generating KSv2<commit_after># ===================================================================================================
#                           _  __     _ _
#                          | |/ /__ _| | |_ _  _ _ _ __ _
#                          | ' </ _` | |  _| || | '_/ _` |
#                          |_|\_\__,_|_|\__|\_,_|_| \__,_|
#
# This file is part of the Kaltura Collaborative Media Suite which allows users
# to do with audio, video, and animation what Wiki platfroms allow them to do with
# text.
#
# Copyright (C) 2006-2011  Kaltura Inc.
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http:#www.gnu.org/licenses/>.
#
# @ignore
# ===================================================================================================
from KalturaCoreClient import *
from KalturaClientBase import *
from xml.parsers.expat import ExpatError
from xml.dom import minidom
from threading import Timer
import hashlib
import random
import base64
import socket
import urllib
import time
import sys
import os

from poster.streaminghttp import register_openers
from poster.encode import multipart_encode
import urllib2

try:
    from Crypto import Random
    from Crypto.Cipher import AES
except ImportError:
    pass            # PyCrypto is required only for creating KS V2

# Register the streaming http handlers with urllib2
register_openers()

pluginsFolder = os.path.normpath(os.path.join(os.path.dirname(__file__), 'KalturaPlugins'))
if not pluginsFolder in sys.path:
    sys.path.append(pluginsFolder)

class MultiRequestSubResult:
    def __init__(self, value):
        self.value = value
    def __str__(self):
        return '{%s}' % self.value
    def __repr__(self):
        return '{%s}' % self.value
    def __getattr__(self, name):
        if name.startswith('__') or name.endswith('__'):
            raise AttributeError
        return MultiRequestSubResult('%s:%s' % (self.value, name))
    def __getitem__(self, key):
        return MultiRequestSubResult('%s:%s' % (self.value, key))

class PluginServicesProxy:
    def addService(self, serviceName, serviceClass):
        setattr(self, serviceName, serviceClass)

class KalturaClient:
    METHOD_POST = 0
    METHOD_GET = 1

    RANDOM_SIZE = 16

    FIELD_EXPIRY =              '_e'
    FIELD_TYPE =                '_t'
    FIELD_USER =                '_u'

    def __init__(self, config):
        self.apiVersion = API_VERSION
        self.config = None
        self.ks = NotImplemented
        self.shouldLog = False
        self.multiRequest = False
        self.callsQueue = []

        self.method = KalturaClient.METHOD_POST
        self.requestHeaders = {}

        self.config = config
        logger = self.config.getLogger()
        if (logger):
            self.shouldLog = True

        self.loadPlugins()

    def loadPlugins(self):
        if not os.path.isdir(pluginsFolder):
            return

        pluginList = ['KalturaCoreClient']
        for fileName in os.listdir(pluginsFolder):
            (pluginClass, fileExt) = os.path.splitext(fileName)
            if fileExt.lower() != '.py':
                continue
            pluginList.append(pluginClass)

        for pluginClass in pluginList:
            self.loadPlugin(pluginClass)

    def loadPlugin(self, pluginClass):
        pluginModule = __import__(pluginClass)
        if not pluginClass in dir(pluginModule):
            return

        pluginClassType = getattr(pluginModule, pluginClass)

        plugin = pluginClassType.get()
        if not isinstance(plugin, IKalturaClientPlugin):
            return

        self.registerPluginServices(plugin)
        self.registerPluginObjects(plugin)

    def registerPluginServices(self, plugin):
        pluginName = plugin.getName()
        if pluginName != '':
            pluginProxy = PluginServicesProxy()
            setattr(self, pluginName, pluginProxy)

        for (serviceName, serviceFactory) in plugin.getServices().items():
            serviceClass = serviceFactory(self)
            if pluginName == '':
                self.addCoreService(serviceName, serviceClass)
            else:
                pluginProxy.addService(serviceName, serviceClass)

    def registerPluginObjects(self, plugin):
        KalturaEnumsFactory.registerEnums(plugin.getEnums())
        KalturaObjectFactory.registerObjects(plugin.getTypes())

    def addCoreService(self, serviceName, serviceClass):
        setattr(self, serviceName, serviceClass)

    def getServeUrl(self):
        if len(self.callsQueue) != 1:
            return None

        (url, params, _) = self.getRequestParams()

        # reset state
        self.callsQueue = []
        self.multiRequest = False

        result = '%s&%s' % (url, urllib.urlencode(params.get()))
        self.log("Returned url [%s]" % result)
        return result

    def queueServiceActionCall(self, service, action, params = KalturaParams(), files = KalturaFiles()):
        # in start session partner id is optional (default -1). if partner id was not set, use the one in the config
        if not params.get().has_key("partnerId") or params.get()["partnerId"] == -1:
            if self.config.partnerId != None:
                params.put("partnerId", self.config.partnerId)
        params.addStringIfDefined("ks", self.ks)
        call = KalturaServiceActionCall(service, action, params, files)
        self.callsQueue.append(call)

    def getRequestParams(self):
        params = KalturaParams()
        files = KalturaFiles()
        params.put("apiVersion", self.apiVersion)
        params.put("format", self.config.format)
        params.put("clientTag", self.config.clientTag)
        url = self.config.serviceUrl + "/api_v3/index.php?service="
        if self.multiRequest:
            url += "multirequest"
            i = 1
            for call in self.callsQueue:
                callParams = call.getParamsForMultiRequest(i)
                callFiles = call.getFilesForMultiRequest(i)
                params.update(callParams)
                files.update(callFiles)
                i += 1
        else:
            call = self.callsQueue[0]
            url += call.service + "&action=" + call.action
            params.update(call.params)
            files.update(call.files)

        signature = params.signature()
        params.put("kalsig", signature)

        if self.method == KalturaClient.METHOD_GET and len(files.get()) == 0:
            url += '&' + urllib.urlencode(params.get())
            params = None

        self.log("request url: [%s]" % url)

        return (url, params, files)

    @staticmethod
    def closeHandle(fh):
        fh.close()

    @staticmethod
    def openRequestUrl(url, params, files, requestHeaders):
        if len(files.get()) == 0:
            data = None
            if params != None:
                data = urllib.urlencode(params.get())
            request = urllib2.Request(url, data, requestHeaders)
        else:
            fullParams = params
            fullParams.update(files)
            datagen, headers = multipart_encode(fullParams.get())
            headers.update(requestHeaders)
            request = urllib2.Request(url, datagen, headers)

        try:
            f = urllib2.urlopen(request)
        except Exception, e:
            raise KalturaClientException(e, KalturaClientException.ERROR_CONNECTION_FAILED)
        return f

    @staticmethod
    def readHttpResponse(f, requestTimeout):
        if requestTimeout != None:
            readTimer = Timer(requestTimeout, KalturaClient.closeHandle, [f])
            readTimer.start()
        try:
            try:
                data = f.read()
            except AttributeError, e:      # socket was closed while reading
                raise KalturaClientException(e, KalturaClientException.ERROR_READ_TIMEOUT)
            except Exception, e:
                raise KalturaClientException(e, KalturaClientException.ERROR_READ_FAILED)
        finally:
            if requestTimeout != None:
                readTimer.cancel()
        return data

    # Send http request
    def doHttpRequest(self, url, params = KalturaParams(), files = KalturaFiles()):
        if len(files.get()) == 0:
            requestTimeout = self.config.requestTimeout
        else:
            requestTimeout = None

        if requestTimeout != None:
            origSocketTimeout = socket.getdefaulttimeout()
            socket.setdefaulttimeout(requestTimeout)
        try:
            f = self.openRequestUrl(url, params, files, self.requestHeaders)
            data = self.readHttpResponse(f, requestTimeout)
            self.responseHeaders = f.info().headers
        finally:
            if requestTimeout != None:
                socket.setdefaulttimeout(origSocketTimeout)
        return data

    def parsePostResult(self, postResult):
        if len(postResult) > 1024:
            self.log("result (xml): %s bytes" % len(postResult))
        else:
            self.log("result (xml): %s" % postResult)

        try:
            resultXml = minidom.parseString(postResult)
        except ExpatError, e:
            raise KalturaClientException(e, KalturaClientException.ERROR_INVALID_XML)

        resultNode = getChildNodeByXPath(resultXml, 'xml/result')
        if resultNode == None:
            raise KalturaClientException('Could not find result node in response xml', KalturaClientException.ERROR_RESULT_NOT_FOUND)

        execTime = getChildNodeByXPath(resultXml, 'xml/executionTime')
        if execTime != None:
            self.executionTime = getXmlNodeFloat(execTime)

        self.throwExceptionIfError(resultNode)

        return resultNode

    # Call all API services that are in queue
    def doQueue(self):
        self.responseHeaders = None
        self.executionTime = None
        if len(self.callsQueue) == 0:
            self.multiRequest = False
            return None

        if self.config.format != KALTURA_SERVICE_FORMAT_XML:
            raise KalturaClientException("unsupported format: %s" % (postResult), KalturaClientException.ERROR_FORMAT_NOT_SUPPORTED)

        startTime = time.time()

        # get request params
        (url, params, files) = self.getRequestParams()

        # reset state
        self.callsQueue = []
        self.multiRequest = False

        # issue the request
        postResult = self.doHttpRequest(url, params, files)

        # parse the result
        resultNode = self.parsePostResult(postResult)

        endTime = time.time()
        self.log("execution time for [%s]: [%s]" % (url, endTime - startTime))

        return resultNode

    def getKs(self):
        return self.ks

    def setKs(self, ks):
        self.ks = ks

    def getConfig(self):
        return self.config

    def setConfig(self, config):
        self.config = config
        logger = self.config.getLogger()
        if isinstance(logger, IKalturaLogger):
            self.shouldLog = True

    def getExceptionIfError(self, resultNode):
        errorNode = getChildNodeByXPath(resultNode, 'error')
        if errorNode == None:
            return None
        messageNode = getChildNodeByXPath(errorNode, 'message')
        codeNode = getChildNodeByXPath(errorNode, 'code')
        if messageNode == None or codeNode == None:
            return None
        return KalturaException(getXmlNodeText(messageNode), getXmlNodeText(codeNode))

    # Validate the result xml node and raise exception if its an error
    def throwExceptionIfError(self, resultNode):
        exceptionObj = self.getExceptionIfError(resultNode)
        if exceptionObj == None:
            return
        raise exceptionObj

    def startMultiRequest(self):
        self.multiRequest = True

    def doMultiRequest(self):
        resultXml = self.doQueue()
        if resultXml == None:
            return []
        result = []
        for childNode in resultXml.childNodes:
            exceptionObj = self.getExceptionIfError(childNode)
            if exceptionObj != None:
                result.append(exceptionObj)
            elif getChildNodeByXPath(childNode, 'objectType') != None:
                result.append(KalturaObjectFactory.create(childNode, KalturaObjectBase))
            elif getChildNodeByXPath(childNode, 'item/objectType') != None:
                result.append(KalturaObjectFactory.createArray(childNode, KalturaObjectBase))
            else:
                result.append(getXmlNodeText(childNode))
        return result

    def isMultiRequest(self):
        return self.multiRequest

    def getMultiRequestResult(self):
        return MultiRequestSubResult('%s:result' % len(self.callsQueue))

    def log(self, msg):
        if self.shouldLog:
            self.config.getLogger().log(msg)

    @staticmethod
    def generateSession(adminSecretForSigning, userId, type, partnerId, expiry = 86400, privileges = ''):
        rand = random.randint(0, 0x10000)
        expiry = int(time.time()) + expiry
        fields = [partnerId, partnerId, expiry, type, rand, userId, privileges]
        fields = map(lambda x: str(x), fields)
        info = ';'.join(fields)
        signature = KalturaClient.hash(adminSecretForSigning + info).encode('hex')
        decodedKS = signature + "|" + info
        KS = base64.b64encode(decodedKS)
        return KS

    @staticmethod
    def generateSessionV2(adminSecretForSigning, userId, type, partnerId, expiry = 86400, privileges = ''):
        # build fields array
        fields = {}
        for privilege in privileges.split(','):
            privilege = privilege.strip()
            if len(privilege) == 0:
                continue
            if privilege == '*':
                privilege = 'all:*'
            splittedPrivilege = privilege.split(':', 1)
            if len(splittedPrivilege) > 1:
                fields[splittedPrivilege[0]] = splittedPrivilege[1]
            else:
                fields[splittedPrivilege[0]] = ''

        fields[KalturaClient.FIELD_EXPIRY] = str(int(time.time()) + expiry)
        fields[KalturaClient.FIELD_TYPE] = str(type)
        fields[KalturaClient.FIELD_USER] = str(userId)

        # build fields string
        fieldsStr = urllib.urlencode(fields)
        fieldsStr = Random.get_random_bytes(KalturaClient.RANDOM_SIZE) + fieldsStr
        fieldsStr = KalturaClient.hash(fieldsStr) + fieldsStr

        # encrypt and encode
        cipher = AES.new(KalturaClient.hash(adminSecretForSigning)[:16], AES.MODE_CBC, '\0' * 16)
        if len(fieldsStr) % cipher.block_size != 0:
            fieldsStr += '\0' * (cipher.block_size - len(fieldsStr) % cipher.block_size)
        encryptedFields = cipher.encrypt(fieldsStr)
        decodedKs = "v2|%s|%s" % (partnerId, encryptedFields)
        return base64.b64encode(decodedKs).replace('+', '-').replace('/', '_')

    @staticmethod
    def hash(msg):
        m = hashlib.sha1()
        m.update(msg)
        return m.digest()

class KalturaServiceActionCall:
    def __init__(self, service, action, params = KalturaParams(), files = KalturaFiles()):
        self.service = service
        self.action = action
        self.params = params
        self.files = files

    # Return the parameters for a multi request
    def getParamsForMultiRequest(self, multiRequestIndex):
        multiRequestParams = KalturaParams()
        multiRequestParams.put("%s:service" % multiRequestIndex, self.service)
        multiRequestParams.put("%s:action" % multiRequestIndex, self.action)
        for (key, val) in self.params.get().items():
            multiRequestParams.put("%s:%s" % (multiRequestIndex, key), val)
        return multiRequestParams

    def getFilesForMultiRequest(self, multiRequestIndex):
        multiRequestParams = KalturaFiles()
        for (key, val) in self.files.get().items():
            multiRequestParams.put("%s:%s" % (multiRequestIndex, key), val)
        return multiRequestParams
<|endoftext|>


<commit_before>import numpy as np
from scipy import cluster
import pickle, sys

from sklearn.cluster import KMeans
from sklearn.metrics import confusion_matrix
from sklearn.naive_bayes import GaussianNB
from sklearn import svm
from sklearn.linear_model import SGDClassifier
from sklearn.neighbors import KNeighborsClassifier
from sklearn.ensemble import RandomForestClassifier
from sklearn.ensemble import ExtraTreesClassifier
from sklearn.ensemble import GradientBoostingClassifier
from sklearn.decomposition import PCA
from sklearn import preprocessing
from itertools import product
from sklearn.tree import DecisionTreeClassifier
from sklearn.metrics import roc_curve, auc
#import pylab as pl
from sklearn.linear_model import LogisticRegression
from operator import itemgetter

MODELPATH = '/netapp/sali/peterc/cryptosite/src_multichain/'

def get_matrix(inputdata, model='linear'):

    Res = {'CYS': (0, 0, 1, 0, 0), 'ASP': (0, 0, 0, 1, 1), 'SER': (0, 1, 1, 1, 1), 'GLN': (0, 0, 1, 0, 1), 'LYS': (0, 1, 0, 1, 1), 'ILE': (0, 1, 0, 0, 1), 'PRO': (0, 1, 1, 1, 0), 'THR': (1, 0, 0, 0, 0), 'PHE': (0, 1, 1, 0, 1), 'ALA': (0, 0, 0, 0, 0), 'GLY': (0, 0, 1, 1, 1), 'HIS': (0, 1, 0, 0, 0), 'GLU': (0, 0, 1, 1, 0), 'LEU': (0, 1, 0, 1, 0), 'ARG': (0, 0, 0, 0, 1), 'TRP': (1, 0, 0, 0, 1), 'VAL': (1, 0, 0, 1, 1), 'ASN': (0, 0, 0, 1, 0), 'TYR': (1, 0, 0, 1, 0), 'MET': (0, 1, 1, 0, 0)}
    Res = Res.keys()
    SSE = ['B','E','G','H','I','S','T','U']

    data = open(inputdata)
    D = data.readlines()
    data.close()

    Header = D[0].strip().split()

    if model=='poly':
        # the bottom visited for poly SVM
        visited = [Header.index('CNC_mean_300'), Header.index('SQC'), Header.index('D2S')]
        visited+= [Header.index('SQCn'), Header.index('PCKn'), Header.index('Hn')]
        visited+= [Header.index('CN5_std_450'), Header.index('CN5_std_300'), Header.index('CN5_std_350')]
        visited+= [Header.index('CNC'), Header.index('PRT_std_450'), Header.index('CN5_mean_500')]
        visited+= [Header.index('Bn'), Header.index('CHRn'), Header.index('In')]
        visited+= [Header.index('CNC_std_300'), Header.index('CNS_300'),Header.index('SAS14_std_400')]
        visited+= [Header.index('SASn')]

    elif model=='linear':
        # for linear SVM
        visited = [Header.index('CNC_mean_300'), Header.index('SQC'), Header.index('CN5_std_450')]
        visited+= [Header.index('D2S'), Header.index('CNS_300'), Header.index('Hn')]
        visited+= [Header.index('CN5_mean_450'), Header.index('CN5_std_300'), Header.index('SQCn')]
        visited+= [Header.index('CNC_std_350'),Header.index('CNCn'), Header.index('CVX_mean_450')]
        visited+= [Header.index('In')]

    elif model=='final':
        visited = [Header.index('CNC_mean_'), Header.index('SQC'), Header.index('PTM')]

    else:
        print 'Wrong model: ', model
        exit()

    M = []
    Indeces,cnt = {},0
    for d in D[1:]:
        d = d.strip().split('\t')
        LA = []
        for hd in xrange(len(Header)):
            if hd not in visited: pass
            else:
                if hd==1:
                    L = [0.]*len(Res)
                    L[Res.index(d[1])]=1.
                    LA += L
                elif hd in range(4,8): LA += [float(d[hd])]
                elif hd==8:
                    s = [0.]*len(SSE)
                    s[SSE.index(d[8])]=1.
                    LA += s
                else:
                    LA += [float(d[hd])]
        LA += [float(d[-1])]

        M.append([d[0]]+LA)

        Indeces[cnt] = tuple(d[:3])
        cnt += 1
    return M, [Header[j] for j in sorted(visited)], Indeces

def predict(inputdata, model='linear'):

    print 'Reading in the data ...'
    M, Header, Indeces = get_matrix(inputdata, model)

    print 'Processing the data for model %s ...' % model.upper()
    if 1:
        pdb=inputdata.split('.')[0]
        print pdb
        NewIndeces, newcnt = {},0
        X_learn, Y_learn = [],[]

        for r,m in enumerate(M):
            if len(np.argwhere(np.isnan(np.array(m[1:-1]))==True))>0:
                print r,m[0]
                print peter

            X_learn.append(np.array(m[1:-1]))
            Y_learn.append(m[-1])
            NewIndeces[newcnt] = Indeces[r]
            newcnt+=1

        X_learn = np.array(X_learn)
        X_learn = np.vstack(( X_learn[:,0], X_learn[:,2], X_learn[:,1] )).T

    if model=='linear':

        print 'Scaling ...'
        out1 = open(MODELPATH+'LinearScaler_Final.pkl')
        scaler = pickle.load(out1)
        out1.close()
        X_learn = scaler.transform(X_learn)

        outmodel = open(MODELPATH+'LinearSVC_FinalModel.pkl')
        learner = pickle.load(outmodel)
        outmodel.close()

    elif model=='poly':

        print 'Scaling ...'
        out1 = open(MODELPATH+'PolyScaler_Final.pkl')
        scaler = pickle.load(out1)
        out1.close()
        X_learn = scaler.transform(X_learn)

        outmodel = open(MODELPATH+'PolySVC_FinalModel.pkl')
        learner = pickle.load(outmodel)
        outmodel.close()

    elif model=='final':
        print 'Scaling ...'
        out1 = open(MODELPATH+'Scaler_Final_Final.pkl')
        scaler = pickle.load(out1)
        out1.close()
        X_learn = scaler.transform(X_learn)

        outmodel = open(MODELPATH+'SVM_Final_Final.pkl')
        learner = pickle.load(outmodel)
        outmodel.close()

    else:
        print 'Unknown model: ', model
        print peter

    print 'Predicting ...'
    Y_pred = learner.predict(X_learn)
    CM = confusion_matrix(Y_learn,Y_pred)
    print
    print "Confusion matrix for: ",pdb
    print CM
    print

    # output
    Y_pred_prob = learner.predict_proba(X_learn)
    Y_PRED_PROB_ALL = list(Y_pred_prob[:, 1])
    Y_TEST_ALL = list(Y_learn)

    #fprC, tprC, thresholdsC = roc_curve(Y_TEST_ALL, Y_PRED_PROB_ALL)
    #roc_aucC = auc(fprC, tprC)
    #print "Area under the ROC curve : %f" % roc_aucC

    if model=='linear': outn = open(pdb+'.lin.pred','w')
    elif model=='poly' or model=='final': outn = open(pdb+'.pol.pred','w')
    else: print peter

    print 'Writing output files ...'
    outn.write('\t'.join(['PDBID','Res','ResID']+Header+['CryptositeValue'])+'\n')
    for x in xrange(len(Y_PRED_PROB_ALL)):
        outn.write( '\t'.join(list(NewIndeces[x])+[str(i) for i in X_learn[x]]+[str(Y_PRED_PROB_ALL[x])])+'\n' )
    outn.close()

    if model=='linear': write_pdb(pdb,model='linear')
    elif model=='poly' or model=='final': write_pdb(pdb,model='poly')
    else: print peter

    print 'Done!'

def write_pdb(pdb,model='linear'):

    if model=='linear':
        data = open(pdb+'.lin.pred')
        D = data.readlines()
        data.close()
        out = open('%s.lin.pred.pdb' % pdb, 'w')
    elif model=='poly':
        data = open(pdb+'.pol.pred')
        D = data.readlines()
        data.close()
        out = open('%s.pol.pred.pdb' % pdb, 'w')
    else: print peter

    Data = {}
    for d in D:
        d = d.strip().split()
        Data[(d[1],d[2])] = ('0.0',d[-1])

    data = open('%s_mdl.pdb' % pdb.split('/')[-1])
    D = data.readlines()
    data.close()

    for d in D:
        if 'ATOM'==d[:4]:
            p = (d[17:20], str(int(d[22:26])))

            try: pred = Data[p]
            except KeyError: pred = ('0.0', '0.0')

            v = '%.2f' % (float(pred[1])*100) #str(round(100*float(pred[1]),2))
            v = (6-len(v))*' '+v
            line = d[:56]+pred[0]+'0'+v+'\n'
            out.write(line)

        else: out.write(d)
    out.close()

#predict('1NW3A.features', model='poly')
predict(sys.argv[-1]+'.features', model='final')
#predict2('Train_final.features',model='poly')
<commit_msg>Make importable.<commit_after>import numpy as np
from scipy import cluster
import pickle, sys

from sklearn.cluster import KMeans
from sklearn.metrics import confusion_matrix
from sklearn.naive_bayes import GaussianNB
from sklearn import svm
from sklearn.linear_model import SGDClassifier
from sklearn.neighbors import KNeighborsClassifier
from sklearn.ensemble import RandomForestClassifier
from sklearn.ensemble import ExtraTreesClassifier
from sklearn.ensemble import GradientBoostingClassifier
from sklearn.decomposition import PCA
from sklearn import preprocessing
from itertools import product
from sklearn.tree import DecisionTreeClassifier
from sklearn.metrics import roc_curve, auc
#import pylab as pl
from sklearn.linear_model import LogisticRegression
from operator import itemgetter

MODELPATH = '/netapp/sali/peterc/cryptosite/src_multichain/'

def get_matrix(inputdata, model='linear'):

    Res = {'CYS': (0, 0, 1, 0, 0), 'ASP': (0, 0, 0, 1, 1), 'SER': (0, 1, 1, 1, 1), 'GLN': (0, 0, 1, 0, 1), 'LYS': (0, 1, 0, 1, 1), 'ILE': (0, 1, 0, 0, 1), 'PRO': (0, 1, 1, 1, 0), 'THR': (1, 0, 0, 0, 0), 'PHE': (0, 1, 1, 0, 1), 'ALA': (0, 0, 0, 0, 0), 'GLY': (0, 0, 1, 1, 1), 'HIS': (0, 1, 0, 0, 0), 'GLU': (0, 0, 1, 1, 0), 'LEU': (0, 1, 0, 1, 0), 'ARG': (0, 0, 0, 0, 1), 'TRP': (1, 0, 0, 0, 1), 'VAL': (1, 0, 0, 1, 1), 'ASN': (0, 0, 0, 1, 0), 'TYR': (1, 0, 0, 1, 0), 'MET': (0, 1, 1, 0, 0)}
    Res = Res.keys()
    SSE = ['B','E','G','H','I','S','T','U']

    data = open(inputdata)
    D = data.readlines()
    data.close()

    Header = D[0].strip().split()

    if model=='poly':
        # the bottom visited for poly SVM
        visited = [Header.index('CNC_mean_300'), Header.index('SQC'), Header.index('D2S')]
        visited+= [Header.index('SQCn'), Header.index('PCKn'), Header.index('Hn')]
        visited+= [Header.index('CN5_std_450'), Header.index('CN5_std_300'), Header.index('CN5_std_350')]
        visited+= [Header.index('CNC'), Header.index('PRT_std_450'), Header.index('CN5_mean_500')]
        visited+= [Header.index('Bn'), Header.index('CHRn'), Header.index('In')]
        visited+= [Header.index('CNC_std_300'), Header.index('CNS_300'),Header.index('SAS14_std_400')]
        visited+= [Header.index('SASn')]

    elif model=='linear':
        # for linear SVM
        visited = [Header.index('CNC_mean_300'), Header.index('SQC'), Header.index('CN5_std_450')]
        visited+= [Header.index('D2S'), Header.index('CNS_300'), Header.index('Hn')]
        visited+= [Header.index('CN5_mean_450'), Header.index('CN5_std_300'), Header.index('SQCn')]
        visited+= [Header.index('CNC_std_350'),Header.index('CNCn'), Header.index('CVX_mean_450')]
        visited+= [Header.index('In')]

    elif model=='final':
        visited = [Header.index('CNC_mean_'), Header.index('SQC'), Header.index('PTM')]

    else:
        print 'Wrong model: ', model
        exit()

    M = []
    Indeces,cnt = {},0
    for d in D[1:]:
        d = d.strip().split('\t')
        LA = []
        for hd in xrange(len(Header)):
            if hd not in visited: pass
            else:
                if hd==1:
                    L = [0.]*len(Res)
                    L[Res.index(d[1])]=1.
                    LA += L
                elif hd in range(4,8): LA += [float(d[hd])]
                elif hd==8:
                    s = [0.]*len(SSE)
                    s[SSE.index(d[8])]=1.
                    LA += s
                else:
                    LA += [float(d[hd])]
        LA += [float(d[-1])]

        M.append([d[0]]+LA)

        Indeces[cnt] = tuple(d[:3])
        cnt += 1
    return M, [Header[j] for j in sorted(visited)], Indeces

def predict(inputdata, model='linear'):

    print 'Reading in the data ...'
    M, Header, Indeces = get_matrix(inputdata, model)

    print 'Processing the data for model %s ...' % model.upper()
    if 1:
        pdb=inputdata.split('.')[0]
        print pdb
        NewIndeces, newcnt = {},0
        X_learn, Y_learn = [],[]

        for r,m in enumerate(M):
            if len(np.argwhere(np.isnan(np.array(m[1:-1]))==True))>0:
                print r,m[0]
                print peter

            X_learn.append(np.array(m[1:-1]))
            Y_learn.append(m[-1])
            NewIndeces[newcnt] = Indeces[r]
            newcnt+=1

        X_learn = np.array(X_learn)
        X_learn = np.vstack(( X_learn[:,0], X_learn[:,2], X_learn[:,1] )).T

    if model=='linear':

        print 'Scaling ...'
        out1 = open(MODELPATH+'LinearScaler_Final.pkl')
        scaler = pickle.load(out1)
        out1.close()
        X_learn = scaler.transform(X_learn)

        outmodel = open(MODELPATH+'LinearSVC_FinalModel.pkl')
        learner = pickle.load(outmodel)
        outmodel.close()

    elif model=='poly':

        print 'Scaling ...'
        out1 = open(MODELPATH+'PolyScaler_Final.pkl')
        scaler = pickle.load(out1)
        out1.close()
        X_learn = scaler.transform(X_learn)

        outmodel = open(MODELPATH+'PolySVC_FinalModel.pkl')
        learner = pickle.load(outmodel)
        outmodel.close()

    elif model=='final':
        print 'Scaling ...'
        out1 = open(MODELPATH+'Scaler_Final_Final.pkl')
        scaler = pickle.load(out1)
        out1.close()
        X_learn = scaler.transform(X_learn)

        outmodel = open(MODELPATH+'SVM_Final_Final.pkl')
        learner = pickle.load(outmodel)
        outmodel.close()

    else:
        print 'Unknown model: ', model
        print peter

    print 'Predicting ...'
    Y_pred = learner.predict(X_learn)
    CM = confusion_matrix(Y_learn,Y_pred)
    print
    print "Confusion matrix for: ",pdb
    print CM
    print

    # output
    Y_pred_prob = learner.predict_proba(X_learn)
    Y_PRED_PROB_ALL = list(Y_pred_prob[:, 1])
    Y_TEST_ALL = list(Y_learn)

    #fprC, tprC, thresholdsC = roc_curve(Y_TEST_ALL, Y_PRED_PROB_ALL)
    #roc_aucC = auc(fprC, tprC)
    #print "Area under the ROC curve : %f" % roc_aucC

    if model=='linear': outn = open(pdb+'.lin.pred','w')
    elif model=='poly' or model=='final': outn = open(pdb+'.pol.pred','w')
    else: print peter

    print 'Writing output files ...'
    outn.write('\t'.join(['PDBID','Res','ResID']+Header+['CryptositeValue'])+'\n')
    for x in xrange(len(Y_PRED_PROB_ALL)):
        outn.write( '\t'.join(list(NewIndeces[x])+[str(i) for i in X_learn[x]]+[str(Y_PRED_PROB_ALL[x])])+'\n' )
    outn.close()

    if model=='linear': write_pdb(pdb,model='linear')
    elif model=='poly' or model=='final': write_pdb(pdb,model='poly')
    else: print peter

    print 'Done!'

def write_pdb(pdb,model='linear'):

    if model=='linear':
        data = open(pdb+'.lin.pred')
        D = data.readlines()
        data.close()
        out = open('%s.lin.pred.pdb' % pdb, 'w')
    elif model=='poly':
        data = open(pdb+'.pol.pred')
        D = data.readlines()
        data.close()
        out = open('%s.pol.pred.pdb' % pdb, 'w')
    else: print peter

    Data = {}
    for d in D:
        d = d.strip().split()
        Data[(d[1],d[2])] = ('0.0',d[-1])

    data = open('%s_mdl.pdb' % pdb.split('/')[-1])
    D = data.readlines()
    data.close()

    for d in D:
        if 'ATOM'==d[:4]:
            p = (d[17:20], str(int(d[22:26])))

            try: pred = Data[p]
            except KeyError: pred = ('0.0', '0.0')

            v = '%.2f' % (float(pred[1])*100) #str(round(100*float(pred[1]),2))
            v = (6-len(v))*' '+v
            line = d[:56]+pred[0]+'0'+v+'\n'
            out.write(line)

        else: out.write(d)
    out.close()

def main():
    predict(sys.argv[-1]+'.features', model='final')

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before># Copyright (c) 2012, the Dart project authors.  Please see the AUTHORS file
# for details. All rights reserved. Use of this source code is governed by a
# BSD-style license that can be found in the LICENSE file.
"""Top-level presubmit script for Dart.

See http://dev.chromium.org/developers/how-tos/depottools/presubmit-scripts
for more details about the presubmit API built into gcl.
"""

import imp
import os
import os.path
import scm
import subprocess
import tempfile
import platform

def is_cpp_file(path):
    return path.endswith('.cc') or path.endswith('.h')

def _CheckNnbdSdkSync(input_api, output_api):
    files = [git_file.LocalPath() for git_file in input_api.AffectedTextFiles()]
    unsynchronized_files = []
    for file in files:
        if file.startswith('sdk/'):
            nnbd_file = 'sdk_nnbd/' + file[4:]
            if not nnbd_file in files:
                unsynchronized_files.append(nnbd_file)
    if unsynchronized_files:
        return [
            output_api.PresubmitPromptWarning(
                'Changes were made to sdk/ that were not made to sdk_nnbd/\n'
                'Please update these files as well:\n'
                '\n'
                '%s' % ('\n'.join(unsynchronized_files)))
        ]
    return []

def _CheckFormat(input_api,
                 identification,
                 extension,
                 windows,
                 hasFormatErrors,
                 should_skip=lambda path: False):
    local_root = input_api.change.RepositoryRoot()
    upstream = input_api.change._upstream
    unformatted_files = []
    for git_file in input_api.AffectedTextFiles():
        if git_file.LocalPath().startswith("pkg/front_end/testcases/"):
            continue
        if should_skip(git_file.LocalPath()):
            continue
        filename = git_file.AbsoluteLocalPath()
        if filename.endswith(extension) and hasFormatErrors(filename=filename):
            old_version_has_errors = False
            try:
                path = git_file.LocalPath()
                if windows:
                    # Git expects a linux style path.
                    path = path.replace(os.sep, '/')
                old_contents = scm.GIT.Capture(['show', upstream + ':' + path],
                                               cwd=local_root,
                                               strip_out=False)
                if hasFormatErrors(contents=old_contents):
                    old_version_has_errors = True
            except subprocess.CalledProcessError as e:
                old_version_has_errors = False

            if old_version_has_errors:
                print("WARNING: %s has existing and possibly new %s issues" %
                      (git_file.LocalPath(), identification))
            else:
                unformatted_files.append(filename)

    return unformatted_files

def _CheckDartFormat(input_api, output_api):
    local_root = input_api.change.RepositoryRoot()
    upstream = input_api.change._upstream
    utils = imp.load_source('utils',
                            os.path.join(local_root, 'tools', 'utils.py'))

    prebuilt_dartfmt = os.path.join(utils.CheckedInSdkPath(), 'bin', 'dartfmt')

    windows = utils.GuessOS() == 'win32'
    if windows:
        prebuilt_dartfmt += '.bat'

    if not os.path.isfile(prebuilt_dartfmt):
        print('WARNING: dartfmt not found: %s' % (prebuilt_dartfmt))
        return []

    def HasFormatErrors(filename=None, contents=None):
        # Don't look for formatting errors in multitests. Since those are very
        # sensitive to whitespace, many cannot be formatted with dartfmt without
        # breaking them.
        if filename and filename.endswith('_test.dart'):
            with open(filename) as f:
                contents = f.read()
                if '//#' in contents:
                    return False

        args = [prebuilt_dartfmt, '--set-exit-if-changed']
        if not contents:
            args += [filename, '-n']

        process = subprocess.Popen(
            args, stdout=subprocess.PIPE, stdin=subprocess.PIPE)
        process.communicate(input=contents)

        # Check for exit code 1 explicitly to distinguish it from a syntax error
        # in the file (exit code 65). The repo contains many Dart files that are
        # known to have syntax errors for testing purposes and which can't be
        # parsed and formatted. Don't treat those as errors.
        return process.returncode == 1

    unformatted_files = _CheckFormat(input_api, "dartfmt", ".dart", windows,
                                     HasFormatErrors)

    if unformatted_files:
        lineSep = " \\\n"
        if windows:
            lineSep = " ^\n"
        return [
            output_api.PresubmitError(
                'File output does not match dartfmt.\n'
                'Fix these issues with:\n'
                '%s -w%s%s' % (prebuilt_dartfmt, lineSep,
                               lineSep.join(unformatted_files)))
        ]

    return []

def _CheckStatusFiles(input_api, output_api):
    local_root = input_api.change.RepositoryRoot()
    upstream = input_api.change._upstream
    utils = imp.load_source('utils',
                            os.path.join(local_root, 'tools', 'utils.py'))

    dart = os.path.join(utils.CheckedInSdkPath(), 'bin', 'dart')
    lint = os.path.join(local_root, 'pkg', 'status_file', 'bin', 'lint.dart')

    windows = utils.GuessOS() == 'win32'
    if windows:
        dart += '.exe'

    if not os.path.isfile(dart):
        print('WARNING: dart not found: %s' % dart)
        return []

    if not os.path.isfile(lint):
        print('WARNING: Status file linter not found: %s' % lint)
        return []

    def HasFormatErrors(filename=None, contents=None):
        args = [dart, lint] + (['-t'] if contents else [filename])
        process = subprocess.Popen(
            args, stdout=subprocess.PIPE, stdin=subprocess.PIPE)
        process.communicate(input=contents)
        return process.returncode != 0

    def should_skip(path):
        return (path.startswith("pkg/status_file/test/data/") or
                path.startswith("pkg/front_end/"))

    unformatted_files = _CheckFormat(input_api, "status file", ".status",
                                     windows, HasFormatErrors, should_skip)

    if unformatted_files:
        normalize = os.path.join(local_root, 'pkg', 'status_file', 'bin',
                                 'normalize.dart')
        lineSep = " \\\n"
        if windows:
            lineSep = " ^\n"
        return [
            output_api.PresubmitError(
                'Status files are not normalized.\n'
                'Fix these issues with:\n'
                '%s %s -w%s%s' % (dart, normalize, lineSep,
                                  lineSep.join(unformatted_files)))
        ]

    return []

def _CheckValidHostsInDEPS(input_api, output_api):
    """Checks that DEPS file deps are from allowed_hosts."""
    # Run only if DEPS file has been modified to annoy fewer bystanders.
    if all(f.LocalPath() != 'DEPS' for f in input_api.AffectedFiles()):
        return []
    # Outsource work to gclient verify
    try:
        input_api.subprocess.check_output(['gclient', 'verify'])
        return []
    except input_api.subprocess.CalledProcessError, error:
        return [
            output_api.PresubmitError(
                'DEPS file must have only dependencies from allowed hosts.',
                long_text=error.output)
        ]

def _CheckLayering(input_api, output_api):
    """Run VM layering check.

  This check validates that sources from one layer do not reference sources
  from another layer accidentally.
  """

    # Run only if .cc or .h file was modified.
    if all(not is_cpp_file(f.LocalPath()) for f in input_api.AffectedFiles()):
        return []

    local_root = input_api.change.RepositoryRoot()
    compiler_layering_check = imp.load_source(
        'compiler_layering_check',
        os.path.join(local_root, 'runtime', 'tools',
                     'compiler_layering_check.py'))
    errors = compiler_layering_check.DoCheck(local_root)
    embedder_layering_check = imp.load_source(
        'embedder_layering_check',
        os.path.join(local_root, 'runtime', 'tools',
                     'embedder_layering_check.py'))
    errors += embedder_layering_check.DoCheck(local_root)
    if errors:
        return [
            output_api.PresubmitError(
                'Layering check violation for C++ sources.',
                long_text='\n'.join(errors))
        ]

    return []

def _CheckClangTidy(input_api, output_api):
    """Run clang-tidy on VM changes."""

    # Only run clang-tidy on linux x64.
    if platform.system() != 'Linux' or platform.machine() != 'x86_64':
        return []

    # Run only for modified .cc or .h files.
    files = []
    for f in input_api.AffectedFiles():
        path = f.LocalPath()
        if is_cpp_file(path) and os.path.isfile(path): files.append(path)

    if not files:
        return []

    args = [
        'tools/sdks/dart-sdk/bin/dart',
        'runtime/tools/run_clang_tidy.dart',
    ]
    args.extend(files)
    stdout = input_api.subprocess.check_output(args).strip()
    if not stdout:
        return []

    return [
        output_api.PresubmitError(
            'The `clang-tidy` linter revealed issues:',
            long_text=stdout)
    ]

def _CommonChecks(input_api, output_api):
    results = []
    results.extend(_CheckNnbdSdkSync(input_api, output_api))
    results.extend(_CheckValidHostsInDEPS(input_api, output_api))
    results.extend(_CheckDartFormat(input_api, output_api))
    results.extend(_CheckStatusFiles(input_api, output_api))
    results.extend(_CheckLayering(input_api, output_api))
    results.extend(_CheckClangTidy(input_api, output_api))
    results.extend(
        input_api.canned_checks.CheckPatchFormatted(input_api, output_api))
    return results

def CheckChangeOnCommit(input_api, output_api):
    return _CommonChecks(input_api, output_api)

def CheckChangeOnUpload(input_api, output_api):
    return _CommonChecks(input_api, output_api)
<commit_msg>Add some minimal presubmit checks to keep forked SDK tests in sync.<commit_after># Copyright (c) 2012, the Dart project authors.  Please see the AUTHORS file
# for details. All rights reserved. Use of this source code is governed by a
# BSD-style license that can be found in the LICENSE file.
"""Top-level presubmit script for Dart.

See http://dev.chromium.org/developers/how-tos/depottools/presubmit-scripts
for more details about the presubmit API built into gcl.
"""

import imp
import os
import os.path
import scm
import subprocess
import tempfile
import platform

def is_cpp_file(path):
    return path.endswith('.cc') or path.endswith('.h')

def _CheckNnbdSdkSync(input_api, output_api):
    files = [git_file.LocalPath() for git_file in input_api.AffectedTextFiles()]
    unsynchronized_files = []
    for file in files:
        if file.startswith('sdk/'):
            nnbd_file = 'sdk_nnbd/' + file[4:]
            if not nnbd_file in files:
                unsynchronized_files.append(nnbd_file)
    if unsynchronized_files:
        return [
            output_api.PresubmitPromptWarning(
                'Changes were made to sdk/ that were not made to sdk_nnbd/\n'
                'Please update these files as well:\n'
                '\n'
                '%s' % ('\n'.join(unsynchronized_files)))
        ]
    return []

def _CheckNnbdTestSync(input_api, output_api):
    """Make sure that any forked SDK tests are kept in sync. If a CL touches
    a test, the test's counterpart (if it exists at all) should be in the CL
    too.
    """
    DIRS = ["co19", "corelib", "language", "lib", "standalone"]

    files = [git_file.LocalPath() for git_file in input_api.AffectedTextFiles()]
    unsynchronized = []
    for file in files:
        for dir in DIRS:
            legacy_dir = "tests/{}_2/".format(dir)
            nnbd_dir = "tests/{}/".format(dir)

            counterpart = None
            if file.startswith(legacy_dir):
                counterpart = file.replace(legacy_dir, nnbd_dir)
            elif file.startswith(nnbd_dir):
                counterpart = file.replace(nnbd_dir, legacy_dir)

            if counterpart:
                # Changed one file with a potential counterpart. If it exists
                # on disc, make sure it is also in the CL.
                if counterpart not in files and os.path.exists(counterpart):
                    unsynchronized.append("- {} -> {}".format(
                        file, counterpart))
                break

    # TODO(rnystrom): Currently, we only warn if a test does exist in both
    # places on disc but only one is touched by a CL. We don't warn for files
    # that only exist on one side because the migration isn't complete and the
    # CL may be migrating. Once the migration is complete, consider making
    # these checks more rigorous.

    if unsynchronized:
        return [
            output_api.PresubmitPromptWarning(
                'Make sure to keep the forked legacy and NNBD tests in sync.\n'
                'You may need to synchronize these files:\n' +
                '\n'.join(unsynchronized))
        ]

    return []

def _CheckFormat(input_api,
                 identification,
                 extension,
                 windows,
                 hasFormatErrors,
                 should_skip=lambda path: False):
    local_root = input_api.change.RepositoryRoot()
    upstream = input_api.change._upstream
    unformatted_files = []
    for git_file in input_api.AffectedTextFiles():
        if git_file.LocalPath().startswith("pkg/front_end/testcases/"):
            continue
        if should_skip(git_file.LocalPath()):
            continue
        filename = git_file.AbsoluteLocalPath()
        if filename.endswith(extension) and hasFormatErrors(filename=filename):
            old_version_has_errors = False
            try:
                path = git_file.LocalPath()
                if windows:
                    # Git expects a linux style path.
                    path = path.replace(os.sep, '/')
                old_contents = scm.GIT.Capture(['show', upstream + ':' + path],
                                               cwd=local_root,
                                               strip_out=False)
                if hasFormatErrors(contents=old_contents):
                    old_version_has_errors = True
            except subprocess.CalledProcessError as e:
                old_version_has_errors = False

            if old_version_has_errors:
                print("WARNING: %s has existing and possibly new %s issues" %
                      (git_file.LocalPath(), identification))
            else:
                unformatted_files.append(filename)

    return unformatted_files

def _CheckDartFormat(input_api, output_api):
    local_root = input_api.change.RepositoryRoot()
    upstream = input_api.change._upstream
    utils = imp.load_source('utils',
                            os.path.join(local_root, 'tools', 'utils.py'))

    prebuilt_dartfmt = os.path.join(utils.CheckedInSdkPath(), 'bin', 'dartfmt')

    windows = utils.GuessOS() == 'win32'
    if windows:
        prebuilt_dartfmt += '.bat'

    if not os.path.isfile(prebuilt_dartfmt):
        print('WARNING: dartfmt not found: %s' % (prebuilt_dartfmt))
        return []

    def HasFormatErrors(filename=None, contents=None):
        # Don't look for formatting errors in multitests. Since those are very
        # sensitive to whitespace, many cannot be formatted with dartfmt without
        # breaking them.
        if filename and filename.endswith('_test.dart'):
            with open(filename) as f:
                contents = f.read()
                if '//#' in contents:
                    return False

        args = [prebuilt_dartfmt, '--set-exit-if-changed']
        if not contents:
            args += [filename, '-n']

        process = subprocess.Popen(
            args, stdout=subprocess.PIPE, stdin=subprocess.PIPE)
        process.communicate(input=contents)

        # Check for exit code 1 explicitly to distinguish it from a syntax error
        # in the file (exit code 65). The repo contains many Dart files that are
        # known to have syntax errors for testing purposes and which can't be
        # parsed and formatted. Don't treat those as errors.
        return process.returncode == 1

    unformatted_files = _CheckFormat(input_api, "dartfmt", ".dart", windows,
                                     HasFormatErrors)

    if unformatted_files:
        lineSep = " \\\n"
        if windows:
            lineSep = " ^\n"
        return [
            output_api.PresubmitError(
                'File output does not match dartfmt.\n'
                'Fix these issues with:\n'
                '%s -w%s%s' % (prebuilt_dartfmt, lineSep,
                               lineSep.join(unformatted_files)))
        ]

    return []

def _CheckStatusFiles(input_api, output_api):
    local_root = input_api.change.RepositoryRoot()
    upstream = input_api.change._upstream
    utils = imp.load_source('utils',
                            os.path.join(local_root, 'tools', 'utils.py'))

    dart = os.path.join(utils.CheckedInSdkPath(), 'bin', 'dart')
    lint = os.path.join(local_root, 'pkg', 'status_file', 'bin', 'lint.dart')

    windows = utils.GuessOS() == 'win32'
    if windows:
        dart += '.exe'

    if not os.path.isfile(dart):
        print('WARNING: dart not found: %s' % dart)
        return []

    if not os.path.isfile(lint):
        print('WARNING: Status file linter not found: %s' % lint)
        return []

    def HasFormatErrors(filename=None, contents=None):
        args = [dart, lint] + (['-t'] if contents else [filename])
        process = subprocess.Popen(
            args, stdout=subprocess.PIPE, stdin=subprocess.PIPE)
        process.communicate(input=contents)
        return process.returncode != 0

    def should_skip(path):
        return (path.startswith("pkg/status_file/test/data/") or
                path.startswith("pkg/front_end/"))

    unformatted_files = _CheckFormat(input_api, "status file", ".status",
                                     windows, HasFormatErrors, should_skip)

    if unformatted_files:
        normalize = os.path.join(local_root, 'pkg', 'status_file', 'bin',
                                 'normalize.dart')
        lineSep = " \\\n"
        if windows:
            lineSep = " ^\n"
        return [
            output_api.PresubmitError(
                'Status files are not normalized.\n'
                'Fix these issues with:\n'
                '%s %s -w%s%s' % (dart, normalize, lineSep,
                                  lineSep.join(unformatted_files)))
        ]

    return []

def _CheckValidHostsInDEPS(input_api, output_api):
    """Checks that DEPS file deps are from allowed_hosts."""
    # Run only if DEPS file has been modified to annoy fewer bystanders.
    if all(f.LocalPath() != 'DEPS' for f in input_api.AffectedFiles()):
        return []
    # Outsource work to gclient verify
    try:
        input_api.subprocess.check_output(['gclient', 'verify'])
        return []
    except input_api.subprocess.CalledProcessError, error:
        return [
            output_api.PresubmitError(
                'DEPS file must have only dependencies from allowed hosts.',
                long_text=error.output)
        ]

def _CheckLayering(input_api, output_api):
    """Run VM layering check.

  This check validates that sources from one layer do not reference sources
  from another layer accidentally.
  """

    # Run only if .cc or .h file was modified.
    if all(not is_cpp_file(f.LocalPath()) for f in input_api.AffectedFiles()):
        return []

    local_root = input_api.change.RepositoryRoot()
    compiler_layering_check = imp.load_source(
        'compiler_layering_check',
        os.path.join(local_root, 'runtime', 'tools',
                     'compiler_layering_check.py'))
    errors = compiler_layering_check.DoCheck(local_root)
    embedder_layering_check = imp.load_source(
        'embedder_layering_check',
        os.path.join(local_root, 'runtime', 'tools',
                     'embedder_layering_check.py'))
    errors += embedder_layering_check.DoCheck(local_root)
    if errors:
        return [
            output_api.PresubmitError(
                'Layering check violation for C++ sources.',
                long_text='\n'.join(errors))
        ]

    return []

def _CheckClangTidy(input_api, output_api):
    """Run clang-tidy on VM changes."""

    # Only run clang-tidy on linux x64.
    if platform.system() != 'Linux' or platform.machine() != 'x86_64':
        return []

    # Run only for modified .cc or .h files.
    files = []
    for f in input_api.AffectedFiles():
        path = f.LocalPath()
        if is_cpp_file(path) and os.path.isfile(path): files.append(path)

    if not files:
        return []

    args = [
        'tools/sdks/dart-sdk/bin/dart',
        'runtime/tools/run_clang_tidy.dart',
    ]
    args.extend(files)
    stdout = input_api.subprocess.check_output(args).strip()
    if not stdout:
        return []

    return [
        output_api.PresubmitError(
            'The `clang-tidy` linter revealed issues:',
            long_text=stdout)
    ]

def _CommonChecks(input_api, output_api):
    results = []
    results.extend(_CheckNnbdSdkSync(input_api, output_api))
    results.extend(_CheckNnbdTestSync(input_api, output_api))
    results.extend(_CheckValidHostsInDEPS(input_api, output_api))
    results.extend(_CheckDartFormat(input_api, output_api))
    results.extend(_CheckStatusFiles(input_api, output_api))
    results.extend(_CheckLayering(input_api, output_api))
    results.extend(_CheckClangTidy(input_api, output_api))
    results.extend(
        input_api.canned_checks.CheckPatchFormatted(input_api, output_api))
    return results

def CheckChangeOnCommit(input_api, output_api):
    return _CommonChecks(input_api, output_api)

def CheckChangeOnUpload(input_api, output_api):
    return _CommonChecks(input_api, output_api)
<|endoftext|>


<commit_before># Copyright (c) 2012, the Dart project authors.  Please see the AUTHORS file
# for details. All rights reserved. Use of this source code is governed by a
# BSD-style license that can be found in the LICENSE file.
"""Top-level presubmit script for Dart.

See http://dev.chromium.org/developers/how-tos/depottools/presubmit-scripts
for more details about the presubmit API built into gcl.
"""

import imp
import os
import os.path
import scm
import subprocess
import tempfile
import platform

def is_cpp_file(path):
    return path.endswith('.cc') or path.endswith('.h')

def _CheckNnbdTestSync(input_api, output_api):
    """Make sure that any forked SDK tests are kept in sync. If a CL touches
    a test, the test's counterpart (if it exists at all) should be in the CL
    too.
    """
    DIRS = [
        "tests/co19", "tests/corelib", "tests/ffi", "tests/language",
        "tests/lib", "tests/standalone", "runtime/tests/vm/dart"
    ]

    files = [git_file.LocalPath() for git_file in input_api.AffectedTextFiles()]
    unsynchronized = []
    for file in files:
        if file.endswith('.status'): continue

        for dir in DIRS:
            legacy_dir = "{}_2/".format(dir)
            nnbd_dir = "{}/".format(dir)

            counterpart = None
            if file.startswith(legacy_dir):
                counterpart = file.replace(legacy_dir, nnbd_dir)
            elif file.startswith(nnbd_dir):
                counterpart = file.replace(nnbd_dir, legacy_dir)

            if counterpart:
                # Changed one file with a potential counterpart.
                if counterpart not in files:
                    missing = '' if os.path.exists(
                        counterpart) else ' (missing)'
                    unsynchronized.append("- {} -> {}{}".format(
                        file, counterpart, missing))
                break

    if unsynchronized:
        return [
            output_api.PresubmitPromptWarning(
                'Make sure to keep the forked legacy and NNBD tests in sync.\n'
                'You may need to synchronize these files:\n' +
                '\n'.join(unsynchronized))
        ]

    return []

def _CheckFormat(input_api,
                 identification,
                 extension,
                 windows,
                 hasFormatErrors,
                 should_skip=lambda path: False):
    local_root = input_api.change.RepositoryRoot()
    upstream = input_api.change._upstream
    unformatted_files = []
    for git_file in input_api.AffectedTextFiles():
        if git_file.LocalPath().startswith("pkg/front_end/testcases/"):
            continue
        if should_skip(git_file.LocalPath()):
            continue
        filename = git_file.AbsoluteLocalPath()
        if filename.endswith(extension) and hasFormatErrors(filename=filename):
            old_version_has_errors = False
            try:
                path = git_file.LocalPath()
                if windows:
                    # Git expects a linux style path.
                    path = path.replace(os.sep, '/')
                old_contents = scm.GIT.Capture(['show', upstream + ':' + path],
                                               cwd=local_root,
                                               strip_out=False)
                if hasFormatErrors(contents=old_contents):
                    old_version_has_errors = True
            except subprocess.CalledProcessError as e:
                old_version_has_errors = False

            if old_version_has_errors:
                print("WARNING: %s has existing and possibly new %s issues" %
                      (git_file.LocalPath(), identification))
            else:
                unformatted_files.append(filename)

    return unformatted_files

def _CheckDartFormat(input_api, output_api):
    local_root = input_api.change.RepositoryRoot()
    upstream = input_api.change._upstream
    utils = imp.load_source('utils',
                            os.path.join(local_root, 'tools', 'utils.py'))

    prebuilt_dartfmt = os.path.join(utils.CheckedInSdkPath(), 'bin', 'dartfmt')

    windows = utils.GuessOS() == 'win32'
    if windows:
        prebuilt_dartfmt += '.bat'

    if not os.path.isfile(prebuilt_dartfmt):
        print('WARNING: dartfmt not found: %s' % (prebuilt_dartfmt))
        return []

    def HasFormatErrors(filename=None, contents=None):
        # Don't look for formatting errors in multitests. Since those are very
        # sensitive to whitespace, many cannot be formatted with dartfmt without
        # breaking them.
        if filename and filename.endswith('_test.dart'):
            with open(filename) as f:
                contents = f.read()
                if '//#' in contents:
                    return False

        args = [prebuilt_dartfmt, '--set-exit-if-changed']
        if not contents:
            args += [filename, '-n']

        process = subprocess.Popen(
            args, stdout=subprocess.PIPE, stdin=subprocess.PIPE)
        process.communicate(input=contents)

        # Check for exit code 1 explicitly to distinguish it from a syntax error
        # in the file (exit code 65). The repo contains many Dart files that are
        # known to have syntax errors for testing purposes and which can't be
        # parsed and formatted. Don't treat those as errors.
        return process.returncode == 1

    unformatted_files = _CheckFormat(input_api, "dartfmt", ".dart", windows,
                                     HasFormatErrors)

    if unformatted_files:
        lineSep = " \\\n"
        if windows:
            lineSep = " ^\n"
        return [
            output_api.PresubmitError(
                'File output does not match dartfmt.\n'
                'Fix these issues with:\n'
                '%s -w%s%s' % (prebuilt_dartfmt, lineSep,
                               lineSep.join(unformatted_files)))
        ]

    return []

def _CheckStatusFiles(input_api, output_api):
    local_root = input_api.change.RepositoryRoot()
    upstream = input_api.change._upstream
    utils = imp.load_source('utils',
                            os.path.join(local_root, 'tools', 'utils.py'))

    dart = os.path.join(utils.CheckedInSdkPath(), 'bin', 'dart')
    lint = os.path.join(local_root, 'pkg', 'status_file', 'bin', 'lint.dart')

    windows = utils.GuessOS() == 'win32'
    if windows:
        dart += '.exe'

    if not os.path.isfile(dart):
        print('WARNING: dart not found: %s' % dart)
        return []

    if not os.path.isfile(lint):
        print('WARNING: Status file linter not found: %s' % lint)
        return []

    def HasFormatErrors(filename=None, contents=None):
        args = [dart, lint] + (['-t'] if contents else [filename])
        process = subprocess.Popen(
            args, stdout=subprocess.PIPE, stdin=subprocess.PIPE)
        process.communicate(input=contents)
        return process.returncode != 0

    def should_skip(path):
        return (path.startswith("pkg/status_file/test/data/") or
                path.startswith("pkg/front_end/"))

    unformatted_files = _CheckFormat(input_api, "status file", ".status",
                                     windows, HasFormatErrors, should_skip)

    if unformatted_files:
        normalize = os.path.join(local_root, 'pkg', 'status_file', 'bin',
                                 'normalize.dart')
        lineSep = " \\\n"
        if windows:
            lineSep = " ^\n"
        return [
            output_api.PresubmitError(
                'Status files are not normalized.\n'
                'Fix these issues with:\n'
                '%s %s -w%s%s' % (dart, normalize, lineSep,
                                  lineSep.join(unformatted_files)))
        ]

    return []

def _CheckValidHostsInDEPS(input_api, output_api):
    """Checks that DEPS file deps are from allowed_hosts."""
    # Run only if DEPS file has been modified to annoy fewer bystanders.
    if all(f.LocalPath() != 'DEPS' for f in input_api.AffectedFiles()):
        return []
    # Outsource work to gclient verify
    try:
        input_api.subprocess.check_output(['gclient', 'verify'])
        return []
    except input_api.subprocess.CalledProcessError, error:
        return [
            output_api.PresubmitError(
                'DEPS file must have only dependencies from allowed hosts.',
                long_text=error.output)
        ]

def _CheckLayering(input_api, output_api):
    """Run VM layering check.

  This check validates that sources from one layer do not reference sources
  from another layer accidentally.
  """

    # Run only if .cc or .h file was modified.
    if all(not is_cpp_file(f.LocalPath()) for f in input_api.AffectedFiles()):
        return []

    local_root = input_api.change.RepositoryRoot()
    compiler_layering_check = imp.load_source(
        'compiler_layering_check',
        os.path.join(local_root, 'runtime', 'tools',
                     'compiler_layering_check.py'))
    errors = compiler_layering_check.DoCheck(local_root)
    embedder_layering_check = imp.load_source(
        'embedder_layering_check',
        os.path.join(local_root, 'runtime', 'tools',
                     'embedder_layering_check.py'))
    errors += embedder_layering_check.DoCheck(local_root)
    if errors:
        return [
            output_api.PresubmitError(
                'Layering check violation for C++ sources.',
                long_text='\n'.join(errors))
        ]

    return []

def _CheckTestPlan(input_api, output_api):
    """Run test plan check.

  Each change that touches specified directories in the checkout are required
  to have TEST= line explaining how this change was validated.
  """

    DIRS = [
        'runtime/vm',
        'runtime/bin',
        'runtime/lib',
        'runtime/include',
        'runtime/observatory',
        'runtime/observatory_2',
        'runtime/platform',
        'pkg/vm',
        'sdk/lib/_internal/vm',
    ]

    # Run only if directory was affected.
    files = [f.LocalPath() for f in input_api.AffectedFiles()]
    affected = filter(lambda dir: any(f.startswith(dir) for f in files), DIRS)
    if len(affected) != 0 and input_api.change.tags.get('TEST', '') == '':
        return [
            output_api.PresubmitError('Change is missing TEST= line',
                                      long_text="""
When changing files in one of the following directories you
must include TEST= line at the end of your change description.

    %s

This line is expected to explain in a free form the kind of testing
that was performed to validate effect of the change. For example,
it can list newly added tests or already existing tests which are assumed
to cover the change.
""" % ('    '.join(affected)))
        ]

    return []

def _CheckClangTidy(input_api, output_api):
    """Run clang-tidy on VM changes."""

    # Only run clang-tidy on linux x64.
    if platform.system() != 'Linux' or platform.machine() != 'x86_64':
        return []

    # Run only for modified .cc or .h files.
    files = []
    for f in input_api.AffectedFiles():
        path = f.LocalPath()
        if is_cpp_file(path) and os.path.isfile(path): files.append(path)

    if not files:
        return []

    args = [
        'tools/sdks/dart-sdk/bin/dart',
        'runtime/tools/run_clang_tidy.dart',
    ]
    args.extend(files)
    stdout = input_api.subprocess.check_output(args).strip()
    if not stdout:
        return []

    return [
        output_api.PresubmitError(
            'The `clang-tidy` linter revealed issues:',
            long_text=stdout)
    ]

def _CheckTestMatrixValid(input_api, output_api):
    """Run script to check that the test matrix has no errors."""

    def test_matrix_filter(affected_file):
        """Only run test if either the test matrix or the code that
           validates it was modified."""
        path = affected_file.LocalPath()
        return (path == 'tools/bots/test_matrix.json' or
                path == 'tools/validate_test_matrix.dart' or
                path.startswith('pkg/smith/'))

    if len(
            input_api.AffectedFiles(
                include_deletes=False, file_filter=test_matrix_filter)) == 0:
        return []

    command = [
        'tools/sdks/dart-sdk/bin/dart',
        'tools/validate_test_matrix.dart',
    ]
    stdout = input_api.subprocess.check_output(command).strip()
    if not stdout:
        return []
    else:
        return [
            output_api.PresubmitError(
                'The test matrix is not valid:', long_text=stdout)
        ]

def _CommonChecks(input_api, output_api):
    results = []
    results.extend(_CheckNnbdTestSync(input_api, output_api))
    results.extend(_CheckValidHostsInDEPS(input_api, output_api))
    results.extend(_CheckDartFormat(input_api, output_api))
    results.extend(_CheckStatusFiles(input_api, output_api))
    results.extend(_CheckLayering(input_api, output_api))
    results.extend(_CheckClangTidy(input_api, output_api))
    results.extend(_CheckTestMatrixValid(input_api, output_api))
    results.extend(_CheckTestPlan(input_api, output_api))
    results.extend(
        input_api.canned_checks.CheckPatchFormatted(input_api, output_api))
    return results

def CheckChangeOnCommit(input_api, output_api):
    return _CommonChecks(input_api, output_api)

def CheckChangeOnUpload(input_api, output_api):
    return _CommonChecks(input_api, output_api)
<commit_msg>Revert "Require that all changes to VM have TEST line"<commit_after># Copyright (c) 2012, the Dart project authors.  Please see the AUTHORS file
# for details. All rights reserved. Use of this source code is governed by a
# BSD-style license that can be found in the LICENSE file.
"""Top-level presubmit script for Dart.

See http://dev.chromium.org/developers/how-tos/depottools/presubmit-scripts
for more details about the presubmit API built into gcl.
"""

import imp
import os
import os.path
import scm
import subprocess
import tempfile
import platform

def is_cpp_file(path):
    return path.endswith('.cc') or path.endswith('.h')

def _CheckNnbdTestSync(input_api, output_api):
    """Make sure that any forked SDK tests are kept in sync. If a CL touches
    a test, the test's counterpart (if it exists at all) should be in the CL
    too.
    """
    DIRS = [
        "tests/co19", "tests/corelib", "tests/ffi", "tests/language",
        "tests/lib", "tests/standalone", "runtime/tests/vm/dart"
    ]

    files = [git_file.LocalPath() for git_file in input_api.AffectedTextFiles()]
    unsynchronized = []
    for file in files:
        if file.endswith('.status'): continue

        for dir in DIRS:
            legacy_dir = "{}_2/".format(dir)
            nnbd_dir = "{}/".format(dir)

            counterpart = None
            if file.startswith(legacy_dir):
                counterpart = file.replace(legacy_dir, nnbd_dir)
            elif file.startswith(nnbd_dir):
                counterpart = file.replace(nnbd_dir, legacy_dir)

            if counterpart:
                # Changed one file with a potential counterpart.
                if counterpart not in files:
                    missing = '' if os.path.exists(
                        counterpart) else ' (missing)'
                    unsynchronized.append("- {} -> {}{}".format(
                        file, counterpart, missing))
                break

    if unsynchronized:
        return [
            output_api.PresubmitPromptWarning(
                'Make sure to keep the forked legacy and NNBD tests in sync.\n'
                'You may need to synchronize these files:\n' +
                '\n'.join(unsynchronized))
        ]

    return []

def _CheckFormat(input_api,
                 identification,
                 extension,
                 windows,
                 hasFormatErrors,
                 should_skip=lambda path: False):
    local_root = input_api.change.RepositoryRoot()
    upstream = input_api.change._upstream
    unformatted_files = []
    for git_file in input_api.AffectedTextFiles():
        if git_file.LocalPath().startswith("pkg/front_end/testcases/"):
            continue
        if should_skip(git_file.LocalPath()):
            continue
        filename = git_file.AbsoluteLocalPath()
        if filename.endswith(extension) and hasFormatErrors(filename=filename):
            old_version_has_errors = False
            try:
                path = git_file.LocalPath()
                if windows:
                    # Git expects a linux style path.
                    path = path.replace(os.sep, '/')
                old_contents = scm.GIT.Capture(['show', upstream + ':' + path],
                                               cwd=local_root,
                                               strip_out=False)
                if hasFormatErrors(contents=old_contents):
                    old_version_has_errors = True
            except subprocess.CalledProcessError as e:
                old_version_has_errors = False

            if old_version_has_errors:
                print("WARNING: %s has existing and possibly new %s issues" %
                      (git_file.LocalPath(), identification))
            else:
                unformatted_files.append(filename)

    return unformatted_files

def _CheckDartFormat(input_api, output_api):
    local_root = input_api.change.RepositoryRoot()
    upstream = input_api.change._upstream
    utils = imp.load_source('utils',
                            os.path.join(local_root, 'tools', 'utils.py'))

    prebuilt_dartfmt = os.path.join(utils.CheckedInSdkPath(), 'bin', 'dartfmt')

    windows = utils.GuessOS() == 'win32'
    if windows:
        prebuilt_dartfmt += '.bat'

    if not os.path.isfile(prebuilt_dartfmt):
        print('WARNING: dartfmt not found: %s' % (prebuilt_dartfmt))
        return []

    def HasFormatErrors(filename=None, contents=None):
        # Don't look for formatting errors in multitests. Since those are very
        # sensitive to whitespace, many cannot be formatted with dartfmt without
        # breaking them.
        if filename and filename.endswith('_test.dart'):
            with open(filename) as f:
                contents = f.read()
                if '//#' in contents:
                    return False

        args = [prebuilt_dartfmt, '--set-exit-if-changed']
        if not contents:
            args += [filename, '-n']

        process = subprocess.Popen(
            args, stdout=subprocess.PIPE, stdin=subprocess.PIPE)
        process.communicate(input=contents)

        # Check for exit code 1 explicitly to distinguish it from a syntax error
        # in the file (exit code 65). The repo contains many Dart files that are
        # known to have syntax errors for testing purposes and which can't be
        # parsed and formatted. Don't treat those as errors.
        return process.returncode == 1

    unformatted_files = _CheckFormat(input_api, "dartfmt", ".dart", windows,
                                     HasFormatErrors)

    if unformatted_files:
        lineSep = " \\\n"
        if windows:
            lineSep = " ^\n"
        return [
            output_api.PresubmitError(
                'File output does not match dartfmt.\n'
                'Fix these issues with:\n'
                '%s -w%s%s' % (prebuilt_dartfmt, lineSep,
                               lineSep.join(unformatted_files)))
        ]

    return []

def _CheckStatusFiles(input_api, output_api):
    local_root = input_api.change.RepositoryRoot()
    upstream = input_api.change._upstream
    utils = imp.load_source('utils',
                            os.path.join(local_root, 'tools', 'utils.py'))

    dart = os.path.join(utils.CheckedInSdkPath(), 'bin', 'dart')
    lint = os.path.join(local_root, 'pkg', 'status_file', 'bin', 'lint.dart')

    windows = utils.GuessOS() == 'win32'
    if windows:
        dart += '.exe'

    if not os.path.isfile(dart):
        print('WARNING: dart not found: %s' % dart)
        return []

    if not os.path.isfile(lint):
        print('WARNING: Status file linter not found: %s' % lint)
        return []

    def HasFormatErrors(filename=None, contents=None):
        args = [dart, lint] + (['-t'] if contents else [filename])
        process = subprocess.Popen(
            args, stdout=subprocess.PIPE, stdin=subprocess.PIPE)
        process.communicate(input=contents)
        return process.returncode != 0

    def should_skip(path):
        return (path.startswith("pkg/status_file/test/data/") or
                path.startswith("pkg/front_end/"))

    unformatted_files = _CheckFormat(input_api, "status file", ".status",
                                     windows, HasFormatErrors, should_skip)

    if unformatted_files:
        normalize = os.path.join(local_root, 'pkg', 'status_file', 'bin',
                                 'normalize.dart')
        lineSep = " \\\n"
        if windows:
            lineSep = " ^\n"
        return [
            output_api.PresubmitError(
                'Status files are not normalized.\n'
                'Fix these issues with:\n'
                '%s %s -w%s%s' % (dart, normalize, lineSep,
                                  lineSep.join(unformatted_files)))
        ]

    return []

def _CheckValidHostsInDEPS(input_api, output_api):
    """Checks that DEPS file deps are from allowed_hosts."""
    # Run only if DEPS file has been modified to annoy fewer bystanders.
    if all(f.LocalPath() != 'DEPS' for f in input_api.AffectedFiles()):
        return []
    # Outsource work to gclient verify
    try:
        input_api.subprocess.check_output(['gclient', 'verify'])
        return []
    except input_api.subprocess.CalledProcessError, error:
        return [
            output_api.PresubmitError(
                'DEPS file must have only dependencies from allowed hosts.',
                long_text=error.output)
        ]

def _CheckLayering(input_api, output_api):
    """Run VM layering check.

  This check validates that sources from one layer do not reference sources
  from another layer accidentally.
  """

    # Run only if .cc or .h file was modified.
    if all(not is_cpp_file(f.LocalPath()) for f in input_api.AffectedFiles()):
        return []

    local_root = input_api.change.RepositoryRoot()
    compiler_layering_check = imp.load_source(
        'compiler_layering_check',
        os.path.join(local_root, 'runtime', 'tools',
                     'compiler_layering_check.py'))
    errors = compiler_layering_check.DoCheck(local_root)
    embedder_layering_check = imp.load_source(
        'embedder_layering_check',
        os.path.join(local_root, 'runtime', 'tools',
                     'embedder_layering_check.py'))
    errors += embedder_layering_check.DoCheck(local_root)
    if errors:
        return [
            output_api.PresubmitError(
                'Layering check violation for C++ sources.',
                long_text='\n'.join(errors))
        ]

    return []

def _CheckClangTidy(input_api, output_api):
    """Run clang-tidy on VM changes."""

    # Only run clang-tidy on linux x64.
    if platform.system() != 'Linux' or platform.machine() != 'x86_64':
        return []

    # Run only for modified .cc or .h files.
    files = []
    for f in input_api.AffectedFiles():
        path = f.LocalPath()
        if is_cpp_file(path) and os.path.isfile(path): files.append(path)

    if not files:
        return []

    args = [
        'tools/sdks/dart-sdk/bin/dart',
        'runtime/tools/run_clang_tidy.dart',
    ]
    args.extend(files)
    stdout = input_api.subprocess.check_output(args).strip()
    if not stdout:
        return []

    return [
        output_api.PresubmitError(
            'The `clang-tidy` linter revealed issues:',
            long_text=stdout)
    ]

def _CheckTestMatrixValid(input_api, output_api):
    """Run script to check that the test matrix has no errors."""

    def test_matrix_filter(affected_file):
        """Only run test if either the test matrix or the code that
           validates it was modified."""
        path = affected_file.LocalPath()
        return (path == 'tools/bots/test_matrix.json' or
                path == 'tools/validate_test_matrix.dart' or
                path.startswith('pkg/smith/'))

    if len(
            input_api.AffectedFiles(
                include_deletes=False, file_filter=test_matrix_filter)) == 0:
        return []

    command = [
        'tools/sdks/dart-sdk/bin/dart',
        'tools/validate_test_matrix.dart',
    ]
    stdout = input_api.subprocess.check_output(command).strip()
    if not stdout:
        return []
    else:
        return [
            output_api.PresubmitError(
                'The test matrix is not valid:', long_text=stdout)
        ]

def _CommonChecks(input_api, output_api):
    results = []
    results.extend(_CheckNnbdTestSync(input_api, output_api))
    results.extend(_CheckValidHostsInDEPS(input_api, output_api))
    results.extend(_CheckDartFormat(input_api, output_api))
    results.extend(_CheckStatusFiles(input_api, output_api))
    results.extend(_CheckLayering(input_api, output_api))
    results.extend(_CheckClangTidy(input_api, output_api))
    results.extend(_CheckTestMatrixValid(input_api, output_api))
    results.extend(
        input_api.canned_checks.CheckPatchFormatted(input_api, output_api))
    return results

def CheckChangeOnCommit(input_api, output_api):
    return _CommonChecks(input_api, output_api)

def CheckChangeOnUpload(input_api, output_api):
    return _CommonChecks(input_api, output_api)
<|endoftext|>


<commit_before>__author__ = 'xu'

class PVariable:
    val = 0
    type = 0
    name = ''

<commit_msg>remove PVariable.py<commit_after><|endoftext|>