<commit_before>#! /usr/bin/env python
"""Unit tests for smartcard.ExclusiveTransmitCardConnection.

This test case can be executed individually, or with all other test cases
thru testsuite_framework.py.

__author__ = "http://www.gemalto.com"

Copyright 2001-2008 gemalto
Author: Jean-Daniel Aussel, mailto:jean-daniel.aussel@gemalto.com

This file is part of pyscard.

pyscard is free software; you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation; either version 2.1 of the License, or
(at your option) any later version.

pyscard is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
GNU Lesser General Public License for more details.

You should have received a copy of the GNU Lesser General Public License
along with pyscard; if not, write to the Free Software
Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
"""

import sys
import random
import threading
import time
import unittest

# define the apdus used in this script
GET_RESPONSE = [0XA0, 0XC0, 00, 00 ]
SELECT = [0xA0, 0xA4, 0x00, 0x00, 0x02]
DF_TELECOM = [0x7F, 0x10]

from smartcard.CardConnection import CardConnection
from smartcard.CardConnectionObserver import ConsoleCardConnectionObserver
from smartcard.CardMonitoring import CardMonitor, CardObserver
from smartcard.CardRequest import CardRequest
from smartcard.CardType import AnyCardType
from smartcard.ExclusiveTransmitCardConnection import ExclusiveTransmitCardConnection
from smartcard.util import toHexString

def signalEvent( evt ):
    '''A simple callback that signals an event.'''
    evt.set()

class testthread( threading.Thread ):
    '''A test thread that repetitevely sends APDUs to a card within a transaction.'''
    def __init__(self, threadindex ):
        '''Connect to a card with an ExclusiveTransmitCardConnection.'''
        threading.Thread.__init__(self)

        self.threadindex=threadindex

        # request any card type
        cardtype = AnyCardType()
        cardrequest = CardRequest( timeout=5, cardType=cardtype )
        cardservice = cardrequest.waitforcard()

        # attach our decorator
        cardservice.connection = ExclusiveTransmitCardConnection( cardservice.connection )

        # uncomment to attach the console tracer
        #observer=ConsoleCardConnectionObserver()
        #cardservice.connection.addObserver( observer )

        # connect to the card
        cardservice.connection.connect()

        self.cardservice = cardservice

        # this event will signal the end of the thread
        self.evtStop = threading.Event()

        # this timer will set the event stop event in 30s
        timer = threading.Timer( 30, signalEvent, [self.evtStop] )
        timer.start()
        self.countTransmitted=0

    def run(self):
        '''Transmit APDUS with a random interval to the card.'''
        connection = self.cardservice.connection
        while not self.evtStop.isSet():
            try:
                connection.lock()

                apdu = SELECT+DF_TELECOM
                response, sw1, sw2 = connection.transmit( apdu )

                if  0x90==(sw1 & 0xF0):
                    apdu = GET_RESPONSE + [sw2]
                    response, sw1, sw2 = connection.transmit( apdu )
            finally:
                connection.unlock()
            self.countTransmitted=self.countTransmitted+1
            time.sleep( float(random.uniform( 1, 3 ))*0.01)

class testcase_cardmonitor(unittest.TestCase):
    """Test smartcard framework card monitoring classes"""

    def testcase_cardmonitorthread(self):
        threads = []
        for i in range( 0, 4 ):
            t = testthread( i )
            threads.append( t )
        for t in threads:
            t.start()
        for t in threads:
            t.join()
        for t in threads:
            print 'Thread %d: transmitted %ld apdus.' % (t.threadindex, t.countTransmitted )

def suite():
    suite1 = unittest.makeSuite(testcase_cardmonitorthread)
    return unittest.TestSuite((suite1))

if __name__ == '__main__':
    unittest.main()

<commit_msg>commented test output messasge<commit_after>#! /usr/bin/env python
"""Unit tests for smartcard.ExclusiveTransmitCardConnection.

This test case can be executed individually, or with all other test cases
thru testsuite_framework.py.

__author__ = "http://www.gemalto.com"

Copyright 2001-2008 gemalto
Author: Jean-Daniel Aussel, mailto:jean-daniel.aussel@gemalto.com

This file is part of pyscard.

pyscard is free software; you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation; either version 2.1 of the License, or
(at your option) any later version.

pyscard is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
GNU Lesser General Public License for more details.

You should have received a copy of the GNU Lesser General Public License
along with pyscard; if not, write to the Free Software
Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
"""

import sys
import random
import threading
import time
import unittest

# define the apdus used in this script
GET_RESPONSE = [0XA0, 0XC0, 00, 00 ]
SELECT = [0xA0, 0xA4, 0x00, 0x00, 0x02]
DF_TELECOM = [0x7F, 0x10]

from smartcard.CardConnection import CardConnection
from smartcard.CardConnectionObserver import ConsoleCardConnectionObserver
from smartcard.CardMonitoring import CardMonitor, CardObserver
from smartcard.CardRequest import CardRequest
from smartcard.CardType import AnyCardType
from smartcard.ExclusiveTransmitCardConnection import ExclusiveTransmitCardConnection
from smartcard.util import toHexString

def signalEvent( evt ):
    '''A simple callback that signals an event.'''
    evt.set()

class testthread( threading.Thread ):
    '''A test thread that repetitevely sends APDUs to a card within a transaction.'''
    def __init__(self, threadindex ):
        '''Connect to a card with an ExclusiveTransmitCardConnection.'''
        threading.Thread.__init__(self)

        self.threadindex=threadindex

        # request any card type
        cardtype = AnyCardType()
        cardrequest = CardRequest( timeout=5, cardType=cardtype )
        cardservice = cardrequest.waitforcard()

        # attach our decorator
        cardservice.connection = ExclusiveTransmitCardConnection( cardservice.connection )

        # uncomment to attach the console tracer
        #observer=ConsoleCardConnectionObserver()
        #cardservice.connection.addObserver( observer )

        # connect to the card
        cardservice.connection.connect()

        self.cardservice = cardservice

        # this event will signal the end of the thread
        self.evtStop = threading.Event()

        # this timer will set the event stop event in 30s
        timer = threading.Timer( 30, signalEvent, [self.evtStop] )
        timer.start()
        self.countTransmitted=0

    def run(self):
        '''Transmit APDUS with a random interval to the card.'''
        connection = self.cardservice.connection
        while not self.evtStop.isSet():
            try:
                connection.lock()

                apdu = SELECT+DF_TELECOM
                response, sw1, sw2 = connection.transmit( apdu )

                if  0x90==(sw1 & 0xF0):
                    apdu = GET_RESPONSE + [sw2]
                    response, sw1, sw2 = connection.transmit( apdu )
            finally:
                connection.unlock()
            self.countTransmitted=self.countTransmitted+1
            time.sleep( float(random.uniform( 1, 3 ))*0.01)

class testcase_cardmonitor(unittest.TestCase):
    """Test smartcard framework card monitoring classes"""

    def testcase_cardmonitorthread(self):
        threads = []
        for i in range( 0, 4 ):
            t = testthread( i )
            threads.append( t )
        for t in threads:
            t.start()
        for t in threads:
            t.join()
        for t in threads:
            if 0==1:
                print 'Thread %d: transmitted %ld apdus.' % (t.threadindex, t.countTransmitted )

def suite():
    suite1 = unittest.makeSuite(testcase_cardmonitorthread)
    return unittest.TestSuite((suite1))

if __name__ == '__main__':
    unittest.main()

<|endoftext|>


<commit_before>'''
/**
 * *  Copyright (C) 2011 Citrix Systems, Inc.  All rights reserved
*
 *
 * This software is licensed under the GNU General Public License v3 or later.
 *
 * It is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or any later version.
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 *
 */
'''
from optparse import OptionParser
import sys
import os, os.path
import fnmatch

class CopyRightDetecter(object):
    def isCopyRightLine(self, txt):
        return False

class KeyWordCopyRightDetecter(CopyRightDetecter):
    keywords = ['Cloud.com', 'Copyright', '(C)', '2011', 'Citrix', 'Systems', 'Inc', 'All', 'rights', 'reserved', 'This', 'software', 'is', 'licensed', 'under', 'the', 'GNU', 'General', 'Public', 'License', 'v3', 'or', 'later', 'It', 'is', 'free', 'software:', 'you', 'can', 'redistribute', 'it', 'and/or', 'modify', 'it', 'under', 'the', 'terms', 'of', 'the', 'GNU', 'General', 'Public', 'License', 'as', 'published', 'by', 'the', 'Free', 'Software', 'Foundation', 'either', 'version', '3', 'of', 'the', 'License', 'or', 'any', 'later', 'version', 'This', 'program', 'is', 'distributed', 'in', 'the', 'hope', 'that', 'it', 'will', 'be', 'useful', 'but', 'WITHOUT', 'ANY', 'WARRANTY;', 'without', 'even', 'the', 'implied', 'warranty', 'of', 'MERCHANTABILITY', 'or', 'FITNESS', 'FOR', 'A', 'PARTICULAR', 'PURPOSE', 'See', 'the', 'GNU', 'General', 'Public', 'License', 'for', 'more', 'details', 'You', 'should', 'have', 'received', 'a', 'copy', 'of', 'the', 'GNU', 'General', 'Public', 'License', 'along', 'with', 'this', 'program', 'If', 'not', 'see', '<http://www.gnu.org/licenses/>']

    def isCopyRightLine(self, txt):
        words = [ c.strip().strip('.').strip('\n').strip(',') for c in txt.split(" ") ]
        total = len(words)
        if total == 0: return False

        numKeyWord = 0
        for w in words:
            if w in self.keywords: numKeyWord+=1

        if float(numKeyWord)/float(total) > float(1)/float(3): return True
        return False

copyRightDetectingFactory = {"KeyWord":KeyWordCopyRightDetecter.__name__}

class Logger(object):
    @staticmethod
    def info(msg):
        sys.stdout.write("INFO: %s"%msg)
        sys.stdout.write("\n")
        sys.stdout.flush()

    @staticmethod
    def debug(msg):
        sys.stdout.write("DEBUG: %s"%msg)
        sys.stdout.write("\n")
        sys.stdout.flush()

    @staticmethod
    def warn(msg):
        sys.stdout.write("WARNNING: %s"%msg)
        sys.stdout.write("\n")
        sys.stdout.flush()

    @staticmethod
    def error(msg):
        sys.stderr.write("ERROR: %s"%msg)
        sys.stderr.write("\n")
        sys.stderr.flush()

class Adder(object):
    defaultTxt = '''Copyright (C) 2011 Citrix Systems, Inc.  All rights reserved

This software is licensed under the GNU General Public License v3 or later.

It is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
GNU General Public License for more details.

You should have received a copy of the GNU General Public License
along with this program.  If not, see <http://www.gnu.org/licenses/>.
'''
    copyRightTxt = None
    targetFile = None
    fileBody = None
    decter = None

    COMMENT_NOTATION = ""

    def __init__(self):
        self.decter = eval(copyRightDetectingFactory["KeyWord"])()

    def setTargetFile(self, fpath):
        self.targetFile = fpath
        if not os.path.exists(self.targetFile):
            raise Exception("Cannot find %s"%self.targetFile)

    def setCopyRightTxt(self, txt):
        self.copyRightTxt = txt.split("\n")

    def checkParams(self):
        assert self.targetFile != None, "Target file not set"
        if self.copyRightTxt == None:
            self.copyRightTxt = self.defaultTxt.split("\n")

    def pasteCopyRight(self):
        self.fileBody = self.copyRightTxt + self.fileBody
        file(self.targetFile, 'w').write("".join(self.fileBody))
        Logger.info("Added copyright header to %s"%self.targetFile)

    def composeCopyRight(self):
        l = self.copyRightTxt
        l = [ self.COMMENT_NOTATION + " " + line + "\n" for line in l ]
        self.copyRightTxt = l

    def getFileBody(self):
        self.fileBody = file(self.targetFile).readlines()

    def isCommentLine(self, line):
        if line.lstrip().startswith(self.COMMENT_NOTATION):
            return True
        return False

    def removeOldCopyRight(self):
        newBody = []
        removed = False
        for line in self.fileBody:
            if self.isCommentLine(line) and self.decter.isCopyRightLine(line): removed = True; continue
            newBody.append(line)

        self.fileBody = newBody
        if removed:
            Logger.info("Removed old copyright header of %s"%self.targetFile)

    def cleanBlankComment(self):
        newBody = []
        for l in self.fileBody:
            if self.isCommentLine(l) and l.strip(self.COMMENT_NOTATION).strip().strip('\n') == "":
                continue
            newBody.append(l)
        self.fileBody = newBody

    def doWork(self):
        self.checkParams()
        self.getFileBody()
        self.removeOldCopyRight()
        self.composeCopyRight()
        self.cleanBlankComment()
        self.pasteCopyRight()

class SqlAdder(Adder):
    def __init__(self):
        super(SqlAdder, self).__init__()
        self.COMMENT_NOTATION = "#"

copyRightAdderFactory = {".sql":SqlAdder.__name__}
class CopyRightAdder(object):
    parser = None
    options = None
    args = None
    targetFiles = None
    excludeFiles = None
    copyRightFileTxt = None
    rootDir = None

    def errAndExit(self, msg):
        Logger.error(msg)
        Logger.info("addcopyright -h for help")
        sys.exit(1)

    def __init__(self):
        usage = '''Usage: addcopyright [file_name_pattern] [--exculdes=file_name_pattern] [--file=copyright_file] [--root=root_dir_of_files_to_add_header]
Examples:
    addcopyright
    addcopyright *.sql
    addcopyright *.sql --excludes="*.sql~"
    addcopyright *.sql --file=/root/Citrix.copyright
    addcopyright *.sql --file=/root/Citrix.copyright --root=~/cloudstack-oss
        '''
        self.parser = OptionParser(usage=usage)
        self.parser.add_option("", "--excludes", action="store", type="string", dest="excludes", default="",
                               help="Exclude these files when adding copyright header")
        self.parser.add_option("", "--file", action="store", type="string", dest="copyRightFile", default="",
                               help="Path to copyright header file. Default to Citrix copyright header")
        self.parser.add_option("", "--root", action="store", type="string", dest="rootDir", default="",
                               help="Root folder where files being added copyright header locate. Default to current directory")
        (self.options, self.args) = self.parser.parse_args()
        if len(self.args) > 1:
            self.errAndExit("Invalid arguments:%s" % self.args)

        if len(self.args) == 1:
            self.targetFiles = self.args[0]
        if self.options.excludes != "":
            self.excludeFiles = self.options.excludes
        if self.options.copyRightFile != "":
            self.copyRightFileTxt = file(self.options.copyRightFile).read()
        if self.options.rootDir != "":
            self.rootDir = os.path.expanduser(self.options.rootDir)
            if not os.path.isdir(self.rootDir): raise Exception("Cannot find directory %s"%self.rootDir)
        else:
            self.rootDir = os.getcwd()

    def createConcreteAdder(self, filename):
        (x, ext) = os.path.splitext(filename)
        if not copyRightAdderFactory.has_key(ext): return None
        return eval(copyRightAdderFactory[ext])()

    def run(self):
        for root, dirs, files in os.walk(self.rootDir):
            for f in files:
                fpath = os.path.join(root, f)
                if self.excludeFiles != None and fnmatch.fnmatch(f, self.excludeFiles):
                    Logger.info("Skipping excluded file %s" % fpath)
                    continue
                if self.targetFiles != None and not fnmatch.fnmatch(f, self.targetFiles):
                    Logger.info("Skipping %s not matching our file name pattern" % fpath)
                    continue

                adder = self.createConcreteAdder(f)
                if adder == None:
                    Logger.warn("Cannot find a proper copyright Adder for %s, skip it" % fpath)
                    continue

                adder.setTargetFile(fpath)
                if self.copyRightFileTxt != None:
                    adder.setCopyRightTxt(self.copyRightFileTxt)
                adder.doWork()

if __name__ == '__main__':
    task = CopyRightAdder()
    task.run()<commit_msg>add shell/python support in copyright header tool<commit_after>'''
/**
 * *  Copyright (C) 2011 Citrix Systems, Inc.  All rights reserved
*
 *
 * This software is licensed under the GNU General Public License v3 or later.
 *
 * It is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or any later version.
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 *
 */
'''
from optparse import OptionParser
import sys
import os, os.path
import fnmatch

class CopyRightDetecter(object):
    def isCopyRightLine(self, txt):
        return False

# if more than 1/2 words in one line are keyword, we treat it as copyright line
class KeyWordCopyRightDetecter(CopyRightDetecter):
    keywords = ['Cloud.com', 'Copyright', '(C)', '2011', 'Citrix', 'Systems', 'Inc', 'All', 'rights', 'reserved', 'This', 'software', 'is', 'licensed', 'under', 'the', 'GNU', 'General', 'Public', 'License', 'v3', 'or', 'later', 'It', 'is', 'free', 'software:', 'you', 'can', 'redistribute', 'it', 'and/or', 'modify', 'it', 'under', 'the', 'terms', 'of', 'the', 'GNU', 'General', 'Public', 'License', 'as', 'published', 'by', 'the', 'Free', 'Software', 'Foundation', 'either', 'version', '3', 'of', 'the', 'License', 'or', 'any', 'later', 'version', 'This', 'program', 'is', 'distributed', 'in', 'the', 'hope', 'that', 'it', 'will', 'be', 'useful', 'but', 'WITHOUT', 'ANY', 'WARRANTY;', 'without', 'even', 'the', 'implied', 'warranty', 'of', 'MERCHANTABILITY', 'or', 'FITNESS', 'FOR', 'A', 'PARTICULAR', 'PURPOSE', 'See', 'the', 'GNU', 'General', 'Public', 'License', 'for', 'more', 'details', 'You', 'should', 'have', 'received', 'a', 'copy', 'of', 'the', 'GNU', 'General', 'Public', 'License', 'along', 'with', 'this', 'program', 'If', 'not', 'see', '<http://www.gnu.org/licenses/>']

    def isCopyRightLine(self, txt):
        words = [ c.strip().strip('.').strip('\n').strip(',').strip() for c in txt.split(" ") ]
        total = len(words)
        if total == 0: return False

        numKeyWord = 0
        for w in words:
            if w == "": continue
            if w in self.keywords: numKeyWord+=1

        if float(numKeyWord)/float(total) >= float(1)/float(2): return True
        return False

copyRightDetectingFactory = {"KeyWord":KeyWordCopyRightDetecter.__name__}

logfd = open("/tmp/addcopyright.log", 'w')
class Logger(object):
    @staticmethod
    def info(msg):
        sys.stdout.write("INFO: %s"%msg)
        sys.stdout.write("\n")
        sys.stdout.flush()

    @staticmethod
    def debug(msg):
        logfd.write("DEBUG: %s"%msg)
        logfd.write("\n")

    @staticmethod
    def warn(msg):
        sys.stdout.write("WARNNING: %s"%msg)
        sys.stdout.write("\n")
        sys.stdout.flush()

    @staticmethod
    def error(msg):
        sys.stderr.write("ERROR: %s"%msg)
        sys.stderr.write("\n")
        sys.stderr.flush()

class Adder(object):
    defaultTxt = '''Copyright (C) 2011 Citrix Systems, Inc.  All rights reserved

This software is licensed under the GNU General Public License v3 or later.

It is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
GNU General Public License for more details.

You should have received a copy of the GNU General Public License
along with this program.  If not, see <http://www.gnu.org/licenses/>.
'''
    copyRightTxt = None
    targetFile = None
    fileBody = None
    decter = None

    COMMENT_NOTATION = ""

    def __init__(self):
        self.decter = eval(copyRightDetectingFactory["KeyWord"])()

    def setTargetFile(self, fpath):
        self.targetFile = fpath
        if not os.path.exists(self.targetFile):
            raise Exception("Cannot find %s"%self.targetFile)

    def setCopyRightTxt(self, txt):
        self.copyRightTxt = txt.split("\n")

    def checkParams(self):
        assert self.targetFile != None, "Target file not set"
        if self.copyRightTxt == None:
            self.copyRightTxt = self.defaultTxt.split("\n")

    def pasteCopyRight(self):
        self.fileBody = self.copyRightTxt + self.fileBody
        file(self.targetFile, 'w').write("".join(self.fileBody))
        Logger.info("Added copyright header to %s"%self.targetFile)

    def composeCopyRight(self):
        l = self.copyRightTxt
        l = [ self.COMMENT_NOTATION + " " + line + "\n" for line in l ]
        self.copyRightTxt = l

    def getFileBody(self):
        self.fileBody = file(self.targetFile).readlines()

    def isCommentLine(self, line):
        if line.strip().startswith(self.COMMENT_NOTATION):
            return True
        return False

    def removeOldCopyRight(self):
        newBody = []
        removed = False
        for line in self.fileBody:
            if self.isCommentLine(line) and self.decter.isCopyRightLine(line):
                removed = True
                Logger.debug("remove old copyright: %s" % line)
                continue
            newBody.append(line)

        self.fileBody = newBody
        if removed:
            Logger.info("Removed old copyright header of %s"%self.targetFile)

    def cleanBlankComment(self):
        newBody = []
        for l in self.fileBody:
            if self.isCommentLine(l) and l.strip().strip(self.COMMENT_NOTATION).strip().strip('\n') == "":
                Logger.debug("Blank Comment: %s" % l)
                continue
            newBody.append(l)
        self.fileBody = newBody

    def doWork(self):
        self.checkParams()
        self.getFileBody()
        self.removeOldCopyRight()
        self.composeCopyRight()
        self.cleanBlankComment()
        self.pasteCopyRight()

class SqlAdder(Adder):
    def __init__(self):
        super(SqlAdder, self).__init__()
        self.COMMENT_NOTATION = "#"

class InterpreterAdder(Adder):
    def __init__(self):
        super(InterpreterAdder, self).__init__()
        self.COMMENT_NOTATION = "#"

    def pasteCopyRight(self):
        if len(self.fileBody) > 0 and self.isCommentLine(self.fileBody[0]):
            # Don't cover the first line of interpreter comment
            self.fileBody = [self.fileBody[0]] + self.copyRightTxt + self.fileBody[1:]
        else:
            self.fileBody = self.copyRightTxt + self.fileBody
        file(self.targetFile, 'w').write("".join(self.fileBody))
        Logger.info("Added copyright header to %s"%self.targetFile)

copyRightAdderFactory = {".sql":SqlAdder.__name__, ".sh":InterpreterAdder.__name__, ".py":InterpreterAdder.__name__}
class CopyRightAdder(object):
    parser = None
    options = None
    args = None
    targetFiles = None
    excludeFiles = None
    copyRightFileTxt = None
    rootDir = None

    def errAndExit(self, msg):
        Logger.error(msg)
        Logger.info("addcopyright -h for help")
        sys.exit(1)

    def __init__(self):
        usage = '''Usage: addcopyright [file_name_pattern] [--exculdes=file_name_pattern] [--file=copyright_file] [--root=root_dir_of_files_to_add_header]
Examples:
    addcopyright
    addcopyright *.sql
    addcopyright *.sql --excludes="*.sql~"
    addcopyright *.sql --file=/root/Citrix.copyright
    addcopyright *.sql --file=/root/Citrix.copyright --root=~/cloudstack-oss
        '''
        self.parser = OptionParser(usage=usage)
        self.parser.add_option("", "--excludes", action="store", type="string", dest="excludes", default="",
                               help="Exclude these files when adding copyright header")
        self.parser.add_option("", "--file", action="store", type="string", dest="copyRightFile", default="",
                               help="Path to copyright header file. Default to Citrix copyright header")
        self.parser.add_option("", "--root", action="store", type="string", dest="rootDir", default="",
                               help="Root folder where files being added copyright header locate. Default to current directory")
        (self.options, self.args) = self.parser.parse_args()
        if len(self.args) > 1:
            self.errAndExit("Invalid arguments:%s" % self.args)

        if len(self.args) == 1:
            self.targetFiles = self.args[0]
        if self.options.excludes != "":
            self.excludeFiles = self.options.excludes
        if self.options.copyRightFile != "":
            self.copyRightFileTxt = file(self.options.copyRightFile).read()
        if self.options.rootDir != "":
            self.rootDir = os.path.expanduser(self.options.rootDir)
            if not os.path.isdir(self.rootDir): raise Exception("Cannot find directory %s"%self.rootDir)
        else:
            self.rootDir = os.getcwd()

    def createConcreteAdder(self, filename):
        (x, ext) = os.path.splitext(filename)
        if not copyRightAdderFactory.has_key(ext): return None
        return eval(copyRightAdderFactory[ext])()

    def run(self):
        for root, dirs, files in os.walk(self.rootDir):
            for f in files:
                fpath = os.path.join(root, f)
                if self.excludeFiles != None and fnmatch.fnmatch(f, self.excludeFiles):
                    Logger.info("Skipping excluded file %s" % fpath)
                    continue
                if self.targetFiles != None and not fnmatch.fnmatch(f, self.targetFiles):
                    Logger.info("Skipping %s not matching our file name pattern" % fpath)
                    continue

                adder = self.createConcreteAdder(f)
                if adder == None:
                    Logger.warn("Cannot find a proper copyright Adder for %s, skip it" % fpath)
                    continue

                adder.setTargetFile(fpath)
                if self.copyRightFileTxt != None:
                    adder.setCopyRightTxt(self.copyRightFileTxt)
                adder.doWork()

if __name__ == '__main__':
    task = CopyRightAdder()
    task.run()<|endoftext|>


<commit_before>#!/usr/bin/python2.5
#
# Copyright 2008 the Melange authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Helpers functions for updating different kinds of models in datastore.
"""

__authors__ = [
  '"Todd Larsen" <tlarsen@google.com>',
  '"Sverre Rabbelier" <sverre@rabbelier.nl>',
  '"Lennard de Rijk" <ljvderijk@gmail.com>',
  '"Pawel Solyga" <pawel.solyga@gmail.com>',
  ]

from google.appengine.ext import db

from django.utils.translation import ugettext

from soc.cache import sidebar
from soc.logic import dicts
from soc.views import out_of_band

class Error(Exception):
  """Base class for all exceptions raised by this module.
  """

  pass

class InvalidArgumentError(Error):
  """Raised when an invalid argument is passed to a method.

  For example, if an argument is None, but must always be non-False.
  """

  pass

class NoEntityError(InvalidArgumentError):
  """Raised when no entity is passed to a method that requires one.
  """

  pass

class Logic(object):
  """Base logic for entity classes.

  The BaseLogic class functions specific to Entity classes by relying
  on arguments passed to __init__.
  """

  def __init__(self, model, base_model=None, scope_logic=None,
               name=None, skip_properties=None, id_based=False):
    """Defines the name, key_name and model for this entity.
    """

    self._model = model
    self._base_model = base_model
    self._scope_logic = scope_logic
    self._id_based = id_based

    if name:
      self._name = name
    else:
      self._name =  self._model.__name__

    if skip_properties:
      self._skip_properties = skip_properties
    else:
      self._skip_properties = []

  def getModel(self):
    """Returns the model this logic class uses.
    """

    return self._model

  def getScopeLogic(self):
    """Returns the logic of the enclosing scope.
    """

    return self._scope_logic

  def getScopeDepth(self):
    """Returns the scope depth for this entity.

    Returns None if any of the parent scopes return None.
    """

    if not self._scope_logic:
      return 0

    depth = self._scope_logic.logic.getScopeDepth()
    return None if (depth is None) else (depth + 1)

  def getKeyNameFromFields(self, fields):
    """Returns the KeyName constructed from fields dict for this type of entity.

    The KeyName is in the following format:
    <key_value1>/<key_value2>/.../<key_valueN>
    """

    if not fields:
      raise InvalidArgumentError

    key_field_names = self.getKeyFieldNames()

    # check if all key_field_names for this entity are present in fields
    if not all(field in fields.keys() for field in key_field_names):
      raise InvalidArgumentError("Not all the required key fields are present")

    if not all(fields.get(field) for field in key_field_names):
      raise InvalidArgumentError("Not all KeyValues are non-false")

    # construct the KeyValues in the order given by getKeyFieldNames()
    keyvalues = []
    for key_field_name in key_field_names:
      keyvalues.append(fields[key_field_name])

    # construct the KeyName in the appropriate format
    return '/'.join(keyvalues)

  def getFullModelClassName(self):
    """Returns fully-qualified model module.class name string.
    """

    return '%s.%s' % (self._model.__module__, self._model.__name__)

  def getKeyValuesFromEntity(self, entity):
    """Extracts the key values from entity and returns them.

    The default implementation uses the scope and link_id as key values.

    Args:
      entity: the entity from which to extract the key values
    """

    if not entity:
      raise NoEntityError

    return [entity.scope_path, entity.link_id]

  def getKeyValuesFromFields(self, fields):
    """Extracts the key values from a dict and returns them.

    The default implementation uses the scope and link_id as key values.

    Args:
      fields: the dict from which to extract the key values
    """

    if ('scope_path' not in fields) or ('link_id' not in fields):
      raise InvalidArgumentError

    return [fields['scope_path'], fields['link_id']]

  def getKeyFieldNames(self):
    """Returns an array with the names of the Key Fields.

    The default implementation uses the scope and link_id as key values.
    """

    return ['scope_path', 'link_id']

  def getKeyFieldsFromFields(self, dictionary):
    """Does any required massaging and filtering of dictionary.

    The resulting dictionary contains just the key names, and has any
    required translations/modifications performed.

    Args:
      dictionary: The arguments to massage
    """

    if not dictionary:
      raise InvalidArgumentError

    keys = self.getKeyFieldNames()
    values = self.getKeyValuesFromFields(dictionary)
    key_fields = dicts.zip(keys, values)

    return key_fields

  def getFromKeyName(self, key_name):
    """"Returns entity for key_name or None if not found.

    Args:
      key_name: key name of entity
    """

    if self._id_based:
      raise Error("getFromKeyName called on an id based logic")

    if not key_name:
      raise InvalidArgumentError

    return self._model.get_by_key_name(key_name)

  def getFromID(self, id):
    """Returns entity for id or None if not found.

    Args:
      id: id of entity
    """

    if not self._id_based:
      raise Error("getFromID called on a not id based logic")

    if not id:
      raise InvalidArgumentError

    return self._model.get_by_id(id)

  def getFromKeyNameOr404(self, key_name):
    """Like getFromKeyName but expects to find an entity.

    Raises:
      out_of_band.Error if no entity is found
    """

    entity = self.getFromKeyName(key_name)

    if entity:
      return entity

    msg = ugettext('There is no "%(name)s" named %(key_name)s.') % {
        'name': self._name, 'key_name': key_name}

    raise out_of_band.Error(msg, status=404)

  def getFromIDOr404(self, id):
    """Like getFromID but expects to find an entity.

    Raises:
      out_of_band.Error if no entity is found
    """

    entity = self.getFromID(id)

    if entity:
      return entity

    msg = ugettext('There is no "%(name)s" with id %(id)s.') % {
        'name': self._name, 'id': id}

    raise out_of_band.Error(msg, status=404)

  def getFromKeyFields(self, fields):
    """Returns the entity for the specified key names, or None if not found.

    Args:
      fields: a dict containing the fields of the entity that
        uniquely identifies it
    """

    if not fields:
      raise InvalidArgumentError

    key_fields = self.getKeyFieldsFromFields(fields)

    if all(key_fields.values()):
      key_name = self.getKeyNameFromFields(key_fields)
      entity = self.getFromKeyName(key_name)
    else:
      entity = None

    return entity

  def getFromKeyFieldsOr404(self, fields):
    """Like getFromKeyFields but expects to find an entity.

    Raises:
      out_of_band.Error if no entity is found
    """

    entity = self.getFromKeyFields(fields)

    if entity:
      return entity

    key_fields = self.getKeyFieldsFromFields(fields)
    format_text = ugettext('"%(key)s" is "%(value)s"')

    msg_pairs = [format_text % {'key': key, 'value': value}
      for key, value in key_fields.iteritems()]

    joined_pairs = ' and '.join(msg_pairs)

    msg = ugettext(
      'There is no "%(name)s" where %(pairs)s.') % {
        'name': self._name, 'pairs': joined_pairs}

    raise out_of_band.Error(msg, status=404)

  def getForFields(self, filter=None, unique=False,
                   limit=1000, offset=0, order=None):
    """Returns all entities that have the specified properties.

    Args:
      filter: a dict for the properties that the entities should have
      unique: if set, only the first item from the resultset will be returned
      limit: the amount of entities to fetch at most
      offset: the position to start at
      order: a list with the sort order
    """

    if unique:
      limit = 1

    query = self.getQueryForFields(filter=filter, order=order)

    result = query.fetch(limit, offset)

    if unique:
      return result[0] if result else None

    return result

  def getQueryForFields(self, filter=None, order=None):
    """Returns a query with the specified properties.

    Args:
      filter: a dict for the properties that the entities should have
      order: a list with the sort order

    Returns:
      - Query object instantiated with the given properties
    """

    if not filter:
      filter = {}

    if not order:
      order = []

    orderset = set([i.strip('-') for i in order])
    if len(orderset) != len(order):
      raise InvalidArgumentError

    query = db.Query(self._model)

    for key, value in filter.iteritems():
      if isinstance(value, list) and len(value) == 1:
        value = value[0]
      if isinstance(value, list):
        op = '%s IN' % key
        query.filter(op, value)
      else:
        query.filter(key, value)

    for key in order:
      query.order(key)

    return query

  def updateEntityProperties(self, entity, entity_properties, silent=False):
    """Update existing entity using supplied properties.

    Args:
      entity: a model entity
      entity_properties: keyword arguments that correspond to entity
        properties and their values
      silent: iff True does not call _onUpdate method

    Returns:
      The original entity with any supplied properties changed.
    """

    if not entity:
      raise NoEntityError

    if not entity_properties:
      raise InvalidArgumentError

    properties = self._model.properties()

    for name, prop in properties.iteritems():
      # if the property is not updateable or is not updated, skip it
      if name in self._skip_properties or (name not in entity_properties):
        continue

      if self._updateField(entity, entity_properties, name):
        value = entity_properties[name]
        prop.__set__(entity, value)

    entity.put()

    # call the _onUpdate method
    if not silent:
      self._onUpdate(entity)

    return entity

  def updateOrCreateFromKeyName(self, properties, key_name):
    """Update existing entity, or create new one with supplied properties.

    Args:
      properties: dict with entity properties and their values
      key_name: the key_name of the entity that uniquely identifies it

    Returns:
      the entity corresponding to the key_name, with any supplied
      properties changed, or a new entity now associated with the
      supplied key_name and properties.
    """

    entity = self.getFromKeyName(key_name)

    create_entity = not entity

    if create_entity:
      for property_name in properties:
        self._createField(properties, property_name)

      # entity did not exist, so create one in a transaction
      entity = self._model.get_or_insert(key_name, **properties)
    else:
      # If someone else already created the entity (due to a race), we
      # should not update the propties (as they 'won' the race).
      entity = self.updateEntityProperties(entity, properties, silent=True)

    if create_entity:
      # a new entity has been created call _onCreate
      self._onCreate(entity)
    else:
      # the entity has been updated call _onUpdate
      self._onUpdate(entity)

    return entity

  def updateOrCreateFromFields(self, properties):
    """Creates a new entity with the supplied properties.

    Args:
      properties: dict with entity properties and their values
    """

    for property_name in properties:
      self._createField(properties, property_name)

    if self._id_based:
      entity = self._model(**properties)
      entity.put()
    else:
      key_name = self.getKeyNameFromFields(properties)
      entity = self._model.get_or_insert(key_name, **properties)

    self._onCreate(entity)
    return entity

  def isDeletable(self, entity):
    """Returns whether the specified entity can be deleted.

    Args:
      entity: an existing entity in datastore
    """

    return True

  def delete(self, entity):
    """Delete existing entity from datastore.

    Args:
      entity: an existing entity in datastore
    """

    entity.delete()
    # entity has been deleted call _onDelete
    self._onDelete(entity)

  def getAll(self, query):
    """Retrieves all entities for the specified query.
    """

    chunk = 999
    offset = 0
    result = []
    more = True

    while(more):
      data = query.fetch(chunk+1, offset)

      more = len(data) > chunk

      if more:
        del data[chunk]

      result.extend(data)
      offset = offset + chunk

    return result

  def _createField(self, entity_properties, name):
    """Hook called when a field is created.

    To be exact, this method is called for each field (that has a value
    specified) on an entity that is being created.

    Base classes should override if any special actions need to be
    taken when a field is created.

    Args:
      entity_properties: keyword arguments that correspond to entity
        properties and their values
      name: the name of the field to be created
    """

    if not entity_properties or (name not in entity_properties):
      raise InvalidArgumentError

  def _updateField(self, entity, entity_properties, name):
    """Hook called when a field is updated.

    Base classes should override if any special actions need to be
    taken when a field is updated. The field is not updated if the
    method does not return a True value.

    Args:
      entity: the unaltered entity
      entity_properties: keyword arguments that correspond to entity
        properties and their values
      name: the name of the field to be changed
    """

    if not entity:
      raise NoEntityError

    if not entity_properties or (name not in entity_properties):
      raise InvalidArgumentError

    return True

  def _onCreate(self, entity):
    """Called when an entity has been created.

    Classes that override this can use it to do any post-creation operations.
    """

    if not entity:
      raise NoEntityError

    sidebar.flush()

  def _onUpdate(self, entity):
    """Called when an entity has been updated.

    Classes that override this can use it to do any post-update operations.
    """

    if not entity:
      raise NoEntityError

  def _onDelete(self, entity):
    """Called when an entity has been deleted.

    Classes that override this can use it to do any post-deletion operations.
    """

    if not entity:
      raise NoEntityError
<commit_msg>Add a silent option to updateOrCreateFromFields<commit_after>#!/usr/bin/python2.5
#
# Copyright 2008 the Melange authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Helpers functions for updating different kinds of models in datastore.
"""

__authors__ = [
  '"Todd Larsen" <tlarsen@google.com>',
  '"Sverre Rabbelier" <sverre@rabbelier.nl>',
  '"Lennard de Rijk" <ljvderijk@gmail.com>',
  '"Pawel Solyga" <pawel.solyga@gmail.com>',
  ]

from google.appengine.ext import db

from django.utils.translation import ugettext

from soc.cache import sidebar
from soc.logic import dicts
from soc.views import out_of_band

class Error(Exception):
  """Base class for all exceptions raised by this module.
  """

  pass

class InvalidArgumentError(Error):
  """Raised when an invalid argument is passed to a method.

  For example, if an argument is None, but must always be non-False.
  """

  pass

class NoEntityError(InvalidArgumentError):
  """Raised when no entity is passed to a method that requires one.
  """

  pass

class Logic(object):
  """Base logic for entity classes.

  The BaseLogic class functions specific to Entity classes by relying
  on arguments passed to __init__.
  """

  def __init__(self, model, base_model=None, scope_logic=None,
               name=None, skip_properties=None, id_based=False):
    """Defines the name, key_name and model for this entity.
    """

    self._model = model
    self._base_model = base_model
    self._scope_logic = scope_logic
    self._id_based = id_based

    if name:
      self._name = name
    else:
      self._name =  self._model.__name__

    if skip_properties:
      self._skip_properties = skip_properties
    else:
      self._skip_properties = []

  def getModel(self):
    """Returns the model this logic class uses.
    """

    return self._model

  def getScopeLogic(self):
    """Returns the logic of the enclosing scope.
    """

    return self._scope_logic

  def getScopeDepth(self):
    """Returns the scope depth for this entity.

    Returns None if any of the parent scopes return None.
    """

    if not self._scope_logic:
      return 0

    depth = self._scope_logic.logic.getScopeDepth()
    return None if (depth is None) else (depth + 1)

  def getKeyNameFromFields(self, fields):
    """Returns the KeyName constructed from fields dict for this type of entity.

    The KeyName is in the following format:
    <key_value1>/<key_value2>/.../<key_valueN>
    """

    if not fields:
      raise InvalidArgumentError

    key_field_names = self.getKeyFieldNames()

    # check if all key_field_names for this entity are present in fields
    if not all(field in fields.keys() for field in key_field_names):
      raise InvalidArgumentError("Not all the required key fields are present")

    if not all(fields.get(field) for field in key_field_names):
      raise InvalidArgumentError("Not all KeyValues are non-false")

    # construct the KeyValues in the order given by getKeyFieldNames()
    keyvalues = []
    for key_field_name in key_field_names:
      keyvalues.append(fields[key_field_name])

    # construct the KeyName in the appropriate format
    return '/'.join(keyvalues)

  def getFullModelClassName(self):
    """Returns fully-qualified model module.class name string.
    """

    return '%s.%s' % (self._model.__module__, self._model.__name__)

  def getKeyValuesFromEntity(self, entity):
    """Extracts the key values from entity and returns them.

    The default implementation uses the scope and link_id as key values.

    Args:
      entity: the entity from which to extract the key values
    """

    if not entity:
      raise NoEntityError

    return [entity.scope_path, entity.link_id]

  def getKeyValuesFromFields(self, fields):
    """Extracts the key values from a dict and returns them.

    The default implementation uses the scope and link_id as key values.

    Args:
      fields: the dict from which to extract the key values
    """

    if ('scope_path' not in fields) or ('link_id' not in fields):
      raise InvalidArgumentError

    return [fields['scope_path'], fields['link_id']]

  def getKeyFieldNames(self):
    """Returns an array with the names of the Key Fields.

    The default implementation uses the scope and link_id as key values.
    """

    return ['scope_path', 'link_id']

  def getKeyFieldsFromFields(self, dictionary):
    """Does any required massaging and filtering of dictionary.

    The resulting dictionary contains just the key names, and has any
    required translations/modifications performed.

    Args:
      dictionary: The arguments to massage
    """

    if not dictionary:
      raise InvalidArgumentError

    keys = self.getKeyFieldNames()
    values = self.getKeyValuesFromFields(dictionary)
    key_fields = dicts.zip(keys, values)

    return key_fields

  def getFromKeyName(self, key_name):
    """"Returns entity for key_name or None if not found.

    Args:
      key_name: key name of entity
    """

    if self._id_based:
      raise Error("getFromKeyName called on an id based logic")

    if not key_name:
      raise InvalidArgumentError

    return self._model.get_by_key_name(key_name)

  def getFromID(self, id):
    """Returns entity for id or None if not found.

    Args:
      id: id of entity
    """

    if not self._id_based:
      raise Error("getFromID called on a not id based logic")

    if not id:
      raise InvalidArgumentError

    return self._model.get_by_id(id)

  def getFromKeyNameOr404(self, key_name):
    """Like getFromKeyName but expects to find an entity.

    Raises:
      out_of_band.Error if no entity is found
    """

    entity = self.getFromKeyName(key_name)

    if entity:
      return entity

    msg = ugettext('There is no "%(name)s" named %(key_name)s.') % {
        'name': self._name, 'key_name': key_name}

    raise out_of_band.Error(msg, status=404)

  def getFromIDOr404(self, id):
    """Like getFromID but expects to find an entity.

    Raises:
      out_of_band.Error if no entity is found
    """

    entity = self.getFromID(id)

    if entity:
      return entity

    msg = ugettext('There is no "%(name)s" with id %(id)s.') % {
        'name': self._name, 'id': id}

    raise out_of_band.Error(msg, status=404)

  def getFromKeyFields(self, fields):
    """Returns the entity for the specified key names, or None if not found.

    Args:
      fields: a dict containing the fields of the entity that
        uniquely identifies it
    """

    if not fields:
      raise InvalidArgumentError

    key_fields = self.getKeyFieldsFromFields(fields)

    if all(key_fields.values()):
      key_name = self.getKeyNameFromFields(key_fields)
      entity = self.getFromKeyName(key_name)
    else:
      entity = None

    return entity

  def getFromKeyFieldsOr404(self, fields):
    """Like getFromKeyFields but expects to find an entity.

    Raises:
      out_of_band.Error if no entity is found
    """

    entity = self.getFromKeyFields(fields)

    if entity:
      return entity

    key_fields = self.getKeyFieldsFromFields(fields)
    format_text = ugettext('"%(key)s" is "%(value)s"')

    msg_pairs = [format_text % {'key': key, 'value': value}
      for key, value in key_fields.iteritems()]

    joined_pairs = ' and '.join(msg_pairs)

    msg = ugettext(
      'There is no "%(name)s" where %(pairs)s.') % {
        'name': self._name, 'pairs': joined_pairs}

    raise out_of_band.Error(msg, status=404)

  def getForFields(self, filter=None, unique=False,
                   limit=1000, offset=0, order=None):
    """Returns all entities that have the specified properties.

    Args:
      filter: a dict for the properties that the entities should have
      unique: if set, only the first item from the resultset will be returned
      limit: the amount of entities to fetch at most
      offset: the position to start at
      order: a list with the sort order
    """

    if unique:
      limit = 1

    query = self.getQueryForFields(filter=filter, order=order)

    result = query.fetch(limit, offset)

    if unique:
      return result[0] if result else None

    return result

  def getQueryForFields(self, filter=None, order=None):
    """Returns a query with the specified properties.

    Args:
      filter: a dict for the properties that the entities should have
      order: a list with the sort order

    Returns:
      - Query object instantiated with the given properties
    """

    if not filter:
      filter = {}

    if not order:
      order = []

    orderset = set([i.strip('-') for i in order])
    if len(orderset) != len(order):
      raise InvalidArgumentError

    query = db.Query(self._model)

    for key, value in filter.iteritems():
      if isinstance(value, list) and len(value) == 1:
        value = value[0]
      if isinstance(value, list):
        op = '%s IN' % key
        query.filter(op, value)
      else:
        query.filter(key, value)

    for key in order:
      query.order(key)

    return query

  def updateEntityProperties(self, entity, entity_properties, silent=False):
    """Update existing entity using supplied properties.

    Args:
      entity: a model entity
      entity_properties: keyword arguments that correspond to entity
        properties and their values
      silent: iff True does not call _onUpdate method

    Returns:
      The original entity with any supplied properties changed.
    """

    if not entity:
      raise NoEntityError

    if not entity_properties:
      raise InvalidArgumentError

    properties = self._model.properties()

    for name, prop in properties.iteritems():
      # if the property is not updateable or is not updated, skip it
      if name in self._skip_properties or (name not in entity_properties):
        continue

      if self._updateField(entity, entity_properties, name):
        value = entity_properties[name]
        prop.__set__(entity, value)

    entity.put()

    # call the _onUpdate method
    if not silent:
      self._onUpdate(entity)

    return entity

  def updateOrCreateFromKeyName(self, properties, key_name):
    """Update existing entity, or create new one with supplied properties.

    Args:
      properties: dict with entity properties and their values
      key_name: the key_name of the entity that uniquely identifies it

    Returns:
      the entity corresponding to the key_name, with any supplied
      properties changed, or a new entity now associated with the
      supplied key_name and properties.
    """

    entity = self.getFromKeyName(key_name)

    create_entity = not entity

    if create_entity:
      for property_name in properties:
        self._createField(properties, property_name)

      # entity did not exist, so create one in a transaction
      entity = self._model.get_or_insert(key_name, **properties)
    else:
      # If someone else already created the entity (due to a race), we
      # should not update the propties (as they 'won' the race).
      entity = self.updateEntityProperties(entity, properties, silent=True)

    if create_entity:
      # a new entity has been created call _onCreate
      self._onCreate(entity)
    else:
      # the entity has been updated call _onUpdate
      self._onUpdate(entity)

    return entity

  def updateOrCreateFromFields(self, properties, silent=False):
    """Creates a new entity with the supplied properties.

    Args:
      properties: dict with entity properties and their values
      silent: if True, do not run the _onCreate hook
    """

    for property_name in properties:
      self._createField(properties, property_name)

    if self._id_based:
      entity = self._model(**properties)
      entity.put()
    else:
      key_name = self.getKeyNameFromFields(properties)
      entity = self._model.get_or_insert(key_name, **properties)

    if not silent:
      self._onCreate(entity)

    return entity

  def isDeletable(self, entity):
    """Returns whether the specified entity can be deleted.

    Args:
      entity: an existing entity in datastore
    """

    return True

  def delete(self, entity):
    """Delete existing entity from datastore.

    Args:
      entity: an existing entity in datastore
    """

    entity.delete()
    # entity has been deleted call _onDelete
    self._onDelete(entity)

  def getAll(self, query):
    """Retrieves all entities for the specified query.
    """

    chunk = 999
    offset = 0
    result = []
    more = True

    while(more):
      data = query.fetch(chunk+1, offset)

      more = len(data) > chunk

      if more:
        del data[chunk]

      result.extend(data)
      offset = offset + chunk

    return result

  def _createField(self, entity_properties, name):
    """Hook called when a field is created.

    To be exact, this method is called for each field (that has a value
    specified) on an entity that is being created.

    Base classes should override if any special actions need to be
    taken when a field is created.

    Args:
      entity_properties: keyword arguments that correspond to entity
        properties and their values
      name: the name of the field to be created
    """

    if not entity_properties or (name not in entity_properties):
      raise InvalidArgumentError

  def _updateField(self, entity, entity_properties, name):
    """Hook called when a field is updated.

    Base classes should override if any special actions need to be
    taken when a field is updated. The field is not updated if the
    method does not return a True value.

    Args:
      entity: the unaltered entity
      entity_properties: keyword arguments that correspond to entity
        properties and their values
      name: the name of the field to be changed
    """

    if not entity:
      raise NoEntityError

    if not entity_properties or (name not in entity_properties):
      raise InvalidArgumentError

    return True

  def _onCreate(self, entity):
    """Called when an entity has been created.

    Classes that override this can use it to do any post-creation operations.
    """

    if not entity:
      raise NoEntityError

    sidebar.flush()

  def _onUpdate(self, entity):
    """Called when an entity has been updated.

    Classes that override this can use it to do any post-update operations.
    """

    if not entity:
      raise NoEntityError

  def _onDelete(self, entity):
    """Called when an entity has been deleted.

    Classes that override this can use it to do any post-deletion operations.
    """

    if not entity:
      raise NoEntityError
<|endoftext|>


<commit_before>#!/usr/bin/python2.5
#
# Copyright 2008 the Melange authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""User (Model) query functions.
"""

__authors__ = [
  '"Sverre Rabbelier" <sverre@rabbelier.nl>',
  ]

from google.appengine.api import users
from google.appengine.ext import db

from soc.cache import sidebar
from soc.logic.helper import notifications
from soc.logic.models import base
from soc.logic.models.site import logic as site_logic

import soc.models.user

class Logic(base.Logic):
  """Logic methods for the User model.
  """

  def __init__(self):
    """Defines the name, key_name and model for this entity.
    """

    super(Logic, self).__init__(soc.models.user.User,
                        skip_properties=['former_accounts'])

  def isFormerAccount(self, account):
    """Returns true if account is a former account of some User.
    """

    # TODO(pawel.solyga): replace 1000 with solution that works for any
    #   number of queries
    users_with_former_accounts = soc.models.user.User.gql(
        'WHERE former_accounts != :1', None).fetch(1000)

    for former_account_user in users_with_former_accounts:
      if account in former_account_user.former_accounts:
        return True

    return False

  def getForCurrentAccount(self):
    """Retrieves the user entity for the currently logged in account.

    If there is no user logged in, or they have no associated User
    entity, None is returend.
    """

    account = users.get_current_user()

    if not account:
      return None

    user = self.getForFields({'account': account, 'status':'valid'},
        unique=True)

    return user

  def agreesToSiteToS(self, entity):
    """Returns indication of User's answer to the site-wide Terms of Service.

    Args:
      entity: User entity to check for agreement to site-wide ToS

    Returns:
      True: no site-wide ToS is currently in effect on the site
      True: site-wide ToS is in effect *and* User agrees to it
        (User explicitly answered "Yes")
      False: site-wide ToS is in effect but User did not agree to it yet
    """
    if not site_logic.getToS(site_logic.getSingleton()):
      # no site-wide ToS in effect, so let the User slide for now
      return True

    try:
      agreed_on = entity.agreed_to_tos_on
    except db.Error:
      # return False indicating that answer is missing
      return False

    # user has not agreed yet
    if not agreed_on:
      return False

    return True

  def getKeyValuesFromEntity(self, entity):
    """See base.Logic.getKeyValuesFromEntity.
    """

    return [entity.link_id]

  def getSuffixValues(self, entity):
    """See base.Logic.getSuffixValues.
    """

    return [entity.link_id]

  def getKeyValuesFromFields(self, fields):
    """See base.Logic.getKeyValuesFromFields.
    """

    return [fields['link_id']]

  def getKeyFieldNames(self):
    """See base.Logic.getKeyFieldNames.
    """

    return ['link_id']

  def _updateField(self, entity, name, value):
    """Special case logic for account.

    When the account is changed, the former_accounts field should be appended
    with the old account.
    Also, if either is_developer or agrees_to_tos change, the user's
    rights have changed, so we need to flush the sidebar.
    Make sure once the user agreed ToS, the ToS fields can't be changed.
    """

    # iff the agreed_to_tos is True and we want to set it to False
    if (name == 'agreed_to_tos') and (not value) and entity.agreed_to_tos:
      return False

    # iff the agreed_to_tos_on has a value and we want to change it
    if (name == 'agreed_to_tos_on') and entity.agreed_to_tos_on and (
        value != entity.agreed_to_tos_on):
      return False

    if (name == 'is_developer') and (entity.is_developer != value):
      sidebar.flush(entity.account)

    if (name == 'agreed_to_tos') and (entity.agreed_to_tos != value):
      sidebar.flush(entity.account)

    if (name == 'account') and (entity.account != value):
      entity.former_accounts.append(entity.account)

    return True

  def _onCreate(self, entity):
    """Send out a message to welcome the new user.
    """

    notifications.sendWelcomeMessage(entity)

    super(Logic, self)._onCreate(entity)

logic = Logic()
<commit_msg>Fixed a typo in getForCurrentAccount comment.<commit_after>#!/usr/bin/python2.5
#
# Copyright 2008 the Melange authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""User (Model) query functions.
"""

__authors__ = [
  '"Sverre Rabbelier" <sverre@rabbelier.nl>',
  ]

from google.appengine.api import users
from google.appengine.ext import db

from soc.cache import sidebar
from soc.logic.helper import notifications
from soc.logic.models import base
from soc.logic.models.site import logic as site_logic

import soc.models.user

class Logic(base.Logic):
  """Logic methods for the User model.
  """

  def __init__(self):
    """Defines the name, key_name and model for this entity.
    """

    super(Logic, self).__init__(soc.models.user.User,
                        skip_properties=['former_accounts'])

  def isFormerAccount(self, account):
    """Returns true if account is a former account of some User.
    """

    # TODO(pawel.solyga): replace 1000 with solution that works for any
    #   number of queries
    users_with_former_accounts = soc.models.user.User.gql(
        'WHERE former_accounts != :1', None).fetch(1000)

    for former_account_user in users_with_former_accounts:
      if account in former_account_user.former_accounts:
        return True

    return False

  def getForCurrentAccount(self):
    """Retrieves the user entity for the currently logged in account.

    If there is no user logged in, or they have no valid associated User
    entity, None is returned.
    """

    account = users.get_current_user()

    if not account:
      return None

    user = self.getForFields({'account': account, 'status':'valid'},
        unique=True)

    return user

  def agreesToSiteToS(self, entity):
    """Returns indication of User's answer to the site-wide Terms of Service.

    Args:
      entity: User entity to check for agreement to site-wide ToS

    Returns:
      True: no site-wide ToS is currently in effect on the site
      True: site-wide ToS is in effect *and* User agrees to it
        (User explicitly answered "Yes")
      False: site-wide ToS is in effect but User did not agree to it yet
    """
    if not site_logic.getToS(site_logic.getSingleton()):
      # no site-wide ToS in effect, so let the User slide for now
      return True

    try:
      agreed_on = entity.agreed_to_tos_on
    except db.Error:
      # return False indicating that answer is missing
      return False

    # user has not agreed yet
    if not agreed_on:
      return False

    return True

  def getKeyValuesFromEntity(self, entity):
    """See base.Logic.getKeyValuesFromEntity.
    """

    return [entity.link_id]

  def getSuffixValues(self, entity):
    """See base.Logic.getSuffixValues.
    """

    return [entity.link_id]

  def getKeyValuesFromFields(self, fields):
    """See base.Logic.getKeyValuesFromFields.
    """

    return [fields['link_id']]

  def getKeyFieldNames(self):
    """See base.Logic.getKeyFieldNames.
    """

    return ['link_id']

  def _updateField(self, entity, name, value):
    """Special case logic for account.

    When the account is changed, the former_accounts field should be appended
    with the old account.
    Also, if either is_developer or agrees_to_tos change, the user's
    rights have changed, so we need to flush the sidebar.
    Make sure once the user agreed ToS, the ToS fields can't be changed.
    """

    # iff the agreed_to_tos is True and we want to set it to False
    if (name == 'agreed_to_tos') and (not value) and entity.agreed_to_tos:
      return False

    # iff the agreed_to_tos_on has a value and we want to change it
    if (name == 'agreed_to_tos_on') and entity.agreed_to_tos_on and (
        value != entity.agreed_to_tos_on):
      return False

    if (name == 'is_developer') and (entity.is_developer != value):
      sidebar.flush(entity.account)

    if (name == 'agreed_to_tos') and (entity.agreed_to_tos != value):
      sidebar.flush(entity.account)

    if (name == 'account') and (entity.account != value):
      entity.former_accounts.append(entity.account)

    return True

  def _onCreate(self, entity):
    """Send out a message to welcome the new user.
    """

    notifications.sendWelcomeMessage(entity)

    super(Logic, self)._onCreate(entity)

logic = Logic()
<|endoftext|>


<commit_before>#!/usr/bin/python2.5
#
# Copyright 2008 the Melange authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Views for Host profiles.
"""

__authors__ = [
    '"Sverre Rabbelier" <sverre@rabbelier.nl>',
    '"Lennard de Rijk" <ljvderijk@gmail.com>',
  ]

from django import forms

from soc.logic import cleaning
from soc.logic import accounts
from soc.logic import dicts
from soc.logic.models import host as host_logic
from soc.logic.models import user as user_logic
from soc.logic.models import sponsor as sponsor_logic
from soc.views.helper import access
from soc.views.helper import dynaform
from soc.views.helper import widgets
from soc.views.models import role
from soc.views.models import sponsor as sponsor_view

import soc.models.host
import soc.logic.models.host
import soc.views.helper
import soc.views.models.sponsor

# TODO(pawel.solyga): Rename all list methods and functions to something else
# and remove this tolist assignment
tolist = list

class View(role.View):
  """View methods for the Host model.
  """

  def __init__(self, params=None):
    """Defines the fields and methods required for the base View class
    to provide the user with list, public, create, edit and delete views.

    Params:
      params: a dict with params for this View
    """

    rights = {}
    rights['create'] = [access.checkIsHost]
    rights['edit'] = [access.checkIsMyActiveRole(soc.logic.models.host)]
    rights['invite'] = [access.checkIsHost]
    rights['list'] = [access.checkIsHost]
    rights['accept_invite'] = [access.checkCanCreateFromRequest('host')]
    rights['process_request'] = [access.checkIsHost,
        access.checkCanProcessRequest('host')]

    new_params = {}
    new_params['rights'] = rights
    new_params['logic'] = soc.logic.models.host.logic
    new_params['group_logic'] = sponsor_logic.logic

    new_params['scope_view'] = sponsor_view

    new_params['group_view'] = soc.views.models.sponsor.view
    new_params['invite_filter'] = {'group_ln': 'link_id'}

    new_params['name'] = "Program Administrator"
    new_params['module_name'] = "host"

    new_params['extra_dynaexclude'] = ['user', 'state']

    new_params['create_extra_dynafields'] = {
       'scope_path': forms.CharField(widget=forms.HiddenInput,
                                  required=True),
       'clean_link_id': cleaning.clean_existing_user('link_id'),
       'clean_home_page': cleaning.clean_url('home_page'),
       'clean_blog': cleaning.clean_url('blog'),
       'clean_photo_url': cleaning.clean_url('photo_url')}

    new_params['allow_invites'] = True

    params = dicts.merge(params, new_params)

    super(View, self).__init__(params=params)

    # create and store the special form for invited users
    updated_fields = {
        'link_id': forms.CharField(widget=widgets.ReadOnlyInput(),
            required=False)}

    invited_create_form = dynaform.extendDynaForm(
        dynaform = self._params['create_form'],
        dynafields = updated_fields)

    params['invited_create_form'] = invited_create_form

  def list(self, request, access_type, page_name=None,
           params=None, filter=None):
    """See base.View.list.

    Passes a filter to base.View.list so that only hosts from a sponsor
    that this user is host for are listed.
    """

    user = user_logic.logic.getForCurrentAccount()

    # Don't bother looking up everything if there's no user
    if user and (not accounts.isDeveloper(user=user)):
      hosts = host_logic.logic.getForFields({'user': user})
      sponsors = tolist((host.scope for host in hosts))

      new_filter = {'scope': sponsors}
      filter = dicts.merge(filter, new_filter)

    return super(View, self).list(request, access_type, page_name=page_name,
                                  params=params, filter=filter)

  def _editPost(self, request, entity, fields):
    """See base.View._editPost().
    """
    if not entity
      fields['user'] = fields['link_id']
      fields['link_id'] = fields['link_id'].link_id

    super(View, self)._editPost(request, entity, fields)

  def _acceptInvitePost(self, fields, request, context, params, **kwargs):
    """Fills in the fields that were missing in the invited_created_form.

    For params see base.View._acceptInvitePost()
    """
    # fill in the appropriate fields that were missing in the form
    fields['user'] = fields['link_id']
    fields['link_id'] = fields['link_id'].link_id

view = View()

accept_invite = view.acceptInvite
create = view.create
delete = view.delete
edit = view.edit
invite = view.invite
list = view.list
process_request = view.processRequest
public = view.public
export = view.export
<commit_msg>Brown paper bag fix.<commit_after>#!/usr/bin/python2.5
#
# Copyright 2008 the Melange authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Views for Host profiles.
"""

__authors__ = [
    '"Sverre Rabbelier" <sverre@rabbelier.nl>',
    '"Lennard de Rijk" <ljvderijk@gmail.com>',
  ]

from django import forms

from soc.logic import cleaning
from soc.logic import accounts
from soc.logic import dicts
from soc.logic.models import host as host_logic
from soc.logic.models import user as user_logic
from soc.logic.models import sponsor as sponsor_logic
from soc.views.helper import access
from soc.views.helper import dynaform
from soc.views.helper import widgets
from soc.views.models import role
from soc.views.models import sponsor as sponsor_view

import soc.models.host
import soc.logic.models.host
import soc.views.helper
import soc.views.models.sponsor

# TODO(pawel.solyga): Rename all list methods and functions to something else
# and remove this tolist assignment
tolist = list

class View(role.View):
  """View methods for the Host model.
  """

  def __init__(self, params=None):
    """Defines the fields and methods required for the base View class
    to provide the user with list, public, create, edit and delete views.

    Params:
      params: a dict with params for this View
    """

    rights = {}
    rights['create'] = [access.checkIsHost]
    rights['edit'] = [access.checkIsMyActiveRole(soc.logic.models.host)]
    rights['invite'] = [access.checkIsHost]
    rights['list'] = [access.checkIsHost]
    rights['accept_invite'] = [access.checkCanCreateFromRequest('host')]
    rights['process_request'] = [access.checkIsHost,
        access.checkCanProcessRequest('host')]

    new_params = {}
    new_params['rights'] = rights
    new_params['logic'] = soc.logic.models.host.logic
    new_params['group_logic'] = sponsor_logic.logic

    new_params['scope_view'] = sponsor_view

    new_params['group_view'] = soc.views.models.sponsor.view
    new_params['invite_filter'] = {'group_ln': 'link_id'}

    new_params['name'] = "Program Administrator"
    new_params['module_name'] = "host"

    new_params['extra_dynaexclude'] = ['user', 'state']

    new_params['create_extra_dynafields'] = {
       'scope_path': forms.CharField(widget=forms.HiddenInput,
                                  required=True),
       'clean_link_id': cleaning.clean_existing_user('link_id'),
       'clean_home_page': cleaning.clean_url('home_page'),
       'clean_blog': cleaning.clean_url('blog'),
       'clean_photo_url': cleaning.clean_url('photo_url')}

    new_params['allow_invites'] = True

    params = dicts.merge(params, new_params)

    super(View, self).__init__(params=params)

    # create and store the special form for invited users
    updated_fields = {
        'link_id': forms.CharField(widget=widgets.ReadOnlyInput(),
            required=False)}

    invited_create_form = dynaform.extendDynaForm(
        dynaform = self._params['create_form'],
        dynafields = updated_fields)

    params['invited_create_form'] = invited_create_form

  def list(self, request, access_type, page_name=None,
           params=None, filter=None):
    """See base.View.list.

    Passes a filter to base.View.list so that only hosts from a sponsor
    that this user is host for are listed.
    """

    user = user_logic.logic.getForCurrentAccount()

    # Don't bother looking up everything if there's no user
    if user and (not accounts.isDeveloper(user=user)):
      hosts = host_logic.logic.getForFields({'user': user})
      sponsors = tolist((host.scope for host in hosts))

      new_filter = {'scope': sponsors}
      filter = dicts.merge(filter, new_filter)

    return super(View, self).list(request, access_type, page_name=page_name,
                                  params=params, filter=filter)

  def _editPost(self, request, entity, fields):
    """See base.View._editPost().
    """
    if not entity:
      fields['user'] = fields['link_id']
      fields['link_id'] = fields['link_id'].link_id

    super(View, self)._editPost(request, entity, fields)

  def _acceptInvitePost(self, fields, request, context, params, **kwargs):
    """Fills in the fields that were missing in the invited_created_form.

    For params see base.View._acceptInvitePost()
    """
    # fill in the appropriate fields that were missing in the form
    fields['user'] = fields['link_id']
    fields['link_id'] = fields['link_id'].link_id

view = View()

accept_invite = view.acceptInvite
create = view.create
delete = view.delete
edit = view.edit
invite = view.invite
list = view.list
process_request = view.processRequest
public = view.public
export = view.export
<|endoftext|>


<commit_before>import CFG_pb2
try:
    import r2pipe
except:
    print("Install the radare2 python bindings. (https://github.com/radare/radare2-r2pipe)")

#
# r2 is a r2pipe.open instance
#
def analyze(r2):

    # Get list of json dicts that have function information.
    # An entry in the list is like:
    # {
    #   "offset":4196100,
    #   "name":"sym._fini",
    #   "size":9,
    #   "realsz":9,
    #   "cc":1,
    #   "nbbs":1,
    #   "edges":0,
    #   "ebbs":1,
    #   "calltype":"amd64",
    #   "type":"sym",
    #   "diff":"NEW",
    #   "callrefs":[{ "address":"", "type" : "", "at" : ""},...],
    #   "datarefs":[NNN, NNN...],
    #   "codexrefs":[],
    #   "dataxrefs":[],
    #   "difftype":"new",
    #   "indegree":0,
    #   "outdegree":0,
    #   "nargs":0,
    #   "nlocals":0
    # }
    fn_list = r2.cmdj("aflj")

    mod = CFG_pb2.Module()
    for fne in fn_list:
        name = fne["name"]
        new_block = True

        # Having each function be a named block. Is enough?
        named_block = CFG_pb2.NamedBlock()
        named_block.name = name
        named_block.address = fne["offset"]
        named_block.visibility = 1	# XXX: Ehm; must revisit.
        mod.named_blocks.extend([named_block])

        # Analyze the current function
        r2.cmd("af@{0}".format(name))

        # Disassemble function and return in json, which
        # will look like:
        # {
        #   "name":"sym.main",
        #   "size":261,
        #   "addr":4195709,
        #   "ops": [
        #      {
        #        "offset":4195709,
        #        "esil":"rbp,8,rsp,-=,rsp,=[8]",
        #        "refptr":false,
        #        "fcn_addr":4195709,
        #        "fcn_last":4195969,
        #        "size":1,
        #        "opcode":"push rbp",
        #        "bytes":"55",
        #        "family":"cpu",
        #        "type":"upush",
        #        "type_num":12,
        #        "type2_num":0,
        #        "flags":["main","sym.main"],
        #        "xrefs":[{"addr":4195501,"type":"DATA"}]
        #      }, ... ]
        # }
        dis = r2.cmdj("pdfj@{0}".format(name))
        if dis is None:
            print("Unable to disassemble {0}".format(name))
            continue

        for op in dis["ops"]:
            if new_block == True:
                if "bytes" not in op.keys():
                    continue
                blk = CFG_pb2.Block()
                blk.address = op["offset"]
                new_block = False

            if "bytes" not in op.keys():
                if new_block == False:
                    mod.blocks.extend([blk])
                    new_block = True
                    blk = None
                continue

            ii = CFG_pb2.Instr()

            # I was using python3 before so had bytes.fromhex() API call,
            # but I believe this is the correct juju for Python2
            btmp = [op["bytes"][i:i+2] for i in range(0, len(op["bytes"]), 2)]
            ii.bytes = ''.join([chr(int(x, 16)) for x in btmp])
            ii.address = op["offset"]
            oc = op["opcode"]
            if oc.startswith("ud2") == True:
                print("Undefined (ud2) {1}@{0}. Ignoring block, skipping to next function!".format(op["offset"], name))
                new_block = True
                blk = None
                break

            blk.instructions.extend([ii])
            blk_terminators = ["call", "ret", "int", "j", "iret", "hlt", "nop", \
              "lcall", "ljmp", "syscall" ]
            for bt in blk_terminators:
                if oc.startswith(bt) == True:
                    mod.blocks.extend([blk])
                    new_block = True
                    blk = None
                    break
    return mod
<commit_msg>Mark imported symbols as such. Add block even if no terminating instruction encountered.<commit_after>import CFG_pb2
try:
    import r2pipe
except:
    print("Install the radare2 python bindings. (https://github.com/radare/radare2-r2pipe)")

#
# r2 is a r2pipe.open instance
#
def analyze(r2):

    # Get list of json dicts that have function information.
    # An entry in the list is like:
    # {
    #   "offset":4196100,
    #   "name":"sym._fini",
    #   "size":9,
    #   "realsz":9,
    #   "cc":1,
    #   "nbbs":1,
    #   "edges":0,
    #   "ebbs":1,
    #   "calltype":"amd64",
    #   "type":"sym",
    #   "diff":"NEW",
    #   "callrefs":[{ "address":"", "type" : "", "at" : ""},...],
    #   "datarefs":[NNN, NNN...],
    #   "codexrefs":[],
    #   "dataxrefs":[],
    #   "difftype":"new",
    #   "indegree":0,
    #   "outdegree":0,
    #   "nargs":0,
    #   "nlocals":0
    # }
    fn_list = r2.cmdj("aflj")

    mod = CFG_pb2.Module()
    for fne in fn_list:
        name = fne["name"]
        new_block = True

        # Having each function be a named block. Is enough?
        named_block = CFG_pb2.NamedBlock()
        named_block.name = name
        named_block.address = fne["offset"]
	if name.startswith("sym.imp.") == True:
		named_block.visibility = 1
	else:
	        named_block.visibility = 0
        mod.named_blocks.extend([named_block])

        # Analyze the current function
        r2.cmd("af@{0}".format(name))

        # Disassemble function and return in json, which
        # will look like:
        # {
        #   "name":"sym.main",
        #   "size":261,
        #   "addr":4195709,
        #   "ops": [
        #      {
        #        "offset":4195709,
        #        "esil":"rbp,8,rsp,-=,rsp,=[8]",
        #        "refptr":false,
        #        "fcn_addr":4195709,
        #        "fcn_last":4195969,
        #        "size":1,
        #        "opcode":"push rbp",
        #        "bytes":"55",
        #        "family":"cpu",
        #        "type":"upush",
        #        "type_num":12,
        #        "type2_num":0,
        #        "flags":["main","sym.main"],
        #        "xrefs":[{"addr":4195501,"type":"DATA"}]
        #      }, ... ]
        # }
        dis = r2.cmdj("pdfj@{0}".format(name))
        if dis is None:
            print("Unable to disassemble {0}".format(name))
            continue

        for op in dis["ops"]:
            if new_block == True:
                if "bytes" not in op.keys():
                    continue
                blk = CFG_pb2.Block()
                blk.address = op["offset"]
                new_block = False

            if "bytes" not in op.keys():
                if new_block == False:
                    mod.blocks.extend([blk])
                    new_block = True
                    blk = None
                continue

            ii = CFG_pb2.Instr()

            # I was using python3 before so had bytes.fromhex() API call,
            # but I believe this is the correct juju for Python2
            btmp = [op["bytes"][i:i+2] for i in range(0, len(op["bytes"]), 2)]
            ii.bytes = ''.join([chr(int(x, 16)) for x in btmp])
            ii.address = op["offset"]
            oc = op["opcode"]
            if oc.startswith("ud2") == True:
                print("Undefined (ud2) {1}@{0}. Ignoring block, skipping to next function!".format(op["offset"], name))
                new_block = True
                blk = None
                break

            blk.instructions.extend([ii])
            blk_terminators = ["call", "ret", "int", "j", "iret", "hlt", "nop", \
              "lcall", "ljmp", "syscall" ]
            for bt in blk_terminators:
                if oc.startswith(bt) == True:
                    mod.blocks.extend([blk])
                    new_block = True
                    blk = None
                    break
        if blk:
            print("Reach non-block terminated set of instructions... inserting block anyway")
            mod.blocks.extend([blk])

    return mod
<|endoftext|>