<commit_before>#!/usr/bin/env python

# Copyright (c) 2001, MetaSlash Inc.  All rights reserved.

"""
Check python source code files for possible errors and print warnings

Contact Info:
  http://pychecker.sourceforge.net/
  pychecker@metaslash.com
"""

import string
import types
import sys
import imp
import os
import glob

# see __init__.py for meaning, this must match the version there
LOCAL_MAIN_VERSION = 1

def setupNamespace(path) :
    # remove pychecker if it's the first component, it needs to be last
    if sys.path[0][-9:] == 'pychecker' :
        del sys.path[0]

    # make sure pychecker is last in path, so we can import
    checker_path = os.path.dirname(os.path.dirname(path))
    if checker_path not in sys.path :
        sys.path.append(checker_path)

if __name__ == '__main__' :
    setupNamespace(sys.argv[0])

from pychecker import printer
from pychecker import warn
from pychecker import OP
from pychecker import Config
from pychecker import function
from pychecker.Warning import Warning

# Globals for storing a dictionary of info about modules and classes
_allModules = {}
_cfg = None

# Constants
_DEFAULT_MODULE_TOKENS = ('__builtins__', '__doc__', '__file__', '__name__',
                          '__path__')
_DEFAULT_CLASS_TOKENS = ('__doc__', '__name__', '__module__')

_VERSION_MISMATCH_ERROR = '''
There seem to be two versions of PyChecker being used.
One is probably in python/site-packages, the other in a local directory.
If you want to run the local version, you must remove the version
from site-packages.  Or you can install the current version
by doing python setup.py install.
'''

def _flattenList(list) :
    "Returns a list which contains no lists"

    new_list = []
    for element in list :
        if type(element) == types.ListType :
            new_list.extend(_flattenList(element))
        else :
            new_list.append(element)

    return new_list

def getModules(arg_list) :
    "Returns a list of module names that can be imported"

    new_arguments = []
    for arg in arg_list :
        # is this a wildcard filespec? (necessary for windows)
        if '*' in arg or '?' in arg or '[' in arg :
            arg = glob.glob(arg)
        new_arguments.append(arg)

    PY_SUFFIX = '.py'
    PY_SUFFIX_LEN = len(PY_SUFFIX)

    modules = []
    for arg in _flattenList(new_arguments) :
        # is it a .py file?
        if len(arg) > PY_SUFFIX_LEN and arg[-PY_SUFFIX_LEN:] == PY_SUFFIX:
            arg_dir = os.path.dirname(arg)
            if arg_dir and not os.path.exists(arg) :
                print 'File or pathname element does not exist: "%s"' % arg
                continue

            module_name = os.path.basename(arg)[:-PY_SUFFIX_LEN]
            if arg_dir not in sys.path :
                sys.path.insert(1, arg_dir)
	    arg = module_name
        modules.append(arg)

    return modules

def _findModule(name, path = sys.path) :
    """Returns the result of an imp.find_module(), ie, (file, filename, smt)
       name can be a module or a package name.  It is *not* a filename."""

    packages = string.split(name, '.')
    for p in packages :
        # smt = (suffix, mode, type)
        file, filename, smt = imp.find_module(p, path)
        if smt[-1] == imp.PKG_DIRECTORY :
            try :
                # package found - read path info from init file
                m = imp.load_module(p, file, filename, smt)
            finally :
                if file is not None :
                    file.close()

            # importing xml plays a trick, which replaces itself with _xmlplus
            # both have subdirs w/same name, but different modules in them
            # we need to choose the real (replaced) version
            if m.__name__ != p :
                try :
                    file, filename, smt = imp.find_module(m.__name__, path)
                    m = imp.load_module(p, file, filename, smt)
                finally :
                    if file is not None :
                        file.close()

	    new_path = m.__path__
	    if type(new_path) == types.ListType :
	        new_path = filename
            if new_path not in path :
                path.insert(1, new_path)
        else:
            if p is not packages[-1] :
                if file is not None :
                    file.close()
                raise ImportError, "No module named %s" % packages[-1]
            return file, filename, smt

    # in case we have been given a package to check
    return file, filename, smt

class Variable :
    "Class to hold all information about a variable"

    def __init__(self, name, type):
        self.name = name
        self.type = type
        self.value = None

def _filterDir(object, ignoreList) :
    "Return a list of tokens (attributes) in a class, except for ignoreList"

    tokens = dir(object)
    for token in ignoreList :
        if token in tokens :
            tokens.remove(token)
    return tokens

def _getClassTokens(c) :
    return _filterDir(c, _DEFAULT_CLASS_TOKENS)

class Class :
    "Class to hold all information about a class"

    def __init__(self, name, module) :
        self.name = name
        self.module = module
        self.classObject = getattr(module, name)
        self.ignoreAttrs = 0
        self.methods = {}
        self.members = { '__class__': types.ClassType,
                         '__doc__': types.StringType,
                         '__dict__': types.DictType, }

    def getFirstLine(self) :
        "Return first line we can find in THIS class, not any base classes"

        lineNums = []
        classDir = dir(self.classObject)
        for m in self.methods.values() :
            if m != None and m.function.func_code.co_name in classDir:
                lineNums.append(m.function.func_code.co_firstlineno)
        if lineNums :
            return min(lineNums)
        return 0

    def allBaseClasses(self, c = None) :
        "Return a list of all base classes for this class and it's subclasses"

        baseClasses = []
        if c == None :
            c = self.classObject
        for base in c.__bases__ :
            baseClasses = baseClasses + [ base ] + self.allBaseClasses(base)
        return baseClasses

    def __getMethodName(self, func_name, className = None) :
        if func_name[0:2] == '__' and func_name[-2:] != '__' :
            if className == None :
                className = self.name
            if className[0] != '_' :
                className = '_' + className
            func_name = className + func_name
        return func_name

    def addMethod(self, method, className = None, methodName = None) :
        if type(method) == types.StringType :
            self.methods[method] = None
            return
        if not hasattr(method, "func_name") :
            return

        if not methodName :
            methodName = self.__getMethodName(method.func_name, className)
        self.methods[methodName] = function.Function(method)

    def addMethods(self, classObject) :
        for classToken in _getClassTokens(classObject) :
            token = getattr(classObject, classToken)
            if type(token) == types.MethodType :
                self.addMethod(token.im_func, classObject.__name__, classToken)
            else :
                self.members[classToken] = type(token)

        # add standard methods
        for methodName in ('__class__',) :
            self.addMethod(methodName, classObject.__name__)

    def addMembers(self, classObject) :
        if not _cfg.onlyCheckInitForMembers :
            for classToken in _getClassTokens(classObject) :
                method = getattr(classObject, classToken)
                if type(method) == types.MethodType :
                    self.addMembersFromMethod(method.im_func)
        elif hasattr(classObject, "__init__") :
            self.addMembersFromMethod(classObject.__init__.im_func)

    def addMembersFromMethod(self, method) :
        if not hasattr(method, 'func_code') :
            return

        func_code, code, i, maxCode, extended_arg = OP.initFuncCode(method)
        stack = []
        while i < maxCode :
            op, oparg, i, extended_arg = OP.getInfo(code, i, extended_arg)
            if op >= OP.HAVE_ARGUMENT :
                operand = OP.getOperand(op, func_code, oparg)
                if OP.LOAD_CONST(op) or OP.LOAD_FAST(op) :
                    stack.append(operand)
                elif OP.STORE_ATTR(op) :
                    if len(stack) > 0 :
                        if stack[-1] == _cfg.methodArgName :
                            value = None
                            if len(stack) > 1 :
                                value = type(stack[-2])
                            self.members[operand] = value
                        stack = []

def importError(moduleName, info):
    # detail may contain a newline replace with -
    # use str to avoid undestanding the tuple structure in the exception
    info = string.join(string.split(str(info), '\n' ), ' - ')
    sys.stderr.write("  Problem importing module %s - %s\n" % (moduleName, info))

class Module :
    "Class to hold all information for a module"

    def __init__(self, moduleName, check = 1) :
        self.moduleName = moduleName
        self.variables = {}
        self.functions = {}
        self.classes = {}
        self.modules = {}
        self.moduleLineNums = {}
        self.attributes = [ '__dict__' ]
        self.main_code = None
        self.module = None
        self.check = check
        global _allModules
        _allModules[moduleName] = self

    def addVariable(self, var, varType) :
        self.variables[var] = Variable(var, varType)

    def addFunction(self, func) :
        self.functions[func.__name__] = function.Function(func)

    def __addAttributes(self, c, classObject) :
        for base in classObject.__bases__ :
            self.__addAttributes(c, base)
        c.addMethods(classObject)
        c.addMembers(classObject)

    def addClass(self, name) :
        self.classes[name] = c = Class(name, self.module)
        packages = string.split(str(c.classObject), '.')
        c.ignoreAttrs = packages[0] in _cfg.blacklist
        if not c.ignoreAttrs :
            self.__addAttributes(c, c.classObject)

    def addModule(self, name) :
        module = _allModules.get(name, None)
        if module is None :
            self.modules[name] = module = Module(name, 0)
            if imp.is_builtin(name) == 0 :
                module.load()
            else :
                globalModule = globals().get(name)
                if globalModule :
                    module.attributes.extend(dir(globalModule))
        else :
            self.modules[name] = module

    def filename(self) :
        if not self.module :
            return self.moduleName
        filename = self.module.__file__
        if string.lower(filename[-4:]) == '.pyc' :
            filename = filename[:-4] + '.py'
        return filename

    def load(self) :
        try :
            # there's no need to reload modules we already have
            if sys.modules.has_key(self.moduleName) :
                if not _allModules[self.moduleName].module :
                    return self.initModule(sys.modules[self.moduleName])
                return 1

	    file, filename, smt = _findModule(self.moduleName)
            # FIXME: if the smt[-1] == imp.PKG_DIRECTORY : load __all__
            try :
                module = imp.load_module(self.moduleName, file, filename, smt)
                self.setupMainCode(file, filename, module)
            finally :
                if file != None :
                    file.close()
            return self.initModule(module)
        except (SystemExit, KeyboardInterrupt) :
            exc_type, exc_value, exc_tb = sys.exc_info()
            raise exc_type, exc_value
        except :
            return importError(self.moduleName, sys.exc_info()[1])

    def initModule(self, module) :
        self.module = module
        self.attributes = dir(self.module)
        for tokenName in _filterDir(self.module, _DEFAULT_MODULE_TOKENS) :
            token = getattr(self.module, tokenName)
            tokenType = type(token)
            if tokenType == types.ModuleType :
                # get the real module name, tokenName could be an alias
                moduleName = getattr(self.module, tokenName).__name__
                self.addModule(moduleName)
            elif tokenType == types.FunctionType :
                self.addFunction(token)
            elif tokenType == types.ClassType :
                self.addClass(tokenName)
            else :
                self.addVariable(tokenName, tokenType)

        return 1

    def setupMainCode(self, file, filename, module) :
        if file :
            self.main_code = function.create_from_file(file, filename, module)

def getAllModules() :
    "Returns a list of all modules that should be checked."
    modules = []
    for module in _allModules.values() :
        if module.check :
            modules.append(module)
    return modules

_BUILTIN_MODULE_ATTRS = { 'sys': [ 'ps1', 'ps2', 'tracebacklimit',
                                   'exc_type', 'exc_value', 'exc_traceback',
                                   'last_type', 'last_value', 'last_traceback',
                                 ],
                        }

def fixupBuiltinModules() :
    for moduleName in sys.builtin_module_names :
        module = _allModules.get(moduleName, None)
        if module is not None :
            try :
                m = imp.init_builtin(moduleName)
            except ImportError :
                pass
            else :
                extra_attrs = _BUILTIN_MODULE_ATTRS.get(moduleName, [])
                module.attributes = [ '__dict__' ] + dir(m) + extra_attrs

def _printWarnings(warnings, stream = sys.stdout) :
    warnings.sort()
    lastWarning = None
    for warning in warnings :
        if lastWarning != None :
            # ignore duplicate warnings
            if cmp(lastWarning, warning) == 0 :
                continue
            # print blank line between files
            if lastWarning.file != warning.file :
                stream.write("\n")

        lastWarning = warning
        warning.output(stream)

def processFiles(files, cfg = None, pre_process_cb = None) :
    # insert this here, so we find files in the local dir before std library
    if sys.path[0] != '' :
        sys.path.insert(0, '')

    # ensure we have a config object, it's necessary
    global _cfg
    if cfg is not None :
        _cfg = cfg
    elif _cfg is None :
        _cfg = Config.Config()

    warnings = []
    for moduleName in getModules(files) :
        if callable(pre_process_cb) :
            pre_process_cb(moduleName)
        module = Module(moduleName)
        if not module.load() :
            w = Warning(module.filename(), 1, "NOT PROCESSED UNABLE TO IMPORT")
            warnings.append(w)
    return warnings

def getWarnings(files, cfg = None, suppressions = None):
    warnings = processFiles(files, cfg)
    fixupBuiltinModules()
    return warnings + warn.find(getAllModules(), _cfg, suppressions)

def _print_processing(name) :
    if not _cfg.quiet :
        sys.stderr.write("Processing %s...\n" % name)

def main(argv) :
    import pychecker
    if LOCAL_MAIN_VERSION != pychecker.MAIN_MODULE_VERSION :
        sys.stderr.write(_VERSION_MISMATCH_ERROR)
        sys.exit(100)

    global _cfg
    _cfg, files, suppressions = Config.setupFromArgs(argv[1:])
    if not files :
        return 0

    # insert this here, so we find files in the local dir before std library
    sys.path.insert(0, '')

    importWarnings = processFiles(files, _cfg, _print_processing)
    fixupBuiltinModules()
    if _cfg.printParse :
        for module in getAllModules() :
            printer.module(module)

    warnings = warn.find(getAllModules(), _cfg, suppressions)
    if not _cfg.quiet :
        print "\nWarnings...\n"
    if warnings or importWarnings :
        _printWarnings(importWarnings + warnings)
        return 1

    if not _cfg.quiet :
        print "None"
    return 0

if __name__ == '__main__' :
    try :
        sys.exit(main(sys.argv))
    except Config.UsageError :
        sys.exit(127)

<commit_msg>we already have the token, no need to get it again<commit_after>#!/usr/bin/env python

# Copyright (c) 2001, MetaSlash Inc.  All rights reserved.

"""
Check python source code files for possible errors and print warnings

Contact Info:
  http://pychecker.sourceforge.net/
  pychecker@metaslash.com
"""

import string
import types
import sys
import imp
import os
import glob

# see __init__.py for meaning, this must match the version there
LOCAL_MAIN_VERSION = 1

def setupNamespace(path) :
    # remove pychecker if it's the first component, it needs to be last
    if sys.path[0][-9:] == 'pychecker' :
        del sys.path[0]

    # make sure pychecker is last in path, so we can import
    checker_path = os.path.dirname(os.path.dirname(path))
    if checker_path not in sys.path :
        sys.path.append(checker_path)

if __name__ == '__main__' :
    setupNamespace(sys.argv[0])

from pychecker import printer
from pychecker import warn
from pychecker import OP
from pychecker import Config
from pychecker import function
from pychecker.Warning import Warning

# Globals for storing a dictionary of info about modules and classes
_allModules = {}
_cfg = None

# Constants
_DEFAULT_MODULE_TOKENS = ('__builtins__', '__doc__', '__file__', '__name__',
                          '__path__')
_DEFAULT_CLASS_TOKENS = ('__doc__', '__name__', '__module__')

_VERSION_MISMATCH_ERROR = '''
There seem to be two versions of PyChecker being used.
One is probably in python/site-packages, the other in a local directory.
If you want to run the local version, you must remove the version
from site-packages.  Or you can install the current version
by doing python setup.py install.
'''

def _flattenList(list) :
    "Returns a list which contains no lists"

    new_list = []
    for element in list :
        if type(element) == types.ListType :
            new_list.extend(_flattenList(element))
        else :
            new_list.append(element)

    return new_list

def getModules(arg_list) :
    "Returns a list of module names that can be imported"

    new_arguments = []
    for arg in arg_list :
        # is this a wildcard filespec? (necessary for windows)
        if '*' in arg or '?' in arg or '[' in arg :
            arg = glob.glob(arg)
        new_arguments.append(arg)

    PY_SUFFIX = '.py'
    PY_SUFFIX_LEN = len(PY_SUFFIX)

    modules = []
    for arg in _flattenList(new_arguments) :
        # is it a .py file?
        if len(arg) > PY_SUFFIX_LEN and arg[-PY_SUFFIX_LEN:] == PY_SUFFIX:
            arg_dir = os.path.dirname(arg)
            if arg_dir and not os.path.exists(arg) :
                print 'File or pathname element does not exist: "%s"' % arg
                continue

            module_name = os.path.basename(arg)[:-PY_SUFFIX_LEN]
            if arg_dir not in sys.path :
                sys.path.insert(1, arg_dir)
	    arg = module_name
        modules.append(arg)

    return modules

def _findModule(name, path = sys.path) :
    """Returns the result of an imp.find_module(), ie, (file, filename, smt)
       name can be a module or a package name.  It is *not* a filename."""

    packages = string.split(name, '.')
    for p in packages :
        # smt = (suffix, mode, type)
        file, filename, smt = imp.find_module(p, path)
        if smt[-1] == imp.PKG_DIRECTORY :
            try :
                # package found - read path info from init file
                m = imp.load_module(p, file, filename, smt)
            finally :
                if file is not None :
                    file.close()

            # importing xml plays a trick, which replaces itself with _xmlplus
            # both have subdirs w/same name, but different modules in them
            # we need to choose the real (replaced) version
            if m.__name__ != p :
                try :
                    file, filename, smt = imp.find_module(m.__name__, path)
                    m = imp.load_module(p, file, filename, smt)
                finally :
                    if file is not None :
                        file.close()

	    new_path = m.__path__
	    if type(new_path) == types.ListType :
	        new_path = filename
            if new_path not in path :
                path.insert(1, new_path)
        else:
            if p is not packages[-1] :
                if file is not None :
                    file.close()
                raise ImportError, "No module named %s" % packages[-1]
            return file, filename, smt

    # in case we have been given a package to check
    return file, filename, smt

class Variable :
    "Class to hold all information about a variable"

    def __init__(self, name, type):
        self.name = name
        self.type = type
        self.value = None

def _filterDir(object, ignoreList) :
    "Return a list of tokens (attributes) in a class, except for ignoreList"

    tokens = dir(object)
    for token in ignoreList :
        if token in tokens :
            tokens.remove(token)
    return tokens

def _getClassTokens(c) :
    return _filterDir(c, _DEFAULT_CLASS_TOKENS)

class Class :
    "Class to hold all information about a class"

    def __init__(self, name, module) :
        self.name = name
        self.module = module
        self.classObject = getattr(module, name)
        self.ignoreAttrs = 0
        self.methods = {}
        self.members = { '__class__': types.ClassType,
                         '__doc__': types.StringType,
                         '__dict__': types.DictType, }

    def getFirstLine(self) :
        "Return first line we can find in THIS class, not any base classes"

        lineNums = []
        classDir = dir(self.classObject)
        for m in self.methods.values() :
            if m != None and m.function.func_code.co_name in classDir:
                lineNums.append(m.function.func_code.co_firstlineno)
        if lineNums :
            return min(lineNums)
        return 0

    def allBaseClasses(self, c = None) :
        "Return a list of all base classes for this class and it's subclasses"

        baseClasses = []
        if c == None :
            c = self.classObject
        for base in c.__bases__ :
            baseClasses = baseClasses + [ base ] + self.allBaseClasses(base)
        return baseClasses

    def __getMethodName(self, func_name, className = None) :
        if func_name[0:2] == '__' and func_name[-2:] != '__' :
            if className == None :
                className = self.name
            if className[0] != '_' :
                className = '_' + className
            func_name = className + func_name
        return func_name

    def addMethod(self, method, className = None, methodName = None) :
        if type(method) == types.StringType :
            self.methods[method] = None
            return
        if not hasattr(method, "func_name") :
            return

        if not methodName :
            methodName = self.__getMethodName(method.func_name, className)
        self.methods[methodName] = function.Function(method)

    def addMethods(self, classObject) :
        for classToken in _getClassTokens(classObject) :
            token = getattr(classObject, classToken)
            if type(token) == types.MethodType :
                self.addMethod(token.im_func, classObject.__name__, classToken)
            else :
                self.members[classToken] = type(token)

        # add standard methods
        for methodName in ('__class__',) :
            self.addMethod(methodName, classObject.__name__)

    def addMembers(self, classObject) :
        if not _cfg.onlyCheckInitForMembers :
            for classToken in _getClassTokens(classObject) :
                method = getattr(classObject, classToken)
                if type(method) == types.MethodType :
                    self.addMembersFromMethod(method.im_func)
        elif hasattr(classObject, "__init__") :
            self.addMembersFromMethod(classObject.__init__.im_func)

    def addMembersFromMethod(self, method) :
        if not hasattr(method, 'func_code') :
            return

        func_code, code, i, maxCode, extended_arg = OP.initFuncCode(method)
        stack = []
        while i < maxCode :
            op, oparg, i, extended_arg = OP.getInfo(code, i, extended_arg)
            if op >= OP.HAVE_ARGUMENT :
                operand = OP.getOperand(op, func_code, oparg)
                if OP.LOAD_CONST(op) or OP.LOAD_FAST(op) :
                    stack.append(operand)
                elif OP.STORE_ATTR(op) :
                    if len(stack) > 0 :
                        if stack[-1] == _cfg.methodArgName :
                            value = None
                            if len(stack) > 1 :
                                value = type(stack[-2])
                            self.members[operand] = value
                        stack = []

def importError(moduleName, info):
    # detail may contain a newline replace with -
    # use str to avoid undestanding the tuple structure in the exception
    info = string.join(string.split(str(info), '\n' ), ' - ')
    sys.stderr.write("  Problem importing module %s - %s\n" % (moduleName, info))

class Module :
    "Class to hold all information for a module"

    def __init__(self, moduleName, check = 1) :
        self.moduleName = moduleName
        self.variables = {}
        self.functions = {}
        self.classes = {}
        self.modules = {}
        self.moduleLineNums = {}
        self.attributes = [ '__dict__' ]
        self.main_code = None
        self.module = None
        self.check = check
        global _allModules
        _allModules[moduleName] = self

    def addVariable(self, var, varType) :
        self.variables[var] = Variable(var, varType)

    def addFunction(self, func) :
        self.functions[func.__name__] = function.Function(func)

    def __addAttributes(self, c, classObject) :
        for base in classObject.__bases__ :
            self.__addAttributes(c, base)
        c.addMethods(classObject)
        c.addMembers(classObject)

    def addClass(self, name) :
        self.classes[name] = c = Class(name, self.module)
        packages = string.split(str(c.classObject), '.')
        c.ignoreAttrs = packages[0] in _cfg.blacklist
        if not c.ignoreAttrs :
            self.__addAttributes(c, c.classObject)

    def addModule(self, name) :
        module = _allModules.get(name, None)
        if module is None :
            self.modules[name] = module = Module(name, 0)
            if imp.is_builtin(name) == 0 :
                module.load()
            else :
                globalModule = globals().get(name)
                if globalModule :
                    module.attributes.extend(dir(globalModule))
        else :
            self.modules[name] = module

    def filename(self) :
        if not self.module :
            return self.moduleName
        filename = self.module.__file__
        if string.lower(filename[-4:]) == '.pyc' :
            filename = filename[:-4] + '.py'
        return filename

    def load(self) :
        try :
            # there's no need to reload modules we already have
            if sys.modules.has_key(self.moduleName) :
                if not _allModules[self.moduleName].module :
                    return self.initModule(sys.modules[self.moduleName])
                return 1

	    file, filename, smt = _findModule(self.moduleName)
            # FIXME: if the smt[-1] == imp.PKG_DIRECTORY : load __all__
            try :
                module = imp.load_module(self.moduleName, file, filename, smt)
                self.setupMainCode(file, filename, module)
            finally :
                if file != None :
                    file.close()
            return self.initModule(module)
        except (SystemExit, KeyboardInterrupt) :
            exc_type, exc_value, exc_tb = sys.exc_info()
            raise exc_type, exc_value
        except :
            return importError(self.moduleName, sys.exc_info()[1])

    def initModule(self, module) :
        self.module = module
        self.attributes = dir(self.module)
        for tokenName in _filterDir(self.module, _DEFAULT_MODULE_TOKENS) :
            token = getattr(self.module, tokenName)
            tokenType = type(token)
            if tokenType == types.ModuleType :
                # get the real module name, tokenName could be an alias
                self.addModule(token.__name__)
            elif tokenType == types.FunctionType :
                self.addFunction(token)
            elif tokenType == types.ClassType :
                self.addClass(tokenName)
            else :
                self.addVariable(tokenName, tokenType)

        return 1

    def setupMainCode(self, file, filename, module) :
        if file :
            self.main_code = function.create_from_file(file, filename, module)

def getAllModules() :
    "Returns a list of all modules that should be checked."
    modules = []
    for module in _allModules.values() :
        if module.check :
            modules.append(module)
    return modules

_BUILTIN_MODULE_ATTRS = { 'sys': [ 'ps1', 'ps2', 'tracebacklimit',
                                   'exc_type', 'exc_value', 'exc_traceback',
                                   'last_type', 'last_value', 'last_traceback',
                                 ],
                        }

def fixupBuiltinModules() :
    for moduleName in sys.builtin_module_names :
        module = _allModules.get(moduleName, None)
        if module is not None :
            try :
                m = imp.init_builtin(moduleName)
            except ImportError :
                pass
            else :
                extra_attrs = _BUILTIN_MODULE_ATTRS.get(moduleName, [])
                module.attributes = [ '__dict__' ] + dir(m) + extra_attrs

def _printWarnings(warnings, stream = sys.stdout) :
    warnings.sort()
    lastWarning = None
    for warning in warnings :
        if lastWarning != None :
            # ignore duplicate warnings
            if cmp(lastWarning, warning) == 0 :
                continue
            # print blank line between files
            if lastWarning.file != warning.file :
                stream.write("\n")

        lastWarning = warning
        warning.output(stream)

def processFiles(files, cfg = None, pre_process_cb = None) :
    # insert this here, so we find files in the local dir before std library
    if sys.path[0] != '' :
        sys.path.insert(0, '')

    # ensure we have a config object, it's necessary
    global _cfg
    if cfg is not None :
        _cfg = cfg
    elif _cfg is None :
        _cfg = Config.Config()

    warnings = []
    for moduleName in getModules(files) :
        if callable(pre_process_cb) :
            pre_process_cb(moduleName)
        module = Module(moduleName)
        if not module.load() :
            w = Warning(module.filename(), 1, "NOT PROCESSED UNABLE TO IMPORT")
            warnings.append(w)
    return warnings

def getWarnings(files, cfg = None, suppressions = None):
    warnings = processFiles(files, cfg)
    fixupBuiltinModules()
    return warnings + warn.find(getAllModules(), _cfg, suppressions)

def _print_processing(name) :
    if not _cfg.quiet :
        sys.stderr.write("Processing %s...\n" % name)

def main(argv) :
    import pychecker
    if LOCAL_MAIN_VERSION != pychecker.MAIN_MODULE_VERSION :
        sys.stderr.write(_VERSION_MISMATCH_ERROR)
        sys.exit(100)

    global _cfg
    _cfg, files, suppressions = Config.setupFromArgs(argv[1:])
    if not files :
        return 0

    # insert this here, so we find files in the local dir before std library
    sys.path.insert(0, '')

    importWarnings = processFiles(files, _cfg, _print_processing)
    fixupBuiltinModules()
    if _cfg.printParse :
        for module in getAllModules() :
            printer.module(module)

    warnings = warn.find(getAllModules(), _cfg, suppressions)
    if not _cfg.quiet :
        print "\nWarnings...\n"
    if warnings or importWarnings :
        _printWarnings(importWarnings + warnings)
        return 1

    if not _cfg.quiet :
        print "None"
    return 0

if __name__ == '__main__' :
    try :
        sys.exit(main(sys.argv))
    except Config.UsageError :
        sys.exit(127)

<|endoftext|>


<commit_before># The MIT License (MIT)
#
# Copyright (c) 2014 Richard Moore
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.

import random
import time
import sys

from .basenode import BaseNode

from .. import blockchain
from .. import coins
from .. import protocol

class Node(BaseNode):

    # maximum number of pending getdata requests for a peer to have in-flight
    MAX_INCOMPLETE_INFLIGHT = 10000

    # maximum number of incomplete blocks to track at a time (ish)
    MAX_INCOMPLETE_BLOCKS = 50000

    # maximum query limit for incomplete blocks
    MAX_INCOMPLETE_FETCH = 10000

    def __init__(self, data_dir = None, address = None, seek_peers = 16, max_peers = 125, bootstrap = True, log = sys.stdout, coin = coins.Bitcoin):
        BaseNode.__init__(self, data_dir, address, seek_peers, max_peers, bootstrap, log, coin)

        # blockchain database
        self._blocks = blockchain.block.Database(self.data_dir, self._coin)
        self._txns = self._blocks._txns

        # memory pool; circular buffer of 30,000 most recent seen transactions
        self._mempool_index = 0
        self._mempool = []
        self._prime_mempool()

        # how long since we last asked for headers or blocks
        self._last_get_headers = 0

        # maps blockhash to last request time
        self._incomplete_blocks = dict()
        self._last_incomplete_block = None

        # how many in-flight block requests per peer
        self._inflight_blocks = dict()

        # last time headers were requested from a peer
        self._inflight_headers = dict()

    @property
    def blockchain_height(self):
        return self._blocks[-1].height

    def _prime_mempool(self):
        # @TODO: on start-up pull in last couple of blocks' transactions
        pass

    def _add_mempool(self, txn):
        if len(self._mempool) > 30000:
            self._mempool[self._mempool_index] = txn
            self._mempool_index = (self._mempool_index + 1) % 30000
        else:
            self._mempool.append(txn)

    def _search_mempool(self, txid):
         txns = [t for t in self._mempool if t.hash == txid]
         if txns:
             return txns[0]
         return None

    def command_block(self, peer, version, prev_block, merkle_root, timestamp, bits, nonce, txns):

        try:
            # get the block
            header = protocol.BlockHeader(version, prev_block, merkle_root,
                                          timestamp, bits, nonce, 0)
            block = self._blocks.get(header.hash)
            if not block:
                raise blockchain.block.InvalidBlockException('block header not found')

            # add the transactions
            self._txns.add(block, txns)

            # update the memory pool
            for txn in txns:
                self._add_mempool(txn)

            # it is no longer incomplete
            if block.hash in self._incomplete_blocks:
                del self._incomplete_blocks[block.hash]

        except blockchain.block.InvalidBlockException, e:
            self.log('invalid block header: %s (%s)' % (header.hash.encode('hex'), e.message), level = self.LOG_LEVEL_DEBUG)
            self.punish_peer(peer, str(e))

        # give the peer more room to request blocks
        if peer in self._inflight_blocks:
            self._inflight_blocks[peer] -= 1
            if self._inflight_blocks[peer] < 0:
                self._inflight_blocks[peer] = 0

    def command_get_blocks(self, peer, version, block_locator_hashes, hash_stop):
        blocks = self._blocks.locate_blocks(block_locator_hashes, 500, hash_stop)

        # we found their place on the blockchain
        if blocks:
            inv = [protocol.InventoryVector(protocol.OBJECT_TYPE_MSG_BLOCK, b.hash) for b in blocks]
            self.send_message(protocol.Inventory(inv))

        # we didn't find anything that matched their locator... What to do? not_found?
        else:
            self.send_message(protocol.NotFound(block_locator_hashes))

    def command_get_data(self, peer, inventory):

        # look up each block and transaction requested
        notfound = [ ]
        for iv in inventory:

            if iv.type == protocol.OBJECT_TYPE_MSG_BLOCK:

                # search the database
                block = self._blocks.get(iv.hash)

                # if we found one, return it
                if block:
                    peer.send_message(protocol.Block(block.block_message()))
                else:
                    notfound.append(iv)

            elif iv.type == protocol.OBJECT_TYPE_MSG_TX:

                # search the memory pool and database
                txn = self._search_mempool(iv.hash)
                if not txn:
                    tx = self._txns.get(iv.hash)
                    if tx: txn = tx.txn

                # if we found one, return it
                if txn:
                    peer.send_message(txn)
                else:
                    notfound.append(iv)

        # anything not found?
        if notfound:
            peer.send_message(protocol.NotFound(notfound))

    def command_get_headers(self, peer, version, block_locator_hashes, hash_stop):
        # Send the list of headers
        blocks = self._blocks.locate_blocks(block_locator_hashes, 2000, hash_stop)
        peer.send_message(protocol.Headers([protocol.BlockHeader.from_block(b) for b in blocks]))

    def command_headers(self, peer, headers):

        # no longer a get_header in-flight for this peer
        if peer in self._inflight_headers:
            del self._inflight_headers[peer]

        # nothing to do
        if len(headers) == 0: return

        # Add the headers to the database (we fill in the transactions later)
        new_headers = False
        for header in headers:
            try:
                added = self._blocks.add_header(header)
                if added:
                    new_headers = True
                else:
                    self.log('block header already exists: %s' % header.hash.encode('hex'), level = self.LOG_LEVEL_DEBUG)
            except blockchain.block.InvalidBlockException, e:
                self.log('invalid block header: %s (%s)' % (header.hash.encode('hex'), e.message), level = self.LOG_LEVEL_DEBUG)
                self.punish_peer(peer, str(e))

        # we got some headers, so we can request the next batch now
        self.sync_blockchain_headers(new_headers = new_headers)

    def command_inventory(self, peer, inventory):
        pass

        # look for new blocks being advertised by peers
        #def useful_block(iv):
        #    if iv.object_type != protocol.OBJECT_TYPE_MSG_BLOCK:
        #        return False
        #    if self._blockchain.get(iv.hash):
        #        return False
        #    return True

        #block_inventory = [iv for iv in inventory if useful_block(iv)]

        # @TODO:
        # new block
        #if block_inventory:
        #    peer.send_message(protocol.GetData(block_inventory))

    def command_memory_pool(self, peer):
        inv = [protocol.InventoryVector(protocol.OBJECT_TYPE_MSG_TX, t.hash) for t in self._mempool]
        per.send_message(inv)

    def command_not_found(self, peer, inventory):

        # the peer did not have the blocks we were looking for, so we can send more
        block_count = len(b for b in inventory if b.object_type == protocol.OBJECT_TYPE_MSG_BLOCK)
        if peer in self._inflight_blocks:
            self._inflight_blocks[peer] -= block_count
            if self._inflight_blocks[peer] < 0:
                self._inflight_blocks[peer] = 0

    def command_version_ack(self, peer):
        BaseNode.command_version_ack(self, peer)

        # might be the first peer, see if we can sync some blockchain
        self.sync_blockchain_headers()
        self.sync_blockchain_blocks()

    def disconnected(self, peer):
        'Called by a peer after it has been closed.'

        BaseNode.disconnected(self, peer)

        if peer in self._inflight_blocks:
            del self._inflight_blocks[peer]

    def heartbeat(self):
        BaseNode.heartbeat(self)

        # if we have peers, poke them to sync the blockchain
        if self.peers:
            self.sync_blockchain_headers()
            self.sync_blockchain_blocks()

    def close(self):
        self._blocks.close()
        BaseNode.close(self)

    def sync_blockchain_headers(self, new_headers = False):

        # give getheaders at least 30 seconds to respond (new_headers means
        # it already did and we are ready to ask for more)
        if not new_headers and time.time() - self._last_get_headers < 30:
            return
        self._last_get_headers = time.time()

        # after 15 minutes, let's assume they forgot (but with a slap on the wrist)
        now = time.time()
        for peer in list(self._inflight_headers):
            if now - self._inflight_headers[peer] > 900:
                del self._inflight_headers[peer]
                self.punish_peer(peer, 'no response for get_headers')

        # pick a peer that's ready
        peers = [p for p in self.peers if (p.verack and p not in self._inflight_headers)]
        if not peers: return
        peer = random.choice(peers)
        self._inflight_headers[peer] = now

        # request the next block headers (if any)
        locator = self._blocks.block_locator_hashes()
        getheaders = protocol.GetHeaders(self.coin.protocol_version, locator, chr(0) * 32)
        peer.send_message(getheaders)

    def sync_blockchain_blocks(self):

        # we can handle more incomplete blocks
        if len(self._incomplete_blocks) < self.MAX_INCOMPLETE_BLOCKS:
            incomplete = self._blocks.incomplete_blocks(from_block = self._last_incomplete_block, max_count = self.MAX_INCOMPLETE_FETCH)
            if incomplete:
                for block in incomplete:
                    if block.hash in self._incomplete_blocks: continue
                    self._incomplete_blocks[block.hash] = 0
                self._last_incomplete_block = incomplete[-1]

        # we have incomplete blocks, so request data from our peers
        if self._incomplete_blocks:
            now = time.time()

            peers = [p for p in self.peers if p.verack]
            random.shuffle(peers)
            for peer in peers:

                # this peer is already full
                inflight = self._inflight_blocks.get(peer, 0)
                if inflight >= self.MAX_INCOMPLETE_INFLIGHT:
                    continue

                # find some not-recently-requested blocks (over 15 minutes ago)
                getdata = []
                for hash in self._incomplete_blocks:
                    if now - self._incomplete_blocks[hash] < 900:
                        continue
                    self._incomplete_blocks[hash] = now

                    getdata.append(protocol.InventoryVector(protocol.OBJECT_TYPE_MSG_BLOCK, hash))
                    if len(getdata) + inflight >= self.MAX_INCOMPLETE_INFLIGHT:
                        break

                # nothing to request
                if not getdata:
                    break

                # ask the peer
                peer.send_message(protocol.GetData(getdata))

                # track how many inflight block requests this peer has
                if peer not in self._inflight_blocks:
                    self._inflight_blocks[peer] = 0
                self._inflight_blocks[peer] += len(getdata)

<commit_msg>small memory pool issue (one entry could be immortal).<commit_after># The MIT License (MIT)
#
# Copyright (c) 2014 Richard Moore
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.

import random
import time
import sys

from .basenode import BaseNode

from .. import blockchain
from .. import coins
from .. import protocol

class Node(BaseNode):

    # maximum number of pending getdata requests for a peer to have in-flight
    MAX_INCOMPLETE_INFLIGHT = 10000

    # maximum number of incomplete blocks to track at a time (ish)
    MAX_INCOMPLETE_BLOCKS = 50000

    # maximum query limit for incomplete blocks
    MAX_INCOMPLETE_FETCH = 10000

    # maximum number of entries in the memory pool
    MEMORY_POOL_SIZE = 30000

    def __init__(self, data_dir = None, address = None, seek_peers = 16, max_peers = 125, bootstrap = True, log = sys.stdout, coin = coins.Bitcoin):
        BaseNode.__init__(self, data_dir, address, seek_peers, max_peers, bootstrap, log, coin)

        # blockchain database
        self._blocks = blockchain.block.Database(self.data_dir, self._coin)
        self._txns = self._blocks._txns

        # memory pool; circular buffer of 30,000 most recent seen transactions
        self._mempool_index = 0
        self._mempool = []
        self._prime_mempool()

        # how long since we last asked for headers or blocks
        self._last_get_headers = 0

        # maps blockhash to last request time
        self._incomplete_blocks = dict()
        self._last_incomplete_block = None

        # how many in-flight block requests per peer
        self._inflight_blocks = dict()

        # last time headers were requested from a peer
        self._inflight_headers = dict()

    @property
    def blockchain_height(self):
        return self._blocks[-1].height

    def _prime_mempool(self):
        # @TODO: on start-up pull in last couple of blocks' transactions
        pass

    def _add_mempool(self, txn):
        if len(self._mempool) >= self.MEMORY_POOL_SIZE:
            self._mempool[self._mempool_index] = txn
            self._mempool_index = (self._mempool_index + 1) % self.MEMORY_POOL_SIZE
        else:
            self._mempool.append(txn)

    def _search_mempool(self, txid):
         txns = [t for t in self._mempool if t.hash == txid]
         if txns:
             return txns[0]
         return None

    def command_block(self, peer, version, prev_block, merkle_root, timestamp, bits, nonce, txns):

        try:
            # get the block
            header = protocol.BlockHeader(version, prev_block, merkle_root,
                                          timestamp, bits, nonce, 0)
            block = self._blocks.get(header.hash)
            if not block:
                raise blockchain.block.InvalidBlockException('block header not found')

            # add the transactions
            self._txns.add(block, txns)

            # update the memory pool
            for txn in txns:
                self._add_mempool(txn)

            # it is no longer incomplete
            if block.hash in self._incomplete_blocks:
                del self._incomplete_blocks[block.hash]

        except blockchain.block.InvalidBlockException, e:
            self.log('invalid block header: %s (%s)' % (header.hash.encode('hex'), e.message), level = self.LOG_LEVEL_DEBUG)
            self.punish_peer(peer, str(e))

        # give the peer more room to request blocks
        if peer in self._inflight_blocks:
            self._inflight_blocks[peer] -= 1
            if self._inflight_blocks[peer] < 0:
                self._inflight_blocks[peer] = 0

    def command_get_blocks(self, peer, version, block_locator_hashes, hash_stop):
        blocks = self._blocks.locate_blocks(block_locator_hashes, 500, hash_stop)

        # we found their place on the blockchain
        if blocks:
            inv = [protocol.InventoryVector(protocol.OBJECT_TYPE_MSG_BLOCK, b.hash) for b in blocks]
            self.send_message(protocol.Inventory(inv))

        # we didn't find anything that matched their locator... What to do? not_found?
        else:
            self.send_message(protocol.NotFound(block_locator_hashes))

    def command_get_data(self, peer, inventory):

        # look up each block and transaction requested
        notfound = [ ]
        for iv in inventory:

            if iv.type == protocol.OBJECT_TYPE_MSG_BLOCK:

                # search the database
                block = self._blocks.get(iv.hash)

                # if we found one, return it
                if block:
                    peer.send_message(protocol.Block(block.block_message()))
                else:
                    notfound.append(iv)

            elif iv.type == protocol.OBJECT_TYPE_MSG_TX:

                # search the memory pool and database
                txn = self._search_mempool(iv.hash)
                if not txn:
                    tx = self._txns.get(iv.hash)
                    if tx: txn = tx.txn

                # if we found one, return it
                if txn:
                    peer.send_message(txn)
                else:
                    notfound.append(iv)

        # anything not found?
        if notfound:
            peer.send_message(protocol.NotFound(notfound))

    def command_get_headers(self, peer, version, block_locator_hashes, hash_stop):
        # Send the list of headers
        blocks = self._blocks.locate_blocks(block_locator_hashes, 2000, hash_stop)
        peer.send_message(protocol.Headers([protocol.BlockHeader.from_block(b) for b in blocks]))

    def command_headers(self, peer, headers):

        # no longer a get_header in-flight for this peer
        if peer in self._inflight_headers:
            del self._inflight_headers[peer]

        # nothing to do
        if len(headers) == 0: return

        # Add the headers to the database (we fill in the transactions later)
        new_headers = False
        for header in headers:
            try:
                added = self._blocks.add_header(header)
                if added:
                    new_headers = True
                else:
                    self.log('block header already exists: %s' % header.hash.encode('hex'), level = self.LOG_LEVEL_DEBUG)
            except blockchain.block.InvalidBlockException, e:
                self.log('invalid block header: %s (%s)' % (header.hash.encode('hex'), e.message), level = self.LOG_LEVEL_DEBUG)
                self.punish_peer(peer, str(e))

        # we got some headers, so we can request the next batch now
        self.sync_blockchain_headers(new_headers = new_headers)

    def command_inventory(self, peer, inventory):
        pass

        # look for new blocks being advertised by peers
        #def useful_block(iv):
        #    if iv.object_type != protocol.OBJECT_TYPE_MSG_BLOCK:
        #        return False
        #    if self._blockchain.get(iv.hash):
        #        return False
        #    return True

        #block_inventory = [iv for iv in inventory if useful_block(iv)]

        # @TODO:
        # new block
        #if block_inventory:
        #    peer.send_message(protocol.GetData(block_inventory))

    def command_memory_pool(self, peer):
        inv = [protocol.InventoryVector(protocol.OBJECT_TYPE_MSG_TX, t.hash) for t in self._mempool]
        per.send_message(inv)

    def command_not_found(self, peer, inventory):

        # the peer did not have the blocks we were looking for, so we can send more
        block_count = len(b for b in inventory if b.object_type == protocol.OBJECT_TYPE_MSG_BLOCK)
        if peer in self._inflight_blocks:
            self._inflight_blocks[peer] -= block_count
            if self._inflight_blocks[peer] < 0:
                self._inflight_blocks[peer] = 0

    def command_version_ack(self, peer):
        BaseNode.command_version_ack(self, peer)

        # might be the first peer, see if we can sync some blockchain
        self.sync_blockchain_headers()
        self.sync_blockchain_blocks()

    def disconnected(self, peer):
        'Called by a peer after it has been closed.'

        BaseNode.disconnected(self, peer)

        if peer in self._inflight_blocks:
            del self._inflight_blocks[peer]

    def heartbeat(self):
        BaseNode.heartbeat(self)

        # if we have peers, poke them to sync the blockchain
        if self.peers:
            self.sync_blockchain_headers()
            self.sync_blockchain_blocks()

    def close(self):
        self._blocks.close()
        BaseNode.close(self)

    def sync_blockchain_headers(self, new_headers = False):

        # give getheaders at least 30 seconds to respond (new_headers means
        # it already did and we are ready to ask for more)
        if not new_headers and time.time() - self._last_get_headers < 30:
            return
        self._last_get_headers = time.time()

        # after 15 minutes, let's assume they forgot (but with a slap on the wrist)
        now = time.time()
        for peer in list(self._inflight_headers):
            if now - self._inflight_headers[peer] > 900:
                del self._inflight_headers[peer]
                self.punish_peer(peer, 'no response for get_headers')

        # pick a peer that's ready
        peers = [p for p in self.peers if (p.verack and p not in self._inflight_headers)]
        if not peers: return
        peer = random.choice(peers)
        self._inflight_headers[peer] = now

        # request the next block headers (if any)
        locator = self._blocks.block_locator_hashes()
        getheaders = protocol.GetHeaders(self.coin.protocol_version, locator, chr(0) * 32)
        peer.send_message(getheaders)

    def sync_blockchain_blocks(self):

        # we can handle more incomplete blocks
        if len(self._incomplete_blocks) < self.MAX_INCOMPLETE_BLOCKS:
            incomplete = self._blocks.incomplete_blocks(from_block = self._last_incomplete_block, max_count = self.MAX_INCOMPLETE_FETCH)
            if incomplete:
                for block in incomplete:
                    if block.hash in self._incomplete_blocks: continue
                    self._incomplete_blocks[block.hash] = 0
                self._last_incomplete_block = incomplete[-1]

        # we have incomplete blocks, so request data from our peers
        if self._incomplete_blocks:
            now = time.time()

            peers = [p for p in self.peers if p.verack]
            random.shuffle(peers)
            for peer in peers:

                # this peer is already full
                inflight = self._inflight_blocks.get(peer, 0)
                if inflight >= self.MAX_INCOMPLETE_INFLIGHT:
                    continue

                # find some not-recently-requested blocks (over 15 minutes ago)
                getdata = []
                for hash in self._incomplete_blocks:
                    if now - self._incomplete_blocks[hash] < 900:
                        continue
                    self._incomplete_blocks[hash] = now

                    getdata.append(protocol.InventoryVector(protocol.OBJECT_TYPE_MSG_BLOCK, hash))
                    if len(getdata) + inflight >= self.MAX_INCOMPLETE_INFLIGHT:
                        break

                # nothing to request
                if not getdata:
                    break

                # ask the peer
                peer.send_message(protocol.GetData(getdata))

                # track how many inflight block requests this peer has
                if peer not in self._inflight_blocks:
                    self._inflight_blocks[peer] = 0
                self._inflight_blocks[peer] += len(getdata)

<|endoftext|>


<commit_before>__version__ = '0.12.0.dev0+git'
<commit_msg>Bump version<commit_after>__version__ = '0.13.0.dev0+git'
<|endoftext|>


<commit_before>'''
    This module provides utilities to get the absolute filenames so that we can be sure that:
        - The case of a file will match the actual file in the filesystem (otherwise breakpoints won't be hit).
        - Providing means for the user to make path conversions when doing a remote debugging session in
          one machine and debugging in another.

    To do that, the PATHS_FROM_ECLIPSE_TO_PYTHON constant must be filled with the appropriate paths.

    @note:
        in this context, the server is where your python process is running
        and the client is where eclipse is running.

    E.g.:
        If the server (your python process) has the structure
            /user/projects/my_project/src/package/module1.py

        and the client has:
            c:\my_project\src\package\module1.py

        the PATHS_FROM_ECLIPSE_TO_PYTHON would have to be:
            PATHS_FROM_ECLIPSE_TO_PYTHON = [(r'c:\my_project\src', r'/user/projects/my_project/src')]

    @note: DEBUG_CLIENT_SERVER_TRANSLATION can be set to True to debug the result of those translations

    @note: the case of the paths is important! Note that this can be tricky to get right when one machine
    uses a case-independent filesystem and the other uses a case-dependent filesystem (if the system being
    debugged is case-independent, 'normcase()' should be used on the paths defined in PATHS_FROM_ECLIPSE_TO_PYTHON).

    @note: all the paths with breakpoints must be translated (otherwise they won't be found in the server)

    @note: to enable remote debugging in the target machine (pydev extensions in the eclipse installation)
        import pydevd;pydevd.settrace(host, stdoutToServer, stderrToServer, port, suspend)

        see parameter docs on pydevd.py

    @note: for doing a remote debugging session, all the pydevd_ files must be on the server accessible
        through the PYTHONPATH (and the PATHS_FROM_ECLIPSE_TO_PYTHON only needs to be set on the target
        machine for the paths that'll actually have breakpoints).
'''

from _pydevd_bundle.pydevd_constants import IS_PY2, IS_PY3K
from _pydev_bundle._pydev_filesystem_encoding import getfilesystemencoding
import os.path
import sys
import traceback

os_normcase = os.path.normcase
basename = os.path.basename
exists = os.path.exists
join = os.path.join

try:
    rPath = os.path.realpath  #@UndefinedVariable
except:
    # jython does not support os.path.realpath
    # realpath is a no-op on systems without islink support
    rPath = os.path.abspath

#defined as a list of tuples where the 1st element of the tuple is the path in the client machine
#and the 2nd element is the path in the server machine.
#see module docstring for more details.
PATHS_FROM_ECLIPSE_TO_PYTHON = []

#example:
#PATHS_FROM_ECLIPSE_TO_PYTHON = [
#  (r'd:\temp\temp_workspace_2\test_python\src\yyy\yyy',
#   r'd:\temp\temp_workspace_2\test_python\src\hhh\xxx')
#]

normcase = os_normcase # May be rebound on set_ide_os

convert_to_long_pathname = None
if sys.platform == 'win32':
    try:
        import ctypes
    except ImportError:
        pass
    else:
        def convert_to_long_pathname(filename):
            buf = ctypes.create_unicode_buffer(260)
            GetLongPathName = ctypes.windll.kernel32.GetLongPathNameW
            if IS_PY2:
                filename = unicode(filename, getfilesystemencoding())
            rv = GetLongPathName(filename, buf, 260)
            if rv != 0 and rv <= 260:
                return buf.value
            return filename

def norm_case(filename):
    # `normcase` doesn't lower case on Python 2 for non-English locale, but Java side does it,
    # so we should do it manually
    if '~' in filename and convert_to_long_pathname:
        filename = convert_to_long_pathname(filename)

    filename = os_normcase(filename)
    enc = getfilesystemencoding()
    if IS_PY3K or enc is None or enc.lower() == "utf-8":
        return filename
    try:
        return filename.decode(enc).lower().encode(enc)
    except:
        return filename

def set_ide_os(os):
    '''
    We need to set the IDE os because the host where the code is running may be
    actually different from the client (and the point is that we want the proper
    paths to translate from the client to the server).
    '''
    global normcase
    if os == 'UNIX':
        normcase = lambda f:f #Change to no-op if the client side is on unix/mac.
    else:
        if sys.platform == 'win32':
            normcase = norm_case
        else:
            normcase = os_normcase

    # After setting the ide OS, apply the normcase to the existing paths.

    # Note: not using enumerate nor list comprehension because it may not be available in older python versions...
    i = 0
    for path in PATHS_FROM_ECLIPSE_TO_PYTHON[:]:
        PATHS_FROM_ECLIPSE_TO_PYTHON[i] = (normcase(path[0]), normcase(path[1]))
        i += 1

DEBUG_CLIENT_SERVER_TRANSLATION = False

#caches filled as requested during the debug session
NORM_PATHS_CONTAINER = {}
NORM_PATHS_AND_BASE_CONTAINER = {}
NORM_FILENAME_TO_SERVER_CONTAINER = {}
NORM_FILENAME_TO_CLIENT_CONTAINER = {}

def _NormFile(filename):
    abs_path, real_path = _NormPaths(filename)
    return real_path

def _AbsFile(filename):
    abs_path, real_path = _NormPaths(filename)
    return abs_path

# Returns tuple of absolute path and real path for given filename
def _NormPaths(filename):
    try:
        return NORM_PATHS_CONTAINER[filename]
    except KeyError:
        abs_path = _NormPath(filename, os.path.abspath)
        real_path = _NormPath(filename, rPath)

        NORM_PATHS_CONTAINER[filename] = abs_path, real_path
        return abs_path, real_path

def _NormPath(filename, normpath):
    r = normpath(filename)
    #cache it for fast access later
    ind = r.find('.zip')
    if ind == -1:
        ind = r.find('.egg')
    if ind != -1:
        ind+=4
        zip_path = r[:ind]
        if r[ind] == "!":
            ind+=1
        inner_path = r[ind:]
        if inner_path.startswith('/') or inner_path.startswith('\\'):
            inner_path = inner_path[1:]
        r = join(normcase(zip_path), inner_path)
    else:
        r = normcase(r)
    return r

ZIP_SEARCH_CACHE = {}
def exists(file):
    if os.path.exists(file):
        return file

    ind = file.find('.zip')
    if ind == -1:
        ind = file.find('.egg')

    if ind != -1:
        ind+=4
        zip_path = file[:ind]
        if file[ind] == "!":
            ind+=1
        inner_path = file[ind:]
        try:
            zip = ZIP_SEARCH_CACHE[zip_path]
        except KeyError:
            try:
                import zipfile
                zip = zipfile.ZipFile(zip_path, 'r')
                ZIP_SEARCH_CACHE[zip_path] = zip
            except :
                return None

        try:
            if inner_path.startswith('/') or inner_path.startswith('\\'):
                inner_path = inner_path[1:]

            info = zip.getinfo(inner_path.replace('\\', '/'))

            return join(zip_path, inner_path)
        except KeyError:
            return None
    return None

#Now, let's do a quick test to see if we're working with a version of python that has no problems
#related to the names generated...
try:
    try:
        code = rPath.func_code
    except AttributeError:
        code = rPath.__code__
    if not exists(_NormFile(code.co_filename)):
        sys.stderr.write('-------------------------------------------------------------------------------\n')
        sys.stderr.write('pydev debugger: CRITICAL WARNING: This version of python seems to be incorrectly compiled (internal generated filenames are not absolute)\n')
        sys.stderr.write('pydev debugger: The debugger may still function, but it will work slower and may miss breakpoints.\n')
        sys.stderr.write('pydev debugger: Related bug: http://bugs.python.org/issue1666807\n')
        sys.stderr.write('-------------------------------------------------------------------------------\n')
        sys.stderr.flush()

        NORM_SEARCH_CACHE = {}

        initial_norm_paths = _NormPaths
        def _NormPaths(filename):  #Let's redefine _NormPaths to work with paths that may be incorrect
            try:
                return NORM_SEARCH_CACHE[filename]
            except KeyError:
                abs_path, real_path = initial_norm_paths(filename)
                if not exists(real_path):
                    #We must actually go on and check if we can find it as if it was a relative path for some of the paths in the pythonpath
                    for path in sys.path:
                        abs_path, real_path = initial_norm_paths(join(path, filename))
                        if exists(real_path):
                            break
                    else:
                        sys.stderr.write('pydev debugger: Unable to find real location for: %s\n' % (filename,))
                        abs_path = filename
                        real_path = filename

                NORM_SEARCH_CACHE[filename] = abs_path, real_path
                return abs_path, real_path

except:
    #Don't fail if there's something not correct here -- but at least print it to the user so that we can correct that
    traceback.print_exc()

norm_file_to_client = _AbsFile
norm_file_to_server = _NormFile

def setup_client_server_paths(paths):
    '''paths is the same format as PATHS_FROM_ECLIPSE_TO_PYTHON'''

    global NORM_FILENAME_TO_SERVER_CONTAINER
    global NORM_FILENAME_TO_CLIENT_CONTAINER
    global PATHS_FROM_ECLIPSE_TO_PYTHON
    global norm_file_to_client
    global norm_file_to_server

    NORM_FILENAME_TO_SERVER_CONTAINER = {}
    NORM_FILENAME_TO_CLIENT_CONTAINER = {}
    PATHS_FROM_ECLIPSE_TO_PYTHON = paths[:]

    if not PATHS_FROM_ECLIPSE_TO_PYTHON:
        #no translation step needed (just inline the calls)
        norm_file_to_client = _AbsFile
        norm_file_to_server = _NormFile
        return

    #Work on the client and server slashes.
    eclipse_sep = None
    python_sep = None
    for eclipse_prefix, server_prefix in PATHS_FROM_ECLIPSE_TO_PYTHON:
        if eclipse_sep is not None and python_sep is not None:
            break

        if eclipse_sep is None:
            for c in eclipse_prefix:
                if c in ('/', '\\'):
                    eclipse_sep = c
                    break

        if python_sep is None:
            for c in server_prefix:
                if c in ('/', '\\'):
                    python_sep = c
                    break

    #If they're the same or one of them cannot be determined, just make it all None.
    if eclipse_sep == python_sep or eclipse_sep is None or python_sep is None:
        eclipse_sep = python_sep = None

    #only setup translation functions if absolutely needed!
    def _norm_file_to_server(filename):
        #Eclipse will send the passed filename to be translated to the python process
        #So, this would be 'NormFileFromEclipseToPython'
        try:
            return NORM_FILENAME_TO_SERVER_CONTAINER[filename]
        except KeyError:
            #used to translate a path from the client to the debug server
            translated = normcase(filename)
            for eclipse_prefix, server_prefix in PATHS_FROM_ECLIPSE_TO_PYTHON:
                if translated.startswith(eclipse_prefix):
                    if DEBUG_CLIENT_SERVER_TRANSLATION:
                        sys.stderr.write('pydev debugger: replacing to server: %s\n' % (translated,))
                    translated = translated.replace(eclipse_prefix, server_prefix)
                    if DEBUG_CLIENT_SERVER_TRANSLATION:
                        sys.stderr.write('pydev debugger: sent to server: %s\n' % (translated,))
                    break
            else:
                if DEBUG_CLIENT_SERVER_TRANSLATION:
                    sys.stderr.write('pydev debugger: to server: unable to find matching prefix for: %s in %s\n' % \
                        (translated, [x[0] for x in PATHS_FROM_ECLIPSE_TO_PYTHON]))

            #Note that when going to the server, we do the replace first and only later do the norm file.
            if eclipse_sep is not None:
                translated = translated.replace(eclipse_sep, python_sep)
            translated = _NormFile(translated)

            NORM_FILENAME_TO_SERVER_CONTAINER[filename] = translated
            return translated

    def _norm_file_to_client(filename):
        #The result of this method will be passed to eclipse
        #So, this would be 'NormFileFromPythonToEclipse'
        try:
            return NORM_FILENAME_TO_CLIENT_CONTAINER[filename]
        except KeyError:
            #used to translate a path from the debug server to the client
            translated = _NormFile(filename)
            for eclipse_prefix, python_prefix in PATHS_FROM_ECLIPSE_TO_PYTHON:
                if translated.startswith(python_prefix):
                    if DEBUG_CLIENT_SERVER_TRANSLATION:
                        sys.stderr.write('pydev debugger: replacing to client: %s\n' % (translated,))
                    translated = translated.replace(python_prefix, eclipse_prefix)
                    if DEBUG_CLIENT_SERVER_TRANSLATION:
                        sys.stderr.write('pydev debugger: sent to client: %s\n' % (translated,))
                    break
            else:
                if DEBUG_CLIENT_SERVER_TRANSLATION:
                    sys.stderr.write('pydev debugger: to client: unable to find matching prefix for: %s in %s\n' % \
                        (translated, [x[1] for x in PATHS_FROM_ECLIPSE_TO_PYTHON]))

            if eclipse_sep is not None:
                translated = translated.replace(python_sep, eclipse_sep)

            #The resulting path is not in the python process, so, we cannot do a _NormFile here,
            #only at the beginning of this method.
            NORM_FILENAME_TO_CLIENT_CONTAINER[filename] = translated
            return translated

    norm_file_to_server = _norm_file_to_server
    norm_file_to_client = _norm_file_to_client

setup_client_server_paths(PATHS_FROM_ECLIPSE_TO_PYTHON)

# For given file f returns tuple of its absolute path, real path and base name
def get_abs_path_real_path_and_base_from_file(f):
    try:
        return NORM_PATHS_AND_BASE_CONTAINER[f]
    except:
        abs_path, real_path = _NormPaths(f)
        base = basename(real_path)
        ret = abs_path, real_path, base
        NORM_PATHS_AND_BASE_CONTAINER[f] = ret
        return ret

def get_abs_path_real_path_and_base_from_frame(frame):
    try:
        return NORM_PATHS_AND_BASE_CONTAINER[frame.f_code.co_filename]
    except:
        #This one is just internal (so, does not need any kind of client-server translation)
        f = frame.f_code.co_filename
        if f is not None and f.startswith (('build/bdist.','build\\bdist.')):
            # files from eggs in Python 2.7 have paths like build/bdist.linux-x86_64/egg/<path-inside-egg>
            f = frame.f_globals['__file__']
        if f is not None and f.endswith('.pyc'):
            f = f[:-1]
        ret = get_abs_path_real_path_and_base_from_file(f)
        # Also cache based on the frame.f_code.co_filename (if we had it inside build/bdist it can make a difference).
        NORM_PATHS_AND_BASE_CONTAINER[frame.f_code.co_filename] = ret
        return ret
<commit_msg>Get `PATHS_FROM_ECLIPSE_TO_PYTHON` from env.<commit_after>'''
    This module provides utilities to get the absolute filenames so that we can be sure that:
        - The case of a file will match the actual file in the filesystem (otherwise breakpoints won't be hit).
        - Providing means for the user to make path conversions when doing a remote debugging session in
          one machine and debugging in another.

    To do that, the PATHS_FROM_ECLIPSE_TO_PYTHON constant must be filled with the appropriate paths.

    @note:
        in this context, the server is where your python process is running
        and the client is where eclipse is running.

    E.g.:
        If the server (your python process) has the structure
            /user/projects/my_project/src/package/module1.py

        and the client has:
            c:\my_project\src\package\module1.py

        the PATHS_FROM_ECLIPSE_TO_PYTHON would have to be:
            PATHS_FROM_ECLIPSE_TO_PYTHON = [(r'c:\my_project\src', r'/user/projects/my_project/src')]

        alternatively, this can be set with an environment variable from the command line:
           set PATHS_FROM_ECLIPSE_TO_PYTHON=[['c:\my_project\src','/user/projects/my_project/src']]

    @note: DEBUG_CLIENT_SERVER_TRANSLATION can be set to True to debug the result of those translations

    @note: the case of the paths is important! Note that this can be tricky to get right when one machine
    uses a case-independent filesystem and the other uses a case-dependent filesystem (if the system being
    debugged is case-independent, 'normcase()' should be used on the paths defined in PATHS_FROM_ECLIPSE_TO_PYTHON).

    @note: all the paths with breakpoints must be translated (otherwise they won't be found in the server)

    @note: to enable remote debugging in the target machine (pydev extensions in the eclipse installation)
        import pydevd;pydevd.settrace(host, stdoutToServer, stderrToServer, port, suspend)

        see parameter docs on pydevd.py

    @note: for doing a remote debugging session, all the pydevd_ files must be on the server accessible
        through the PYTHONPATH (and the PATHS_FROM_ECLIPSE_TO_PYTHON only needs to be set on the target
        machine for the paths that'll actually have breakpoints).
'''

from _pydevd_bundle.pydevd_constants import IS_PY2, IS_PY3K
from _pydev_bundle._pydev_filesystem_encoding import getfilesystemencoding
import json
import os
import os.path
import sys
import traceback

os_normcase = os.path.normcase
basename = os.path.basename
exists = os.path.exists
join = os.path.join

try:
    rPath = os.path.realpath  #@UndefinedVariable
except:
    # jython does not support os.path.realpath
    # realpath is a no-op on systems without islink support
    rPath = os.path.abspath

#defined as a list of tuples where the 1st element of the tuple is the path in the client machine
#and the 2nd element is the path in the server machine.
#see module docstring for more details.
PATHS_FROM_ECLIPSE_TO_PYTHON = json.loads(os.environ.get('PATHS_FROM_ECLIPSE_TO_PYTHON', '[]'))

#example:
#PATHS_FROM_ECLIPSE_TO_PYTHON = [
#  (r'd:\temp\temp_workspace_2\test_python\src\yyy\yyy',
#   r'd:\temp\temp_workspace_2\test_python\src\hhh\xxx')
#]

normcase = os_normcase # May be rebound on set_ide_os

convert_to_long_pathname = None
if sys.platform == 'win32':
    try:
        import ctypes
    except ImportError:
        pass
    else:
        def convert_to_long_pathname(filename):
            buf = ctypes.create_unicode_buffer(260)
            GetLongPathName = ctypes.windll.kernel32.GetLongPathNameW
            if IS_PY2:
                filename = unicode(filename, getfilesystemencoding())
            rv = GetLongPathName(filename, buf, 260)
            if rv != 0 and rv <= 260:
                return buf.value
            return filename

def norm_case(filename):
    # `normcase` doesn't lower case on Python 2 for non-English locale, but Java side does it,
    # so we should do it manually
    if '~' in filename and convert_to_long_pathname:
        filename = convert_to_long_pathname(filename)

    filename = os_normcase(filename)
    enc = getfilesystemencoding()
    if IS_PY3K or enc is None or enc.lower() == "utf-8":
        return filename
    try:
        return filename.decode(enc).lower().encode(enc)
    except:
        return filename

def set_ide_os(os):
    '''
    We need to set the IDE os because the host where the code is running may be
    actually different from the client (and the point is that we want the proper
    paths to translate from the client to the server).
    '''
    global normcase
    if os == 'UNIX':
        normcase = lambda f:f #Change to no-op if the client side is on unix/mac.
    else:
        if sys.platform == 'win32':
            normcase = norm_case
        else:
            normcase = os_normcase

    # After setting the ide OS, apply the normcase to the existing paths.

    # Note: not using enumerate nor list comprehension because it may not be available in older python versions...
    i = 0
    for path in PATHS_FROM_ECLIPSE_TO_PYTHON[:]:
        PATHS_FROM_ECLIPSE_TO_PYTHON[i] = (normcase(path[0]), normcase(path[1]))
        i += 1

DEBUG_CLIENT_SERVER_TRANSLATION = False

#caches filled as requested during the debug session
NORM_PATHS_CONTAINER = {}
NORM_PATHS_AND_BASE_CONTAINER = {}
NORM_FILENAME_TO_SERVER_CONTAINER = {}
NORM_FILENAME_TO_CLIENT_CONTAINER = {}

def _NormFile(filename):
    abs_path, real_path = _NormPaths(filename)
    return real_path

def _AbsFile(filename):
    abs_path, real_path = _NormPaths(filename)
    return abs_path

# Returns tuple of absolute path and real path for given filename
def _NormPaths(filename):
    try:
        return NORM_PATHS_CONTAINER[filename]
    except KeyError:
        abs_path = _NormPath(filename, os.path.abspath)
        real_path = _NormPath(filename, rPath)

        NORM_PATHS_CONTAINER[filename] = abs_path, real_path
        return abs_path, real_path

def _NormPath(filename, normpath):
    r = normpath(filename)
    #cache it for fast access later
    ind = r.find('.zip')
    if ind == -1:
        ind = r.find('.egg')
    if ind != -1:
        ind+=4
        zip_path = r[:ind]
        if r[ind] == "!":
            ind+=1
        inner_path = r[ind:]
        if inner_path.startswith('/') or inner_path.startswith('\\'):
            inner_path = inner_path[1:]
        r = join(normcase(zip_path), inner_path)
    else:
        r = normcase(r)
    return r

ZIP_SEARCH_CACHE = {}
def exists(file):
    if os.path.exists(file):
        return file

    ind = file.find('.zip')
    if ind == -1:
        ind = file.find('.egg')

    if ind != -1:
        ind+=4
        zip_path = file[:ind]
        if file[ind] == "!":
            ind+=1
        inner_path = file[ind:]
        try:
            zip = ZIP_SEARCH_CACHE[zip_path]
        except KeyError:
            try:
                import zipfile
                zip = zipfile.ZipFile(zip_path, 'r')
                ZIP_SEARCH_CACHE[zip_path] = zip
            except :
                return None

        try:
            if inner_path.startswith('/') or inner_path.startswith('\\'):
                inner_path = inner_path[1:]

            info = zip.getinfo(inner_path.replace('\\', '/'))

            return join(zip_path, inner_path)
        except KeyError:
            return None
    return None

#Now, let's do a quick test to see if we're working with a version of python that has no problems
#related to the names generated...
try:
    try:
        code = rPath.func_code
    except AttributeError:
        code = rPath.__code__
    if not exists(_NormFile(code.co_filename)):
        sys.stderr.write('-------------------------------------------------------------------------------\n')
        sys.stderr.write('pydev debugger: CRITICAL WARNING: This version of python seems to be incorrectly compiled (internal generated filenames are not absolute)\n')
        sys.stderr.write('pydev debugger: The debugger may still function, but it will work slower and may miss breakpoints.\n')
        sys.stderr.write('pydev debugger: Related bug: http://bugs.python.org/issue1666807\n')
        sys.stderr.write('-------------------------------------------------------------------------------\n')
        sys.stderr.flush()

        NORM_SEARCH_CACHE = {}

        initial_norm_paths = _NormPaths
        def _NormPaths(filename):  #Let's redefine _NormPaths to work with paths that may be incorrect
            try:
                return NORM_SEARCH_CACHE[filename]
            except KeyError:
                abs_path, real_path = initial_norm_paths(filename)
                if not exists(real_path):
                    #We must actually go on and check if we can find it as if it was a relative path for some of the paths in the pythonpath
                    for path in sys.path:
                        abs_path, real_path = initial_norm_paths(join(path, filename))
                        if exists(real_path):
                            break
                    else:
                        sys.stderr.write('pydev debugger: Unable to find real location for: %s\n' % (filename,))
                        abs_path = filename
                        real_path = filename

                NORM_SEARCH_CACHE[filename] = abs_path, real_path
                return abs_path, real_path

except:
    #Don't fail if there's something not correct here -- but at least print it to the user so that we can correct that
    traceback.print_exc()

norm_file_to_client = _AbsFile
norm_file_to_server = _NormFile

def setup_client_server_paths(paths):
    '''paths is the same format as PATHS_FROM_ECLIPSE_TO_PYTHON'''

    global NORM_FILENAME_TO_SERVER_CONTAINER
    global NORM_FILENAME_TO_CLIENT_CONTAINER
    global PATHS_FROM_ECLIPSE_TO_PYTHON
    global norm_file_to_client
    global norm_file_to_server

    NORM_FILENAME_TO_SERVER_CONTAINER = {}
    NORM_FILENAME_TO_CLIENT_CONTAINER = {}
    PATHS_FROM_ECLIPSE_TO_PYTHON = paths[:]

    if not PATHS_FROM_ECLIPSE_TO_PYTHON:
        #no translation step needed (just inline the calls)
        norm_file_to_client = _AbsFile
        norm_file_to_server = _NormFile
        return

    #Work on the client and server slashes.
    eclipse_sep = None
    python_sep = None
    for eclipse_prefix, server_prefix in PATHS_FROM_ECLIPSE_TO_PYTHON:
        if eclipse_sep is not None and python_sep is not None:
            break

        if eclipse_sep is None:
            for c in eclipse_prefix:
                if c in ('/', '\\'):
                    eclipse_sep = c
                    break

        if python_sep is None:
            for c in server_prefix:
                if c in ('/', '\\'):
                    python_sep = c
                    break

    #If they're the same or one of them cannot be determined, just make it all None.
    if eclipse_sep == python_sep or eclipse_sep is None or python_sep is None:
        eclipse_sep = python_sep = None

    #only setup translation functions if absolutely needed!
    def _norm_file_to_server(filename):
        #Eclipse will send the passed filename to be translated to the python process
        #So, this would be 'NormFileFromEclipseToPython'
        try:
            return NORM_FILENAME_TO_SERVER_CONTAINER[filename]
        except KeyError:
            #used to translate a path from the client to the debug server
            translated = normcase(filename)
            for eclipse_prefix, server_prefix in PATHS_FROM_ECLIPSE_TO_PYTHON:
                if translated.startswith(eclipse_prefix):
                    if DEBUG_CLIENT_SERVER_TRANSLATION:
                        sys.stderr.write('pydev debugger: replacing to server: %s\n' % (translated,))
                    translated = translated.replace(eclipse_prefix, server_prefix)
                    if DEBUG_CLIENT_SERVER_TRANSLATION:
                        sys.stderr.write('pydev debugger: sent to server: %s\n' % (translated,))
                    break
            else:
                if DEBUG_CLIENT_SERVER_TRANSLATION:
                    sys.stderr.write('pydev debugger: to server: unable to find matching prefix for: %s in %s\n' % \
                        (translated, [x[0] for x in PATHS_FROM_ECLIPSE_TO_PYTHON]))

            #Note that when going to the server, we do the replace first and only later do the norm file.
            if eclipse_sep is not None:
                translated = translated.replace(eclipse_sep, python_sep)
            translated = _NormFile(translated)

            NORM_FILENAME_TO_SERVER_CONTAINER[filename] = translated
            return translated

    def _norm_file_to_client(filename):
        #The result of this method will be passed to eclipse
        #So, this would be 'NormFileFromPythonToEclipse'
        try:
            return NORM_FILENAME_TO_CLIENT_CONTAINER[filename]
        except KeyError:
            #used to translate a path from the debug server to the client
            translated = _NormFile(filename)
            for eclipse_prefix, python_prefix in PATHS_FROM_ECLIPSE_TO_PYTHON:
                if translated.startswith(python_prefix):
                    if DEBUG_CLIENT_SERVER_TRANSLATION:
                        sys.stderr.write('pydev debugger: replacing to client: %s\n' % (translated,))
                    translated = translated.replace(python_prefix, eclipse_prefix)
                    if DEBUG_CLIENT_SERVER_TRANSLATION:
                        sys.stderr.write('pydev debugger: sent to client: %s\n' % (translated,))
                    break
            else:
                if DEBUG_CLIENT_SERVER_TRANSLATION:
                    sys.stderr.write('pydev debugger: to client: unable to find matching prefix for: %s in %s\n' % \
                        (translated, [x[1] for x in PATHS_FROM_ECLIPSE_TO_PYTHON]))

            if eclipse_sep is not None:
                translated = translated.replace(python_sep, eclipse_sep)

            #The resulting path is not in the python process, so, we cannot do a _NormFile here,
            #only at the beginning of this method.
            NORM_FILENAME_TO_CLIENT_CONTAINER[filename] = translated
            return translated

    norm_file_to_server = _norm_file_to_server
    norm_file_to_client = _norm_file_to_client

setup_client_server_paths(PATHS_FROM_ECLIPSE_TO_PYTHON)

# For given file f returns tuple of its absolute path, real path and base name
def get_abs_path_real_path_and_base_from_file(f):
    try:
        return NORM_PATHS_AND_BASE_CONTAINER[f]
    except:
        abs_path, real_path = _NormPaths(f)
        base = basename(real_path)
        ret = abs_path, real_path, base
        NORM_PATHS_AND_BASE_CONTAINER[f] = ret
        return ret

def get_abs_path_real_path_and_base_from_frame(frame):
    try:
        return NORM_PATHS_AND_BASE_CONTAINER[frame.f_code.co_filename]
    except:
        #This one is just internal (so, does not need any kind of client-server translation)
        f = frame.f_code.co_filename
        if f is not None and f.startswith (('build/bdist.','build\\bdist.')):
            # files from eggs in Python 2.7 have paths like build/bdist.linux-x86_64/egg/<path-inside-egg>
            f = frame.f_globals['__file__']
        if f is not None and f.endswith('.pyc'):
            f = f[:-1]
        ret = get_abs_path_real_path_and_base_from_file(f)
        # Also cache based on the frame.f_code.co_filename (if we had it inside build/bdist it can make a difference).
        NORM_PATHS_AND_BASE_CONTAINER[frame.f_code.co_filename] = ret
        return ret
<|endoftext|>


<commit_before>import pyethereum
import shutil
import tempfile
import time
import logging
import sys

serpent = None

u = pyethereum.utils
t = pyethereum.transactions
b = pyethereum.blocks
pb = pyethereum.processblock

accounts = []
keys = []

for i in range(10):
    keys.append(u.sha3(str(i)))
    accounts.append(u.privtoaddr(keys[-1]))

k0, k1, k2, k3, k4, k5, k6, k7, k8, k9 = keys[:10]
a0, a1, a2, a3, a4, a5, a6, a7, a8, a9 = accounts[:10]

seed = 3**160

def rand():
    global seed
    seed = pow(seed, 2, 2**512)
    return seed % 2**256

class state():
    def __init__(self, num_accounts=len(keys)):
        global serpent
        if not serpent:
            serpent = __import__('serpent')

        self.temp_data_dir = tempfile.mkdtemp()
        u.data_dir.set(self.temp_data_dir)

        o = {}
        for i in range(num_accounts):
            o[accounts[i]] = 10**24
        self.block = b.genesis(o)
        self.block.timestamp = 1410973349
        self.block.coinbase = a0

    def __del__(self):
        shutil.rmtree(self.temp_data_dir)

    def contract(self, code, sender=k0, endowment=0):
        sendnonce = self.block.get_nonce(u.privtoaddr(sender))
        evm = serpent.compile(code)
        tx = t.contract(sendnonce, 1, gas_limit, endowment, evm)
        tx.sign(sender)
        (s, a) = pb.apply_transaction(self.block, tx)
        if not s:
            raise Exception("Contract creation failed")
        return a

    def send(self, sender, to, value, data=[]):
        sendnonce = self.block.get_nonce(u.privtoaddr(sender))
        evmdata = serpent.encode_datalist(data)
        tx = t.Transaction(sendnonce, 1, gas_limit, to, value, evmdata)
        self.last_tx = tx
        tx.sign(sender)
        (s, r) = pb.apply_transaction(self.block, tx)
        if not s:
            raise Exception("Transaction failed")
        o = serpent.decode_datalist(r)
        return map(lambda x: x-2**256 if x > 2**255 else x, o)

    def profile(self, sender, to, value, data=[]):
        tm, g = time.time(), self.block.gas_used
        o = self.send(sender, to, value, data)
        intrinsic_gas_used = pb.GTXDATA * len(self.last_tx.data) + pb.GTXCOST
        return {
            "time": time.time() - tm,
            "gas": self.block.gas_used - g - intrinsic_gas_used,
            "output": o
        }

    def mkspv(self, sender, to, value, data=[]):
        sendnonce = self.block.get_nonce(u.privtoaddr(sender))
        evmdata = serpent.encode_datalist(data)
        tx = t.Transaction(sendnonce, 1, gas_limit, to, value, evmdata)
        self.last_tx = tx
        tx.sign(sender)
        return pb.mk_transaction_spv_proof(self.block, tx)

    def verifyspv(self, sender, to, value, data=[], proof=[]):
        sendnonce = self.block.get_nonce(u.privtoaddr(sender))
        evmdata = serpent.encode_datalist(data)
        tx = t.Transaction(sendnonce, 1, gas_limit, to, value, evmdata)
        self.last_tx = tx
        tx.sign(sender)
        return pb.verify_transaction_spv_proof(self.block, tx, proof)

    def mine(self, n=1, coinbase=a0):
        for i in range(n):
            self.block.finalize()
            t = self.block.timestamp + 6 + rand() % 12
            self.block = b.Block.init_from_parent(self.block, coinbase, '', t)

    def snapshot(self):
        return self.block.serialize()

    def revert(self, data):
        self.block = b.Block.deserialize(data)

def enable_logging():
    logging.basicConfig(level=logging.DEBUG, format='%(message)s')

gas_limit = 100000
<commit_msg>comment<commit_after>import pyethereum
import shutil
import tempfile
import time
import logging
import sys

serpent = None

u = pyethereum.utils
t = pyethereum.transactions
b = pyethereum.blocks
pb = pyethereum.processblock

accounts = []
keys = []

for i in range(10):
    keys.append(u.sha3(str(i)))
    accounts.append(u.privtoaddr(keys[-1]))

k0, k1, k2, k3, k4, k5, k6, k7, k8, k9 = keys[:10]
a0, a1, a2, a3, a4, a5, a6, a7, a8, a9 = accounts[:10]

seed = 3**160

# Pseudo-RNG (deterministic for now for testing purposes)
def rand():
    global seed
    seed = pow(seed, 2, 2**512)
    return seed % 2**256

class state():
    def __init__(self, num_accounts=len(keys)):
        global serpent
        if not serpent:
            serpent = __import__('serpent')

        self.temp_data_dir = tempfile.mkdtemp()
        u.data_dir.set(self.temp_data_dir)

        o = {}
        for i in range(num_accounts):
            o[accounts[i]] = 10**24
        self.block = b.genesis(o)
        self.block.timestamp = 1410973349
        self.block.coinbase = a0

    def __del__(self):
        shutil.rmtree(self.temp_data_dir)

    def contract(self, code, sender=k0, endowment=0):
        sendnonce = self.block.get_nonce(u.privtoaddr(sender))
        evm = serpent.compile(code)
        tx = t.contract(sendnonce, 1, gas_limit, endowment, evm)
        tx.sign(sender)
        (s, a) = pb.apply_transaction(self.block, tx)
        if not s:
            raise Exception("Contract creation failed")
        return a

    def send(self, sender, to, value, data=[]):
        sendnonce = self.block.get_nonce(u.privtoaddr(sender))
        evmdata = serpent.encode_datalist(data)
        tx = t.Transaction(sendnonce, 1, gas_limit, to, value, evmdata)
        self.last_tx = tx
        tx.sign(sender)
        (s, r) = pb.apply_transaction(self.block, tx)
        if not s:
            raise Exception("Transaction failed")
        o = serpent.decode_datalist(r)
        return map(lambda x: x-2**256 if x > 2**255 else x, o)

    def profile(self, sender, to, value, data=[]):
        tm, g = time.time(), self.block.gas_used
        o = self.send(sender, to, value, data)
        intrinsic_gas_used = pb.GTXDATA * len(self.last_tx.data) + pb.GTXCOST
        return {
            "time": time.time() - tm,
            "gas": self.block.gas_used - g - intrinsic_gas_used,
            "output": o
        }

    def mkspv(self, sender, to, value, data=[]):
        sendnonce = self.block.get_nonce(u.privtoaddr(sender))
        evmdata = serpent.encode_datalist(data)
        tx = t.Transaction(sendnonce, 1, gas_limit, to, value, evmdata)
        self.last_tx = tx
        tx.sign(sender)
        return pb.mk_transaction_spv_proof(self.block, tx)

    def verifyspv(self, sender, to, value, data=[], proof=[]):
        sendnonce = self.block.get_nonce(u.privtoaddr(sender))
        evmdata = serpent.encode_datalist(data)
        tx = t.Transaction(sendnonce, 1, gas_limit, to, value, evmdata)
        self.last_tx = tx
        tx.sign(sender)
        return pb.verify_transaction_spv_proof(self.block, tx, proof)

    def mine(self, n=1, coinbase=a0):
        for i in range(n):
            self.block.finalize()
            t = self.block.timestamp + 6 + rand() % 12
            self.block = b.Block.init_from_parent(self.block, coinbase, '', t)

    def snapshot(self):
        return self.block.serialize()

    def revert(self, data):
        self.block = b.Block.deserialize(data)

def enable_logging():
    logging.basicConfig(level=logging.DEBUG, format='%(message)s')

gas_limit = 100000
<|endoftext|>