<commit_before>"""Provide access to Python's configuration information.  The specific names
defined in the module depend heavily on the platform and configuration.

Written by:   Fred L. Drake, Jr.
Email:        <fdrake@acm.org>
Initial date: 17-Dec-1998
"""

__revision__ = "$Id$"

import os
import re
import string
import sys

from errors import DistutilsPlatformError

# These are needed in a couple of spots, so just compute them once.
PREFIX = os.path.normpath(sys.prefix)
EXEC_PREFIX = os.path.normpath(sys.exec_prefix)

def get_python_inc(plat_specific=0, prefix=None):
    """Return the directory containing installed Python header files.

    If 'plat_specific' is false (the default), this is the path to the
    non-platform-specific header files, i.e. Python.h and so on;
    otherwise, this is the path to platform-specific header files
    (namely config.h).

    If 'prefix' is supplied, use it instead of sys.prefix or
    sys.exec_prefix -- i.e., ignore 'plat_specific'.
    """
    if prefix is None:
        prefix = (plat_specific and EXEC_PREFIX or PREFIX)
    if os.name == "posix":
        return os.path.join(prefix, "include", "python" + sys.version[:3])
    elif os.name == "nt":
        return os.path.join(prefix, "Include") # include or Include?
    elif os.name == "mac":
        return os.path.join(prefix, "Include")
    else:
        raise DistutilsPlatformError, \
              ("I don't know where Python installs its C header files " +
               "on platform '%s'") % os.name

def get_python_lib(plat_specific=0, standard_lib=0, prefix=None):
    """Return the directory containing the Python library (standard or
    site additions).

    If 'plat_specific' is true, return the directory containing
    platform-specific modules, i.e. any module from a non-pure-Python
    module distribution; otherwise, return the platform-shared library
    directory.  If 'standard_lib' is true, return the directory
    containing standard Python library modules; otherwise, return the
    directory for site-specific modules.

    If 'prefix' is supplied, use it instead of sys.prefix or
    sys.exec_prefix -- i.e., ignore 'plat_specific'.
    """
    if prefix is None:
        prefix = (plat_specific and EXEC_PREFIX or PREFIX)

    if os.name == "posix":
        libpython = os.path.join(prefix,
                                 "lib", "python" + sys.version[:3])
        if standard_lib:
            return libpython
        else:
            return os.path.join(libpython, "site-packages")

    elif os.name == "nt":
        if standard_lib:
            return os.path.join(PREFIX, "Lib")
        else:
            return prefix

    elif os.name == "mac":
        if plat_specific:
            if standard_lib:
                return os.path.join(EXEC_PREFIX, "Mac", "Plugins")
            else:
                raise DistutilsPlatformError, \
                      "OK, where DO site-specific extensions go on the Mac?"
        else:
            if standard_lib:
                return os.path.join(PREFIX, "Lib")
            else:
                raise DistutilsPlatformError, \
                      "OK, where DO site-specific modules go on the Mac?"
    else:
        raise DistutilsPlatformError, \
              ("I don't know where Python installs its library " +
               "on platform '%s'") % os.name

# get_python_lib()

def customize_compiler (compiler):
    """Do any platform-specific customization of the CCompiler instance
    'compiler'.  Mainly needed on Unix, so we can plug in the information
    that varies across Unices and is stored in Python's Makefile.
    """
    if compiler.compiler_type == "unix":
        (cc, opt, ccshared, ldshared, so_ext) = \
            get_config_vars('CC', 'OPT', 'CCSHARED', 'LDSHARED', 'SO')

        cc_cmd = cc + ' ' + opt
        compiler.set_executables(
            preprocessor=cc + " -E",    # not always!
            compiler=cc_cmd,
            compiler_so=cc_cmd + ' ' + ccshared,
            linker_so=ldshared,
            linker_exe=cc)

        compiler.shared_lib_extension = so_ext

def get_config_h_filename():
    """Return full pathname of installed config.h file."""
    inc_dir = get_python_inc(plat_specific=1)
    return os.path.join(inc_dir, "config.h")

def get_makefile_filename():
    """Return full pathname of installed Makefile from the Python build."""
    lib_dir = get_python_lib(plat_specific=1, standard_lib=1)
    return os.path.join(lib_dir, "config", "Makefile")

def parse_config_h(fp, g=None):
    """Parse a config.h-style file.

    A dictionary containing name/value pairs is returned.  If an
    optional dictionary is passed in as the second argument, it is
    used instead of a new dictionary.
    """
    if g is None:
        g = {}
    define_rx = re.compile("#define ([A-Z][A-Z0-9_]+) (.*)\n")
    undef_rx = re.compile("/[*] #undef ([A-Z][A-Z0-9_]+) [*]/\n")
    #
    while 1:
        line = fp.readline()
        if not line:
            break
        m = define_rx.match(line)
        if m:
            n, v = m.group(1, 2)
            try: v = string.atoi(v)
            except ValueError: pass
            g[n] = v
        else:
            m = undef_rx.match(line)
            if m:
                g[m.group(1)] = 0
    return g

# Regexes needed for parsing Makefile (and similar syntaxes,
# like old-style Setup files).
_variable_rx = re.compile("([a-zA-Z][a-zA-Z0-9_]+)\s*=\s*(.*)")
_findvar1_rx = re.compile(r"\$\(([A-Za-z][A-Za-z0-9_]*)\)")
_findvar2_rx = re.compile(r"\${([A-Za-z][A-Za-z0-9_]*)}")

def parse_makefile(fn, g=None):
    """Parse a Makefile-style file.

    A dictionary containing name/value pairs is returned.  If an
    optional dictionary is passed in as the second argument, it is
    used instead of a new dictionary.

    """
    from distutils.text_file import TextFile
    fp = TextFile(fn, strip_comments=1, skip_blanks=1, join_lines=1)

    if g is None:
        g = {}
    done = {}
    notdone = {}

    while 1:
        line = fp.readline()
        if line is None:                # eof
            break
        m = _variable_rx.match(line)
        if m:
            n, v = m.group(1, 2)
            v = string.strip(v)
            if "$" in v:
                notdone[n] = v
            else:
                try: v = string.atoi(v)
                except ValueError: pass
                done[n] = v

    # do variable interpolation here
    while notdone:
        for name in notdone.keys():
            value = notdone[name]
            m = _findvar1_rx.search(value) or _findvar2_rx.search(value)
            if m:
                n = m.group(1)
                if done.has_key(n):
                    after = value[m.end():]
                    value = value[:m.start()] + done[n] + after
                    if "$" in after:
                        notdone[name] = value
                    else:
                        try: value = string.atoi(value)
                        except ValueError: pass
                        done[name] = string.strip(value)
                        del notdone[name]
                elif notdone.has_key(n):
                    # get it on a subsequent round
                    pass
                else:
                    done[n] = ""
                    after = value[m.end():]
                    value = value[:m.start()] + after
                    if "$" in after:
                        notdone[name] = value
                    else:
                        try: value = string.atoi(value)
                        except ValueError: pass
                        done[name] = string.strip(value)
                        del notdone[name]
            else:
                # bogus variable reference; just drop it since we can't deal
                del notdone[name]

    fp.close()

    # save the results in the global dictionary
    g.update(done)
    return g

def expand_makefile_vars(s, vars):
    """Expand Makefile-style variables -- "${foo}" or "$(foo)" -- in
    'string' according to 'vars' (a dictionary mapping variable names to
    values).  Variables not present in 'vars' are silently expanded to the
    empty string.  The variable values in 'vars' should not contain further
    variable expansions; if 'vars' is the output of 'parse_makefile()',
    you're fine.  Returns a variable-expanded version of 's'.
    """

    # This algorithm does multiple expansion, so if vars['foo'] contains
    # "${bar}", it will expand ${foo} to ${bar}, and then expand
    # ${bar}... and so forth.  This is fine as long as 'vars' comes from
    # 'parse_makefile()', which takes care of such expansions eagerly,
    # according to make's variable expansion semantics.

    while 1:
        m = _findvar1_rx.search(s) or _findvar2_rx.search(s)
        if m:
            name = m.group(1)
            (beg, end) = m.span()
            s = s[0:beg] + vars.get(m.group(1)) + s[end:]
        else:
            break
    return s

_config_vars = None

def _init_posix():
    """Initialize the module as appropriate for POSIX systems."""
    g = {}
    # load the installed Makefile:
    try:
        filename = get_makefile_filename()
        parse_makefile(filename, g)
    except IOError, msg:
        my_msg = "invalid Python installation: unable to open %s" % filename
        if hasattr(msg, "strerror"):
            my_msg = my_msg + " (%s)" % msg.strerror

        raise DistutilsPlatformError, my_msg

    # On AIX, there are wrong paths to the linker scripts in the Makefile
    # -- these paths are relative to the Python source, but when installed
    # the scripts are in another directory.
    if sys.platform == 'aix4':          # what about AIX 3.x ?
        # Linker script is in the config directory, not in Modules as the
        # Makefile says.
        python_lib = get_python_lib(standard_lib=1)
        ld_so_aix = os.path.join(python_lib, 'config', 'ld_so_aix')
        python_exp = os.path.join(python_lib, 'config', 'python.exp')

        g['LDSHARED'] = "%s %s -bI:%s" % (ld_so_aix, g['CC'], python_exp)

    elif sys.platform == 'beos':

        # Linker script is in the config directory.  In the Makefile it is
        # relative to the srcdir, which after installation no longer makes
        # sense.
        python_lib = get_python_lib(standard_lib=1)
        linkerscript_name = os.path.basename(string.split(g['LDSHARED'])[0])
        linkerscript = os.path.join(python_lib, 'config', linkerscript_name)

        # XXX this isn't the right place to do this: adding the Python
        # library to the link, if needed, should be in the "build_ext"
        # command.  (It's also needed for non-MS compilers on Windows, and
        # it's taken care of for them by the 'build_ext.get_libraries()'
        # method.)
        g['LDSHARED'] = ("%s -L%s/lib -lpython%s" %
                         (linkerscript, PREFIX, sys.version[0:3]))

    global _config_vars
    _config_vars = g

def _init_nt():
    """Initialize the module as appropriate for NT"""
    g = {}
    # set basic install directories
    g['LIBDEST'] = get_python_lib(plat_specific=0, standard_lib=1)
    g['BINLIBDEST'] = get_python_lib(plat_specific=1, standard_lib=1)

    # XXX hmmm.. a normal install puts include files here
    g['INCLUDEPY'] = get_python_inc(plat_specific=0)

    g['SO'] = '.pyd'
    g['EXE'] = ".exe"

    global _config_vars
    _config_vars = g

def _init_mac():
    """Initialize the module as appropriate for Macintosh systems"""
    g = {}
    # set basic install directories
    g['LIBDEST'] = get_python_lib(plat_specific=0, standard_lib=1)
    g['BINLIBDEST'] = get_python_lib(plat_specific=1, standard_lib=1)

    # XXX hmmm.. a normal install puts include files here
    g['INCLUDEPY'] = get_python_inc(plat_specific=0)

    g['SO'] = '.ppc.slb'

    # XXX are these used anywhere?
    g['install_lib'] = os.path.join(EXEC_PREFIX, "Lib")
    g['install_platlib'] = os.path.join(EXEC_PREFIX, "Mac", "Lib")

    global _config_vars
    _config_vars = g

def get_config_vars(*args):
    """With no arguments, return a dictionary of all configuration
    variables relevant for the current platform.  Generally this includes
    everything needed to build extensions and install both pure modules and
    extensions.  On Unix, this means every variable defined in Python's
    installed Makefile; on Windows and Mac OS it's a much smaller set.

    With arguments, return a list of values that result from looking up
    each argument in the configuration variable dictionary.
    """
    global _config_vars
    if _config_vars is None:
        func = globals().get("_init_" + os.name)
        if func:
            func()
        else:
            _config_vars = {}

        # Normalized versions of prefix and exec_prefix are handy to have;
        # in fact, these are the standard versions used most places in the
        # Distutils.
        _config_vars['prefix'] = PREFIX
        _config_vars['exec_prefix'] = EXEC_PREFIX

    if args:
        vals = []
        for name in args:
            vals.append(_config_vars.get(name))
        return vals
    else:
        return _config_vars

def get_config_var(name):
    """Return the value of a single variable using the dictionary
    returned by 'get_config_vars()'.  Equivalent to
      get_config_vars().get(name)
    """
    return get_config_vars().get(name)
<commit_msg>Fix bugs with integer-valued variables when parsing Makefiles.  Values for done[n] can be integers as well as strings, but the code concatenates them with strings (fixed by adding a str()) and calls string.strip() on them (fixed by rearranging the logic)<commit_after>"""Provide access to Python's configuration information.  The specific names
defined in the module depend heavily on the platform and configuration.

Written by:   Fred L. Drake, Jr.
Email:        <fdrake@acm.org>
Initial date: 17-Dec-1998
"""

__revision__ = "$Id$"

import os
import re
import string
import sys

from errors import DistutilsPlatformError

# These are needed in a couple of spots, so just compute them once.
PREFIX = os.path.normpath(sys.prefix)
EXEC_PREFIX = os.path.normpath(sys.exec_prefix)

def get_python_inc(plat_specific=0, prefix=None):
    """Return the directory containing installed Python header files.

    If 'plat_specific' is false (the default), this is the path to the
    non-platform-specific header files, i.e. Python.h and so on;
    otherwise, this is the path to platform-specific header files
    (namely config.h).

    If 'prefix' is supplied, use it instead of sys.prefix or
    sys.exec_prefix -- i.e., ignore 'plat_specific'.
    """
    if prefix is None:
        prefix = (plat_specific and EXEC_PREFIX or PREFIX)
    if os.name == "posix":
        return os.path.join(prefix, "include", "python" + sys.version[:3])
    elif os.name == "nt":
        return os.path.join(prefix, "Include") # include or Include?
    elif os.name == "mac":
        return os.path.join(prefix, "Include")
    else:
        raise DistutilsPlatformError, \
              ("I don't know where Python installs its C header files " +
               "on platform '%s'") % os.name

def get_python_lib(plat_specific=0, standard_lib=0, prefix=None):
    """Return the directory containing the Python library (standard or
    site additions).

    If 'plat_specific' is true, return the directory containing
    platform-specific modules, i.e. any module from a non-pure-Python
    module distribution; otherwise, return the platform-shared library
    directory.  If 'standard_lib' is true, return the directory
    containing standard Python library modules; otherwise, return the
    directory for site-specific modules.

    If 'prefix' is supplied, use it instead of sys.prefix or
    sys.exec_prefix -- i.e., ignore 'plat_specific'.
    """
    if prefix is None:
        prefix = (plat_specific and EXEC_PREFIX or PREFIX)

    if os.name == "posix":
        libpython = os.path.join(prefix,
                                 "lib", "python" + sys.version[:3])
        if standard_lib:
            return libpython
        else:
            return os.path.join(libpython, "site-packages")

    elif os.name == "nt":
        if standard_lib:
            return os.path.join(PREFIX, "Lib")
        else:
            return prefix

    elif os.name == "mac":
        if plat_specific:
            if standard_lib:
                return os.path.join(EXEC_PREFIX, "Mac", "Plugins")
            else:
                raise DistutilsPlatformError, \
                      "OK, where DO site-specific extensions go on the Mac?"
        else:
            if standard_lib:
                return os.path.join(PREFIX, "Lib")
            else:
                raise DistutilsPlatformError, \
                      "OK, where DO site-specific modules go on the Mac?"
    else:
        raise DistutilsPlatformError, \
              ("I don't know where Python installs its library " +
               "on platform '%s'") % os.name

# get_python_lib()

def customize_compiler (compiler):
    """Do any platform-specific customization of the CCompiler instance
    'compiler'.  Mainly needed on Unix, so we can plug in the information
    that varies across Unices and is stored in Python's Makefile.
    """
    if compiler.compiler_type == "unix":
        (cc, opt, ccshared, ldshared, so_ext) = \
            get_config_vars('CC', 'OPT', 'CCSHARED', 'LDSHARED', 'SO')

        cc_cmd = cc + ' ' + opt
        compiler.set_executables(
            preprocessor=cc + " -E",    # not always!
            compiler=cc_cmd,
            compiler_so=cc_cmd + ' ' + ccshared,
            linker_so=ldshared,
            linker_exe=cc)

        compiler.shared_lib_extension = so_ext

def get_config_h_filename():
    """Return full pathname of installed config.h file."""
    inc_dir = get_python_inc(plat_specific=1)
    return os.path.join(inc_dir, "config.h")

def get_makefile_filename():
    """Return full pathname of installed Makefile from the Python build."""
    lib_dir = get_python_lib(plat_specific=1, standard_lib=1)
    return os.path.join(lib_dir, "config", "Makefile")

def parse_config_h(fp, g=None):
    """Parse a config.h-style file.

    A dictionary containing name/value pairs is returned.  If an
    optional dictionary is passed in as the second argument, it is
    used instead of a new dictionary.
    """
    if g is None:
        g = {}
    define_rx = re.compile("#define ([A-Z][A-Z0-9_]+) (.*)\n")
    undef_rx = re.compile("/[*] #undef ([A-Z][A-Z0-9_]+) [*]/\n")
    #
    while 1:
        line = fp.readline()
        if not line:
            break
        m = define_rx.match(line)
        if m:
            n, v = m.group(1, 2)
            try: v = string.atoi(v)
            except ValueError: pass
            g[n] = v
        else:
            m = undef_rx.match(line)
            if m:
                g[m.group(1)] = 0
    return g

# Regexes needed for parsing Makefile (and similar syntaxes,
# like old-style Setup files).
_variable_rx = re.compile("([a-zA-Z][a-zA-Z0-9_]+)\s*=\s*(.*)")
_findvar1_rx = re.compile(r"\$\(([A-Za-z][A-Za-z0-9_]*)\)")
_findvar2_rx = re.compile(r"\${([A-Za-z][A-Za-z0-9_]*)}")

def parse_makefile(fn, g=None):
    """Parse a Makefile-style file.

    A dictionary containing name/value pairs is returned.  If an
    optional dictionary is passed in as the second argument, it is
    used instead of a new dictionary.

    """
    from distutils.text_file import TextFile
    fp = TextFile(fn, strip_comments=1, skip_blanks=1, join_lines=1)

    if g is None:
        g = {}
    done = {}
    notdone = {}

    while 1:
        line = fp.readline()
        if line is None:                # eof
            break
        m = _variable_rx.match(line)
        if m:
            n, v = m.group(1, 2)
            v = string.strip(v)
            if "$" in v:
                notdone[n] = v
            else:
                try: v = string.atoi(v)
                except ValueError: pass
                done[n] = v

    # do variable interpolation here
    while notdone:
        for name in notdone.keys():
            value = notdone[name]
            m = _findvar1_rx.search(value) or _findvar2_rx.search(value)
            if m:
                n = m.group(1)
                if done.has_key(n):
                    after = value[m.end():]
                    value = value[:m.start()] + str(done[n]) + after
                    if "$" in after:
                        notdone[name] = value
                    else:
                        try: value = string.atoi(value)
                        except ValueError:
                            done[name] = string.strip(value)
                        else:
                            done[name] = value
                        del notdone[name]
                elif notdone.has_key(n):
                    # get it on a subsequent round
                    pass
                else:
                    done[n] = ""
                    after = value[m.end():]
                    value = value[:m.start()] + after
                    if "$" in after:
                        notdone[name] = value
                    else:
                        try: value = string.atoi(value)
                        except ValueError:
                            done[name] = string.strip(value)
                        else:
                            done[name] = value
                        del notdone[name]
            else:
                # bogus variable reference; just drop it since we can't deal
                del notdone[name]

    fp.close()

    # save the results in the global dictionary
    g.update(done)
    return g

def expand_makefile_vars(s, vars):
    """Expand Makefile-style variables -- "${foo}" or "$(foo)" -- in
    'string' according to 'vars' (a dictionary mapping variable names to
    values).  Variables not present in 'vars' are silently expanded to the
    empty string.  The variable values in 'vars' should not contain further
    variable expansions; if 'vars' is the output of 'parse_makefile()',
    you're fine.  Returns a variable-expanded version of 's'.
    """

    # This algorithm does multiple expansion, so if vars['foo'] contains
    # "${bar}", it will expand ${foo} to ${bar}, and then expand
    # ${bar}... and so forth.  This is fine as long as 'vars' comes from
    # 'parse_makefile()', which takes care of such expansions eagerly,
    # according to make's variable expansion semantics.

    while 1:
        m = _findvar1_rx.search(s) or _findvar2_rx.search(s)
        if m:
            name = m.group(1)
            (beg, end) = m.span()
            s = s[0:beg] + vars.get(m.group(1)) + s[end:]
        else:
            break
    return s

_config_vars = None

def _init_posix():
    """Initialize the module as appropriate for POSIX systems."""
    g = {}
    # load the installed Makefile:
    try:
        filename = get_makefile_filename()
        parse_makefile(filename, g)
    except IOError, msg:
        my_msg = "invalid Python installation: unable to open %s" % filename
        if hasattr(msg, "strerror"):
            my_msg = my_msg + " (%s)" % msg.strerror

        raise DistutilsPlatformError, my_msg

    # On AIX, there are wrong paths to the linker scripts in the Makefile
    # -- these paths are relative to the Python source, but when installed
    # the scripts are in another directory.
    if sys.platform == 'aix4':          # what about AIX 3.x ?
        # Linker script is in the config directory, not in Modules as the
        # Makefile says.
        python_lib = get_python_lib(standard_lib=1)
        ld_so_aix = os.path.join(python_lib, 'config', 'ld_so_aix')
        python_exp = os.path.join(python_lib, 'config', 'python.exp')

        g['LDSHARED'] = "%s %s -bI:%s" % (ld_so_aix, g['CC'], python_exp)

    elif sys.platform == 'beos':

        # Linker script is in the config directory.  In the Makefile it is
        # relative to the srcdir, which after installation no longer makes
        # sense.
        python_lib = get_python_lib(standard_lib=1)
        linkerscript_name = os.path.basename(string.split(g['LDSHARED'])[0])
        linkerscript = os.path.join(python_lib, 'config', linkerscript_name)

        # XXX this isn't the right place to do this: adding the Python
        # library to the link, if needed, should be in the "build_ext"
        # command.  (It's also needed for non-MS compilers on Windows, and
        # it's taken care of for them by the 'build_ext.get_libraries()'
        # method.)
        g['LDSHARED'] = ("%s -L%s/lib -lpython%s" %
                         (linkerscript, PREFIX, sys.version[0:3]))

    global _config_vars
    _config_vars = g

def _init_nt():
    """Initialize the module as appropriate for NT"""
    g = {}
    # set basic install directories
    g['LIBDEST'] = get_python_lib(plat_specific=0, standard_lib=1)
    g['BINLIBDEST'] = get_python_lib(plat_specific=1, standard_lib=1)

    # XXX hmmm.. a normal install puts include files here
    g['INCLUDEPY'] = get_python_inc(plat_specific=0)

    g['SO'] = '.pyd'
    g['EXE'] = ".exe"

    global _config_vars
    _config_vars = g

def _init_mac():
    """Initialize the module as appropriate for Macintosh systems"""
    g = {}
    # set basic install directories
    g['LIBDEST'] = get_python_lib(plat_specific=0, standard_lib=1)
    g['BINLIBDEST'] = get_python_lib(plat_specific=1, standard_lib=1)

    # XXX hmmm.. a normal install puts include files here
    g['INCLUDEPY'] = get_python_inc(plat_specific=0)

    g['SO'] = '.ppc.slb'

    # XXX are these used anywhere?
    g['install_lib'] = os.path.join(EXEC_PREFIX, "Lib")
    g['install_platlib'] = os.path.join(EXEC_PREFIX, "Mac", "Lib")

    global _config_vars
    _config_vars = g

def get_config_vars(*args):
    """With no arguments, return a dictionary of all configuration
    variables relevant for the current platform.  Generally this includes
    everything needed to build extensions and install both pure modules and
    extensions.  On Unix, this means every variable defined in Python's
    installed Makefile; on Windows and Mac OS it's a much smaller set.

    With arguments, return a list of values that result from looking up
    each argument in the configuration variable dictionary.
    """
    global _config_vars
    if _config_vars is None:
        func = globals().get("_init_" + os.name)
        if func:
            func()
        else:
            _config_vars = {}

        # Normalized versions of prefix and exec_prefix are handy to have;
        # in fact, these are the standard versions used most places in the
        # Distutils.
        _config_vars['prefix'] = PREFIX
        _config_vars['exec_prefix'] = EXEC_PREFIX

    if args:
        vals = []
        for name in args:
            vals.append(_config_vars.get(name))
        return vals
    else:
        return _config_vars

def get_config_var(name):
    """Return the value of a single variable using the dictionary
    returned by 'get_config_vars()'.  Equivalent to
      get_config_vars().get(name)
    """
    return get_config_vars().get(name)
<|endoftext|>


<commit_before>class Query(object):
    """
    Models a Query object for use with ifind SearchEngine interface.

    Query Attributes:
        terms:  a string of search terms
        lang:   a string defining the language of the results (varies on engine)
        format: a string defining the result format (json, xml)
        top:   an integer specifying the number of results to return
        skip:  an integer specifying the offset of starting point for results returned
        result_type: a string defining the type of query (varies on engine)

    """
    def __init__(self, terms, lang='', format='JSON',
                 top=10, skip=0, result_type='', **kwargs):
        """
        Constructs Query object, creating instance attributes from optional keyword parameters (**kwargs).

        """
        self.terms = terms
        self.result_type = result_type.lower()
        self.format = format.upper()
        self.lang = lang
        self.top = top
        self.skip = skip

        for key, value in kwargs.iteritems():
            setattr(self, key, value)

    def __str__(self):
        """
        Returns human-readable string representation of query object.

        """
        return '\n'.join(['{0}: {1}'.format(key, value)
                          for (key, value) in self.__dict__.items()])

    def __eq__(self, other):
        """
        Returns True if both query's attributes are identical, False otherwise.

        """
        return tuple(self.__dict__.items()) == tuple(other.__dict__.items())<commit_msg>.lowers() all attributes at construction<commit_after>class Query(object):
    """
    Models a Query object for use with ifind SearchEngine interface.

    Query Attributes:
        terms:  a string of search terms
        lang:   a string defining the language of the results (varies on engine)
        format: a string defining the result format (json, xml)
        top:   an integer specifying the number of results to return
        skip:  an integer specifying the offset of starting point for results returned
        result_type: a string defining the type of query (varies on engine)

    """
    def __init__(self, terms, lang='', format='JSON',
                 top=10, skip=0, result_type='', **kwargs):
        """
        Constructs Query object, creating instance attributes from optional keyword parameters (**kwargs).

        """
        self.terms = terms
        self.result_type = result_type.lower()
        self.format = format.lower()
        self.lang = lang
        self.top = top
        self.skip = skip

        for key, value in kwargs.iteritems():
            setattr(self, key, value)

    def __str__(self):
        """
        Returns human-readable string representation of query object.

        """
        return '\n'.join(['{0}: {1}'.format(key, value)
                          for (key, value) in self.__dict__.items()])

    def __eq__(self, other):
        """
        Returns True if both query's attributes are identical, False otherwise.

        """
        return tuple(self.__dict__.items()) == tuple(other.__dict__.items())<|endoftext|>


<commit_before>#!/usr/bin/env python
"""igcollect - JSON via HTTP

Copyright (c) 2018 InnoGames GmbH
"""

from argparse import ArgumentParser
from time import time
from urllib.request import urlopen

import json

def parse_args():
    parser = ArgumentParser()
    parser.add_argument('--prefix', default='url_json')
    parser.add_argument('--url', default='http://localhost/')
    parser.add_argument(
        '--key',
        nargs='+',
        dest='keys', )
    return parser.parse_args()

def main():
    args = parse_args()
    response = urlopen(args.url)
    data = json.loads(response.read().decode('utf-8'))

    template = args.prefix + '.{} {} ' + str(int(time()))

    for key, value in data.items():
        if key in args.keys or not args.keys:
            print(template.format(key, value))

if __name__ == '__main__':
    main()
<commit_msg>feat(url_json): Add (optional) JSON path support<commit_after>#!/usr/bin/env python
"""igcollect - JSON via HTTP

Copyright (c) 2018 InnoGames GmbH
"""

from argparse import ArgumentParser
from time import time
from urllib.request import urlopen

import json

def parse_args():
    parser = ArgumentParser(usage=(
        'python3 igcollect/url_json.py'
        ' --prefix servers.HOSTNAME.software.uwsgi'
        ' --url http://localhost:8080'
        ' --key pid'
        ' --json-path workers.[*].requests workers.[*].respawn_count'
    ), description=(
        'Query JSON from URL and extract metrics for graphite text protocol'
    ))
    parser.add_argument('--prefix', default='url_json')
    parser.add_argument('--url', default='http://localhost/')
    parser.add_argument(
        '--key',
        nargs='+',
        dest='keys',
        help='Plain string to match JSON key e.g.: workers.requests',
    )
    parser.add_argument(
        '--json-path',
        nargs='+',
        dest='jsonpaths',
        help=(
            'JSON path to match key e.g.: workers[*].requests '
            '(requires: jsonpath-rw module)')
    )
    return parser.parse_args()

def main():
    args = parse_args()
    response = urlopen(args.url)
    data = json.loads(response.read().decode('utf-8'))

    template = args.prefix + '.{} {} ' + str(int(time()))

    if args.keys:
        for key, value in data.items():
            if key in args.keys:
                print(template.format(key, value))

    if args.jsonpaths:
        from jsonpath_rw import parse

        for path in args.jsonpaths:
            for match in parse(path).find(data):
                # Graphite text protocol can not process square brackets
                graphite_path = str(match.full_path).replace('[', '').replace(
                    ']', '')
                print(template.format(graphite_path, match.value))

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

__author__ = 'Steve Cook'
__email__ = 'steve@snarkout.org'
__version__ = '0.1.0'

from . import git
<commit_msg>Flake8 tweak in __init__.py<commit_after># -*- coding: utf-8 -*-

__author__ = 'Steve Cook'
__email__ = 'steve@snarkout.org'
__version__ = '0.1.0'

__all__ = ['git']
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
# Copyright 2013 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Setup installation module for Image Bundle."""

import os
import distribute_setup
distribute_setup.use_setuptools()

from setuptools import find_packages
from setuptools import setup

CURDIR = os.path.abspath(os.path.dirname(__file__))

def Read(file_name):
  with open(os.path.join(CURDIR, file_name), 'r') as f:
    return f.read().strip()

setup(
    name='gcimagebundle',
    version=Read('VERSION'),
    url='https://github.com/GoogleCloudPlatform/compute-image-packages/tree/master/image-bundle',
    download_url='https://github.com/GoogleCloudPlatform/compute-image-packages/releases',
    license='Apache 2.0',
    author='Google Inc.',
    author_email='gc-team@google.com',
    description=('Image bundling tool for root file system.'),
    long_description=Read('README.md'),
    zip_safe=False,
    classifiers=[
        'Development Status :: 4 - Beta',
        'Environment :: Console',
        'Intended Audience :: Developers',
        'Intended Audience :: System Administrators',
        'License :: OSI Approved :: Apache Software License',
        'Natural Language :: English',
        'Topic :: System :: Filesystems',
        'Topic :: Utilities',
    ],
    platforms='any',
    include_package_data=True,
    packages=find_packages(exclude=['distribute_setup']),
    scripts=['gcimagebundle'],
    tests_suite='gcimagebundlelib.tests',
)
<commit_msg>test_suite<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
# Copyright 2013 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Setup installation module for Image Bundle."""

import os
import distribute_setup
distribute_setup.use_setuptools()

from setuptools import find_packages
from setuptools import setup

CURDIR = os.path.abspath(os.path.dirname(__file__))

def Read(file_name):
  with open(os.path.join(CURDIR, file_name), 'r') as f:
    return f.read().strip()

setup(
    name='gcimagebundle',
    version=Read('VERSION'),
    url='https://github.com/GoogleCloudPlatform/compute-image-packages/tree/master/image-bundle',
    download_url='https://github.com/GoogleCloudPlatform/compute-image-packages/releases',
    license='Apache 2.0',
    author='Google Inc.',
    author_email='gc-team@google.com',
    description=('Image bundling tool for root file system.'),
    long_description=Read('README.md'),
    zip_safe=False,
    classifiers=[
        'Development Status :: 4 - Beta',
        'Environment :: Console',
        'Intended Audience :: Developers',
        'Intended Audience :: System Administrators',
        'License :: OSI Approved :: Apache Software License',
        'Natural Language :: English',
        'Topic :: System :: Filesystems',
        'Topic :: Utilities',
    ],
    platforms='any',
    include_package_data=True,
    packages=find_packages(exclude=['distribute_setup']),
    scripts=['gcimagebundle'],
    test_suite='gcimagebundlelib.tests',
)
<|endoftext|>


<commit_before>import websocket
import json
import zmq
import collections
import threading

def assureNotClosed(self, ws, timeout = 1):
	ws.timeout = timeout
	with self.assertRaises(websocket.WebSocketTimeoutException):
		ws.recv()

def assureIsClosed(self, ws):
	with self.assertRaises(websocket.WebSocketConnectionClosedException):
		ws.recv()

def registerGateway(self, ws, gatewayID):
	msg = json.dumps(
		{
			"gateway_id" : gatewayID,
			"ip_address" : "192.168.1.1",
			"message_type" : "gateway_register",
			"version" : "v1.0"
		}
	)

	ws.send(msg)
	msg = json.loads(ws.recv())

	self.assertEqual("gateway_accepted", msg["message_type"])
	assureNotClosed(self, ws)

class ZMQConnection:
	def __init__(self, endpoint):
		self._endpoint = endpoint
		self._ctx = zmq.Context()
		self._data = collections.deque()
		self._lock = threading.Lock()
		self._active = threading.Event()
		self._finish = threading.Event()
		self._stop = False

	def open(self):
		self._sock = self._ctx.socket(zmq.PULL)
		self._sock.setsockopt(zmq.LINGER, 0) # avoid 'Address already in use'
		self._sock.bind(self._endpoint)
		self.start()

	def close(self):
		self.stop()
		self._sock.close()

	def __enter__(self):
		self.open()

	def __exit__(self, type, value, traceback):
		self.close()

	def should_stop(self):
		with self._lock:
			return self._stop

	def stop(self, timeout = 5):
		with self._lock:
			self._stop = True

		self._finish.wait(timeout)

	def push_data(self, data):
		with self._lock:
			self._data.append(data)

		self._active.set()

	def _pop_data(self):
		with self._lock:
			return self._data.popleft()

	def pop_data(self, timeout = 10):
		if timeout == 0:
			return self._pop_data()
		else:
			self.wait_for_data(timeout)
			return self._pop_data()

	def has_data(self):
		with self._lock:
			return len(self._data) > 0

	def wait_for_data(self, timeout = 10):
		self._active.wait(timeout)
		self._active.clear()
		return self.has_data()

	def accept(self, f):
		self.f = f

	def start(self):
		self._thread = threading.Thread(target = self.run)
		self._thread.start()

	def run(self):
		self._run()
		self._finish.set()

	def _run(self):
		poller = zmq.Poller()
		poller.register(self._sock, zmq.POLLIN)

		while not self.should_stop():
			self._do_poll(poller)

	def _do_poll(self, poller):
		for sock, event in dict(poller.poll(100)).items():
			if event != zmq.POLLIN:
				continue

			try:
				msg = sock.recv_json(zmq.NOBLOCK)

				if self.f is not None:
					if not self.f(msg):
						continue

				self.push_data(msg)
			except Exception as e:
				self.push_data(e)
<commit_msg>T: gws: increase the default zmq data timeout to 20 seconds<commit_after>import websocket
import json
import zmq
import collections
import threading

def assureNotClosed(self, ws, timeout = 1):
	ws.timeout = timeout
	with self.assertRaises(websocket.WebSocketTimeoutException):
		ws.recv()

def assureIsClosed(self, ws):
	with self.assertRaises(websocket.WebSocketConnectionClosedException):
		ws.recv()

def registerGateway(self, ws, gatewayID):
	msg = json.dumps(
		{
			"gateway_id" : gatewayID,
			"ip_address" : "192.168.1.1",
			"message_type" : "gateway_register",
			"version" : "v1.0"
		}
	)

	ws.send(msg)
	msg = json.loads(ws.recv())

	self.assertEqual("gateway_accepted", msg["message_type"])
	assureNotClosed(self, ws)

class ZMQConnection:
	def __init__(self, endpoint):
		self._endpoint = endpoint
		self._ctx = zmq.Context()
		self._data = collections.deque()
		self._lock = threading.Lock()
		self._active = threading.Event()
		self._finish = threading.Event()
		self._stop = False

	def open(self):
		self._sock = self._ctx.socket(zmq.PULL)
		self._sock.setsockopt(zmq.LINGER, 0) # avoid 'Address already in use'
		self._sock.bind(self._endpoint)
		self.start()

	def close(self):
		self.stop()
		self._sock.close()

	def __enter__(self):
		self.open()

	def __exit__(self, type, value, traceback):
		self.close()

	def should_stop(self):
		with self._lock:
			return self._stop

	def stop(self, timeout = 5):
		with self._lock:
			self._stop = True

		self._finish.wait(timeout)

	def push_data(self, data):
		with self._lock:
			self._data.append(data)

		self._active.set()

	def _pop_data(self):
		with self._lock:
			return self._data.popleft()

	def pop_data(self, timeout = 20):
		if timeout == 0:
			return self._pop_data()
		else:
			self.wait_for_data(timeout)
			return self._pop_data()

	def has_data(self):
		with self._lock:
			return len(self._data) > 0

	def wait_for_data(self, timeout = 20):
		self._active.wait(timeout)
		self._active.clear()
		return self.has_data()

	def accept(self, f):
		self.f = f

	def start(self):
		self._thread = threading.Thread(target = self.run)
		self._thread.start()

	def run(self):
		self._run()
		self._finish.set()

	def _run(self):
		poller = zmq.Poller()
		poller.register(self._sock, zmq.POLLIN)

		while not self.should_stop():
			self._do_poll(poller)

	def _do_poll(self, poller):
		for sock, event in dict(poller.poll(100)).items():
			if event != zmq.POLLIN:
				continue

			try:
				msg = sock.recv_json(zmq.NOBLOCK)

				if self.f is not None:
					if not self.f(msg):
						continue

				self.push_data(msg)
			except Exception as e:
				self.push_data(e)
<|endoftext|>


<commit_before># All the functions here are translations of ESRI Terraformer library from JS and are used for converting
# ESRI geometry JSON to GeoJSON
# TODO look for optimization possibilities
import json
import re

WKT_GEOM_REGEX = re.compile('((?:-?\d+(?:.\d+(?:[eE][-+]?\d+)?)?) (?:-?\d+(?:.\d+(?:[eE][-+]?\d+)?)?))')
ESRI_GEOM_REGEX = re.compile('(\[)(?:-?\d+(?:.\d+)?)(, ?)(?:-?\d+(?:.\d+)?)(\])')

def to_esri_feature(wkt):
    geom_type = wkt[:wkt.find('(')]

    def _geom_repl():
        bracket_multiplier = 1
        if geom_type == 'LINESTRING':
            bracket_multiplier = 2
        return json.loads(WKT_GEOM_REGEX.sub(
            lambda m: '[{}]'.format(m.group().replace(' ', ',')),
            wkt.replace(geom_type, '')
        ).replace('(', '[' * bracket_multiplier).replace(')', ']' * bracket_multiplier))

    if geom_type == 'POINT' or geom_type == 'MULTIPOINT':
        match = WKT_GEOM_REGEX.findall(wkt)
        if len(match) != 1:
            raise ValueError('Invalid Point Geometry: {0}'.format(wkt))
        try:
            x, y = map(float, match[0].split())
        except ValueError:
            raise ValueError('Invalid Point Geometry: {0}'.format(wkt))
        return {'x': x, 'y': y}
    elif geom_type in ['LINESTRING', 'MULTILINESTRING']:
        return {'paths': _geom_repl()}
    elif geom_type == 'POLYGON' or geom_type == 'MULTIPOLYGON':
        return {'rings': _geom_repl()}

def from_esri_feature(feature, geom_type):

    def _get_geom_text(geom_text):
        return ESRI_GEOM_REGEX.sub(
            lambda m: m.group().replace('[', '').replace(']', '').replace(',', ' '),
            geom_text
        ).replace('[', '(').replace(']', ')')

    srid = 'SRID={0};'.format(feature.get('spatialReference', {}).get('wkid', 3857))
    if geom_type == 'esriGeometryPoint':
        return '{srid}POINT({x} {y})'.format(srid=srid, x=feature['x'], y=feature['y'])
    elif geom_type == 'esriGeometryPolyline':
        paths_text = json.dumps(feature['paths'])
        return '{srid}MULTILINESTRING{lines}'.format(srid=srid, lines=_get_geom_text(paths_text.replace(' ', '')))
    elif geom_type == 'esriGeometryPolygon':
        rings_text = json.dumps(from_rings(feature['rings'])['coordinates'])
        return '{srid}MULTIPOLYGON{polygons}'.format(srid=srid, polygons=_get_geom_text(rings_text.replace(' ', '')))
    raise ValueError('Unsupported geometry type')

def from_rings(rings):
    """
    Converts an ESRI polygon to GeoJSON.
    :param rings: a collection of rings from an ESRI geometry JSON similar to the following:
        .. code-block:: json

            {
                geometry: {
                    rings: [
                        [[5, 2], [6, 5], [2, 9], [-2, 5], [1, 2], [2, 3], [5, 2]],
                        [[0, 0], [-5, 5], [0, 10], [5, 10], [10, 5], [5, 0], [0, 0]],
                        [[1, 7], [3, 7], [3, 4], [1, 4], [1, 7]]
                    ],
                    'spatialReference': {'wkid': 4326}
                }
            }
        see here for more info: http://help.arcgis.com/en/arcgisserver/10.0/apis/rest/geometry.html
    :return: A GeoJSON object
    """
    outer_rings = []
    holes = []

    for r in rings:
        ring = close_ring(r)
        if len(ring) < 4:
            # A ring with length of less than 4 is not a polygon and can be ignored.
            # close_ring makes sure the first and the last points are the same; that's why minimum length is 4 and not 3
            continue

        if is_clockwise(ring):
            # a clockwise ring means its an outer ring
            outer_rings.append([ring])
        else:
            # otherwise it's a hole!
            holes.append(ring)

    uncontained_holes = []
    reversed_outer_rings = reversed(outer_rings)  # TODO why esri loops over a reversed rings?

    while holes:
        hole = holes.pop()

        # loop over all outer rings and see if they contain our hole.
        contained = False
        for r in reversed_outer_rings:
            outer_ring = r[0]
            if coordinates_contain_coordinates(outer_ring, hole):
                r.append(hole)
                contained = True
                break

        # sometimes a ring may not be contained in any outer ring (https://github.com/Esri/esri-leaflet/issues/320)
        if not contained:
            uncontained_holes.append(hole)

    # check and see if uncontained holes intersect with any outer rings
    while uncontained_holes:
        hole = uncontained_holes.pop()

        intersects = False
        for r in reversed_outer_rings:
            outer_ring = r[0]
            if arrays_intersect(outer_ring, hole):
                outer_rings[r].append(hole)
                intersects = True
                break

        # hole does not intersect ANY outer ring at this point; make it an outer ring.
        if not intersects:
            hole.reverse()
            outer_rings.append([hole])

    return {
        'type': 'MultiPolygon',
        'coordinates': outer_rings
    }

def close_ring(coordinates):
    """
    Makes sure the ring is closed; if it is not, then closes the ring.
    :param coordinates: a collection of coordinates representing a polygon
    :return: a closed ring
    """
    if coordinates[0] != coordinates[-1]:
        coordinates.append(coordinates[0])

    return coordinates

def is_clockwise(ring_to_test):
    """
    A clockwise ring means it is an outer ring; counter-clockwise means the opposite.
    For determining the direction of a ring, we need to calculate the sum of the areas under each side.
    A positive sum indicates a clockwise polygon. See here for more info:
    http://stackoverflow.com/questions/1165647/how-to-determine-if-a-list-of-polygon-points-are-in-clockwise-order
    :param ring_to_test:
    :return: a boolean
    """
    total = 0
    r_length = len(ring_to_test)
    for idx, pt1 in enumerate(ring_to_test):
        # pt1 and pt1 are arrays in form of [x, y] representing the beginning and end points of sides of the ring.
        if idx + 1 != r_length:
            pt2 = ring_to_test[idx + 1]
            total += (pt2[0] - pt1[0]) * (pt2[1] + pt1[1])
    return total >= 0

def edges_intersect(a1, a2, b1, b2):
    """
    The algorithm for determining whether two lines intersect or not:
    https://www.geometrictools.com/Documentation/IntersectionLine2Circle2.pdf
    This one is the same, with visualization: http://geomalgorithms.com/a05-_intersect-1.html
    a and b are vector representation of the received edges and u is the vector from point a1 to point b1
    """

    a = {'x': a2[0] - a1[0], 'y': a2[1] - a1[1]}
    b = {'x': b2[0] - b1[0], 'y': b2[1] - b1[1]}
    u = {'x': a1[0] - b1[0], 'y': a1[1] - b1[1]}
    a_dot_b = (b['y'] * a['x']) - (b['x'] * a['y'])

    if a_dot_b == 0:
        return False

    b_dot_u = (b['x'] * u['y']) - (b['y'] * u['x'])
    a_dot_u = (a['x'] * u['y']) - (a['y'] * u['x'])

    ua = b_dot_u / a_dot_b
    ub = a_dot_u / a_dot_b

    return 0 <= ua <= 1 and 0 <= ub <= 1

def arrays_intersect(a, b):
    try:
        # test for numbers
        int(a[0][0])
        try:
            int(b[0][0])
            for i, _ in enumerate(a[:-1]):
                for j, __ in enumerate(b[:-1]):
                    if edges_intersect(a[i], a[i + 1], b[j], b[j + 1]):
                        return True
        except TypeError:
            return any(arrays_intersect(a, k) for k in b)
    except TypeError:
        return any(arrays_intersect(l, b) for l in a)
    return False

def coordinates_contain_point(coordinates, point):
    """
    This function uses `Crossing number method` (http:#geomalgorithms.com/a03-_inclusion.html) to check whether
    a polygon contains a point or not.
    For each segment with the coordinates `pt1` and `pt2`, we check to see if y coordinate of `point` is
    between y of `pt1` and `pt2`.
    If so, then we check to see if `point` is to the left of the edge; i.e. a line drawn from `point` to the right
    will intersect the edge. If the line intersects the polygon an odd number of times, it is inside.
    """

    contains = False
    j = - 1
    for i, pt1 in enumerate(coordinates):
        pt2 = coordinates[j]
        check_y = lambda: (pt1[1] <= point[1] < pt2[1]) or (pt2[1] <= point[1] < pt1[1])
        # The following checks if `point` is to the left of the current segment
        check_x = lambda: point[0] < (point[1] - pt1[1]) * (pt2[0] - pt1[0]) / (pt2[1] - pt1[1]) + pt1[0]

        if check_y() and check_x():
            contains = not contains
        j = i

    return contains

def coordinates_contain_coordinates(outer, inner):
    intersects = arrays_intersect(outer, inner)
    contains = coordinates_contain_point(outer, inner[0])
    return not intersects and contains
<commit_msg>Fixed esri polygons having too much nesting<commit_after># All the functions here are translations of ESRI Terraformer library from JS and are used for converting
# ESRI geometry JSON to GeoJSON
# TODO look for optimization possibilities
import json
import re

WKT_GEOM_REGEX = re.compile('((?:-?\d+(?:.\d+(?:[eE][-+]?\d+)?)?) (?:-?\d+(?:.\d+(?:[eE][-+]?\d+)?)?))')
ESRI_GEOM_REGEX = re.compile('(\[)(?:-?\d+(?:.\d+)?)(, ?)(?:-?\d+(?:.\d+)?)(\])')

def to_esri_feature(wkt):
    geom_type = wkt[:wkt.find('(')]

    def _geom_repl():
        bracket_multiplier = 1
        if geom_type == 'LINESTRING':
            bracket_multiplier = 2
        return json.loads(WKT_GEOM_REGEX.sub(
            lambda m: '[{}]'.format(m.group().replace(' ', ',')),
            wkt.replace(geom_type, '')
        ).replace('(', '[' * bracket_multiplier).replace(')', ']' * bracket_multiplier))

    if geom_type == 'POINT' or geom_type == 'MULTIPOINT':
        match = WKT_GEOM_REGEX.findall(wkt)
        if len(match) != 1:
            raise ValueError('Invalid Point Geometry: {0}'.format(wkt))
        try:
            x, y = map(float, match[0].split())
        except ValueError:
            raise ValueError('Invalid Point Geometry: {0}'.format(wkt))
        return {'x': x, 'y': y}
    elif geom_type in ['LINESTRING', 'MULTILINESTRING']:
        return {'paths': _geom_repl()}
    elif geom_type == 'POLYGON' or geom_type == 'MULTIPOLYGON':
        return {'rings': _geom_repl()[0]}

def from_esri_feature(feature, geom_type):

    def _get_geom_text(geom_text):
        return ESRI_GEOM_REGEX.sub(
            lambda m: m.group().replace('[', '').replace(']', '').replace(',', ' '),
            geom_text
        ).replace('[', '(').replace(']', ')')

    srid = 'SRID={0};'.format(feature.get('spatialReference', {}).get('wkid', 3857))
    if geom_type == 'esriGeometryPoint':
        return '{srid}POINT({x} {y})'.format(srid=srid, x=feature['x'], y=feature['y'])
    elif geom_type == 'esriGeometryPolyline':
        paths_text = json.dumps(feature['paths'])
        return '{srid}MULTILINESTRING{lines}'.format(srid=srid, lines=_get_geom_text(paths_text.replace(' ', '')))
    elif geom_type == 'esriGeometryPolygon':
        rings_text = json.dumps(from_rings(feature['rings'])['coordinates'])
        return '{srid}MULTIPOLYGON{polygons}'.format(srid=srid, polygons=_get_geom_text(rings_text.replace(' ', '')))
    raise ValueError('Unsupported geometry type')

def from_rings(rings):
    """
    Converts an ESRI polygon to GeoJSON.
    :param rings: a collection of rings from an ESRI geometry JSON similar to the following:
        .. code-block:: json

            {
                geometry: {
                    rings: [
                        [[5, 2], [6, 5], [2, 9], [-2, 5], [1, 2], [2, 3], [5, 2]],
                        [[0, 0], [-5, 5], [0, 10], [5, 10], [10, 5], [5, 0], [0, 0]],
                        [[1, 7], [3, 7], [3, 4], [1, 4], [1, 7]]
                    ],
                    'spatialReference': {'wkid': 4326}
                }
            }
        see here for more info: http://help.arcgis.com/en/arcgisserver/10.0/apis/rest/geometry.html
    :return: A GeoJSON object
    """
    outer_rings = []
    holes = []

    for r in rings:
        ring = close_ring(r)
        if len(ring) < 4:
            # A ring with length of less than 4 is not a polygon and can be ignored.
            # close_ring makes sure the first and the last points are the same; that's why minimum length is 4 and not 3
            continue

        if is_clockwise(ring):
            # a clockwise ring means its an outer ring
            outer_rings.append([ring])
        else:
            # otherwise it's a hole!
            holes.append(ring)

    uncontained_holes = []
    reversed_outer_rings = reversed(outer_rings)  # TODO why esri loops over a reversed rings?

    while holes:
        hole = holes.pop()

        # loop over all outer rings and see if they contain our hole.
        contained = False
        for r in reversed_outer_rings:
            outer_ring = r[0]
            if coordinates_contain_coordinates(outer_ring, hole):
                r.append(hole)
                contained = True
                break

        # sometimes a ring may not be contained in any outer ring (https://github.com/Esri/esri-leaflet/issues/320)
        if not contained:
            uncontained_holes.append(hole)

    # check and see if uncontained holes intersect with any outer rings
    while uncontained_holes:
        hole = uncontained_holes.pop()

        intersects = False
        for r in reversed_outer_rings:
            outer_ring = r[0]
            if arrays_intersect(outer_ring, hole):
                outer_rings[r].append(hole)
                intersects = True
                break

        # hole does not intersect ANY outer ring at this point; make it an outer ring.
        if not intersects:
            hole.reverse()
            outer_rings.append([hole])

    return {
        'type': 'MultiPolygon',
        'coordinates': outer_rings
    }

def close_ring(coordinates):
    """
    Makes sure the ring is closed; if it is not, then closes the ring.
    :param coordinates: a collection of coordinates representing a polygon
    :return: a closed ring
    """
    if coordinates[0] != coordinates[-1]:
        coordinates.append(coordinates[0])

    return coordinates

def is_clockwise(ring_to_test):
    """
    A clockwise ring means it is an outer ring; counter-clockwise means the opposite.
    For determining the direction of a ring, we need to calculate the sum of the areas under each side.
    A positive sum indicates a clockwise polygon. See here for more info:
    http://stackoverflow.com/questions/1165647/how-to-determine-if-a-list-of-polygon-points-are-in-clockwise-order
    :param ring_to_test:
    :return: a boolean
    """
    total = 0
    r_length = len(ring_to_test)
    for idx, pt1 in enumerate(ring_to_test):
        # pt1 and pt1 are arrays in form of [x, y] representing the beginning and end points of sides of the ring.
        if idx + 1 != r_length:
            pt2 = ring_to_test[idx + 1]
            total += (pt2[0] - pt1[0]) * (pt2[1] + pt1[1])
    return total >= 0

def edges_intersect(a1, a2, b1, b2):
    """
    The algorithm for determining whether two lines intersect or not:
    https://www.geometrictools.com/Documentation/IntersectionLine2Circle2.pdf
    This one is the same, with visualization: http://geomalgorithms.com/a05-_intersect-1.html
    a and b are vector representation of the received edges and u is the vector from point a1 to point b1
    """

    a = {'x': a2[0] - a1[0], 'y': a2[1] - a1[1]}
    b = {'x': b2[0] - b1[0], 'y': b2[1] - b1[1]}
    u = {'x': a1[0] - b1[0], 'y': a1[1] - b1[1]}
    a_dot_b = (b['y'] * a['x']) - (b['x'] * a['y'])

    if a_dot_b == 0:
        return False

    b_dot_u = (b['x'] * u['y']) - (b['y'] * u['x'])
    a_dot_u = (a['x'] * u['y']) - (a['y'] * u['x'])

    ua = b_dot_u / a_dot_b
    ub = a_dot_u / a_dot_b

    return 0 <= ua <= 1 and 0 <= ub <= 1

def arrays_intersect(a, b):
    try:
        # test for numbers
        int(a[0][0])
        try:
            int(b[0][0])
            for i, _ in enumerate(a[:-1]):
                for j, __ in enumerate(b[:-1]):
                    if edges_intersect(a[i], a[i + 1], b[j], b[j + 1]):
                        return True
        except TypeError:
            return any(arrays_intersect(a, k) for k in b)
    except TypeError:
        return any(arrays_intersect(l, b) for l in a)
    return False

def coordinates_contain_point(coordinates, point):
    """
    This function uses `Crossing number method` (http:#geomalgorithms.com/a03-_inclusion.html) to check whether
    a polygon contains a point or not.
    For each segment with the coordinates `pt1` and `pt2`, we check to see if y coordinate of `point` is
    between y of `pt1` and `pt2`.
    If so, then we check to see if `point` is to the left of the edge; i.e. a line drawn from `point` to the right
    will intersect the edge. If the line intersects the polygon an odd number of times, it is inside.
    """

    contains = False
    j = - 1
    for i, pt1 in enumerate(coordinates):
        pt2 = coordinates[j]
        check_y = lambda: (pt1[1] <= point[1] < pt2[1]) or (pt2[1] <= point[1] < pt1[1])
        # The following checks if `point` is to the left of the current segment
        check_x = lambda: point[0] < (point[1] - pt1[1]) * (pt2[0] - pt1[0]) / (pt2[1] - pt1[1]) + pt1[0]

        if check_y() and check_x():
            contains = not contains
        j = i

    return contains

def coordinates_contain_coordinates(outer, inner):
    intersects = arrays_intersect(outer, inner)
    contains = coordinates_contain_point(outer, inner[0])
    return not intersects and contains
<|endoftext|>


<commit_before>__author__ = '@abhinavbom a.k.a Darkl0rd'

#local import
from lib.feeds import *

#stdlib import
import sys
import requests

# for traffic inspection from eclipse only
DEBUG=False

# Proxy Support. Added local proxy for debugging purpose.
if DEBUG:
    HTTP_PROXY = '127.0.0.1:80'
    HTTPS_PROXY = '127.0.0.1:443'
# Add your own proxy server to pass traffic through it
else:
    HTTP_PROXY = ''  # Enter your proxy address
    HTTPS_PROXY = HTTP_PROXY    #enter HTTPS proxy address(remove the assigned HTTPS_PROXY variable)

USER_AGENT = 'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:31.0) Gecko/20100101 Firefox/31.0'

#Creating basic HTTP headers to avoid traffic being dropped by filters.
def create_basic_headers():
    headers = {}
    headers['User-Agent'] = USER_AGENT
    headers['Accept-Language'] = 'en-US,en;q=0.5'
    headers['Content-Type'] = 'application/x-www-form-urlencoded; charset=UTF-8'
    return headers
# parse function calls feeds.sources and traverses each of them to look for the input vector.

def url_param():
    params={}
    return params

def connect(url,params):
    print "Connecting with", url
    try:
        r = requests.get(url,
                        headers = create_basic_headers(),
                        proxies = {'http': HTTP_PROXY, 'https': HTTPS_PROXY},
                        params=url_param())
        return r
    except Exception as exc:
        sys.stdout.write('[!] Could not connect to: %s\n' % url)
#        sys.stdout.write('Exception: %s' % exc)

def parse_ip(ip):
    counter = 0
    ioc_list = []
    for filename, source in OSINT_IP.iteritems():
        c = connect(source,params=url_param())
        for line in c:
            if line.startswith("/") or line.startswith('\n') or line.startswith("#"):
                pass
            else:
                counter += 1
                d = 0
                while d < len(ip):
                    if ip[d] in line:
                        print ip[d], source

                    d +=1

    print "Total scanned indicators", counter

def parse_ipList(list):
    counter = 0
    ioc_list = []
    for filename, source in OSINT_ip.iteritems():
        c = connect(source)
        list = open("items.txt", "r")
        for items in list.readlines():
            for line in c:
                if line.startswith("/") or line.startswith('\n') or line.startswith("#"):
                    pass
                else:
                    counter += 1
                    if items in line:
                        print items, filename

    print "Total scanned indicators", counter
<commit_msg>Added function to match patterns using regex<commit_after>__author__ = '@abhinavbom a.k.a Darkl0rd'

#local import
from lib.feeds import *

#stdlib import
import sys
import requests

# for traffic inspection from eclipse only
DEBUG=False

# Proxy Support. Added local proxy for debugging purpose.
if DEBUG:
    HTTP_PROXY = '127.0.0.1:80'
    HTTPS_PROXY = '127.0.0.1:443'
# Add your own proxy server to pass traffic through it
else:
    HTTP_PROXY = ''  # Enter your proxy address
    HTTPS_PROXY = HTTP_PROXY    #enter HTTPS proxy address(remove the assigned HTTPS_PROXY variable)

USER_AGENT = 'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:31.0) Gecko/20100101 Firefox/31.0'

#Creating basic HTTP headers to avoid traffic being dropped by filters.
def create_basic_headers():
    headers = {}
    headers['User-Agent'] = USER_AGENT
    headers['Accept-Language'] = 'en-US,en;q=0.5'
    headers['Content-Type'] = 'application/x-www-form-urlencoded; charset=UTF-8'
    return headers
# parse function calls feeds.sources and traverses each of them to look for the input vector.

def regex(ioc_type):
    ioc_patts = {
        "ip":"((?:(?:[12]\d?\d?|[1-9]\d|[1-9])(?:\[\.\]|\.)){3}(?:[12]\d?\d?|[\d+]{1,2}))",
        "domain":"([A-Za-z0-9]+(?:[\-|\.][A-Za-z0-9]+)*(?:\[\.\]|\.)(?:com|net|edu|ru|org|de|uk|jp|br|pl|info|fr|it|cn|in|su|pw|biz|co|eu|nl|kr|me))",
        "md5":"\W([A-Fa-f0-9]{32})(?:\W|$)",
        "sha1":"\W([A-Fa-f0-9]{40})(?:\W|$)",
        "sha256":"\W([A-Fa-f0-9]{64})(?:\W|$)",
        "email":"[a-zA-Z0-9_]+(?:\.[A-Za-z0-9!#$%&'*+/=?^_`{|}~-]+)*@(?!([a-zA-Z0-9]*\.[a-zA-Z0-9]*\.[a-zA-Z0-9]*\.))(?:[A-Za-z0-9](?:[a-zA-Z0-9-]*[A-Za-z0-9])?\.)+[a-zA-Z0-9](?:[a-zA-Z0-9-]*[a-zA-Z0-9])?",
        "URL":"((?:http|ftp|https)\:\/\/(?:[\w+?\.\w+])+[a-zA-Z0-9\~\!\@\#\$\%\^\&\*\(\)_\-\=\+\\\/\?\.\:\;]+)",
    }

    try:
        pattern = re.compile(ioc_patts[ioc_type])
    except re.error:
        print '[!] Invalid type specified.'
        sys.exit(0)

    return pattern

def url_param():
    params={}
    return params

def connect(url,params):
    print "Connecting with", url
    try:
        r = requests.get(url,
                        headers = create_basic_headers(),
                        proxies = {'http': HTTP_PROXY, 'https': HTTPS_PROXY},
                        params=url_param())
        return r
    except Exception as exc:
        sys.stdout.write('[!] Could not connect to: %s\n' %url)
#        sys.stdout.write('Exception: %s' % exc)

def parse_ip(ip):
    counter = 0
    ioc_list = []
    for filename, source in OSINT_IP.iteritems():
        c = connect(source,params=url_param())
        for line in c:
            if line.startswith("/") or line.startswith('\n') or line.startswith("#"):
                pass
            else:
                counter += 1
                d = 0
                while d < len(ip):
                    if ip[d] in line:
                        print ip[d], source

                    d +=1

    print "Total scanned indicators", counter

def parse_ipList(list):
    counter = 0
    ioc_list = []
    for filename, source in OSINT_ip.iteritems():
        c = connect(source)
        list = open("items.txt", "r")
        for items in list.readlines():
            for line in c:
                if line.startswith("/") or line.startswith('\n') or line.startswith("#"):
                    pass
                else:
                    counter += 1
                    if items in line:
                        print items, filename

    print "Total scanned indicators", counter
<|endoftext|>


<commit_before>#Copyright (c) 2007-9, Playful Invention Company.

#Permission is hereby granted, free of charge, to any person obtaining a copy
#of this software and associated documentation files (the "Software"), to deal
#in the Software without restriction, including without limitation the rights
#to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
#copies of the Software, and to permit persons to whom the Software is
#furnished to do so, subject to the following conditions:

#The above copyright notice and this permission notice shall be included in
#all copies or substantial portions of the Software.

#THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
#IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
#FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
#AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
#LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
#OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
#THE SOFTWARE.

import pygtk
pygtk.require('2.0')
import gtk
import pickle
import cjson
import os.path

from tasprites import *
from taturtle import *
from talogo import stop_logo
from sugar.datastore import datastore

def new_project(tw):
    stop_logo(tw)
    for b in blocks(tw): hide(b)
    setlayer(tw.turtle.canvas, 600)
    clearscreen(tw.turtle)
    tw.save_file_name = None

def load_file(tw):
    fname = get_load_name(tw)
    if fname==None: return
    if fname[-3:]=='.ta': fname=fname[0:-3]
    load_files(tw,fname+'.ta', fname+'.png')
    tw.save_file_name = os.path.basename(fname)

def load_files(tw,ta_file, png_file=''):
    f = open(ta_file, "r")
    try:
        data = pickle.load(f) # old-style data format
    except:
        # print "reading saved json data"
        f.seek(0) # rewind necessary because of pickle.load
        text = f.read()
        listdata = cjson.decode(text)
        data = tuplify(listdata) # json converts tuples to lists
    f.close()
    new_project(tw)
    read_data(tw,data)
    # don't load the png_file -- we run the program instead
    # if png_file != '':
    #     try:
    #         load_pict(tw,png_file)
    #     except:
    #         pass
    #     inval(tw.turtle.canvas)

def get_load_name(tw):
    dialog = gtk.FileChooserDialog("Load...", None, \
        gtk.FILE_CHOOSER_ACTION_OPEN, \
        (gtk.STOCK_CANCEL, gtk.RESPONSE_CANCEL,gtk.STOCK_OPEN, gtk.RESPONSE_OK))
    dialog.set_default_response(gtk.RESPONSE_OK)
    return do_dialog(tw,dialog)

# unpack serialized data sent across a share
def load_string(tw,text):
    listdata = cjson.decode(text)
    data = tuplify(listdata) # json converts tuples to lists
    new_project(tw)
    read_data(tw,data)

def tuplify(t):
    if type(t) is not list:
        return t
    return tuple(map(tuplify, t))

def read_data(tw,data):
    blocks = []
    for b in data:
        if b[1]=='turtle':
            load_turtle(tw,b)
        else: spr = load_spr(tw,b); blocks.append(spr)
    for i in range(len(blocks)):
        cons=[]
        for c in data[i][4]:
            if c==None: cons.append(None)
            else: cons.append(blocks[c])
        blocks[i].connections = cons

def load_spr(tw,b):
    media = None
    btype, label = b[1],None
#    print btype
    if type(btype)==type((1,2)):
        btype, label = btype
#        print "found a label: " + label
    if btype == 'title':  # for backward compatibility
        btype = 'string'
    if btype == 'journal' or btype == 'audiooff':
        media = label
        label = None
    proto = tw.protodict[btype]
    spr = sprNew(tw,b[2]+tw.turtle.canvas.x,b[3]+tw.turtle.canvas.y, \
        proto.image)
    spr.type = 'block'
    spr.proto = proto
    if label!=None: spr.label=label
    if media!=None and media!="media_None" and media!="audio_None":
        try:
            dsobject = datastore.get(media)
            spr.ds_id = dsobject.object_id
            if spr.proto.name == 'journal':
                from talogo import get_pixbuf_from_journal
                pixbuf = get_pixbuf_from_journal \
                    (dsobject,spr.width,spr.height)
                if pixbuf != None:
                    setimage(spr, pixbuf)
                else:
                    setimage(spr, tw.media_shapes['texton'])
            elif spr.proto.name == 'audiooff':
                setimage(spr, tw.media_shapes['audioon'])
            dsobject.destroy()
        except:
            print "couldn't open dsobject (" + str(spr.ds_id) + ")"
    setlayer(spr,650)
    return spr

def load_turtle(tw,b):
    id, name, xcor, ycor, heading, color, shade, pensize = b
    setxy(tw.turtle, xcor, ycor)
    seth(tw.turtle, heading)
    setcolor(tw.turtle, color)
    setshade(tw.turtle, shade)
    setpensize(tw.turtle, pensize)

def load_pict(tw,fname):
    pict = gtk.gdk.pixbuf_new_from_file(fname)
    tw.turtle.canvas.image.draw_pixbuf(tw.turtle.gc, pict, 0, 0, 0, 0)

def save_file(tw):
    if tw.save_folder != None: tw.load_save_folder = tw.save_folder
    fname = get_save_name(tw)
    if fname==None: return
    if fname[-3:]=='.ta': fname=fname[0:-3]
    save_data(tw,fname+".ta")
    save_pict(tw,fname+".png")
    tw.save_file_name = os.path.basename(fname)

def get_save_name(tw):
    dialog = gtk.FileChooserDialog("Save..", None,
               gtk.FILE_CHOOSER_ACTION_SAVE,
               (gtk.STOCK_CANCEL, gtk.RESPONSE_CANCEL,gtk.STOCK_SAVE, \
                   gtk.RESPONSE_OK))
    dialog.set_default_response(gtk.RESPONSE_OK)
    if tw.save_file_name!=None: dialog.set_current_name( \
        tw.save_file_name+'.ta')
    return do_dialog(tw,dialog)

def save_data(tw,fname):
    f = file(fname, "w")
    data = assemble_data_to_save(tw)
    text = cjson.encode(data)
    f.write(text)
    f.close()

# used to send data across a shared session
def save_string(tw):
    data = assemble_data_to_save(tw)
    # encode it for sending across the network
    text = cjson.encode(data)
    return text

def assemble_data_to_save(tw):
    bs = blocks(tw)
    data = []
    for i in range(len(bs)): bs[i].id=i
    for b in bs:
        name = b.proto.name
        if tw.defdict.has_key(name) or name == 'journal' or \
            name == 'audiooff':
            if b.ds_id != None:
                name=(name,str(b.ds_id))
            else:
                name=(name,b.label)
        if hasattr(b,'connections'):
            connections = [get_id(x) for x in b.connections]
        else:
            connections = None
        data.append((b.id,name,b.x-tw.turtle.canvas.x, \
            b.y-tw.turtle.canvas.y,connections))
    data.append((-1,'turtle',
                  tw.turtle.xcor,tw.turtle.ycor,tw.turtle.heading,
                  tw.turtle.color,tw.turtle.shade,tw.turtle.pensize))
    return data

def save_pict(tw,fname):
    tc = tw.turtle.canvas
    pixbuf = gtk.gdk.Pixbuf(gtk.gdk.COLORSPACE_RGB, False, 8, tc.width, \
        tc.height)
    pixbuf.get_from_drawable(tc.image, tc.image.get_colormap(), 0, 0, 0, 0, \
        tc.width, tc.height)
    pixbuf.save(fname, 'png')

def get_id(x):
    if x==None: return None
    return x.id

def do_dialog(tw,dialog):
    result = None
    filter = gtk.FileFilter()
    filter.add_pattern("*.ta")
    filter.set_name("Turtle Art")
    dialog.add_filter(filter)
    dialog.set_current_folder(tw.load_save_folder)
    response = dialog.run()
    if response == gtk.RESPONSE_OK:
        result = dialog.get_filename()
        tw.load_save_folder = dialog.get_current_folder()
    dialog.destroy()
    return result

def blocks(tw): return [spr for spr in tw.sprites if spr.type == 'block']
<commit_msg>simplejson<commit_after>#Copyright (c) 2007-9, Playful Invention Company.

#Permission is hereby granted, free of charge, to any person obtaining a copy
#of this software and associated documentation files (the "Software"), to deal
#in the Software without restriction, including without limitation the rights
#to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
#copies of the Software, and to permit persons to whom the Software is
#furnished to do so, subject to the following conditions:

#The above copyright notice and this permission notice shall be included in
#all copies or substantial portions of the Software.

#THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
#IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
#FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
#AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
#LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
#OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
#THE SOFTWARE.

import pygtk
pygtk.require('2.0')
import gtk
import pickle
#import cjson
try:
    import json
    json.dumps
except (ImportError, AttributeError):
    import simplejson as json
import os.path

from tasprites import *
from taturtle import *
from talogo import stop_logo
from sugar.datastore import datastore

def new_project(tw):
    stop_logo(tw)
    for b in blocks(tw): hide(b)
    setlayer(tw.turtle.canvas, 600)
    clearscreen(tw.turtle)
    tw.save_file_name = None

def load_file(tw):
    fname = get_load_name(tw)
    if fname==None: return
    if fname[-3:]=='.ta': fname=fname[0:-3]
    load_files(tw,fname+'.ta', fname+'.png')
    tw.save_file_name = os.path.basename(fname)

def load_files(tw,ta_file, png_file=''):
    f = open(ta_file, "r")
    try:
        data = pickle.load(f) # old-style data format
    except:
        # print "reading saved json data"
        f.seek(0) # rewind necessary because of pickle.load
        text = f.read()
#        listdata = cjson.decode(text)
        listdata = json.decode(text)
        data = tuplify(listdata) # json converts tuples to lists
    f.close()
    new_project(tw)
    read_data(tw,data)
    # don't load the png_file -- we run the program instead
    # if png_file != '':
    #     try:
    #         load_pict(tw,png_file)
    #     except:
    #         pass
    #     inval(tw.turtle.canvas)

def get_load_name(tw):
    dialog = gtk.FileChooserDialog("Load...", None, \
        gtk.FILE_CHOOSER_ACTION_OPEN, \
        (gtk.STOCK_CANCEL, gtk.RESPONSE_CANCEL,gtk.STOCK_OPEN, gtk.RESPONSE_OK))
    dialog.set_default_response(gtk.RESPONSE_OK)
    return do_dialog(tw,dialog)

# unpack serialized data sent across a share
def load_string(tw,text):
#    listdata = cjson.decode(text)
    listdata = json.decode(text)
    data = tuplify(listdata) # json converts tuples to lists
    new_project(tw)
    read_data(tw,data)

def tuplify(t):
    if type(t) is not list:
        return t
    return tuple(map(tuplify, t))

def read_data(tw,data):
    blocks = []
    for b in data:
        if b[1]=='turtle':
            load_turtle(tw,b)
        else: spr = load_spr(tw,b); blocks.append(spr)
    for i in range(len(blocks)):
        cons=[]
        for c in data[i][4]:
            if c==None: cons.append(None)
            else: cons.append(blocks[c])
        blocks[i].connections = cons

def load_spr(tw,b):
    media = None
    btype, label = b[1],None
#    print btype
    if type(btype)==type((1,2)):
        btype, label = btype
#        print "found a label: " + label
    if btype == 'title':  # for backward compatibility
        btype = 'string'
    if btype == 'journal' or btype == 'audiooff':
        media = label
        label = None
    proto = tw.protodict[btype]
    spr = sprNew(tw,b[2]+tw.turtle.canvas.x,b[3]+tw.turtle.canvas.y, \
        proto.image)
    spr.type = 'block'
    spr.proto = proto
    if label!=None: spr.label=label
    if media!=None and media!="media_None" and media!="audio_None":
        try:
            dsobject = datastore.get(media)
            spr.ds_id = dsobject.object_id
            if spr.proto.name == 'journal':
                from talogo import get_pixbuf_from_journal
                pixbuf = get_pixbuf_from_journal \
                    (dsobject,spr.width,spr.height)
                if pixbuf != None:
                    setimage(spr, pixbuf)
                else:
                    setimage(spr, tw.media_shapes['texton'])
            elif spr.proto.name == 'audiooff':
                setimage(spr, tw.media_shapes['audioon'])
            dsobject.destroy()
        except:
            print "couldn't open dsobject (" + str(spr.ds_id) + ")"
    setlayer(spr,650)
    return spr

def load_turtle(tw,b):
    id, name, xcor, ycor, heading, color, shade, pensize = b
    setxy(tw.turtle, xcor, ycor)
    seth(tw.turtle, heading)
    setcolor(tw.turtle, color)
    setshade(tw.turtle, shade)
    setpensize(tw.turtle, pensize)

def load_pict(tw,fname):
    pict = gtk.gdk.pixbuf_new_from_file(fname)
    tw.turtle.canvas.image.draw_pixbuf(tw.turtle.gc, pict, 0, 0, 0, 0)

def save_file(tw):
    if tw.save_folder != None: tw.load_save_folder = tw.save_folder
    fname = get_save_name(tw)
    if fname==None: return
    if fname[-3:]=='.ta': fname=fname[0:-3]
    save_data(tw,fname+".ta")
    save_pict(tw,fname+".png")
    tw.save_file_name = os.path.basename(fname)

def get_save_name(tw):
    dialog = gtk.FileChooserDialog("Save..", None,
               gtk.FILE_CHOOSER_ACTION_SAVE,
               (gtk.STOCK_CANCEL, gtk.RESPONSE_CANCEL,gtk.STOCK_SAVE, \
                   gtk.RESPONSE_OK))
    dialog.set_default_response(gtk.RESPONSE_OK)
    if tw.save_file_name!=None: dialog.set_current_name( \
        tw.save_file_name+'.ta')
    return do_dialog(tw,dialog)

def save_data(tw,fname):
    f = file(fname, "w")
    data = assemble_data_to_save(tw)
#    text = cjson.encode(data)
    text = json.encode(data)
    f.write(text)
    f.close()

# used to send data across a shared session
def save_string(tw):
    data = assemble_data_to_save(tw)
    # encode it for sending across the network
#    text = cjson.encode(data)
    text = json.encode(data)
    return text

def assemble_data_to_save(tw):
    bs = blocks(tw)
    data = []
    for i in range(len(bs)): bs[i].id=i
    for b in bs:
        name = b.proto.name
        if tw.defdict.has_key(name) or name == 'journal' or \
            name == 'audiooff':
            if b.ds_id != None:
                name=(name,str(b.ds_id))
            else:
                name=(name,b.label)
        if hasattr(b,'connections'):
            connections = [get_id(x) for x in b.connections]
        else:
            connections = None
        data.append((b.id,name,b.x-tw.turtle.canvas.x, \
            b.y-tw.turtle.canvas.y,connections))
    data.append((-1,'turtle',
                  tw.turtle.xcor,tw.turtle.ycor,tw.turtle.heading,
                  tw.turtle.color,tw.turtle.shade,tw.turtle.pensize))
    return data

def save_pict(tw,fname):
    tc = tw.turtle.canvas
    pixbuf = gtk.gdk.Pixbuf(gtk.gdk.COLORSPACE_RGB, False, 8, tc.width, \
        tc.height)
    pixbuf.get_from_drawable(tc.image, tc.image.get_colormap(), 0, 0, 0, 0, \
        tc.width, tc.height)
    pixbuf.save(fname, 'png')

def get_id(x):
    if x==None: return None
    return x.id

def do_dialog(tw,dialog):
    result = None
    filter = gtk.FileFilter()
    filter.add_pattern("*.ta")
    filter.set_name("Turtle Art")
    dialog.add_filter(filter)
    dialog.set_current_folder(tw.load_save_folder)
    response = dialog.run()
    if response == gtk.RESPONSE_OK:
        result = dialog.get_filename()
        tw.load_save_folder = dialog.get_current_folder()
    dialog.destroy()
    return result

def blocks(tw): return [spr for spr in tw.sprites if spr.type == 'block']
<|endoftext|>


<commit_before>"""
Expose each GPU devices directly.

This module implements a API that is like the "CUDA runtime" context manager
for managing CUDA context stack and clean up.  It relies on thread-local globals
to separate the context stack management of each thread. Contexts are also
sharable among threads.  Only the main thread can destroy Contexts.

Note:
- This module must be imported by the main-thread.

"""
from __future__ import print_function, absolute_import, division
import functools
import threading
from numba import servicelib
from .driver import driver

class _DeviceList(object):
    def __getattr__(self, attr):
        # First time looking at "lst" attribute.
        if attr == "lst":
            # Device list is not initialized.
            # Query all CUDA devices.
            numdev = driver.get_device_count()
            gpus = [_DeviceContextManager(driver.get_device(devid))
                    for devid in range(numdev)]
            # Define "lst" to avoid re-initialization
            self.lst = gpus
            return gpus

        # Other attributes
        return super(_DeviceList, self).__getattr__(attr)

    def __getitem__(self, devnum):
        return self.lst[devnum]

    def __str__(self):
        return ', '.join([str(d) for d in self.lst])

    def __iter__(self):
        return iter(self.lst)

    def __len__(self):
        return len(self.lst)

    @property
    def current(self):
        """Returns the active device
        """
        return self.lst[_runtime.current_context.device.id]

class _DeviceContextManager(object):
    """Provides contextmanager functionality to Device objects
    """

    def __init__(self, device):
        self._device = device

    def __getattr__(self, item):
        return getattr(self._device, item)

    def __enter__(self):
        _runtime.push_context(self)

    def __exit__(self, exc_type, exc_val, exc_tb):
        _runtime.pop_context()

    def __str__(self):
        return "<Managed Device {self.id}>".format(self=self)

_MSG_TOO_MANY_CTX_PER_DEV = "Creating more contexts then devices"

class _Runtime(object):
    """Emulate the CUDA runtime context management.

    It owns all Devices and Contexts.
    Keeps at most one Context per Device
    """

    def __init__(self):
        self.gpus = _DeviceList()

        # A thread local stack
        self.context_stack = servicelib.TLStack()
        # Remembers all context
        # key: context.handle.value
        # value: Context
        self._contexts = {}

        # Device's context map
        # key: Device.id
        # value: Context
        self._devctx = {}

        # Remember the main thread
        # Only the main thread can *actually* destroy
        self._mainthread = threading.current_thread()

        # Avoid mutation of runtime state in multithreaded programs
        self._lock = threading.Lock()

    @property
    def current_context(self):
        """Return the active gpu context
        """
        top = self.context_stack.top
        # integrity check
        assert driver.get_context().value == top.handle.value, (
            "An unmanaged CUDA context is active."
        )
        return top

    def _create_context(self, gpu):
        """Create a new context for the given gpu
        """
        # Use lock to prevent internal state mutation
        with self._lock:
            ctx = gpu.create_context()
            self._contexts[ctx.handle.value] = ctx
            self._devctx[gpu.id] = ctx

            assert len(self._contexts) <= len(gpus), _MSG_TOO_MANY_CTX_PER_DEV
            assert len(self._devctx) <= len(gpus), _MSG_TOO_MANY_CTX_PER_DEV
            return ctx

    def _get_or_create_context(self, gpu):
        """Try to use a already created context for the given gpu.  If none
        existed, create a new context.

        Returns the context
        """
        # Get a context from the device
        ctx = self._devctx.get(gpu.id)
        if ctx is None:
            ctx = self._create_context(gpu)
        else:
            # Push the context to the stack
            ctx.push()

        return ctx

    def push_context(self, gpu):
        """Push a context for the given GPU if it is not the current context
        on the context stack.
        """
        # First context
        if not self._contexts:
            # Create a context and quit
            ctx = self._create_context(gpu)

        # Context stack is empty or the active device is not the given gpu
        elif self.context_stack.is_empty or self.current_context.device != gpu:
            ctx = self._get_or_create_context(gpu)

        # Active context is from the gpu
        else:
            ctx = self.current_context

        # Always put the new context on the stack
        self.context_stack.push(ctx)
        return ctx

    def pop_context(self):
        """Pop a context from the context stack if there is more than
        one context in the stack.

        Will not remove the last context in the stack.
        """
        ctx = self.current_context
        # If there is more than one context
        # Do not pop the last context so there is always a active context
        if len(self.context_stack) > 1:
            ctx.pop()
            self.context_stack.pop()
        assert self.context_stack

    def get_or_create_context(self, devnum):
        """Returns the current context or push/create a context for the GPU
        with the given device number.
        """
        if self.context_stack:
            return self.current_context
        else:
            return _runtime.push_context(self.gpus[devnum])

    def reset(self):
        """Clear all contexts in the thread.  Destroy the context if and only
        if we are in the main thread.
        """
        # Clear the context stack
        while self.context_stack:
            ctx = self.context_stack.pop()
            ctx.pop()

        # If it is the main thread
        if threading.current_thread() == self._mainthread:
            self._destroy_all_contexts()

    def _destroy_all_contexts(self):
        # Destroy all contexts
        for ctx in self._contexts.values():
            ctx.reset()
        # Reset all devices
        for gpu in self.gpus:
            gpu.reset()
        # Clear internal references to contexts and devices
        self._contexts.clear()
        self._devctx.clear()

_runtime = _Runtime()

# ================================ PUBLIC API ================================

gpus = _runtime.gpus

def get_context(devnum=0):
    """Get the current device or use a device by device number, and
    return the CUDA context.
    """
    return _runtime.get_or_create_context(devnum)

def require_context(fn):
    """
    A decorator to ensure a context for the CUDA subsystem
    """

    @functools.wraps(fn)
    def _require_cuda_context(*args, **kws):
        get_context()
        return fn(*args, **kws)

    return _require_cuda_context

def reset():
    """Reset the CUDA subsystem for the current thread.

    In the main thread:
    This removes all CUDA contexts.  Only use this at shutdown or for
    cleaning up between tests.

    In non-main threads:
    This clear the CUDA context stack only.

    """
    _runtime.reset()
<commit_msg>Fix docs<commit_after>"""
Expose each GPU devices directly.

This module implements a API that is like the "CUDA runtime" context manager
for managing CUDA context stack and clean up.  It relies on thread-local globals
to separate the context stack management of each thread. Contexts are also
sharable among threads.  Only the main thread can destroy Contexts.

Note:
- This module must be imported by the main-thread.

"""
from __future__ import print_function, absolute_import, division
import functools
import threading
from numba import servicelib
from .driver import driver

class _DeviceList(object):
    def __getattr__(self, attr):
        # First time looking at "lst" attribute.
        if attr == "lst":
            # Device list is not initialized.
            # Query all CUDA devices.
            numdev = driver.get_device_count()
            gpus = [_DeviceContextManager(driver.get_device(devid))
                    for devid in range(numdev)]
            # Define "lst" to avoid re-initialization
            self.lst = gpus
            return gpus

        # Other attributes
        return super(_DeviceList, self).__getattr__(attr)

    def __getitem__(self, devnum):
        return self.lst[devnum]

    def __str__(self):
        return ', '.join([str(d) for d in self.lst])

    def __iter__(self):
        return iter(self.lst)

    def __len__(self):
        return len(self.lst)

    @property
    def current(self):
        """Returns the active device
        """
        return self.lst[_runtime.current_context.device.id]

class _DeviceContextManager(object):
    """Provides contextmanager functionality to Device objects
    """

    def __init__(self, device):
        self._device = device

    def __getattr__(self, item):
        return getattr(self._device, item)

    def __enter__(self):
        _runtime.push_context(self)

    def __exit__(self, exc_type, exc_val, exc_tb):
        _runtime.pop_context()

    def __str__(self):
        return "<Managed Device {self.id}>".format(self=self)

_MSG_TOO_MANY_CTX_PER_DEV = "Creating more contexts then devices"

class _Runtime(object):
    """Emulate the CUDA runtime context management.

    It owns all Devices and Contexts.
    Keeps at most one Context per Device
    """

    def __init__(self):
        self.gpus = _DeviceList()

        # A thread local stack
        self.context_stack = servicelib.TLStack()
        # Remembers all context
        # key: context.handle.value
        # value: Context
        self._contexts = {}

        # Device's context map
        # key: Device.id
        # value: Context
        self._devctx = {}

        # Remember the main thread
        # Only the main thread can *actually* destroy
        self._mainthread = threading.current_thread()

        # Avoid mutation of runtime state in multithreaded programs
        self._lock = threading.Lock()

    @property
    def current_context(self):
        """Return the active gpu context
        """
        top = self.context_stack.top
        # integrity check
        assert driver.get_context().value == top.handle.value, (
            "An unmanaged CUDA context is active."
        )
        return top

    def _create_context(self, gpu):
        """Create a new context for the given gpu
        """
        # Use lock to prevent internal state corruption
        with self._lock:
            ctx = gpu.create_context()
            self._contexts[ctx.handle.value] = ctx
            self._devctx[gpu.id] = ctx

            assert len(self._contexts) <= len(gpus), _MSG_TOO_MANY_CTX_PER_DEV
            assert len(self._devctx) <= len(gpus), _MSG_TOO_MANY_CTX_PER_DEV
            return ctx

    def _get_or_create_context(self, gpu):
        """Try to use a already created context for the given gpu.  If none
        existed, create a new context.

        Returns the context
        """
        # Get a context from the device
        ctx = self._devctx.get(gpu.id)
        if ctx is None:
            ctx = self._create_context(gpu)
        else:
            # Push the context to the stack
            ctx.push()

        return ctx

    def push_context(self, gpu):
        """Push a context for the given GPU or create a new one if no context
        exist for the given GPU.
        """
        # First context
        if not self._contexts:
            # Create a context and quit
            ctx = self._create_context(gpu)

        # Context stack is empty or the active device is not the given gpu
        elif self.context_stack.is_empty or self.current_context.device != gpu:
            ctx = self._get_or_create_context(gpu)

        # Active context is from the gpu
        else:
            ctx = self.current_context

        # Always put the new context on the stack
        self.context_stack.push(ctx)
        return ctx

    def pop_context(self):
        """Pop a context from the context stack if there is more than
        one context in the stack.

        Will not remove the last context in the stack.
        """
        ctx = self.current_context
        # If there is more than one context
        # Do not pop the last context so there is always a active context
        if len(self.context_stack) > 1:
            ctx.pop()
            self.context_stack.pop()
        assert self.context_stack

    def get_or_create_context(self, devnum):
        """Returns the current context or push/create a context for the GPU
        with the given device number.
        """
        if self.context_stack:
            return self.current_context
        else:
            return _runtime.push_context(self.gpus[devnum])

    def reset(self):
        """Clear all contexts in the thread.  Destroy the context if and only
        if we are in the main thread.
        """
        # Clear the context stack
        while self.context_stack:
            ctx = self.context_stack.pop()
            ctx.pop()

        # If it is the main thread
        if threading.current_thread() == self._mainthread:
            self._destroy_all_contexts()

    def _destroy_all_contexts(self):
        # Destroy all contexts
        for ctx in self._contexts.values():
            ctx.reset()
        # Reset all devices
        for gpu in self.gpus:
            gpu.reset()
        # Clear internal references to contexts and devices
        self._contexts.clear()
        self._devctx.clear()

_runtime = _Runtime()

# ================================ PUBLIC API ================================

gpus = _runtime.gpus

def get_context(devnum=0):
    """Get the current device or use a device by device number, and
    return the CUDA context.
    """
    return _runtime.get_or_create_context(devnum)

def require_context(fn):
    """
    A decorator to ensure a context for the CUDA subsystem
    """

    @functools.wraps(fn)
    def _require_cuda_context(*args, **kws):
        get_context()
        return fn(*args, **kws)

    return _require_cuda_context

def reset():
    """Reset the CUDA subsystem for the current thread.

    In the main thread:
    This removes all CUDA contexts.  Only use this at shutdown or for
    cleaning up between tests.

    In non-main threads:
    This clear the CUDA context stack only.

    """
    _runtime.reset()
<|endoftext|>