<commit_before>"""\
Generic (shallow and deep) copying operations
=============================================

Interface summary:

	import copy

	x = copy.copy(y)	# make a shallow copy of y
	x = copy.deepcopy(y)	# make a deep copy of y

For module specific errors, copy.Error is raised.

The difference between shallow and deep copying is only relevant for
compound objects (objects that contain other objects, like lists or
class instances).

- A shallow copy constructs a new compound object and then (to the
  extent possible) inserts *the same objects* into in that the
  original contains.

- A deep copy constructs a new compound object and then, recursively,
  inserts *copies* into it of the objects found in the original.

Two problems often exist with deep copy operations that don't exist
with shallow copy operations:

(a) recursive objects (compound objects that, directly or indirectly,
    contain a reference to themselves) may cause a recursive loop

(b) because deep copy copies *everything* it may copy too much, e.g.
    administrative data structures that should be shared even between
    copies

Python's deep copy operation avoids these problems by:

(a) keeping a table of objects already copied during the current
copying pass

(b) letting user-defined classes override the copying operation or the
    set of components copied

This version does not copy types like module, class, function, method,
nor stack trace, stack frame, nor file, socket, window, nor array, nor
any similar types.

Classes can use the same interfaces to control copying that they use
to control pickling: they can define methods called __getinitargs__(),
__getstate__() and __setstate__().  See the __doc__ string of module
"pickle" for information on these methods.
"""

import types

Error = 'copy.Error'

def copy(x):
	"""Shallow copy operation on arbitrary Python objects.

	See the module's __doc__ string for more info.
	"""

	try:
		copierfunction = _copy_dispatch[type(x)]
	except KeyError:
		try:
			copier = x.__copy__
		except AttributeError:
			raise Error, \
			      "un(shallow)copyable object of type %s" % type(x)
		y = copier()
	else:
		y = copierfunction(x)
	return y

_copy_dispatch = d = {}

def _copy_atomic(x):
	return x
d[types.NoneType] = _copy_atomic
d[types.IntType] = _copy_atomic
d[types.LongType] = _copy_atomic
d[types.FloatType] = _copy_atomic
d[types.StringType] = _copy_atomic
d[types.CodeType] = _copy_atomic
d[types.TypeType] = _copy_atomic
d[types.XRangeType] = _copy_atomic

def _copy_list(x):
	return x[:]
d[types.ListType] = _copy_list

def _copy_tuple(x):
	return x[:]
d[types.TupleType] = _copy_tuple

def _copy_dict(x):
	y = {}
	for key in x.keys():
		y[key] = x[key]
	return y
d[types.DictionaryType] = _copy_dict

def _copy_inst(x):
	if hasattr(x, '__copy__'):
		return x.__copy__()
	if hasattr(x, '__getinitargs__'):
		args = x.__getinitargs__()
	else:
		args = ()
	y = apply(x.__class__, args)
	if hasattr(x, '__getstate__'):
			state = x.__getstate__()
	else:
			state = x.__dict__
	if y.hasattr('__setstate__'):
			y.__setstate__(state)
	else:
			for key in state.keys():
				setattr(y, key, state[key])
	return y
d[types.InstanceType] = _copy_inst

del d

def deepcopy(x, memo = None):
	"""Deep copy operation on arbitrary Python objects.

	See the module's __doc__ string for more info.
	"""

	if memo is None:
		memo = {}
	d = id(x)
	if memo.has_key(d):
		return memo[d]
	try:
		copierfunction = _deepcopy_dispatch[type(x)]
	except KeyError:
		try:
			copier = x.__deepcopy__
		except AttributeError:
			raise Error, \
			      "un-deep-copyable object of type %s" % type(x)
		y = copier(memo)
	else:
		y = copierfunction(x, memo)
	memo[d] = y
	return y

_deepcopy_dispatch = d = {}

def _deepcopy_atomic(x, memo):
	return x
d[types.NoneType] = _deepcopy_atomic
d[types.IntType] = _deepcopy_atomic
d[types.LongType] = _deepcopy_atomic
d[types.FloatType] = _deepcopy_atomic
d[types.StringType] = _deepcopy_atomic
d[types.CodeType] = _deepcopy_atomic
d[types.TypeType] = _deepcopy_atomic
d[types.XRangeType] = _deepcopy_atomic

def _deepcopy_list(x, memo):
	y = []
	memo[id(x)] = y
	for a in x:
		y.append(deepcopy(a, memo))
	return y
d[types.ListType] = _deepcopy_list

def _deepcopy_tuple(x, memo):
	y = []
	for a in x:
		y.append(deepcopy(a, memo))
	d = id(x)
	try:
		return memo[d]
	except KeyError:
		pass
	for i in range(len(x)):
		if x[i] is not y[i]:
			return tuple(y)
	return x
d[types.TupleType] = _deepcopy_tuple

def _deepcopy_dict(x, memo):
	y = {}
	memo[id(x)] = y
	for key in x.keys():
		y[deepcopy(key, memo)] = deepcopy(x[key], memo)
	return y
d[types.DictionaryType] = _deepcopy_dict

def _deepcopy_inst(x, memo):
	if hasattr(x, '__deepcopy__'):
		return x.__deepcopy__()
	if hasattr(x, '__getinitargs__'):
		args = x.__getinitargs__()
		args = deepcopy(args, memo)
	else:
		args = ()
	y = apply(x.__class__, args)
	memo[id(x)] = y
	if hasattr(x, '__getstate__'):
			state = x.__getstate__()
	else:
			state = x.__dict__
	state = deepcopy(state, memo)
	if y.hasattr('__setstate__'):
			y.__setstate__(state)
	else:
			for key in state.keys():
				setattr(y, key, state[key])
	return y
d[types.InstanceType] = _deepcopy_inst

del d

del types

def _test():
	l = [None, 1, 2L, 3.14, 'xyzzy', (1, 2L), [3.14, 'abc'], {'abc': 'ABC'}, (), [], {}]
	l1 = copy(l)
	print l1==l
	l1 = map(copy, l)
	print l1==l
	l1 = deepcopy(l)
	print l1==l
	class C:
		def __init__(self, arg=None):
			self.a = 1
			self.arg = arg
			self.fp = open('copy.py')
			self.fp.close()
		def __getstate__(self):
			return {'a': self.a, 'arg': self.arg}
		def __setstate__(self, state):
			for key in state.keys():
				setattr(self, key, state[key])
		def __deepcopy__(self, memo = None):
			new = self.__class__(deepcopy(self.arg, memo))
			new.a = self.a
			return new
	c = C('argument sketch')
	l.append(c)
	l2 = copy(l)
	print l == l2
	print l
	print l2
	l2 = deepcopy(l)
	print l == l2
	print l
	print l2
	l.append({l[1]: l, 'xyz': l[2]})
	l3 = copy(l)
	import repr
	print map(repr.repr, l)
	print map(repr.repr, l1)
	print map(repr.repr, l2)
	print map(repr.repr, l3)
	l3 = deepcopy(l)
	import repr
	print map(repr.repr, l)
	print map(repr.repr, l1)
	print map(repr.repr, l2)
	print map(repr.repr, l3)

if __name__ == '__main__':
	_test()
<commit_msg>rename copy.Error to copy.error<commit_after>"""\
Generic (shallow and deep) copying operations
=============================================

Interface summary:

	import copy

	x = copy.copy(y)	# make a shallow copy of y
	x = copy.deepcopy(y)	# make a deep copy of y

For module specific errors, copy.error is raised.

The difference between shallow and deep copying is only relevant for
compound objects (objects that contain other objects, like lists or
class instances).

- A shallow copy constructs a new compound object and then (to the
  extent possible) inserts *the same objects* into in that the
  original contains.

- A deep copy constructs a new compound object and then, recursively,
  inserts *copies* into it of the objects found in the original.

Two problems often exist with deep copy operations that don't exist
with shallow copy operations:

(a) recursive objects (compound objects that, directly or indirectly,
    contain a reference to themselves) may cause a recursive loop

(b) because deep copy copies *everything* it may copy too much, e.g.
    administrative data structures that should be shared even between
    copies

Python's deep copy operation avoids these problems by:

(a) keeping a table of objects already copied during the current
copying pass

(b) letting user-defined classes override the copying operation or the
    set of components copied

This version does not copy types like module, class, function, method,
nor stack trace, stack frame, nor file, socket, window, nor array, nor
any similar types.

Classes can use the same interfaces to control copying that they use
to control pickling: they can define methods called __getinitargs__(),
__getstate__() and __setstate__().  See the __doc__ string of module
"pickle" for information on these methods.
"""

import types

error = 'copy.error'
Error = error # backward compatibility

def copy(x):
	"""Shallow copy operation on arbitrary Python objects.

	See the module's __doc__ string for more info.
	"""

	try:
		copierfunction = _copy_dispatch[type(x)]
	except KeyError:
		try:
			copier = x.__copy__
		except AttributeError:
			raise error, \
			      "un(shallow)copyable object of type %s" % type(x)
		y = copier()
	else:
		y = copierfunction(x)
	return y

_copy_dispatch = d = {}

def _copy_atomic(x):
	return x
d[types.NoneType] = _copy_atomic
d[types.IntType] = _copy_atomic
d[types.LongType] = _copy_atomic
d[types.FloatType] = _copy_atomic
d[types.StringType] = _copy_atomic
d[types.CodeType] = _copy_atomic
d[types.TypeType] = _copy_atomic
d[types.XRangeType] = _copy_atomic
d[types.ClassType] = _copy_atomic

def _copy_list(x):
	return x[:]
d[types.ListType] = _copy_list

def _copy_tuple(x):
	return x[:]
d[types.TupleType] = _copy_tuple

def _copy_dict(x):
	y = {}
	for key in x.keys():
		y[key] = x[key]
	return y
d[types.DictionaryType] = _copy_dict

def _copy_inst(x):
	if hasattr(x, '__copy__'):
		return x.__copy__()
	if hasattr(x, '__getinitargs__'):
		args = x.__getinitargs__()
	else:
		args = ()
	y = apply(x.__class__, args)
	if hasattr(x, '__getstate__'):
			state = x.__getstate__()
	else:
			state = x.__dict__
	if y.hasattr('__setstate__'):
			y.__setstate__(state)
	else:
			for key in state.keys():
				setattr(y, key, state[key])
	return y
d[types.InstanceType] = _copy_inst

del d

def deepcopy(x, memo = None):
	"""Deep copy operation on arbitrary Python objects.

	See the module's __doc__ string for more info.
	"""

	if memo is None:
		memo = {}
	d = id(x)
	if memo.has_key(d):
		return memo[d]
	try:
		copierfunction = _deepcopy_dispatch[type(x)]
	except KeyError:
		try:
			copier = x.__deepcopy__
		except AttributeError:
			raise error, \
			      "un-deep-copyable object of type %s" % type(x)
		y = copier(memo)
	else:
		y = copierfunction(x, memo)
	memo[d] = y
	return y

_deepcopy_dispatch = d = {}

def _deepcopy_atomic(x, memo):
	return x
d[types.NoneType] = _deepcopy_atomic
d[types.IntType] = _deepcopy_atomic
d[types.LongType] = _deepcopy_atomic
d[types.FloatType] = _deepcopy_atomic
d[types.StringType] = _deepcopy_atomic
d[types.CodeType] = _deepcopy_atomic
d[types.TypeType] = _deepcopy_atomic
d[types.XRangeType] = _deepcopy_atomic

def _deepcopy_list(x, memo):
	y = []
	memo[id(x)] = y
	for a in x:
		y.append(deepcopy(a, memo))
	return y
d[types.ListType] = _deepcopy_list

def _deepcopy_tuple(x, memo):
	y = []
	for a in x:
		y.append(deepcopy(a, memo))
	d = id(x)
	try:
		return memo[d]
	except KeyError:
		pass
	for i in range(len(x)):
		if x[i] is not y[i]:
			return tuple(y)
	return x
d[types.TupleType] = _deepcopy_tuple

def _deepcopy_dict(x, memo):
	y = {}
	memo[id(x)] = y
	for key in x.keys():
		y[deepcopy(key, memo)] = deepcopy(x[key], memo)
	return y
d[types.DictionaryType] = _deepcopy_dict

def _deepcopy_inst(x, memo):
	if hasattr(x, '__deepcopy__'):
		return x.__deepcopy__()
	if hasattr(x, '__getinitargs__'):
		args = x.__getinitargs__()
		args = deepcopy(args, memo)
	else:
		args = ()
	y = apply(x.__class__, args)
	memo[id(x)] = y
	if hasattr(x, '__getstate__'):
			state = x.__getstate__()
	else:
			state = x.__dict__
	state = deepcopy(state, memo)
	if y.hasattr('__setstate__'):
			y.__setstate__(state)
	else:
			for key in state.keys():
				setattr(y, key, state[key])
	return y
d[types.InstanceType] = _deepcopy_inst

del d

del types

def _test():
	l = [None, 1, 2L, 3.14, 'xyzzy', (1, 2L), [3.14, 'abc'],
	     {'abc': 'ABC'}, (), [], {}]
	l1 = copy(l)
	print l1==l
	l1 = map(copy, l)
	print l1==l
	l1 = deepcopy(l)
	print l1==l
	class C:
		def __init__(self, arg=None):
			self.a = 1
			self.arg = arg
			self.fp = open('copy.py')
			self.fp.close()
		def __getstate__(self):
			return {'a': self.a, 'arg': self.arg}
		def __setstate__(self, state):
			for key in state.keys():
				setattr(self, key, state[key])
		def __deepcopy__(self, memo = None):
			new = self.__class__(deepcopy(self.arg, memo))
			new.a = self.a
			return new
	c = C('argument sketch')
	l.append(c)
	l2 = copy(l)
	print l == l2
	print l
	print l2
	l2 = deepcopy(l)
	print l == l2
	print l
	print l2
	l.append({l[1]: l, 'xyz': l[2]})
	l3 = copy(l)
	import repr
	print map(repr.repr, l)
	print map(repr.repr, l1)
	print map(repr.repr, l2)
	print map(repr.repr, l3)
	l3 = deepcopy(l)
	import repr
	print map(repr.repr, l)
	print map(repr.repr, l1)
	print map(repr.repr, l2)
	print map(repr.repr, l3)

if __name__ == '__main__':
	_test()
<|endoftext|>


<commit_before>"""Append module search paths for third-party packages to sys.path.

****************************************************************
* This module is automatically imported during initialization. *
****************************************************************

In earlier versions of Python (up to 1.5a3), scripts or modules that
needed to use site-specific modules would place ``import site''
somewhere near the top of their code.  Because of the automatic
import, this is no longer necessary (but code that does it still
works).

This will append site-specific paths to to the module search path.  On
Unix, it starts with sys.prefix and sys.exec_prefix (if different) and
appends lib/python<version>/site-packages as well as lib/site-python.
On other platforms (mainly Mac and Windows), it uses just sys.prefix
(and sys.exec_prefix, if different, but this is unlikely).  The
resulting directories, if they exist, are appended to sys.path, and
also inspected for path configuration files.

A path configuration file is a file whose name has the form
<package>.pth; its contents are additional directories (one per line)
to be added to sys.path.  Non-existing directories (or
non-directories) are never added to sys.path; no directory is added to
sys.path more than once.  Blank lines and lines beginning with
'#' are skipped. Lines starting with 'import' are executed.

For example, suppose sys.prefix and sys.exec_prefix are set to
/usr/local and there is a directory /usr/local/lib/python1.5/site-packages
with three subdirectories, foo, bar and spam, and two path
configuration files, foo.pth and bar.pth.  Assume foo.pth contains the
following:

  # foo package configuration
  foo
  bar
  bletch

and bar.pth contains:

  # bar package configuration
  bar

Then the following directories are added to sys.path, in this order:

  /usr/local/lib/python1.5/site-packages/bar
  /usr/local/lib/python1.5/site-packages/foo

Note that bletch is omitted because it doesn't exist; bar precedes foo
because bar.pth comes alphabetically before foo.pth; and spam is
omitted because it is not mentioned in either path configuration file.

After these path manipulations, an attempt is made to import a module
named sitecustomize, which can perform arbitrary additional
site-specific customizations.  If this import fails with an
ImportError exception, it is silently ignored.

"""

import sys, os

def makepath(*paths):
    dir = os.path.abspath(os.path.join(*paths))
    return dir, os.path.normcase(dir)

for m in sys.modules.values():
    if hasattr(m, "__file__") and m.__file__:
        m.__file__ = os.path.abspath(m.__file__)
del m

# This ensures that the initial path provided by the interpreter contains
# only absolute pathnames, even if we're running from the build directory.
L = []
_dirs_in_sys_path = {}
dir = dircase = None  # sys.path may be empty at this point
for dir in sys.path:
    # Filter out duplicate paths (on case-insensitive file systems also
    # if they only differ in case); turn relative paths into absolute
    # paths.
    dir, dircase = makepath(dir)
    if not dircase in _dirs_in_sys_path:
        L.append(dir)
        _dirs_in_sys_path[dircase] = 1
sys.path[:] = L
del dir, dircase, L

# Append ./build/lib.<platform> in case we're running in the build dir
# (especially for Guido :-)
# XXX This should not be part of site.py, since it is needed even when
# using the -S option for Python.  See http://www.python.org/sf/586680
if (os.name == "posix" and sys.path and
    os.path.basename(sys.path[-1]) == "Modules"):
    from distutils.util import get_platform
    s = "build/lib.%s-%.3s" % (get_platform(), sys.version)
    s = os.path.join(os.path.dirname(sys.path[-1]), s)
    sys.path.append(s)
    del get_platform, s

def _init_pathinfo():
    global _dirs_in_sys_path
    _dirs_in_sys_path = d = {}
    for dir in sys.path:
        if dir and not os.path.isdir(dir):
            continue
        dir, dircase = makepath(dir)
        d[dircase] = 1

def addsitedir(sitedir):
    global _dirs_in_sys_path
    if _dirs_in_sys_path is None:
        _init_pathinfo()
        reset = 1
    else:
        reset = 0
    sitedir, sitedircase = makepath(sitedir)
    if not sitedircase in _dirs_in_sys_path:
        sys.path.append(sitedir)        # Add path component
    try:
        names = os.listdir(sitedir)
    except os.error:
        return
    names.sort()
    for name in names:
        if name[-4:] == os.extsep + "pth":
            addpackage(sitedir, name)
    if reset:
        _dirs_in_sys_path = None

def addpackage(sitedir, name):
    global _dirs_in_sys_path
    if _dirs_in_sys_path is None:
        _init_pathinfo()
        reset = 1
    else:
        reset = 0
    fullname = os.path.join(sitedir, name)
    try:
        f = open(fullname)
    except IOError:
        return
    while 1:
        dir = f.readline()
        if not dir:
            break
        if dir[0] == '#':
            continue
        if dir.startswith("import"):
            exec dir
            continue
        if dir[-1] == '\n':
            dir = dir[:-1]
        dir, dircase = makepath(sitedir, dir)
        if not dircase in _dirs_in_sys_path and os.path.exists(dir):
            sys.path.append(dir)
            _dirs_in_sys_path[dircase] = 1
    if reset:
        _dirs_in_sys_path = None

prefixes = [sys.prefix]
if sys.exec_prefix != sys.prefix:
    prefixes.append(sys.exec_prefix)
for prefix in prefixes:
    if prefix:
        if sys.platform in ('os2emx', 'riscos'):
            sitedirs = [os.path.join(prefix, "Lib", "site-packages")]
        elif os.sep == '/':
            sitedirs = [os.path.join(prefix,
                                     "lib",
                                     "python" + sys.version[:3],
                                     "site-packages"),
                        os.path.join(prefix, "lib", "site-python")]
        else:
            sitedirs = [prefix, os.path.join(prefix, "lib", "site-packages")]
        if sys.platform == 'darwin':
            # for framework builds *only* we add the standard Apple
            # locations. Currently only per-user, but /Library and
            # /Network/Library could be added too
            if 'Python.framework' in prefix:
                home = os.environ['HOME']
                if home:
                    sitedirs.append(
                        os.path.join(home,
                                     'Library',
                                     'Python',
                                     sys.version[:3],
                                     'site-packages'))
        for sitedir in sitedirs:
            if os.path.isdir(sitedir):
                addsitedir(sitedir)
del prefix, sitedir

_dirs_in_sys_path = None

# Define new built-ins 'quit' and 'exit'.
# These are simply strings that display a hint on how to exit.
if os.sep == ':':
    exit = 'Use Cmd-Q to quit.'
elif os.sep == '\\':
    exit = 'Use Ctrl-Z plus Return to exit.'
else:
    exit = 'Use Ctrl-D (i.e. EOF) to exit.'
import __builtin__
__builtin__.quit = __builtin__.exit = exit
del exit

# interactive prompt objects for printing the license text, a list of
# contributors and the copyright notice.
class _Printer:
    MAXLINES = 23

    def __init__(self, name, data, files=(), dirs=()):
        self.__name = name
        self.__data = data
        self.__files = files
        self.__dirs = dirs
        self.__lines = None

    def __setup(self):
        if self.__lines:
            return
        data = None
        for dir in self.__dirs:
            for file in self.__files:
                file = os.path.join(dir, file)
                try:
                    fp = open(file)
                    data = fp.read()
                    fp.close()
                    break
                except IOError:
                    pass
            if data:
                break
        if not data:
            data = self.__data
        self.__lines = data.split('\n')
        self.__linecnt = len(self.__lines)

    def __repr__(self):
        self.__setup()
        if len(self.__lines) <= self.MAXLINES:
            return "\n".join(self.__lines)
        else:
            return "Type %s() to see the full %s text" % ((self.__name,)*2)

    def __call__(self):
        self.__setup()
        prompt = 'Hit Return for more, or q (and Return) to quit: '
        lineno = 0
        while 1:
            try:
                for i in range(lineno, lineno + self.MAXLINES):
                    print self.__lines[i]
            except IndexError:
                break
            else:
                lineno += self.MAXLINES
                key = None
                while key is None:
                    key = raw_input(prompt)
                    if key not in ('', 'q'):
                        key = None
                if key == 'q':
                    break

__builtin__.copyright = _Printer("copyright", sys.copyright)
if sys.platform[:4] == 'java':
    __builtin__.credits = _Printer(
        "credits",
        "Jython is maintained by the Jython developers (www.jython.org).")
else:
    __builtin__.credits = _Printer("credits", """\
Thanks to CWI, CNRI, BeOpen.com, Zope Corporation and a cast of thousands
for supporting Python development.  See www.python.org for more information.""")
here = os.path.dirname(os.__file__)
__builtin__.license = _Printer(
    "license", "See http://www.python.org/%.3s/license.html" % sys.version,
    ["LICENSE.txt", "LICENSE"],
    [os.path.join(here, os.pardir), here, os.curdir])

# Define new built-in 'help'.
# This is a wrapper around pydoc.help (with a twist).

class _Helper:
    def __repr__(self):
        return "Type help() for interactive help, " \
               "or help(object) for help about object."
    def __call__(self, *args, **kwds):
        import pydoc
        return pydoc.help(*args, **kwds)

__builtin__.help = _Helper()

# On Windows, some default encodings are not provided
# by Python (e.g. "cp932" in Japanese locale), while they
# are always available as "mbcs" in each locale.
# Make them usable by aliasing to "mbcs" in such a case.

if sys.platform == 'win32':
    import locale, codecs
    enc = locale.getdefaultlocale()[1]
    if enc.startswith('cp'):            # "cp***" ?
        try:
            codecs.lookup(enc)
        except LookupError:
            import encodings
            encodings._cache[enc] = encodings._unknown
            encodings.aliases.aliases[enc] = 'mbcs'

# Set the string encoding used by the Unicode implementation.  The
# default is 'ascii', but if you're willing to experiment, you can
# change this.

encoding = "ascii" # Default value set by _PyUnicode_Init()

if 0:
    # Enable to support locale aware default string encodings.
    import locale
    loc = locale.getdefaultlocale()
    if loc[1]:
        encoding = loc[1]

if 0:
    # Enable to switch off string to Unicode coercion and implicit
    # Unicode to string conversion.
    encoding = "undefined"

if encoding != "ascii":
    # On Non-Unicode builds this will raise an AttributeError...
    sys.setdefaultencoding(encoding) # Needs Python Unicode build !

#
# Run custom site specific code, if available.
#
try:
    import sitecustomize
except ImportError:
    pass

#
# Remove sys.setdefaultencoding() so that users cannot change the
# encoding after initialization.  The test for presence is needed when
# this module is run as a script, because this code is executed twice.
#
if hasattr(sys, "setdefaultencoding"):
    del sys.setdefaultencoding

def _test():
    print "sys.path = ["
    for dir in sys.path:
        print "    %s," % `dir`
    print "]"

if __name__ == '__main__':
    _test()
<commit_msg>In an OSX framework build Python could fail if HOME wasn't set, fixed. Fixes #747954.<commit_after>"""Append module search paths for third-party packages to sys.path.

****************************************************************
* This module is automatically imported during initialization. *
****************************************************************

In earlier versions of Python (up to 1.5a3), scripts or modules that
needed to use site-specific modules would place ``import site''
somewhere near the top of their code.  Because of the automatic
import, this is no longer necessary (but code that does it still
works).

This will append site-specific paths to to the module search path.  On
Unix, it starts with sys.prefix and sys.exec_prefix (if different) and
appends lib/python<version>/site-packages as well as lib/site-python.
On other platforms (mainly Mac and Windows), it uses just sys.prefix
(and sys.exec_prefix, if different, but this is unlikely).  The
resulting directories, if they exist, are appended to sys.path, and
also inspected for path configuration files.

A path configuration file is a file whose name has the form
<package>.pth; its contents are additional directories (one per line)
to be added to sys.path.  Non-existing directories (or
non-directories) are never added to sys.path; no directory is added to
sys.path more than once.  Blank lines and lines beginning with
'#' are skipped. Lines starting with 'import' are executed.

For example, suppose sys.prefix and sys.exec_prefix are set to
/usr/local and there is a directory /usr/local/lib/python1.5/site-packages
with three subdirectories, foo, bar and spam, and two path
configuration files, foo.pth and bar.pth.  Assume foo.pth contains the
following:

  # foo package configuration
  foo
  bar
  bletch

and bar.pth contains:

  # bar package configuration
  bar

Then the following directories are added to sys.path, in this order:

  /usr/local/lib/python1.5/site-packages/bar
  /usr/local/lib/python1.5/site-packages/foo

Note that bletch is omitted because it doesn't exist; bar precedes foo
because bar.pth comes alphabetically before foo.pth; and spam is
omitted because it is not mentioned in either path configuration file.

After these path manipulations, an attempt is made to import a module
named sitecustomize, which can perform arbitrary additional
site-specific customizations.  If this import fails with an
ImportError exception, it is silently ignored.

"""

import sys, os

def makepath(*paths):
    dir = os.path.abspath(os.path.join(*paths))
    return dir, os.path.normcase(dir)

for m in sys.modules.values():
    if hasattr(m, "__file__") and m.__file__:
        m.__file__ = os.path.abspath(m.__file__)
del m

# This ensures that the initial path provided by the interpreter contains
# only absolute pathnames, even if we're running from the build directory.
L = []
_dirs_in_sys_path = {}
dir = dircase = None  # sys.path may be empty at this point
for dir in sys.path:
    # Filter out duplicate paths (on case-insensitive file systems also
    # if they only differ in case); turn relative paths into absolute
    # paths.
    dir, dircase = makepath(dir)
    if not dircase in _dirs_in_sys_path:
        L.append(dir)
        _dirs_in_sys_path[dircase] = 1
sys.path[:] = L
del dir, dircase, L

# Append ./build/lib.<platform> in case we're running in the build dir
# (especially for Guido :-)
# XXX This should not be part of site.py, since it is needed even when
# using the -S option for Python.  See http://www.python.org/sf/586680
if (os.name == "posix" and sys.path and
    os.path.basename(sys.path[-1]) == "Modules"):
    from distutils.util import get_platform
    s = "build/lib.%s-%.3s" % (get_platform(), sys.version)
    s = os.path.join(os.path.dirname(sys.path[-1]), s)
    sys.path.append(s)
    del get_platform, s

def _init_pathinfo():
    global _dirs_in_sys_path
    _dirs_in_sys_path = d = {}
    for dir in sys.path:
        if dir and not os.path.isdir(dir):
            continue
        dir, dircase = makepath(dir)
        d[dircase] = 1

def addsitedir(sitedir):
    global _dirs_in_sys_path
    if _dirs_in_sys_path is None:
        _init_pathinfo()
        reset = 1
    else:
        reset = 0
    sitedir, sitedircase = makepath(sitedir)
    if not sitedircase in _dirs_in_sys_path:
        sys.path.append(sitedir)        # Add path component
    try:
        names = os.listdir(sitedir)
    except os.error:
        return
    names.sort()
    for name in names:
        if name[-4:] == os.extsep + "pth":
            addpackage(sitedir, name)
    if reset:
        _dirs_in_sys_path = None

def addpackage(sitedir, name):
    global _dirs_in_sys_path
    if _dirs_in_sys_path is None:
        _init_pathinfo()
        reset = 1
    else:
        reset = 0
    fullname = os.path.join(sitedir, name)
    try:
        f = open(fullname)
    except IOError:
        return
    while 1:
        dir = f.readline()
        if not dir:
            break
        if dir[0] == '#':
            continue
        if dir.startswith("import"):
            exec dir
            continue
        if dir[-1] == '\n':
            dir = dir[:-1]
        dir, dircase = makepath(sitedir, dir)
        if not dircase in _dirs_in_sys_path and os.path.exists(dir):
            sys.path.append(dir)
            _dirs_in_sys_path[dircase] = 1
    if reset:
        _dirs_in_sys_path = None

prefixes = [sys.prefix]
if sys.exec_prefix != sys.prefix:
    prefixes.append(sys.exec_prefix)
for prefix in prefixes:
    if prefix:
        if sys.platform in ('os2emx', 'riscos'):
            sitedirs = [os.path.join(prefix, "Lib", "site-packages")]
        elif os.sep == '/':
            sitedirs = [os.path.join(prefix,
                                     "lib",
                                     "python" + sys.version[:3],
                                     "site-packages"),
                        os.path.join(prefix, "lib", "site-python")]
        else:
            sitedirs = [prefix, os.path.join(prefix, "lib", "site-packages")]
        if sys.platform == 'darwin':
            # for framework builds *only* we add the standard Apple
            # locations. Currently only per-user, but /Library and
            # /Network/Library could be added too
            if 'Python.framework' in prefix:
                home = os.environ.get('HOME')
                if home:
                    sitedirs.append(
                        os.path.join(home,
                                     'Library',
                                     'Python',
                                     sys.version[:3],
                                     'site-packages'))
        for sitedir in sitedirs:
            if os.path.isdir(sitedir):
                addsitedir(sitedir)
del prefix, sitedir

_dirs_in_sys_path = None

# Define new built-ins 'quit' and 'exit'.
# These are simply strings that display a hint on how to exit.
if os.sep == ':':
    exit = 'Use Cmd-Q to quit.'
elif os.sep == '\\':
    exit = 'Use Ctrl-Z plus Return to exit.'
else:
    exit = 'Use Ctrl-D (i.e. EOF) to exit.'
import __builtin__
__builtin__.quit = __builtin__.exit = exit
del exit

# interactive prompt objects for printing the license text, a list of
# contributors and the copyright notice.
class _Printer:
    MAXLINES = 23

    def __init__(self, name, data, files=(), dirs=()):
        self.__name = name
        self.__data = data
        self.__files = files
        self.__dirs = dirs
        self.__lines = None

    def __setup(self):
        if self.__lines:
            return
        data = None
        for dir in self.__dirs:
            for file in self.__files:
                file = os.path.join(dir, file)
                try:
                    fp = open(file)
                    data = fp.read()
                    fp.close()
                    break
                except IOError:
                    pass
            if data:
                break
        if not data:
            data = self.__data
        self.__lines = data.split('\n')
        self.__linecnt = len(self.__lines)

    def __repr__(self):
        self.__setup()
        if len(self.__lines) <= self.MAXLINES:
            return "\n".join(self.__lines)
        else:
            return "Type %s() to see the full %s text" % ((self.__name,)*2)

    def __call__(self):
        self.__setup()
        prompt = 'Hit Return for more, or q (and Return) to quit: '
        lineno = 0
        while 1:
            try:
                for i in range(lineno, lineno + self.MAXLINES):
                    print self.__lines[i]
            except IndexError:
                break
            else:
                lineno += self.MAXLINES
                key = None
                while key is None:
                    key = raw_input(prompt)
                    if key not in ('', 'q'):
                        key = None
                if key == 'q':
                    break

__builtin__.copyright = _Printer("copyright", sys.copyright)
if sys.platform[:4] == 'java':
    __builtin__.credits = _Printer(
        "credits",
        "Jython is maintained by the Jython developers (www.jython.org).")
else:
    __builtin__.credits = _Printer("credits", """\
Thanks to CWI, CNRI, BeOpen.com, Zope Corporation and a cast of thousands
for supporting Python development.  See www.python.org for more information.""")
here = os.path.dirname(os.__file__)
__builtin__.license = _Printer(
    "license", "See http://www.python.org/%.3s/license.html" % sys.version,
    ["LICENSE.txt", "LICENSE"],
    [os.path.join(here, os.pardir), here, os.curdir])

# Define new built-in 'help'.
# This is a wrapper around pydoc.help (with a twist).

class _Helper:
    def __repr__(self):
        return "Type help() for interactive help, " \
               "or help(object) for help about object."
    def __call__(self, *args, **kwds):
        import pydoc
        return pydoc.help(*args, **kwds)

__builtin__.help = _Helper()

# On Windows, some default encodings are not provided
# by Python (e.g. "cp932" in Japanese locale), while they
# are always available as "mbcs" in each locale.
# Make them usable by aliasing to "mbcs" in such a case.

if sys.platform == 'win32':
    import locale, codecs
    enc = locale.getdefaultlocale()[1]
    if enc.startswith('cp'):            # "cp***" ?
        try:
            codecs.lookup(enc)
        except LookupError:
            import encodings
            encodings._cache[enc] = encodings._unknown
            encodings.aliases.aliases[enc] = 'mbcs'

# Set the string encoding used by the Unicode implementation.  The
# default is 'ascii', but if you're willing to experiment, you can
# change this.

encoding = "ascii" # Default value set by _PyUnicode_Init()

if 0:
    # Enable to support locale aware default string encodings.
    import locale
    loc = locale.getdefaultlocale()
    if loc[1]:
        encoding = loc[1]

if 0:
    # Enable to switch off string to Unicode coercion and implicit
    # Unicode to string conversion.
    encoding = "undefined"

if encoding != "ascii":
    # On Non-Unicode builds this will raise an AttributeError...
    sys.setdefaultencoding(encoding) # Needs Python Unicode build !

#
# Run custom site specific code, if available.
#
try:
    import sitecustomize
except ImportError:
    pass

#
# Remove sys.setdefaultencoding() so that users cannot change the
# encoding after initialization.  The test for presence is needed when
# this module is run as a script, because this code is executed twice.
#
if hasattr(sys, "setdefaultencoding"):
    del sys.setdefaultencoding

def _test():
    print "sys.path = ["
    for dir in sys.path:
        print "    %s," % `dir`
    print "]"

if __name__ == '__main__':
    _test()
<|endoftext|>


<commit_before>"""Convenience functions for creating MCEs"""
import logging
import re
import time
from typing import List, Optional, Type, TypeVar, cast, get_type_hints

import typing_inspect
from avrogen.dict_wrapper import DictWrapper

from datahub.metadata.com.linkedin.pegasus2avro.metadata.key import DatasetKey
from datahub.metadata.schema_classes import (
    DatasetLineageTypeClass,
    DatasetSnapshotClass,
    MetadataChangeEventClass,
    OwnershipTypeClass,
    UpstreamClass,
    UpstreamLineageClass,
)

DEFAULT_ENV = "PROD"
DEFAULT_FLOW_CLUSTER = "prod"
UNKNOWN_USER = "urn:li:corpuser:unknown"

logger = logging.getLogger(__name__)

def get_sys_time() -> int:
    # TODO deprecate this
    return int(time.time() * 1000)

def _check_data_platform_name(platform_name: str) -> None:
    if not platform_name.isalpha():
        logger.warning(f"improperly formatted data platform: {platform_name}")

def make_data_platform_urn(platform: str) -> str:
    if platform.startswith("urn:li:dataPlatform:"):
        return platform
    _check_data_platform_name(platform)
    return f"urn:li:dataPlatform:{platform}"

def make_dataset_urn(platform: str, name: str, env: str = DEFAULT_ENV) -> str:
    return f"urn:li:dataset:({make_data_platform_urn(platform)},{name},{env})"

def dataset_urn_to_key(dataset_urn: str) -> Optional[DatasetKey]:
    pattern = r"urn:li:dataset:\(urn:li:dataPlatform:(.*),(.*),(.*)\)"
    results = re.search(pattern, dataset_urn)
    if results is not None:
        return DatasetKey(
            platform=results.group(1), name=results.group(2), origin=results.group(3)
        )
    return None

def make_user_urn(username: str) -> str:
    return f"urn:li:corpuser:{username}"

def make_group_urn(groupname: str) -> str:
    return f"urn:li:corpGroup:{groupname}"

def make_tag_urn(tag: str) -> str:
    return f"urn:li:tag:{tag}"

def make_data_flow_urn(
    orchestrator: str, flow_id: str, cluster: str = DEFAULT_FLOW_CLUSTER
) -> str:
    return f"urn:li:dataFlow:({orchestrator},{flow_id},{cluster})"

def make_data_job_urn_with_flow(flow_urn: str, job_id: str) -> str:
    return f"urn:li:dataJob:({flow_urn},{job_id})"

def make_data_job_urn(
    orchestrator: str, flow_id: str, job_id: str, cluster: str = DEFAULT_FLOW_CLUSTER
) -> str:
    return make_data_job_urn_with_flow(
        make_data_flow_urn(orchestrator, flow_id, cluster), job_id
    )

def make_dashboard_urn(platform: str, name: str) -> str:
    # FIXME: dashboards don't currently include data platform urn prefixes.
    _check_data_platform_name(platform)
    return f"urn:li:dashboard:({platform},{name})"

def make_chart_urn(platform: str, name: str) -> str:
    # FIXME: charts don't currently include data platform urn prefixes.
    _check_data_platform_name(platform)
    return f"urn:li:chart:({platform},{name})"

def make_ml_primary_key_urn(feature_table_name: str, primary_key_name: str) -> str:

    return f"urn:li:mlPrimaryKey:({feature_table_name},{primary_key_name})"

def make_ml_feature_urn(
    feature_table_name: str,
    feature_name: str,
) -> str:

    return f"urn:li:mlFeature:({feature_table_name},{feature_name})"

def make_ml_feature_table_urn(platform: str, feature_table_name: str) -> str:
    return f"urn:li:mlFeatureTable:({make_data_platform_urn(platform)},{feature_table_name})"

def make_ml_model_urn(platform: str, model_name: str, env: str) -> str:
    return f"urn:li:mlModel:({make_data_platform_urn(platform)},{model_name},{env})"

def make_ml_model_deployment_urn(platform: str, deployment_name: str, env: str) -> str:
    return f"urn:li:mlModelDeployment:({make_data_platform_urn(platform)},{deployment_name},{env})"

def make_ml_model_group_urn(platform: str, group_name: str, env: str) -> str:
    return (
        f"urn:li:mlModelGroup:({make_data_platform_urn(platform)},{group_name},{env})"
    )

def is_valid_ownership_type(ownership_type: Optional[str]) -> bool:
    return ownership_type is not None and ownership_type in [
        OwnershipTypeClass.DEVELOPER,
        OwnershipTypeClass.DATAOWNER,
        OwnershipTypeClass.DELEGATE,
        OwnershipTypeClass.PRODUCER,
        OwnershipTypeClass.CONSUMER,
        OwnershipTypeClass.STAKEHOLDER,
    ]

def validate_ownership_type(ownership_type: Optional[str]) -> str:
    if is_valid_ownership_type(ownership_type):
        return cast(str, ownership_type)
    else:
        raise ValueError(f"Unexpected ownership type: {ownership_type}")

def make_lineage_mce(
    upstream_urns: List[str],
    downstream_urn: str,
    lineage_type: str = DatasetLineageTypeClass.TRANSFORMED,
) -> MetadataChangeEventClass:
    mce = MetadataChangeEventClass(
        proposedSnapshot=DatasetSnapshotClass(
            urn=downstream_urn,
            aspects=[
                UpstreamLineageClass(
                    upstreams=[
                        UpstreamClass(
                            dataset=upstream_urn,
                            type=lineage_type,
                        )
                        for upstream_urn in upstream_urns
                    ]
                )
            ],
        )
    )
    return mce

# This bound isn't tight, but it's better than nothing.
Aspect = TypeVar("Aspect", bound=DictWrapper)

def can_add_aspect(mce: MetadataChangeEventClass, AspectType: Type[Aspect]) -> bool:
    SnapshotType = type(mce.proposedSnapshot)

    constructor_annotations = get_type_hints(SnapshotType.__init__)
    aspect_list_union = typing_inspect.get_args(constructor_annotations["aspects"])[0]
    if not isinstance(aspect_list_union, tuple):
        supported_aspect_types = typing_inspect.get_args(aspect_list_union)
    else:
        # On Python 3.6, the union type is represented as a tuple, where
        # the first item is typing.Union and the subsequent elements are
        # the types within the union.
        supported_aspect_types = aspect_list_union[1:]

    return issubclass(AspectType, supported_aspect_types)

def get_aspect_if_available(
    mce: MetadataChangeEventClass, AspectType: Type[Aspect]
) -> Optional[Aspect]:
    assert can_add_aspect(mce, AspectType)

    all_aspects = mce.proposedSnapshot.aspects
    aspects: List[Aspect] = [
        aspect for aspect in all_aspects if isinstance(aspect, AspectType)
    ]

    if len(aspects) > 1:
        raise ValueError(
            f"MCE contains multiple aspects of type {AspectType}: {aspects}"
        )
    if aspects:
        return aspects[0]
    return None

def get_or_add_aspect(mce: MetadataChangeEventClass, default: Aspect) -> Aspect:
    existing = get_aspect_if_available(mce, type(default))
    if existing is not None:
        return existing
    mce.proposedSnapshot.aspects.append(default)  # type: ignore
    return default
<commit_msg>fix(ingestion): deleted alias function call in mce_builder (#3356)<commit_after>"""Convenience functions for creating MCEs"""
import logging
import re
import time
from typing import List, Optional, Type, TypeVar, cast, get_type_hints

import typing_inspect
from avrogen.dict_wrapper import DictWrapper

from datahub.metadata.schema_classes import (
    DatasetKeyClass,
    DatasetLineageTypeClass,
    DatasetSnapshotClass,
    MetadataChangeEventClass,
    OwnershipTypeClass,
    UpstreamClass,
    UpstreamLineageClass,
)

DEFAULT_ENV = "PROD"
DEFAULT_FLOW_CLUSTER = "prod"
UNKNOWN_USER = "urn:li:corpuser:unknown"

logger = logging.getLogger(__name__)

def get_sys_time() -> int:
    # TODO deprecate this
    return int(time.time() * 1000)

def _check_data_platform_name(platform_name: str) -> None:
    if not platform_name.isalpha():
        logger.warning(f"improperly formatted data platform: {platform_name}")

def make_data_platform_urn(platform: str) -> str:
    if platform.startswith("urn:li:dataPlatform:"):
        return platform
    _check_data_platform_name(platform)
    return f"urn:li:dataPlatform:{platform}"

def make_dataset_urn(platform: str, name: str, env: str = DEFAULT_ENV) -> str:
    return f"urn:li:dataset:({make_data_platform_urn(platform)},{name},{env})"

def dataset_urn_to_key(dataset_urn: str) -> Optional[DatasetKeyClass]:
    pattern = r"urn:li:dataset:\(urn:li:dataPlatform:(.*),(.*),(.*)\)"
    results = re.search(pattern, dataset_urn)
    if results is not None:
        return DatasetKeyClass(
            platform=results.group(1), name=results.group(2), origin=results.group(3)
        )
    return None

def make_user_urn(username: str) -> str:
    return f"urn:li:corpuser:{username}"

def make_group_urn(groupname: str) -> str:
    return f"urn:li:corpGroup:{groupname}"

def make_tag_urn(tag: str) -> str:
    return f"urn:li:tag:{tag}"

def make_data_flow_urn(
    orchestrator: str, flow_id: str, cluster: str = DEFAULT_FLOW_CLUSTER
) -> str:
    return f"urn:li:dataFlow:({orchestrator},{flow_id},{cluster})"

def make_data_job_urn_with_flow(flow_urn: str, job_id: str) -> str:
    return f"urn:li:dataJob:({flow_urn},{job_id})"

def make_data_job_urn(
    orchestrator: str, flow_id: str, job_id: str, cluster: str = DEFAULT_FLOW_CLUSTER
) -> str:
    return make_data_job_urn_with_flow(
        make_data_flow_urn(orchestrator, flow_id, cluster), job_id
    )

def make_dashboard_urn(platform: str, name: str) -> str:
    # FIXME: dashboards don't currently include data platform urn prefixes.
    _check_data_platform_name(platform)
    return f"urn:li:dashboard:({platform},{name})"

def make_chart_urn(platform: str, name: str) -> str:
    # FIXME: charts don't currently include data platform urn prefixes.
    _check_data_platform_name(platform)
    return f"urn:li:chart:({platform},{name})"

def make_ml_primary_key_urn(feature_table_name: str, primary_key_name: str) -> str:

    return f"urn:li:mlPrimaryKey:({feature_table_name},{primary_key_name})"

def make_ml_feature_urn(
    feature_table_name: str,
    feature_name: str,
) -> str:

    return f"urn:li:mlFeature:({feature_table_name},{feature_name})"

def make_ml_feature_table_urn(platform: str, feature_table_name: str) -> str:
    return f"urn:li:mlFeatureTable:({make_data_platform_urn(platform)},{feature_table_name})"

def make_ml_model_urn(platform: str, model_name: str, env: str) -> str:
    return f"urn:li:mlModel:({make_data_platform_urn(platform)},{model_name},{env})"

def make_ml_model_deployment_urn(platform: str, deployment_name: str, env: str) -> str:
    return f"urn:li:mlModelDeployment:({make_data_platform_urn(platform)},{deployment_name},{env})"

def make_ml_model_group_urn(platform: str, group_name: str, env: str) -> str:
    return (
        f"urn:li:mlModelGroup:({make_data_platform_urn(platform)},{group_name},{env})"
    )

def is_valid_ownership_type(ownership_type: Optional[str]) -> bool:
    return ownership_type is not None and ownership_type in [
        OwnershipTypeClass.DEVELOPER,
        OwnershipTypeClass.DATAOWNER,
        OwnershipTypeClass.DELEGATE,
        OwnershipTypeClass.PRODUCER,
        OwnershipTypeClass.CONSUMER,
        OwnershipTypeClass.STAKEHOLDER,
    ]

def validate_ownership_type(ownership_type: Optional[str]) -> str:
    if is_valid_ownership_type(ownership_type):
        return cast(str, ownership_type)
    else:
        raise ValueError(f"Unexpected ownership type: {ownership_type}")

def make_lineage_mce(
    upstream_urns: List[str],
    downstream_urn: str,
    lineage_type: str = DatasetLineageTypeClass.TRANSFORMED,
) -> MetadataChangeEventClass:
    mce = MetadataChangeEventClass(
        proposedSnapshot=DatasetSnapshotClass(
            urn=downstream_urn,
            aspects=[
                UpstreamLineageClass(
                    upstreams=[
                        UpstreamClass(
                            dataset=upstream_urn,
                            type=lineage_type,
                        )
                        for upstream_urn in upstream_urns
                    ]
                )
            ],
        )
    )
    return mce

# This bound isn't tight, but it's better than nothing.
Aspect = TypeVar("Aspect", bound=DictWrapper)

def can_add_aspect(mce: MetadataChangeEventClass, AspectType: Type[Aspect]) -> bool:
    SnapshotType = type(mce.proposedSnapshot)

    constructor_annotations = get_type_hints(SnapshotType.__init__)
    aspect_list_union = typing_inspect.get_args(constructor_annotations["aspects"])[0]
    if not isinstance(aspect_list_union, tuple):
        supported_aspect_types = typing_inspect.get_args(aspect_list_union)
    else:
        # On Python 3.6, the union type is represented as a tuple, where
        # the first item is typing.Union and the subsequent elements are
        # the types within the union.
        supported_aspect_types = aspect_list_union[1:]

    return issubclass(AspectType, supported_aspect_types)

def get_aspect_if_available(
    mce: MetadataChangeEventClass, AspectType: Type[Aspect]
) -> Optional[Aspect]:
    assert can_add_aspect(mce, AspectType)

    all_aspects = mce.proposedSnapshot.aspects
    aspects: List[Aspect] = [
        aspect for aspect in all_aspects if isinstance(aspect, AspectType)
    ]

    if len(aspects) > 1:
        raise ValueError(
            f"MCE contains multiple aspects of type {AspectType}: {aspects}"
        )
    if aspects:
        return aspects[0]
    return None

def get_or_add_aspect(mce: MetadataChangeEventClass, default: Aspect) -> Aspect:
    existing = get_aspect_if_available(mce, type(default))
    if existing is not None:
        return existing
    mce.proposedSnapshot.aspects.append(default)  # type: ignore
    return default
<|endoftext|>


<commit_before>import requests
from bs4 import (
    BeautifulSoup as Beau,
    Tag,
    NavigableString
)
from collections import namedtuple

def main():
    url = 'http://portaldalinguaportuguesa.org/index.php?action=fonetica&act=list&region=lbx'
    p = 'html5lib'

    soup = Beau(requests.get(url).text, p)
    identifiers = ['ortho', 'grammar', 'phono']
    wordtable = soup.find(id='rollovertable')('tr')[1:]
    worddicts = [dict(zip(identifiers, word.contents[1:])) for word in wordtable]
    for word in worddicts:
        word['ortho'] = ortho_parse(word['ortho'])
        word['grammar'] = list(word['grammar'].stripped_strings)[0]
    print()

def ortho_parse(td_tag):
    """Parse a <td> ortho tag into a manageable format

    :param td_tag: <td> Tag of the orthography transcription of a word
    :return: List of Syllable namedtuple
    """
    a_tag = td_tag.find('a')
    Syllable = namedtuple('Syllable', ['ortho', 'strongaccent', 'weakaccent', 'previous'])
    syllables = []
    previous_index = ''.join(a_tag.stripped_strings).find('-')
    for i, content in enumerate(a_tag.contents):
        syl_str = list(content.stripped_strings)[0] if type(content) is Tag else str(content)

        # Filter out '·' and '-'
        syl_str = syl_str.replace('-', '')
        if syl_str in '·':
            continue

        syl_accents = markup_tags(content)
        has_strongaccent = True if 'b' in syl_accents else False
        has_weakaccent = True if 'u' in syl_accents else False
        is_previous = True if i < previous_index else False

        assertion = has_weakaccent if has_strongaccent else True
        assert_msg = 'If the syllable is strongly accented it also must be weakly accented'
        assert assertion, assert_msg

        s = Syllable(syl_str, has_weakaccent, has_strongaccent, is_previous)
        syllables.append(s)

    return syllables

def markup_tags(syltag):
    """

    :param syltag:
    :return:
    """
    if type(syltag) is Tag:
        assert len(list(syltag.children)) == 1, "Tag doesn't have just a sole child"
        return markup_tags(list(syltag.children)[0]) + [syltag.name]
    elif type(syltag) is NavigableString:
        return []
    else:
        return False

def phono_parse(td_tag):
    pass

main()

<commit_msg>Add a hyphen handler for ortho syllables<commit_after>import requests
from bs4 import (
    BeautifulSoup as Beau,
    Tag,
    NavigableString
)
from collections import namedtuple

def main():
    homeurl = 'http://portaldalinguaportuguesa.org/index.php'
    query = '?action=fonetica&act=list&region=lbx'
    p = 'html5lib'

    soup = Beau(requests.get(homeurl+query).text, p)
    identifiers = ['ortho', 'grammar', 'phono']
    wordtable = soup.find(id='rollovertable')('tr')[1:]
    worddicts = [dict(zip(identifiers, word.contents[1:])) for word in wordtable]
    parsed_words = [word_parse(w) for w in worddicts]
    print()

def word_parse(word):
    """Parse a word dictionary.

    A word dictionary contains the word's orthographic and phonemic transcription embedded in
    HTML. The word's syllables are separated by special symbols and the syllabics are denoted
    by HTML tags such as <u> or <b>.

    A namedtuple is used to represent each syllable, its transcriptions and syllabics; this
    structure eases the construction of a Phono object.

    :param word: Dictionary containing a word's ortho and phono transcriptions
    :return: List of namedtuple's
    """
    Syl = namedtuple('Syl', ['ortho', 'phono', 'strongaccent', 'weakaccent', 'previous'])
    otrans = ortho_parse(word['ortho'])
    ptrans = phono_parse(word['phono'])
    assert len(otrans) == len(ptrans), """The orthographic and phonemic transcriptions
    of the word have a different number of syllables.
    """

    parsed = []
    for i, ot in enumerate(otrans):
        s = Syl(
            ot.ortho,
            ptrans[i],
            ot.strongaccent,
            ot.weakaccent,
            ot.previous
        )
        parsed.append(s)

    return parsed

def ortho_parse(td_tag):
    """Parse a <td> ortho tag into a manageable format.

    :param td_tag: <td> Tag of the orthography transcription of a word
    :return: List of OrthoSyl namedtuple
    """

    a_tag = td_tag.find('a')
    OrthoSyl = namedtuple('OrthoSyl', ['ortho', 'strongaccent', 'weakaccent', 'previous'])
    previous_index = ''.join(a_tag.stripped_strings).find('-')

    syllables = []
    for i, content in enumerate(a_tag.contents):
        syl_str = list(content.stripped_strings)[0] if type(content) is Tag else str(content)

        # Hop out the '·'
        if syl_str in '·':
            continue

        syl_accents = markup_tags(content)
        has_strongaccent = True if 'b' in syl_accents else False
        has_weakaccent = True if 'u' in syl_accents else False
        is_previous = True if i < previous_index else False

        assert has_weakaccent if has_strongaccent else True, """If the syllable is strongly
        accented it also must be weakly accented
        """

        s = OrthoSyl(syl_str, has_weakaccent, has_strongaccent, is_previous)
        syllables.append(s)

    return clean_hyphen(syllables)

def markup_tags(syltag):
    """

    :param syltag:
    :return:
    """
    if type(syltag) is Tag:
        assert len(list(syltag.children)) == 1, "Tag doesn't have just a sole child"
        return markup_tags(list(syltag.children)[0]) + [syltag.name]
    elif type(syltag) is NavigableString:
        return []
    else:
        return False

def phono_parse(td_tag):
    """Parse a <td> phono tag into a manageable format.

    :param td_tag: <td> Tag of the phonemic transcription of a word
    :return: List of strings with phonemic transcription of a word
    """
    phono_str = list(td_tag.stripped_strings)[0]
    return phono_str.split('.')

def clean_hyphen(syls):
    """Separate two not HTML markup'd hyphenated syllables

    :param syls: List of syls
    :return: Cleaned list of syls
    """
    cleaned_syls = []
    for syl in syls:
        if '-' in syl:
            partition = syl.partition('-')
            cleaned_syls.append(partition[0])
            cleaned_syls.append(partition[2])
        else:
            cleaned_syls.append()
main()

<|endoftext|>


<commit_before><commit_msg>Added missing imports back in<commit_after>from dsp_process import DSPProcess
from dsp_circuit import DSPCircuit
<|endoftext|>


<commit_before>#!/usr/local/bin/python -t -O
# $Id$

# A more complicated gnuplot interface.

# Written by Michael Haggerty <mhagger@blizzard.harvard.edu>
# Derived from earlier version by Konrad Hinsen <hinsen@ibs.ibs.fr>

# New features:
#  +  A gnuplot session is an instance of class `gnuplot', so multiple
#     sessions can be open at once:
#         g = gnuplot()
#  +  The implicitly-generated gnuplot commands can be stored to a file
#     instead of executed immediately:
#         g = gnuplot("commands.gnuplot")
#  +  Can pass arbitrary commands to the gnuplot command interpreter:
#         g("set pointsize 2")
#  +  A gnuplot object knows how to plot three types of `plotitem':
#     `data', `file', and `func'tion.
#  +  Any plotitem can have optional `title' and/or `with' suboptions
#  +  Builtin plotitem types:
#      *  data(array1) -- data from a Python list or NumPy array
#         (option: `cols')
#      *  file("filename") -- data from an existing data file (option:
#         `using')
#      *  func("exp(4.0 * sin(x))") -- functions (passed as a string
#         for gnuplot to evaluate)
#  +  Communication of data between python and gnuplot is via
#     temporary files, which are deleted automatically when their
#     associated plotitem is deleted.
#  +  Plotitems can be created as objects that are kept around and
#     plotted multiple times---temporary files need only be written
#     once.
#  +  Can use `replot' method to add traces to an existing plot.
#  +  Can make persistent gnuplot windows by using the constructor
#     option `persist=1'.

# Restrictions:
#  -  Only 2-d plots are supported
#  -  There is no provision for missing data points in array data
#  -  Does not check whether persistent windows are really supported
#     by the version of gnuplot used (but persistent windows are
#     anyway not the default).
#  -  Plotting to a postscript file is via new `hardcopy' method
#     instead of a option on the `plot' method as in the older
#     version.
#  -  Doesn't automatically plot using 1:2, 1:3, 1:4, etc for
#     multi-columned data as did the old version.  Instead make a
#     temporary data file then plot that file multiple times with
#     different `using=' options:
#         a = temparrayfile(array_nx3)
#         g.plot(file(a, using=(1,2)), file(a, using=(1,3)))
#  -  All of these classes perform their resource deallocation when
#     __del__ is called.  I understand this is not completely
#     reliable, so it is possible that temporary files will be left
#     around.
#  -  There is no way to change the plotting options of plotitems
#     after they are created.

import sys, os, string, tempfile, Numeric

# raised for unrecognized option(s):
class OptionException(Exception):
    pass

# plotitem represents an item that can be plotted by gnuplot.  For the
# finest control over the output, you can create the plotitems yourself
# with additional options or derive new classes from plotitem.
# Members:
#     basecommand -- a string holding the elementary argument that
#                    must be passed to gnuplot's `plot' command for
#                    this item; e.g., 'sin(x)' or '"filename.dat"'.
#     options -- a list of strings that need to be passed as options
#                to the plot command (in the order required); e.g.,
#                ['title "data"', 'with linespoints'].
#     title -- the title requested (undefined if not requested).  Note
#              that `title=None' implies the `notitle' option, whereas
#              omitting the title option implies no option (the
#              gnuplot default is then used).
#     with -- the string requested as a `with' option (undefined if
#             not requested)
class plotitem:
    def __init__(self, basecommand, **keyw):
        self.basecommand = basecommand
        self.options = []
        if keyw.has_key('title'):
            self.title = keyw['title']
            del keyw['title']
            if self.title is None:
                self.options.append('notitle')
            else:
                self.options.append('title "' + self.title + '"')
        if keyw.has_key('with'):
            self.with = keyw['with']
            del keyw['with']
            self.options.append('with ' + self.with)
        if keyw:
            raise OptionException, keyw

    # return the `plot' command, with options, necessary to print out
    # this item:
    def command(self):
        if self.options:
            return self.basecommand + ' ' + string.join(self.options, ' ')
        else:
            return self.basecommand

    # if the plot command requires data to be put on stdin (i.e.,
    # `plot ""'), this method will put that data there.
    def pipein(self, file):
        pass

# represents a mathematical expression that is to be computed by
# gnuplot itself, as in
#     gnuplot> plot sin(x)
# The argument to the contructor is a string which is a gnuplot
# expression.  Example:
#     g.plot(func("sin(x)", with="line 3"))
# or shorthand
#     g.plot("sin(x)")
class func(plotitem):
    def __init__(self, funcstring, **keyw):
        apply(plotitem.__init__, (self, funcstring), keyw)

# An anyfile represents any kind of file, but presumably one that
# holds data in a format readable by gnuplot.  This class simply
# remembers the filename; the existence and format of the file are not
# checked whatsoever.  Note that this is not a plotitem, though it is
# used by the `file' plotitem.  Members:
#   self.filename -- the filename of the file
class anyfile:
    def __init__(self, filename):
        self.filename = filename

# A temp_file is a file that is deleted when the python object is
# deleted.  WARNING: whatever filename you pass to this constructor
# WILL BE DELETED when the temp_file object is deleted, even if it was
# a pre-existing file!  This is intended to be used as a parent class
# of temparrayfile.
class temp_file(anyfile):
    def __del__(self):
        os.unlink(self.filename)

# When an arrayfile is constructed, it creates a file and fills it
# with the contents of a 2-d Numeric array in the format expected by
# gnuplot.  The filename can be specified, otherwise a random filename
# is chosen.  The file is NOT deleted automatically.
class arrayfile(anyfile):
    def __init__(self, set, filename=None):
        # <set> must be a Numeric array
        assert(len(set.shape) == 2)
        (points, columns) = set.shape
        assert(points > 0)
        assert(columns > 0)
        if not filename:
            filename = tempfile.mktemp()
        f = open(filename, 'w')
        for point in set:
            f.write(string.join(map(repr, point.tolist()), ' ') + '\n')
        f.close()
        anyfile.__init__(self, filename)

# Just like an arrayfile except that the file is deleted automatically
# when the python object is deleted.
class temparrayfile(arrayfile, temp_file):
    def __init__(self, set, filename=None):
        arrayfile.__init__(self, set, filename)

# file is a plotitem that represents a file that should be plotted by
# gnuplot.  <file> can be either a string holding the filename of a
# file that already exists, or it can be any kind of anyfile (such as
# a temparrayfile).  Keyword arguments recognized (in addition to
# those supplied by plotitem):
#     using=<n> -- plot that column against line number
#     using=<tuple> -- plot using a:b:c:d etc.
#     using=<string> -- plot `using <string>' (allows gnuplot's
#                       arbitrary column arithmetic)
# Note that the `using' option is interpreted by gnuplot, so columns
# must be numbered starting with 1.  Other keyword arguments are
# passed along to plotitem.  The default `title' for an anyfile
# plotitem is `notitle'.
class file(plotitem):
    def __init__(self, file, using=None, **keyw):
        if isinstance(file, anyfile):
            self.file = file
            # If no title is specified, then use `notitle' for
            # temp_files (to avoid using the temporary filename as the
            # title.)
            if isinstance(file, temp_file) and not keyw.has_key('title'):
                keyw['title'] = None
        elif type(file) == type(""):
            self.file = anyfile(file)
        else:
            raise OptionException
        apply(plotitem.__init__, (self, '"' + self.file.filename + '"'), keyw)
        self.using = using
        if self.using is None:
            pass
        elif type(self.using) == type(""):
            self.options.insert(0, "using " + self.using)
        elif type(self.using) == type(()):
            self.options.insert(0,
                                "using " +
                                string.join(map(repr, self.using), ':'))
        elif type(self.using) == type(1):
            self.options.insert(0, "using " + `self.using`)
        else:
            raise OptionException, 'using=' + `self.using`

# Create a plotitem out of a Python numeric array (or something that
# can be converted to a Float numeric array).  The array is first
# written to a temporary file, then that file is plotted.  Keyword
# arguments recognized (in addition to those supplied by plotitem):
#     cols=<tuple>
# which outputs only the specified columns of the array to the file.
# Columns are of course numbered in the python style, not the gnuplot
# style.  The data are written to the temp file; no copy is kept in
# memory.
class data(file):
    def __init__(self, set, cols=None, **keyw):
        set = Numeric.asarray(set, Numeric.Float)
        if cols is not None:
            set = Numeric.take(set, cols, 1)
        apply(file.__init__, (self, temparrayfile(set)), keyw)

# gnuplot plotting object.  A gnuplot represents a running gnuplot
# process and a pipe to communicate with it.  It keeps a reference to
# each of the plotitems used in the current plot, so that they (and
# their associated temporary files) are not deleted prematurely.
# Members:
#     gnuplot -- the pipe to gnuplot or a file gathering the commands
#     itemlist -- a list of the plotitems that are associated with the
#                 current plot.  These are deleted whenever a new plot
#                 command is issued via the `plot' method.
# Methods:
#     __init__ -- if a filename argument is specified, the commands
#                 will be written to that file instead of being piped
#                 to gnuplot immediately.
#     __call__ -- pass an arbitrary string to the gnuplot process,
#                 followed by a newline.
#     refresh -- issue (or reissue) the plot command using the current
#                plotitems.
#     plot -- clear the old plot and old plotitems, then plot its
#             arguments in a fresh plot command.  Arguments can be: a
#             plotitem, which is plotted along with its internal
#             options; a string, which is plotted as a func(); or
#             anything else, which is plotted as a data().
#     replot -- replot the old items, adding any arguments as
#               additional items as in the plot method.
#     interact -- read lines from stdin and send them, one by one, to
#                 the gnuplot interpreter.  Basically you can type
#                 commands directly to the gnuplot command processor
#                 (though without command-line editing).
#     load -- load a file (using the gnuplot `load' command).
#     save -- save gnuplot commands to a file (using gnuplot `save'
#             command, not the saved plotitems).
#     xlabel,ylabel --  set xlabel or ylabel to be a string.
#     hardcopy -- replot the plot to a postscript file (if filename
#                 argument is specified) or pipe it to lpr othewise.
#                 If the option `color' is set to true, then output
#                 color postscript.
class gnuplot:
    def __init__(self, filename=None, persist=0, debug=0):
        if filename == None:
            if persist:
                self.gnuplot = os.popen('gnuplot -persist', 'w')
            else:
                self.gnuplot = os.popen('gnuplot', 'w')
        else:
            # put gnuplot commands into a file:
            self.gnuplot = open(filename, 'w')
        self.itemlist = []
        self.debug = debug

    def __del__(self):
        self('quit')
        self.gnuplot.close()

    # send a string to the gnuplot process, followed by a newline:
    def __call__(self, s):
        self.gnuplot.write(s + "\n")
        self.gnuplot.flush()
        if self.debug:
            sys.stderr.write("gnuplot> " + s + "\n")

    # refresh the plot, using the current plotitems:
    def refresh(self):
        plotcmds = []
        for item in self.itemlist:
            plotcmds.append(item.command())
        self('plot ' + string.join(plotcmds, ', '))
        for item in self.itemlist:
            item.pipein(self.gnuplot)

    # call the gnuplot `plot' command:
    def plot(self, *data):
        # remove old files:
        self.itemlist = []
        apply(self.replot, data)

    # replot the data, possibly adding new plotitems:
    def replot(self, *items):
        for item in items:
            if isinstance(item, plotitem):
                self.itemlist.append(item)
            elif type(item) is type(""):
                self.itemlist.append(func(item))
            else:
                # assume data is an array:
                self.itemlist.append(data(item))
        self.refresh()

    def interact(self):
        sys.stderr.write("Press C-d to end interactive input\n")
        while 1:
            sys.stderr.write("gnuplot>>> ")
            line = sys.stdin.readline()
            if line == "":
                break
            if line[-1] == "\n": line = line[:-1]
            self(line)

    def load(self, filename=None):
        if filename is None:
            self.interact()
        else:
            self('load "' + filename + '"')

    def save(self, filename):
        self('save "' + filename + '"')

    def xlabel(self, s=None):
        if s==None:
            self('set ylabel')
        else:
            self('set xlabel "' + s + '"')

    def ylabel(self, s=None):
        if s==None:
            self('set ylabel')
        else:
            self('set ylabel "' + s + '"')

    def hardcopy(self, filename = '| lpr', color = 0):
        if color: self('set term postscript enhanced color')
        else: self('set term postscript enhanced')
        self('set output "' + filename + '"')
        self('replot')
        self('set term x11')
        self('set output')

#
# Demo code
#
if __name__ == '__main__':
    from Numeric import *
    import sys

    g1 = gnuplot(debug=1)
    # List of (x, y) pairs
    g1.plot([(0.,1.1),(1.,5.8),(2.,3.3),(3.,4.2)])

    # Two plots given by arrays and one by a gnuplot function:
    g2 = gnuplot(debug=1)
    x = arange(10)
    y1 = x**2
    y2 = (10-x)**2
    g2.plot(data(transpose((x, y1)), title="calculated by python"),
            func("x**2", title="calculated by gnuplot"),
            data(transpose((x, y2)), with="linesp")
            )
    print "Generating postscript file 'junk.ps'"
    g2.hardcopy('junk.ps', color=1)

    sys.stderr.write("Press return to continue...\n")
    sys.stdin.readline()

    del g1, g2

<commit_msg>Fixed options on #!python line.<commit_after>#!/usr/local/bin/python -tO
# $Id$

# A more complicated gnuplot interface.

# Written by Michael Haggerty <mhagger@blizzard.harvard.edu>
# Derived from earlier version by Konrad Hinsen <hinsen@ibs.ibs.fr>

# New features:
#  +  A gnuplot session is an instance of class `gnuplot', so multiple
#     sessions can be open at once:
#         g = gnuplot()
#  +  The implicitly-generated gnuplot commands can be stored to a file
#     instead of executed immediately:
#         g = gnuplot("commands.gnuplot")
#  +  Can pass arbitrary commands to the gnuplot command interpreter:
#         g("set pointsize 2")
#  +  A gnuplot object knows how to plot three types of `plotitem':
#     `data', `file', and `func'tion.
#  +  Any plotitem can have optional `title' and/or `with' suboptions
#  +  Builtin plotitem types:
#      *  data(array1) -- data from a Python list or NumPy array
#         (option: `cols')
#      *  file("filename") -- data from an existing data file (option:
#         `using')
#      *  func("exp(4.0 * sin(x))") -- functions (passed as a string
#         for gnuplot to evaluate)
#  +  Communication of data between python and gnuplot is via
#     temporary files, which are deleted automatically when their
#     associated plotitem is deleted.
#  +  Plotitems can be created as objects that are kept around and
#     plotted multiple times---temporary files need only be written
#     once.
#  +  Can use `replot' method to add traces to an existing plot.
#  +  Can make persistent gnuplot windows by using the constructor
#     option `persist=1'.

# Restrictions:
#  -  Only 2-d plots are supported
#  -  There is no provision for missing data points in array data
#  -  Does not check whether persistent windows are really supported
#     by the version of gnuplot used (but persistent windows are
#     anyway not the default).
#  -  Plotting to a postscript file is via new `hardcopy' method
#     instead of a option on the `plot' method as in the older
#     version.
#  -  Doesn't automatically plot using 1:2, 1:3, 1:4, etc for
#     multi-columned data as did the old version.  Instead make a
#     temporary data file then plot that file multiple times with
#     different `using=' options:
#         a = temparrayfile(array_nx3)
#         g.plot(file(a, using=(1,2)), file(a, using=(1,3)))
#  -  All of these classes perform their resource deallocation when
#     __del__ is called.  I understand this is not completely
#     reliable, so it is possible that temporary files will be left
#     around.
#  -  There is no way to change the plotting options of plotitems
#     after they are created.

import sys, os, string, tempfile, Numeric

# raised for unrecognized option(s):
class OptionException(Exception):
    pass

# plotitem represents an item that can be plotted by gnuplot.  For the
# finest control over the output, you can create the plotitems yourself
# with additional options or derive new classes from plotitem.
# Members:
#     basecommand -- a string holding the elementary argument that
#                    must be passed to gnuplot's `plot' command for
#                    this item; e.g., 'sin(x)' or '"filename.dat"'.
#     options -- a list of strings that need to be passed as options
#                to the plot command (in the order required); e.g.,
#                ['title "data"', 'with linespoints'].
#     title -- the title requested (undefined if not requested).  Note
#              that `title=None' implies the `notitle' option, whereas
#              omitting the title option implies no option (the
#              gnuplot default is then used).
#     with -- the string requested as a `with' option (undefined if
#             not requested)
class plotitem:
    def __init__(self, basecommand, **keyw):
        self.basecommand = basecommand
        self.options = []
        if keyw.has_key('title'):
            self.title = keyw['title']
            del keyw['title']
            if self.title is None:
                self.options.append('notitle')
            else:
                self.options.append('title "' + self.title + '"')
        if keyw.has_key('with'):
            self.with = keyw['with']
            del keyw['with']
            self.options.append('with ' + self.with)
        if keyw:
            raise OptionException, keyw

    # return the `plot' command, with options, necessary to print out
    # this item:
    def command(self):
        if self.options:
            return self.basecommand + ' ' + string.join(self.options, ' ')
        else:
            return self.basecommand

    # if the plot command requires data to be put on stdin (i.e.,
    # `plot ""'), this method will put that data there.
    def pipein(self, file):
        pass

# represents a mathematical expression that is to be computed by
# gnuplot itself, as in
#     gnuplot> plot sin(x)
# The argument to the contructor is a string which is a gnuplot
# expression.  Example:
#     g.plot(func("sin(x)", with="line 3"))
# or shorthand
#     g.plot("sin(x)")
class func(plotitem):
    def __init__(self, funcstring, **keyw):
        apply(plotitem.__init__, (self, funcstring), keyw)

# An anyfile represents any kind of file, but presumably one that
# holds data in a format readable by gnuplot.  This class simply
# remembers the filename; the existence and format of the file are not
# checked whatsoever.  Note that this is not a plotitem, though it is
# used by the `file' plotitem.  Members:
#   self.filename -- the filename of the file
class anyfile:
    def __init__(self, filename):
        self.filename = filename

# A temp_file is a file that is deleted when the python object is
# deleted.  WARNING: whatever filename you pass to this constructor
# WILL BE DELETED when the temp_file object is deleted, even if it was
# a pre-existing file!  This is intended to be used as a parent class
# of temparrayfile.
class temp_file(anyfile):
    def __del__(self):
        os.unlink(self.filename)

# When an arrayfile is constructed, it creates a file and fills it
# with the contents of a 2-d Numeric array in the format expected by
# gnuplot.  The filename can be specified, otherwise a random filename
# is chosen.  The file is NOT deleted automatically.
class arrayfile(anyfile):
    def __init__(self, set, filename=None):
        # <set> must be a Numeric array
        assert(len(set.shape) == 2)
        (points, columns) = set.shape
        assert(points > 0)
        assert(columns > 0)
        if not filename:
            filename = tempfile.mktemp()
        f = open(filename, 'w')
        for point in set:
            f.write(string.join(map(repr, point.tolist()), ' ') + '\n')
        f.close()
        anyfile.__init__(self, filename)

# Just like an arrayfile except that the file is deleted automatically
# when the python object is deleted.
class temparrayfile(arrayfile, temp_file):
    def __init__(self, set, filename=None):
        arrayfile.__init__(self, set, filename)

# file is a plotitem that represents a file that should be plotted by
# gnuplot.  <file> can be either a string holding the filename of a
# file that already exists, or it can be any kind of anyfile (such as
# a temparrayfile).  Keyword arguments recognized (in addition to
# those supplied by plotitem):
#     using=<n> -- plot that column against line number
#     using=<tuple> -- plot using a:b:c:d etc.
#     using=<string> -- plot `using <string>' (allows gnuplot's
#                       arbitrary column arithmetic)
# Note that the `using' option is interpreted by gnuplot, so columns
# must be numbered starting with 1.  Other keyword arguments are
# passed along to plotitem.  The default `title' for an anyfile
# plotitem is `notitle'.
class file(plotitem):
    def __init__(self, file, using=None, **keyw):
        if isinstance(file, anyfile):
            self.file = file
            # If no title is specified, then use `notitle' for
            # temp_files (to avoid using the temporary filename as the
            # title.)
            if isinstance(file, temp_file) and not keyw.has_key('title'):
                keyw['title'] = None
        elif type(file) == type(""):
            self.file = anyfile(file)
        else:
            raise OptionException
        apply(plotitem.__init__, (self, '"' + self.file.filename + '"'), keyw)
        self.using = using
        if self.using is None:
            pass
        elif type(self.using) == type(""):
            self.options.insert(0, "using " + self.using)
        elif type(self.using) == type(()):
            self.options.insert(0,
                                "using " +
                                string.join(map(repr, self.using), ':'))
        elif type(self.using) == type(1):
            self.options.insert(0, "using " + `self.using`)
        else:
            raise OptionException, 'using=' + `self.using`

# Create a plotitem out of a Python numeric array (or something that
# can be converted to a Float numeric array).  The array is first
# written to a temporary file, then that file is plotted.  Keyword
# arguments recognized (in addition to those supplied by plotitem):
#     cols=<tuple>
# which outputs only the specified columns of the array to the file.
# Columns are of course numbered in the python style, not the gnuplot
# style.  The data are written to the temp file; no copy is kept in
# memory.
class data(file):
    def __init__(self, set, cols=None, **keyw):
        set = Numeric.asarray(set, Numeric.Float)
        if cols is not None:
            set = Numeric.take(set, cols, 1)
        apply(file.__init__, (self, temparrayfile(set)), keyw)

# gnuplot plotting object.  A gnuplot represents a running gnuplot
# process and a pipe to communicate with it.  It keeps a reference to
# each of the plotitems used in the current plot, so that they (and
# their associated temporary files) are not deleted prematurely.
# Members:
#     gnuplot -- the pipe to gnuplot or a file gathering the commands
#     itemlist -- a list of the plotitems that are associated with the
#                 current plot.  These are deleted whenever a new plot
#                 command is issued via the `plot' method.
# Methods:
#     __init__ -- if a filename argument is specified, the commands
#                 will be written to that file instead of being piped
#                 to gnuplot immediately.
#     __call__ -- pass an arbitrary string to the gnuplot process,
#                 followed by a newline.
#     refresh -- issue (or reissue) the plot command using the current
#                plotitems.
#     plot -- clear the old plot and old plotitems, then plot its
#             arguments in a fresh plot command.  Arguments can be: a
#             plotitem, which is plotted along with its internal
#             options; a string, which is plotted as a func(); or
#             anything else, which is plotted as a data().
#     replot -- replot the old items, adding any arguments as
#               additional items as in the plot method.
#     interact -- read lines from stdin and send them, one by one, to
#                 the gnuplot interpreter.  Basically you can type
#                 commands directly to the gnuplot command processor
#                 (though without command-line editing).
#     load -- load a file (using the gnuplot `load' command).
#     save -- save gnuplot commands to a file (using gnuplot `save'
#             command, not the saved plotitems).
#     xlabel,ylabel --  set xlabel or ylabel to be a string.
#     hardcopy -- replot the plot to a postscript file (if filename
#                 argument is specified) or pipe it to lpr othewise.
#                 If the option `color' is set to true, then output
#                 color postscript.
class gnuplot:
    def __init__(self, filename=None, persist=0, debug=0):
        if filename == None:
            if persist:
                self.gnuplot = os.popen('gnuplot -persist', 'w')
            else:
                self.gnuplot = os.popen('gnuplot', 'w')
        else:
            # put gnuplot commands into a file:
            self.gnuplot = open(filename, 'w')
        self.itemlist = []
        self.debug = debug

    def __del__(self):
        self('quit')
        self.gnuplot.close()

    # send a string to the gnuplot process, followed by a newline:
    def __call__(self, s):
        self.gnuplot.write(s + "\n")
        self.gnuplot.flush()
        if self.debug:
            sys.stderr.write("gnuplot> " + s + "\n")

    # refresh the plot, using the current plotitems:
    def refresh(self):
        plotcmds = []
        for item in self.itemlist:
            plotcmds.append(item.command())
        self('plot ' + string.join(plotcmds, ', '))
        for item in self.itemlist:
            item.pipein(self.gnuplot)

    # call the gnuplot `plot' command:
    def plot(self, *data):
        # remove old files:
        self.itemlist = []
        apply(self.replot, data)

    # replot the data, possibly adding new plotitems:
    def replot(self, *items):
        for item in items:
            if isinstance(item, plotitem):
                self.itemlist.append(item)
            elif type(item) is type(""):
                self.itemlist.append(func(item))
            else:
                # assume data is an array:
                self.itemlist.append(data(item))
        self.refresh()

    def interact(self):
        sys.stderr.write("Press C-d to end interactive input\n")
        while 1:
            sys.stderr.write("gnuplot>>> ")
            line = sys.stdin.readline()
            if line == "":
                break
            if line[-1] == "\n": line = line[:-1]
            self(line)

    def load(self, filename=None):
        if filename is None:
            self.interact()
        else:
            self('load "' + filename + '"')

    def save(self, filename):
        self('save "' + filename + '"')

    def xlabel(self, s=None):
        if s==None:
            self('set ylabel')
        else:
            self('set xlabel "' + s + '"')

    def ylabel(self, s=None):
        if s==None:
            self('set ylabel')
        else:
            self('set ylabel "' + s + '"')

    def hardcopy(self, filename = '| lpr', color = 0):
        if color: self('set term postscript enhanced color')
        else: self('set term postscript enhanced')
        self('set output "' + filename + '"')
        self('replot')
        self('set term x11')
        self('set output')

#
# Demo code
#
if __name__ == '__main__':
    from Numeric import *
    import sys

    g1 = gnuplot(debug=1)
    # List of (x, y) pairs
    g1.plot([(0.,1.1),(1.,5.8),(2.,3.3),(3.,4.2)])

    # Two plots given by arrays and one by a gnuplot function:
    g2 = gnuplot(debug=1)
    x = arange(10)
    y1 = x**2
    y2 = (10-x)**2
    g2.plot(data(transpose((x, y1)), title="calculated by python"),
            func("x**2", title="calculated by gnuplot"),
            data(transpose((x, y2)), with="linesp")
            )
    print "Generating postscript file 'junk.ps'"
    g2.hardcopy('junk.ps', color=1)

    sys.stderr.write("Press return to continue...\n")
    sys.stdin.readline()

    del g1, g2

<|endoftext|>


<commit_before>

def test():
    from os.path import split
    import unittest
    test_loader = unittest.TestLoader()
    test_loader = test_loader.discover(split(__file__)[0])
    test_runner = unittest.TextTestRunner()
    test_runner.run( test_loader )

demos = {}

demos['texture'] = """
# Create renderer
import chumpy as ch
from opendr.renderer import TexturedRenderer
rn = TexturedRenderer()

# Assign attributes to renderer
from opendr.test_dr.common import get_earthmesh
m = get_earthmesh(trans=ch.array([0,0,4]), rotation=ch.zeros(3))
w, h = (320, 240)
from opendr.camera import ProjectPoints
rn.camera = ProjectPoints(v=m.v, rt=ch.zeros(3), t=ch.zeros(3), f=ch.array([w,w])/2., c=ch.array([w,h])/2., k=ch.zeros(5))
rn.frustum = {'near': 1., 'far': 10., 'width': w, 'height': h}
rn.set(v=m.v, f=m.f, vc=m.vc, texture_image=m.texture_image, ft=m.ft, vt=m.vt)

# Show it
import matplotlib.pyplot as plt
plt.ion()
plt.imshow(rn.r)
plt.show()

dr = rn.dr_wrt(rn.v) # or rn.vc, or rn.camera.rt, rn.camera.t, rn.camera.f, rn.camera.c, etc
"""

demos['silhouette'] = """
# Create renderer
import chumpy as ch
from opendr.renderer import ColoredRenderer
rn = ColoredRenderer()

# Assign attributes to renderer
from opendr.test_dr.common import get_earthmesh
m = get_earthmesh(trans=ch.array([0,0,4]), rotation=ch.zeros(3))
w, h = (320, 240)
from opendr.camera import ProjectPoints
rn.camera = ProjectPoints(v=m.v, rt=ch.zeros(3), t=ch.zeros(3), f=ch.array([w,w])/2., c=ch.array([w,h])/2., k=ch.zeros(5))
rn.frustum = {'near': 1., 'far': 10., 'width': w, 'height': h}
rn.set(v=m.v, f=m.f, vc=m.vc*0+1, bgcolor=ch.zeros(3))

# Show it
import matplotlib.pyplot as plt
plt.ion()
plt.imshow(rn.r)
plt.show()

dr = rn.dr_wrt(rn.v) # or rn.vc, or rn.camera.rt, rn.camera.t, rn.camera.f, rn.camera.c, etc
"""

demos['boundary'] = """
# Create renderer
import chumpy as ch
from opendr.renderer import BoundaryRenderer
rn = BoundaryRenderer()

# Assign attributes to renderer
from opendr.test_dr.common import get_earthmesh
m = get_earthmesh(trans=ch.array([0,0,4]), rotation=ch.zeros(3))
w, h = (320, 240)
from opendr.camera import ProjectPoints
rn.camera = ProjectPoints(v=m.v, rt=ch.zeros(3), t=ch.zeros(3), f=ch.array([w,w])/2., c=ch.array([w,h])/2., k=ch.zeros(5))
rn.frustum = {'near': 1., 'far': 10., 'width': w, 'height': h}
rn.set(v=m.v, f=m.f, vc=m.vc*0+1, bgcolor=ch.zeros(3), num_channels=3)

# Show it
import matplotlib.pyplot as plt
plt.ion()
plt.imshow(rn.r)
plt.show()

dr = rn.dr_wrt(rn.v) # or rn.vc, or rn.camera.rt, rn.camera.t, rn.camera.f, rn.camera.c, etc
"""

demos['point_light'] = """
# Create renderer
import chumpy as ch
from opendr.renderer import ColoredRenderer
from opendr.lighting import LambertianPointLight
rn = ColoredRenderer()

# Assign attributes to renderer
from opendr.test_dr.common import get_earthmesh
m = get_earthmesh(trans=ch.array([0,0,4]), rotation=ch.zeros(3))
w, h = (320, 240)

from opendr.camera import ProjectPoints
rn.camera = ProjectPoints(v=m.v, rt=ch.zeros(3), t=ch.zeros(3), f=ch.array([w,w])/2., c=ch.array([w,h])/2., k=ch.zeros(5))
rn.frustum = {'near': 1., 'far': 10., 'width': w, 'height': h}
rn.set(v=m.v, f=m.f, bgcolor=ch.zeros(3))

# Construct point light source
rn.vc = LambertianPointLight(
    f=m.f,
    v=rn.v,
    num_verts=len(m.v),
    light_pos=ch.array([-1000,-1000,-1000]),
    vc=m.vc,
    light_color=ch.array([1., 1., 1.]))

# Show it
import matplotlib.pyplot as plt
plt.ion()
plt.imshow(rn.r)
plt.show()

dr = rn.dr_wrt(rn.v) # or rn.vc, or rn.camera.rt, rn.camera.t, rn.camera.f, rn.camera.c, etc
"""

demos['spherical_harmonics'] = """
# Create renderer
import chumpy as ch
from opendr.renderer import ColoredRenderer
from opendr.lighting import SphericalHarmonics
from opendr.geometry import VertNormals

rn = ColoredRenderer()

# Assign attributes to renderer
from opendr.test_dr.common import get_earthmesh
m = get_earthmesh(trans=ch.array([0,0,4]), rotation=ch.zeros(3))
w, h = (320, 240)
from opendr.camera import ProjectPoints
rn.camera = ProjectPoints(v=m.v, rt=ch.zeros(3), t=ch.zeros(3), f=ch.array([w,w])/2., c=ch.array([w,h])/2., k=ch.zeros(5))
rn.frustum = {'near': 1., 'far': 10., 'width': w, 'height': h}
rn.set(v=m.v, f=m.f, bgcolor=ch.zeros(3))

vn = VertNormals(v=rn.v, f=rn.f)
sh_red = SphericalHarmonics(vn=vn, light_color=ch.array([1,0,0]), components=ch.random.randn(9))
sh_green = SphericalHarmonics(vn=vn, light_color=ch.array([0,1,0]), components=ch.random.randn(9))
sh_blue = SphericalHarmonics(vn=vn, light_color=ch.array([0,0,1]), components=ch.random.randn(9))
rn.vc = sh_red + sh_green + sh_blue

# Show it
import matplotlib.pyplot as plt
plt.ion()
plt.imshow(rn.r)
plt.show()

dr = rn.dr_wrt(rn.v) # or rn.vc, or rn.camera.rt, rn.camera.t, rn.camera.f, rn.camera.c, etc
"""

def demo(which=None):
    if which not in demos:
        print 'Please indicate which demo you want, as follows:'
        for key in demos:
            print "\tdemo('%s')" % (key,)
        return

    print '- - - - - - - - - - - <CODE> - - - - - - - - - - - -'
    print demos[which]
    print '- - - - - - - - - - - </CODE> - - - - - - - - - - - -\n'
    exec('global np\n' + demos[which], globals(), locals())
<commit_msg>__init__.py: added demo for per-face normals<commit_after>

def test():
    from os.path import split
    import unittest
    test_loader = unittest.TestLoader()
    test_loader = test_loader.discover(split(__file__)[0])
    test_runner = unittest.TextTestRunner()
    test_runner.run( test_loader )

demos = {}

demos['texture'] = """
# Create renderer
import chumpy as ch
from opendr.renderer import TexturedRenderer
rn = TexturedRenderer()

# Assign attributes to renderer
from opendr.test_dr.common import get_earthmesh
m = get_earthmesh(trans=ch.array([0,0,4]), rotation=ch.zeros(3))
w, h = (320, 240)
from opendr.camera import ProjectPoints
rn.camera = ProjectPoints(v=m.v, rt=ch.zeros(3), t=ch.zeros(3), f=ch.array([w,w])/2., c=ch.array([w,h])/2., k=ch.zeros(5))
rn.frustum = {'near': 1., 'far': 10., 'width': w, 'height': h}
rn.set(v=m.v, f=m.f, vc=m.vc, texture_image=m.texture_image, ft=m.ft, vt=m.vt)

# Show it
import matplotlib.pyplot as plt
plt.ion()
plt.imshow(rn.r)
plt.show()

dr = rn.dr_wrt(rn.v) # or rn.vc, or rn.camera.rt, rn.camera.t, rn.camera.f, rn.camera.c, etc
"""

demos['per_face_normals'] = """
# Create renderer
import chumpy as ch
import numpy as np
from opendr.renderer import ColoredRenderer
from opendr.lighting import LambertianPointLight
rn = ColoredRenderer()

# Assign attributes to renderer
from opendr.test_dr.common import get_earthmesh
m = get_earthmesh(trans=ch.array([0,0,4]), rotation=ch.zeros(3))
w, h = (320, 240)

# THESE ARE THE 3 CRITICAL LINES
m.v = m.v[m.f.ravel()]
m.vc = m.vc[m.f.ravel()]
m.f = np.arange(m.f.size).reshape((-1,3))

from opendr.camera import ProjectPoints
rn.camera = ProjectPoints(v=m.v, rt=ch.zeros(3), t=ch.zeros(3), f=ch.array([w,w])/2., c=ch.array([w,h])/2., k=ch.zeros(5))
rn.frustum = {'near': 1., 'far': 10., 'width': w, 'height': h}
rn.set(v=m.v, f=m.f, bgcolor=ch.zeros(3))

# Construct point light source
rn.vc = LambertianPointLight(
    f=m.f,
    v=rn.v,
    num_verts=len(m.v),
    light_pos=ch.array([-1000,-1000,-1000]),
    vc=m.vc,
    light_color=ch.array([1., 1., 1.]))

# Show it
import matplotlib.pyplot as plt
plt.ion()
plt.imshow(rn.r)
plt.show()

dr = rn.dr_wrt(rn.v) # or rn.vc, or rn.camera.rt, rn.camera.t, rn.camera.f, rn.camera.c, etc
"""

demos['silhouette'] = """
# Create renderer
import chumpy as ch
from opendr.renderer import ColoredRenderer
rn = ColoredRenderer()

# Assign attributes to renderer
from opendr.test_dr.common import get_earthmesh
m = get_earthmesh(trans=ch.array([0,0,4]), rotation=ch.zeros(3))
w, h = (320, 240)
from opendr.camera import ProjectPoints
rn.camera = ProjectPoints(v=m.v, rt=ch.zeros(3), t=ch.zeros(3), f=ch.array([w,w])/2., c=ch.array([w,h])/2., k=ch.zeros(5))
rn.frustum = {'near': 1., 'far': 10., 'width': w, 'height': h}
rn.set(v=m.v, f=m.f, vc=m.vc*0+1, bgcolor=ch.zeros(3))

# Show it
import matplotlib.pyplot as plt
plt.ion()
plt.imshow(rn.r)
plt.show()

dr = rn.dr_wrt(rn.v) # or rn.vc, or rn.camera.rt, rn.camera.t, rn.camera.f, rn.camera.c, etc
"""

demos['boundary'] = """
# Create renderer
import chumpy as ch
from opendr.renderer import BoundaryRenderer
rn = BoundaryRenderer()

# Assign attributes to renderer
from opendr.test_dr.common import get_earthmesh
m = get_earthmesh(trans=ch.array([0,0,4]), rotation=ch.zeros(3))
w, h = (320, 240)
from opendr.camera import ProjectPoints
rn.camera = ProjectPoints(v=m.v, rt=ch.zeros(3), t=ch.zeros(3), f=ch.array([w,w])/2., c=ch.array([w,h])/2., k=ch.zeros(5))
rn.frustum = {'near': 1., 'far': 10., 'width': w, 'height': h}
rn.set(v=m.v, f=m.f, vc=m.vc*0+1, bgcolor=ch.zeros(3), num_channels=3)

# Show it
import matplotlib.pyplot as plt
plt.ion()
plt.imshow(rn.r)
plt.show()

dr = rn.dr_wrt(rn.v) # or rn.vc, or rn.camera.rt, rn.camera.t, rn.camera.f, rn.camera.c, etc
"""

demos['point_light'] = """
# Create renderer
import chumpy as ch
from opendr.renderer import ColoredRenderer
from opendr.lighting import LambertianPointLight
rn = ColoredRenderer()

# Assign attributes to renderer
from opendr.test_dr.common import get_earthmesh
m = get_earthmesh(trans=ch.array([0,0,4]), rotation=ch.zeros(3))
w, h = (320, 240)

from opendr.camera import ProjectPoints
rn.camera = ProjectPoints(v=m.v, rt=ch.zeros(3), t=ch.zeros(3), f=ch.array([w,w])/2., c=ch.array([w,h])/2., k=ch.zeros(5))
rn.frustum = {'near': 1., 'far': 10., 'width': w, 'height': h}
rn.set(v=m.v, f=m.f, bgcolor=ch.zeros(3))

# Construct point light source
rn.vc = LambertianPointLight(
    f=m.f,
    v=rn.v,
    num_verts=len(m.v),
    light_pos=ch.array([-1000,-1000,-1000]),
    vc=m.vc,
    light_color=ch.array([1., 1., 1.]))

# Show it
import matplotlib.pyplot as plt
plt.ion()
plt.imshow(rn.r)
plt.show()

dr = rn.dr_wrt(rn.v) # or rn.vc, or rn.camera.rt, rn.camera.t, rn.camera.f, rn.camera.c, etc
"""

demos['spherical_harmonics'] = """
# Create renderer
import chumpy as ch
from opendr.renderer import ColoredRenderer
from opendr.lighting import SphericalHarmonics
from opendr.geometry import VertNormals

rn = ColoredRenderer()

# Assign attributes to renderer
from opendr.test_dr.common import get_earthmesh
m = get_earthmesh(trans=ch.array([0,0,4]), rotation=ch.zeros(3))
w, h = (320, 240)
from opendr.camera import ProjectPoints
rn.camera = ProjectPoints(v=m.v, rt=ch.zeros(3), t=ch.zeros(3), f=ch.array([w,w])/2., c=ch.array([w,h])/2., k=ch.zeros(5))
rn.frustum = {'near': 1., 'far': 10., 'width': w, 'height': h}
rn.set(v=m.v, f=m.f, bgcolor=ch.zeros(3))

vn = VertNormals(v=rn.v, f=rn.f)
sh_red = SphericalHarmonics(vn=vn, light_color=ch.array([1,0,0]), components=ch.random.randn(9))
sh_green = SphericalHarmonics(vn=vn, light_color=ch.array([0,1,0]), components=ch.random.randn(9))
sh_blue = SphericalHarmonics(vn=vn, light_color=ch.array([0,0,1]), components=ch.random.randn(9))
rn.vc = sh_red + sh_green + sh_blue

# Show it
import matplotlib.pyplot as plt
plt.ion()
plt.imshow(rn.r)
plt.show()

dr = rn.dr_wrt(rn.v) # or rn.vc, or rn.camera.rt, rn.camera.t, rn.camera.f, rn.camera.c, etc
"""

def demo(which=None):
    if which not in demos:
        print 'Please indicate which demo you want, as follows:'
        for key in demos:
            print "\tdemo('%s')" % (key,)
        return

    print '- - - - - - - - - - - <CODE> - - - - - - - - - - - -'
    print demos[which]
    print '- - - - - - - - - - - </CODE> - - - - - - - - - - - -\n'
    exec('global np\n' + demos[which], globals(), locals())
<|endoftext|>


<commit_before>"""login support

Revision ID: 456a945560f6
Revises: 38c4e85512a9
Create Date: 2013-12-29 00:18:35.795259

"""

# revision identifiers, used by Alembic.
revision = '456a945560f6'
down_revision = '38c4e85512a9'

from alembic import op
import sqlalchemy as sa

def upgrade():
    ### commands auto generated by Alembic - please adjust! ###
    op.add_column('users', sa.Column('email', sa.String(length=64), nullable=True))
    op.add_column('users', sa.Column('password_hash', sa.String(length=128), nullable=True))
    op.create_index('ix_users_email', 'users', ['email'], unique=True)
    ### end Alembic commands ###

def downgrade():
    ### commands auto generated by Alembic - please adjust! ###
    op.drop_index('ix_users_email', 'users')
    op.drop_column('users', 'password_hash')
    op.drop_column('users', 'email')
    ### end Alembic commands ###
<commit_msg>Delete 456a945560f6_login_support.py<commit_after><|endoftext|>