<commit_before>"""
Array methods which are called by both the C-code for the method
and the Python code for the NumPy-namespace function

"""
import warnings
from contextlib import nullcontext

from numpy.core import multiarray as mu
from numpy.core import umath as um
from numpy.core.multiarray import asanyarray
from numpy.core import numerictypes as nt
from numpy.core import _exceptions
from numpy._globals import _NoValue
from numpy.compat import pickle, os_fspath

# save those O(100) nanoseconds!
umr_maximum = um.maximum.reduce
umr_minimum = um.minimum.reduce
umr_sum = um.add.reduce
umr_prod = um.multiply.reduce
umr_any = um.logical_or.reduce
umr_all = um.logical_and.reduce

# Complex types to -> (2,)float view for fast-path computation in _var()
_complex_to_float = {
    nt.dtype(nt.csingle) : nt.dtype(nt.single),
    nt.dtype(nt.cdouble) : nt.dtype(nt.double),
}
# Special case for windows: ensure double takes precedence
if nt.dtype(nt.longdouble) != nt.dtype(nt.double):
    _complex_to_float.update({
        nt.dtype(nt.clongdouble) : nt.dtype(nt.longdouble),
    })

# avoid keyword arguments to speed up parsing, saves about 15%-20% for very
# small reductions
def _amax(a, axis=None, out=None, keepdims=False,
          initial=_NoValue, where=True):
    return umr_maximum(a, axis, None, out, keepdims, initial, where)

def _amin(a, axis=None, out=None, keepdims=False,
          initial=_NoValue, where=True):
    return umr_minimum(a, axis, None, out, keepdims, initial, where)

def _sum(a, axis=None, dtype=None, out=None, keepdims=False,
         initial=_NoValue, where=True):
    return umr_sum(a, axis, dtype, out, keepdims, initial, where)

def _prod(a, axis=None, dtype=None, out=None, keepdims=False,
          initial=_NoValue, where=True):
    return umr_prod(a, axis, dtype, out, keepdims, initial, where)

def _any(a, axis=None, dtype=None, out=None, keepdims=False, *, where=True):
    # Parsing keyword arguments is currently fairly slow, so avoid it for now
    if where is True:
        return umr_any(a, axis, dtype, out, keepdims)
    return umr_any(a, axis, dtype, out, keepdims, where=where)

def _all(a, axis=None, dtype=None, out=None, keepdims=False, *, where=True):
    # Parsing keyword arguments is currently fairly slow, so avoid it for now
    if where is True:
        return umr_all(a, axis, dtype, out, keepdims)
    return umr_all(a, axis, dtype, out, keepdims, where=where)

def _count_reduce_items(arr, axis, keepdims=False, where=True):
    # fast-path for the default case
    if where is True:
        # no boolean mask given, calculate items according to axis
        if axis is None:
            axis = tuple(range(arr.ndim))
        elif not isinstance(axis, tuple):
            axis = (axis,)
        items = nt.intp(1)
        for ax in axis:
            items *= arr.shape[mu.normalize_axis_index(ax, arr.ndim)]
    else:
        # TODO: Optimize case when `where` is broadcast along a non-reduction
        # axis and full sum is more excessive than needed.

        # guarded to protect circular imports
        from numpy.lib.stride_tricks import broadcast_to
        # count True values in (potentially broadcasted) boolean mask
        items = umr_sum(broadcast_to(where, arr.shape), axis, nt.intp, None,
                        keepdims)
    return items

# Numpy 1.17.0, 2019-02-24
# Various clip behavior deprecations, marked with _clip_dep as a prefix.

def _clip_dep_is_scalar_nan(a):
    # guarded to protect circular imports
    from numpy.core.fromnumeric import ndim
    if ndim(a) != 0:
        return False
    try:
        return um.isnan(a)
    except TypeError:
        return False

def _clip_dep_is_byte_swapped(a):
    if isinstance(a, mu.ndarray):
        return not a.dtype.isnative
    return False

def _clip_dep_invoke_with_casting(ufunc, *args, out=None, casting=None, **kwargs):
    # normal path
    if casting is not None:
        return ufunc(*args, out=out, casting=casting, **kwargs)

    # try to deal with broken casting rules
    try:
        return ufunc(*args, out=out, **kwargs)
    except _exceptions._UFuncOutputCastingError as e:
        # Numpy 1.17.0, 2019-02-24
        warnings.warn(
            "Converting the output of clip from {!r} to {!r} is deprecated. "
            "Pass `casting=\"unsafe\"` explicitly to silence this warning, or "
            "correct the type of the variables.".format(e.from_, e.to),
            DeprecationWarning,
            stacklevel=2
        )
        return ufunc(*args, out=out, casting="unsafe", **kwargs)

def _clip(a, min=None, max=None, out=None, *, casting=None, **kwargs):
    if min is None and max is None:
        raise ValueError("One of max or min must be given")

    # Numpy 1.17.0, 2019-02-24
    # This deprecation probably incurs a substantial slowdown for small arrays,
    # it will be good to get rid of it.
    if not _clip_dep_is_byte_swapped(a) and not _clip_dep_is_byte_swapped(out):
        using_deprecated_nan = False
        if _clip_dep_is_scalar_nan(min):
            min = -float('inf')
            using_deprecated_nan = True
        if _clip_dep_is_scalar_nan(max):
            max = float('inf')
            using_deprecated_nan = True
        if using_deprecated_nan:
            warnings.warn(
                "Passing `np.nan` to mean no clipping in np.clip has always "
                "been unreliable, and is now deprecated. "
                "In future, this will always return nan, like it already does "
                "when min or max are arrays that contain nan. "
                "To skip a bound, pass either None or an np.inf of an "
                "appropriate sign.",
                DeprecationWarning,
                stacklevel=2
            )

    if min is None:
        return _clip_dep_invoke_with_casting(
            um.minimum, a, max, out=out, casting=casting, **kwargs)
    elif max is None:
        return _clip_dep_invoke_with_casting(
            um.maximum, a, min, out=out, casting=casting, **kwargs)
    else:
        return _clip_dep_invoke_with_casting(
            um.clip, a, min, max, out=out, casting=casting, **kwargs)

def _mean(a, axis=None, dtype=None, out=None, keepdims=False, *, where=True):
    arr = asanyarray(a)

    is_float16_result = False

    rcount = _count_reduce_items(arr, axis, keepdims=keepdims, where=where)
    if rcount == 0 if where is True else umr_any(rcount == 0, axis=None):
        warnings.warn("Mean of empty slice.", RuntimeWarning, stacklevel=2)

    # Cast bool, unsigned int, and int to float64 by default
    if dtype is None:
        if issubclass(arr.dtype.type, (nt.integer, nt.bool_)):
            dtype = mu.dtype('f8')
        elif issubclass(arr.dtype.type, nt.float16):
            dtype = mu.dtype('f4')
            is_float16_result = True

    ret = umr_sum(arr, axis, dtype, out, keepdims, where=where)
    if isinstance(ret, mu.ndarray):
        ret = um.true_divide(
                ret, rcount, out=ret, casting='unsafe', subok=False)
        if is_float16_result and out is None:
            ret = arr.dtype.type(ret)
    elif hasattr(ret, 'dtype'):
        if is_float16_result:
            ret = arr.dtype.type(ret / rcount)
        else:
            ret = ret.dtype.type(ret / rcount)
    else:
        ret = ret / rcount

    return ret

def _var(a, axis=None, dtype=None, out=None, ddof=0, keepdims=False, *,
         where=True):
    arr = asanyarray(a)

    rcount = _count_reduce_items(arr, axis, keepdims=keepdims, where=where)
    # Make this warning show up on top.
    if ddof >= rcount if where is True else umr_any(ddof >= rcount, axis=None):
        warnings.warn("Degrees of freedom <= 0 for slice", RuntimeWarning,
                      stacklevel=2)

    # Cast bool, unsigned int, and int to float64 by default
    if dtype is None and issubclass(arr.dtype.type, (nt.integer, nt.bool_)):
        dtype = mu.dtype('f8')

    # Compute the mean.
    # Note that if dtype is not of inexact type then arraymean will
    # not be either.
    arrmean = umr_sum(arr, axis, dtype, keepdims=True, where=where)
    # The shape of rcount has to match arrmean to not change the shape of out
    # in broadcasting. Otherwise, it cannot be stored back to arrmean.
    if rcount.ndim == 0:
        # fast-path for default case when where is True
        div = rcount
    else:
        # matching rcount to arrmean when where is specified as array
        div = rcount.reshape(arrmean.shape)
    if isinstance(arrmean, mu.ndarray):
        arrmean = um.true_divide(arrmean, div, out=arrmean, casting='unsafe',
                                 subok=False)
    else:
        arrmean = arrmean.dtype.type(arrmean / rcount)

    # Compute sum of squared deviations from mean
    # Note that x may not be inexact and that we need it to be an array,
    # not a scalar.
    x = asanyarray(arr - arrmean)

    if issubclass(arr.dtype.type, (nt.floating, nt.integer)):
        x = um.multiply(x, x, out=x)
    # Fast-paths for built-in complex types
    elif x.dtype in _complex_to_float:
        xv = x.view(dtype=(_complex_to_float[x.dtype], (2,)))
        um.multiply(xv, xv, out=xv)
        x = um.add(xv[..., 0], xv[..., 1], out=x.real).real
    # Most general case; includes handling object arrays containing imaginary
    # numbers and complex types with non-native byteorder
    else:
        x = um.multiply(x, um.conjugate(x), out=x).real

    ret = umr_sum(x, axis, dtype, out, keepdims=keepdims, where=where)

    # Compute degrees of freedom and make sure it is not negative.
    rcount = um.maximum(rcount - ddof, 0)

    # divide by degrees of freedom
    if isinstance(ret, mu.ndarray):
        ret = um.true_divide(
                ret, rcount, out=ret, casting='unsafe', subok=False)
    elif hasattr(ret, 'dtype'):
        ret = ret.dtype.type(ret / rcount)
    else:
        ret = ret / rcount

    return ret

def _std(a, axis=None, dtype=None, out=None, ddof=0, keepdims=False, *,
         where=True):
    ret = _var(a, axis=axis, dtype=dtype, out=out, ddof=ddof,
               keepdims=keepdims, where=where)

    if isinstance(ret, mu.ndarray):
        ret = um.sqrt(ret, out=ret)
    elif hasattr(ret, 'dtype'):
        ret = ret.dtype.type(um.sqrt(ret))
    else:
        ret = um.sqrt(ret)

    return ret

def _ptp(a, axis=None, out=None, keepdims=False):
    return um.subtract(
        umr_maximum(a, axis, None, out, keepdims),
        umr_minimum(a, axis, None, None, keepdims),
        out
    )

def _dump(self, file, protocol=2):
    if hasattr(file, 'write'):
        ctx = nullcontext(file)
    else:
        ctx = open(os_fspath(file), "wb")
    with ctx as f:
        pickle.dump(self, f, protocol=protocol)

def _dumps(self, protocol=2):
    return pickle.dumps(self, protocol=protocol)
<commit_msg>BUG: Fixed an issue wherein `var` would raise for 0d object arrays<commit_after>"""
Array methods which are called by both the C-code for the method
and the Python code for the NumPy-namespace function

"""
import warnings
from contextlib import nullcontext

from numpy.core import multiarray as mu
from numpy.core import umath as um
from numpy.core.multiarray import asanyarray
from numpy.core import numerictypes as nt
from numpy.core import _exceptions
from numpy._globals import _NoValue
from numpy.compat import pickle, os_fspath

# save those O(100) nanoseconds!
umr_maximum = um.maximum.reduce
umr_minimum = um.minimum.reduce
umr_sum = um.add.reduce
umr_prod = um.multiply.reduce
umr_any = um.logical_or.reduce
umr_all = um.logical_and.reduce

# Complex types to -> (2,)float view for fast-path computation in _var()
_complex_to_float = {
    nt.dtype(nt.csingle) : nt.dtype(nt.single),
    nt.dtype(nt.cdouble) : nt.dtype(nt.double),
}
# Special case for windows: ensure double takes precedence
if nt.dtype(nt.longdouble) != nt.dtype(nt.double):
    _complex_to_float.update({
        nt.dtype(nt.clongdouble) : nt.dtype(nt.longdouble),
    })

# avoid keyword arguments to speed up parsing, saves about 15%-20% for very
# small reductions
def _amax(a, axis=None, out=None, keepdims=False,
          initial=_NoValue, where=True):
    return umr_maximum(a, axis, None, out, keepdims, initial, where)

def _amin(a, axis=None, out=None, keepdims=False,
          initial=_NoValue, where=True):
    return umr_minimum(a, axis, None, out, keepdims, initial, where)

def _sum(a, axis=None, dtype=None, out=None, keepdims=False,
         initial=_NoValue, where=True):
    return umr_sum(a, axis, dtype, out, keepdims, initial, where)

def _prod(a, axis=None, dtype=None, out=None, keepdims=False,
          initial=_NoValue, where=True):
    return umr_prod(a, axis, dtype, out, keepdims, initial, where)

def _any(a, axis=None, dtype=None, out=None, keepdims=False, *, where=True):
    # Parsing keyword arguments is currently fairly slow, so avoid it for now
    if where is True:
        return umr_any(a, axis, dtype, out, keepdims)
    return umr_any(a, axis, dtype, out, keepdims, where=where)

def _all(a, axis=None, dtype=None, out=None, keepdims=False, *, where=True):
    # Parsing keyword arguments is currently fairly slow, so avoid it for now
    if where is True:
        return umr_all(a, axis, dtype, out, keepdims)
    return umr_all(a, axis, dtype, out, keepdims, where=where)

def _count_reduce_items(arr, axis, keepdims=False, where=True):
    # fast-path for the default case
    if where is True:
        # no boolean mask given, calculate items according to axis
        if axis is None:
            axis = tuple(range(arr.ndim))
        elif not isinstance(axis, tuple):
            axis = (axis,)
        items = nt.intp(1)
        for ax in axis:
            items *= arr.shape[mu.normalize_axis_index(ax, arr.ndim)]
    else:
        # TODO: Optimize case when `where` is broadcast along a non-reduction
        # axis and full sum is more excessive than needed.

        # guarded to protect circular imports
        from numpy.lib.stride_tricks import broadcast_to
        # count True values in (potentially broadcasted) boolean mask
        items = umr_sum(broadcast_to(where, arr.shape), axis, nt.intp, None,
                        keepdims)
    return items

# Numpy 1.17.0, 2019-02-24
# Various clip behavior deprecations, marked with _clip_dep as a prefix.

def _clip_dep_is_scalar_nan(a):
    # guarded to protect circular imports
    from numpy.core.fromnumeric import ndim
    if ndim(a) != 0:
        return False
    try:
        return um.isnan(a)
    except TypeError:
        return False

def _clip_dep_is_byte_swapped(a):
    if isinstance(a, mu.ndarray):
        return not a.dtype.isnative
    return False

def _clip_dep_invoke_with_casting(ufunc, *args, out=None, casting=None, **kwargs):
    # normal path
    if casting is not None:
        return ufunc(*args, out=out, casting=casting, **kwargs)

    # try to deal with broken casting rules
    try:
        return ufunc(*args, out=out, **kwargs)
    except _exceptions._UFuncOutputCastingError as e:
        # Numpy 1.17.0, 2019-02-24
        warnings.warn(
            "Converting the output of clip from {!r} to {!r} is deprecated. "
            "Pass `casting=\"unsafe\"` explicitly to silence this warning, or "
            "correct the type of the variables.".format(e.from_, e.to),
            DeprecationWarning,
            stacklevel=2
        )
        return ufunc(*args, out=out, casting="unsafe", **kwargs)

def _clip(a, min=None, max=None, out=None, *, casting=None, **kwargs):
    if min is None and max is None:
        raise ValueError("One of max or min must be given")

    # Numpy 1.17.0, 2019-02-24
    # This deprecation probably incurs a substantial slowdown for small arrays,
    # it will be good to get rid of it.
    if not _clip_dep_is_byte_swapped(a) and not _clip_dep_is_byte_swapped(out):
        using_deprecated_nan = False
        if _clip_dep_is_scalar_nan(min):
            min = -float('inf')
            using_deprecated_nan = True
        if _clip_dep_is_scalar_nan(max):
            max = float('inf')
            using_deprecated_nan = True
        if using_deprecated_nan:
            warnings.warn(
                "Passing `np.nan` to mean no clipping in np.clip has always "
                "been unreliable, and is now deprecated. "
                "In future, this will always return nan, like it already does "
                "when min or max are arrays that contain nan. "
                "To skip a bound, pass either None or an np.inf of an "
                "appropriate sign.",
                DeprecationWarning,
                stacklevel=2
            )

    if min is None:
        return _clip_dep_invoke_with_casting(
            um.minimum, a, max, out=out, casting=casting, **kwargs)
    elif max is None:
        return _clip_dep_invoke_with_casting(
            um.maximum, a, min, out=out, casting=casting, **kwargs)
    else:
        return _clip_dep_invoke_with_casting(
            um.clip, a, min, max, out=out, casting=casting, **kwargs)

def _mean(a, axis=None, dtype=None, out=None, keepdims=False, *, where=True):
    arr = asanyarray(a)

    is_float16_result = False

    rcount = _count_reduce_items(arr, axis, keepdims=keepdims, where=where)
    if rcount == 0 if where is True else umr_any(rcount == 0, axis=None):
        warnings.warn("Mean of empty slice.", RuntimeWarning, stacklevel=2)

    # Cast bool, unsigned int, and int to float64 by default
    if dtype is None:
        if issubclass(arr.dtype.type, (nt.integer, nt.bool_)):
            dtype = mu.dtype('f8')
        elif issubclass(arr.dtype.type, nt.float16):
            dtype = mu.dtype('f4')
            is_float16_result = True

    ret = umr_sum(arr, axis, dtype, out, keepdims, where=where)
    if isinstance(ret, mu.ndarray):
        ret = um.true_divide(
                ret, rcount, out=ret, casting='unsafe', subok=False)
        if is_float16_result and out is None:
            ret = arr.dtype.type(ret)
    elif hasattr(ret, 'dtype'):
        if is_float16_result:
            ret = arr.dtype.type(ret / rcount)
        else:
            ret = ret.dtype.type(ret / rcount)
    else:
        ret = ret / rcount

    return ret

def _var(a, axis=None, dtype=None, out=None, ddof=0, keepdims=False, *,
         where=True):
    arr = asanyarray(a)

    rcount = _count_reduce_items(arr, axis, keepdims=keepdims, where=where)
    # Make this warning show up on top.
    if ddof >= rcount if where is True else umr_any(ddof >= rcount, axis=None):
        warnings.warn("Degrees of freedom <= 0 for slice", RuntimeWarning,
                      stacklevel=2)

    # Cast bool, unsigned int, and int to float64 by default
    if dtype is None and issubclass(arr.dtype.type, (nt.integer, nt.bool_)):
        dtype = mu.dtype('f8')

    # Compute the mean.
    # Note that if dtype is not of inexact type then arraymean will
    # not be either.
    arrmean = umr_sum(arr, axis, dtype, keepdims=True, where=where)
    # The shape of rcount has to match arrmean to not change the shape of out
    # in broadcasting. Otherwise, it cannot be stored back to arrmean.
    if rcount.ndim == 0:
        # fast-path for default case when where is True
        div = rcount
    else:
        # matching rcount to arrmean when where is specified as array
        div = rcount.reshape(arrmean.shape)
    if isinstance(arrmean, mu.ndarray):
        arrmean = um.true_divide(arrmean, div, out=arrmean, casting='unsafe',
                                 subok=False)
    elif hasattr(arrmean, "dtype"):
        arrmean = arrmean.dtype.type(arrmean / rcount)
    else:
        arrmean = arrmean / rcount

    # Compute sum of squared deviations from mean
    # Note that x may not be inexact and that we need it to be an array,
    # not a scalar.
    x = asanyarray(arr - arrmean)

    if issubclass(arr.dtype.type, (nt.floating, nt.integer)):
        x = um.multiply(x, x, out=x)
    # Fast-paths for built-in complex types
    elif x.dtype in _complex_to_float:
        xv = x.view(dtype=(_complex_to_float[x.dtype], (2,)))
        um.multiply(xv, xv, out=xv)
        x = um.add(xv[..., 0], xv[..., 1], out=x.real).real
    # Most general case; includes handling object arrays containing imaginary
    # numbers and complex types with non-native byteorder
    else:
        x = um.multiply(x, um.conjugate(x), out=x).real

    ret = umr_sum(x, axis, dtype, out, keepdims=keepdims, where=where)

    # Compute degrees of freedom and make sure it is not negative.
    rcount = um.maximum(rcount - ddof, 0)

    # divide by degrees of freedom
    if isinstance(ret, mu.ndarray):
        ret = um.true_divide(
                ret, rcount, out=ret, casting='unsafe', subok=False)
    elif hasattr(ret, 'dtype'):
        ret = ret.dtype.type(ret / rcount)
    else:
        ret = ret / rcount

    return ret

def _std(a, axis=None, dtype=None, out=None, ddof=0, keepdims=False, *,
         where=True):
    ret = _var(a, axis=axis, dtype=dtype, out=out, ddof=ddof,
               keepdims=keepdims, where=where)

    if isinstance(ret, mu.ndarray):
        ret = um.sqrt(ret, out=ret)
    elif hasattr(ret, 'dtype'):
        ret = ret.dtype.type(um.sqrt(ret))
    else:
        ret = um.sqrt(ret)

    return ret

def _ptp(a, axis=None, out=None, keepdims=False):
    return um.subtract(
        umr_maximum(a, axis, None, out, keepdims),
        umr_minimum(a, axis, None, None, keepdims),
        out
    )

def _dump(self, file, protocol=2):
    if hasattr(file, 'write'):
        ctx = nullcontext(file)
    else:
        ctx = open(os_fspath(file), "wb")
    with ctx as f:
        pickle.dump(self, f, protocol=protocol)

def _dumps(self, protocol=2):
    return pickle.dumps(self, protocol=protocol)
<|endoftext|>


<commit_before>import os
import sys
import traceback
from optparse import OptionParser
import pysam
from quicksect import IntervalTree
import fileinput

# manage option and arguments processing
def main():
	global options
	global args
	usage = '''usage: %prog [options] bamFile

generates (single locus) fragmentCounts and (pairwise) contactCount files
	'''
	parser = OptionParser(usage)
	parser.add_option("-q", "--quiet", action="store_false", dest="verbose", default=True,
					help="don't print status messages to stdout")
	parser.add_option("-v", "--verbose", action="store_true", dest="verbose",
					help="print status messages to stdout")
	parser.add_option("-g", "--genomeFragmentFile", type="string", dest="fragmentFile", default="",
					help="file containing the genome fragments after digestion with the restriction enzyme(s), generated by hicup")
	parser.add_option("-f", "--fragmentAggregation", type="int", dest="fragmentAggregation", default=10,
					help="number of restriction enzyme fragments to concat")
	parser.add_option("-s", "--fragmentSize", type="int", dest="fragmentSize", default=50000,
					help="size of a fragment in bp if no genomeFragmentFile is given")
	parser.add_option("-o", "--outputDir", type="string", dest="outputDir", default="",
					help="output directory [default: %default]")
	parser.add_option("-t", "--tmpDir", type="string", dest="tmpDir", default="/tmp",
					help="directory for temp files [default: %default]")

	(options, args) = parser.parse_args()
	if (len(args) < 1):
		parser.print_help()
		parser.error("[ERROR] Incorrect number of arguments, need a dataset")

	if (options.fragmentAggregation < 1):
		parser.error("[ERROR] fragmentAggregation must be a positive integer, was :"+str(options.fragmentAggregation))
		sys.exit(1)

	if (options. fragmentSize < 1):
		parser.error("[ERROR] fragmentSize must be a positive integer, was :"+str(options.fragmentSize))
		sys.exit(1)

	if (options.outputDir != ""):
		options.outputDir += os.sep

	if (options.verbose):
		print >> sys.stdout, "fragmentFile:          %s" % (options. fragmentFile)
		print >> sys.stdout, "fragmentAggregation:   %s" % (options. fragmentAggregation)
#		print >> sys.stdout, "fragmentSize:          %s" % (options. fragmentSize)
		print >> sys.stdout, "outputDir:             %s" % (options.outputDir)
		print >> sys.stdout, "tmpDir:                %s" % (options.tmpDir)

	process()

def createIntervalTrees():
	'''
		creates one interval tree for quick lookups
		returns
			fragmentsMap[fragmentId] = [tuple(chrom, fragmentMidPoint)]
			intersect_tree - intersect Tree for interval matching

	'''

	intersect_tree= IntervalTree()
	fragmentsCount = 0
	fragmentsMap = {}

	start = 0
	end = 0
	counter = 0
	chrom = ""

	for line in fileinput.input([options.fragmentFile]):
		if (len(line.strip())==0 or line.startswith("Genome") or line.startswith("Chromosome")):
			continue

		cols = line.strip().split("\t")
		try:
			# check if chromosome changed from last
			if (cols[0] != chrom):
				# do we have do finish the last chromosome?
				if (end > 0):
					interval = Interval(chrom, start, end)
					intersect_tree = intersect_tree.insert(interval, fragmentsCount)
					fragmentsMap[fragmentsCount] = tuple([chrom, end-start])
				chrom = cols[0]
				start = int(cols[1])
				end = int(cols[2])
				counter = 0
			# check if fragement aggregation is fulfilled
			if (counter >= options. fragmentAggregation):
				interval = Interval(chrom, start, end)
				intersect_tree = intersect_tree.insert(interval, fragmentsCount)
				fragmentsMap[fragmentsCount] = tuple([chrom, end-start])
				start = int(cols[1])
				end = int(cols[2])
				counter = 0

			# increment counter
			counter += 1
			fragmentsCount += 1

		except Exception, err:
			traceback.print_exc()
			if (options.verbose):
				print >> sys.stderr, 'skipping line in options.fragmentFile: %s' % (line)
				sys.exit(1)

	# handle last fragment
	if (end > 0):
		interval = Interval(chrom, start, end)
		intersect_tree = intersect_tree.insert(interval, fragmentsCount)
		fragmentsMap[fragmentsCount] = tuple([chrom, end-start])

	return [fragmentsMap, intersect_tree]

def getNext(iterator):
	'''
	get the next read and populate object
	'''

	try:
		return acRead(iterator.next())
	except StopIteration:
		return acRead("")

def findNextReadPair(samiter):
	'''
	find the next read pair
	'''
	readpair=[getNext(samiter), getNext(samiter)]

	while(readpair[0].qname!= readpair[1].qname):
		if (args.v):
			print "[WARN] File: drop first from unpaired read %s %s" %(readpair[0].qname, readpair[1].qname)
		readpair.pop(0)
		readpair.append(getNext(arr1))

	return readpair

def find(interval, tree):
    ''' Returns a list with the overlapping intervals '''
    out = []
    tree.intersect( interval, lambda x: out.append(x) )
    return [ (x.start, x.end, x.linenum) for x in out ]

def getFragment(read, fragmentList):
	fragmentID = -1
	try:
		# get fragments for both reads
		interval = Interval(pysam.Samfile.getrname(read.tid), read.pos, read.aend)

		fragments = find(interval , intersect_tree)
		if (len(fragments)> 1):
			print >> sys.stderr, '[WARN] number of fragments > 1 : %s (skipping)' % (read)

		#extract fragmentID
		fragmentID = fragments[2] # corresponds to linenum

		if (not fragmentList.has_key(fragmentID)):
			fragmentList[fragmentID] = 0

		fragmentList[fragmentID] += 1

	except:
		if (options.verbose):
			print >> sys.stderr, '[WARN] problems with interval intersection: %s (skipping)' % (read)

	return fragmentID

def countReadsPerFragment(intersect_tree):
	'''
		counts the reads per fragment and generates appropriate output files
	'''
	samfile = pysam.Samfile(args[1], "rb" )

	samiter = samfile.fetch(until_eof=True)

	fragmentList={}
	fragmentPairs = {}

	while(True):
		readpair = findNextReadPair(samiter)
		# if file contains any more reads, exit
		if (readpair[0].qname=="dummy"):
			break

		fragmentID1 = getFragment(readpair[0], fragmentList)
		fragmentID2 = getFragment(readpair[1], fragmentList)
		f_tuple = tuple([min(fragmentID1, fragmentID2), max(fragmentID1, fragmentID2)])
		if (not fragmentPairs.has_key(f_tuple)):
			fragmentPairs[f_tuple] = 0
		fragmentPairs[f_tuple] += 1

	samfile.close()

	return [ fragmentList, fragmentPairs ]

def output(fragmentsMap , fragmentList, fragmentPairs):
	'''
	outputs 2 files, the first containing
	"chr    extraField      fragmentMid     marginalizedContactCount        mappable? (0/1)"

	and the second containing:
	"chr1   fragmentMid1    chr2    fragmentMid2    contactCount"
	'''

	outfile1 = open(options.outputDir+os.basename(args[1])+".fragmentLists","w")
	for fragmentId, contactCounts in fragmentList.iteritems():
		mappable = 0
		if (contactCounts>0):
			mappable = 1
		chrom = fragmentsMap[fragmentId][0]
		midpoint =  fragmentsMap[fragmentId][1]
		outfile1.write("%s\t%d\t%d\t%d\t%d\n" % (chrom, 0, midpoint, contactCounts, mappable))

	outfile1.close()

	outfile2 = open(options.outputDir+os.basename(args[1])+".contactCounts","w")
	for fragmentIds, contactCounts in fragmentPairs.iteritems():
		chrom1 = fragmentsMap[fragmentId[0]][0]
		midpoint1 =  fragmentsMap[fragmentId[0]][1]

		chrom2 = fragmentsMap[fragmentId[1]][0]
		midpoint2 =  fragmentsMap[fragmentId[1]][1]

		outfile2.write("%s\t%d\t%s\t%d\t%d\n" % (chrom1, midpoint1, chrom2, midpoint2, contactCounts))

	outfile2.close()

def process():
	global options
	global args

	[ fragmentsMap, intersect_tree ] = createIntervalTrees()

	[ fragmentList, fragmentPairs ] = countReadsPerFragment(intersect_tree)

	output(fragmentsMap, fragmentList, fragmentPairs)

######################################
# main
######################################
if __name__ == "__main__":
	main()

######################################
# acRead
######################################

class acRead():
    def __init__(self, read):
        if (read==""):
            self.seq=""
            self.qname="dummy"
            self.is_unmapped=True
            self.X0=1
            self.mapq=0
            self.tid=None
            self.is_read1=None
            self.is_reverse=None
        else:
            self.is_duplicate=read.is_duplicate
            self.is_unmapped=read.is_unmapped
            self.tid=read.tid
            self.qname=read.qname
            self.is_read1=True
            if (read.is_read2):
                self.is_read1=False
            self.cigar=read.cigar
            self.is_reverse=read.is_reverse
            self.seq=read.seq
            self.alen=read.alen
            self.pos=read.pos
            self.mapq=read.mapq
            self.tags=read.tags
            self.X0=max(1,self.getInfo("X0"))
            self.MD=self.getInfo("MD")

    def check(self):
        if(self.is_reverse):
            self.revcomp()

    def isPair(self,number):
        if(self.is_read1 and number==0):
            return True
        if(not(self.is_read1) and number==1):
            return True
        print("Reads not paired up correctly: paired/single ended? not namesorted?")
        print(str(self)+" "+str(number))

    def __str__(self):
        if(self.is_unmapped):
            return "%s read1 %s %s" % (self.qname,self.is_read1,self.seq)
        else:
            return "%s read1 %s %s %s %s %s %i %i" % (self.qname,self.is_read1,self.cigar,self.seq,self.tags,self.tid,self.pos,self.alen)

    def revcomp(self):
        basecomplement = {'N':'N','A': 'T', 'C': 'G', 'G': 'C', 'T': 'A'}
        rc=""
        for i in reversed(self.seq):
            rc+=basecomplement[i]
        self.seq=rc

    def getInfo(self, string):
        for i in self.tags:
            if( i[0]==string):
                return i[1]

######################################
# Interval
######################################
class Interval():
	def __init__(self, chrom, start, end):
		self.chrom=chrom
		self.start=start
		self.end=end

<commit_msg>Bugfix hicup class scope (wip)<commit_after>import os
import sys
import traceback
from optparse import OptionParser
import pysam
from quicksect import IntervalTree
import fileinput

######################################
# Read
######################################

class Read():
    def __init__(self, read):
        if (read==""):
            self.seq=""
            self.qname="dummy"
            self.is_unmapped=True
            self.X0=1
            self.mapq=0
            self.tid=None
            self.is_read1=None
            self.is_reverse=None
        else:
            self.is_duplicate=read.is_duplicate
            self.is_unmapped=read.is_unmapped
            self.tid=read.tid
            self.qname=read.qname
            self.is_read1=True
            if (read.is_read2):
                self.is_read1=False
            self.cigar=read.cigar
            self.is_reverse=read.is_reverse
            self.seq=read.seq
            self.alen=read.alen
            self.pos=read.pos
            self.mapq=read.mapq
            self.tags=read.tags
            self.X0=max(1,self.getInfo("X0"))
            self.MD=self.getInfo("MD")

    def check(self):
        if(self.is_reverse):
            self.revcomp()

    def isPair(self,number):
        if(self.is_read1 and number==0):
            return True
        if(not(self.is_read1) and number==1):
            return True
        print("Reads not paired up correctly: paired/single ended? not namesorted?")
        print(str(self)+" "+str(number))

    def __str__(self):
        if(self.is_unmapped):
            return "%s read1 %s %s" % (self.qname,self.is_read1,self.seq)
        else:
            return "%s read1 %s %s %s %s %s %i %i" % (self.qname,self.is_read1,self.cigar,self.seq,self.tags,self.tid,self.pos,self.alen)

    def revcomp(self):
        basecomplement = {'N':'N','A': 'T', 'C': 'G', 'G': 'C', 'T': 'A'}
        rc=""
        for i in reversed(self.seq):
            rc+=basecomplement[i]
        self.seq=rc

    def getInfo(self, string):
        for i in self.tags:
            if( i[0]==string):
                return i[1]

######################################
# Interval
######################################
class Interval():
	def __init__(self, chrom, start, end):
		self.chrom=chrom
		self.start=start
		self.end=end

# manage option and arguments processing
def main():
	global options
	global args
	usage = '''usage: %prog [options] bamFile

generates (single locus) fragmentCounts and (pairwise) contactCount files
	'''
	parser = OptionParser(usage)
	parser.add_option("-q", "--quiet", action="store_false", dest="verbose", default=True,
					help="don't print status messages to stdout")
	parser.add_option("-v", "--verbose", action="store_true", dest="verbose",
					help="print status messages to stdout")
	parser.add_option("-g", "--genomeFragmentFile", type="string", dest="fragmentFile", default="",
					help="file containing the genome fragments after digestion with the restriction enzyme(s), generated by hicup")
	parser.add_option("-f", "--fragmentAggregation", type="int", dest="fragmentAggregation", default=10,
					help="number of restriction enzyme fragments to concat")
	parser.add_option("-s", "--fragmentSize", type="int", dest="fragmentSize", default=50000,
					help="size of a fragment in bp if no genomeFragmentFile is given")
	parser.add_option("-o", "--outputDir", type="string", dest="outputDir", default="",
					help="output directory [default: %default]")
	parser.add_option("-t", "--tmpDir", type="string", dest="tmpDir", default="/tmp",
					help="directory for temp files [default: %default]")

	(options, args) = parser.parse_args()
	if (len(args) < 1):
		parser.print_help()
		parser.error("[ERROR] Incorrect number of arguments, need a dataset")

	if (options.fragmentAggregation < 1):
		parser.error("[ERROR] fragmentAggregation must be a positive integer, was :"+str(options.fragmentAggregation))
		sys.exit(1)

	if (options. fragmentSize < 1):
		parser.error("[ERROR] fragmentSize must be a positive integer, was :"+str(options.fragmentSize))
		sys.exit(1)

	if (options.outputDir != ""):
		options.outputDir += os.sep

	if (options.verbose):
		print >> sys.stdout, "fragmentFile:          %s" % (options. fragmentFile)
		print >> sys.stdout, "fragmentAggregation:   %s" % (options. fragmentAggregation)
#		print >> sys.stdout, "fragmentSize:          %s" % (options. fragmentSize)
		print >> sys.stdout, "outputDir:             %s" % (options.outputDir)
		print >> sys.stdout, "tmpDir:                %s" % (options.tmpDir)

	process()

def createIntervalTrees():
	'''
		creates one interval tree for quick lookups
		returns
			fragmentsMap[fragmentId] = [tuple(chrom, fragmentMidPoint)]
			intersect_tree - intersect Tree for interval matching

	'''

	intersect_tree= IntervalTree()
	fragmentsCount = 0
	fragmentsMap = {}

	start = 0
	end = 0
	counter = 0
	chrom = ""

	for line in fileinput.input([options.fragmentFile]):
		if (len(line.strip())==0 or line.startswith("Genome") or line.startswith("Chromosome")):
			continue

		cols = line.strip().split("\t")
		try:
			# check if chromosome changed from last
			if (cols[0] != chrom):
				# do we have do finish the last chromosome?
				if (end > 0):
					interval = Interval(chrom, start, end)
					intersect_tree = intersect_tree.insert(interval, fragmentsCount)
					fragmentsMap[fragmentsCount] = tuple([chrom, end-start])
				chrom = cols[0]
				start = int(cols[1])
				end = int(cols[2])
				counter = 0
			# check if fragement aggregation is fulfilled
			if (counter >= options. fragmentAggregation):
				interval = Interval(chrom, start, end)
				intersect_tree = intersect_tree.insert(interval, fragmentsCount)
				fragmentsMap[fragmentsCount] = tuple([chrom, end-start])
				start = int(cols[1])
				end = int(cols[2])
				counter = 0

			# increment counter
			counter += 1
			fragmentsCount += 1

		except Exception, err:
			traceback.print_exc()
			if (options.verbose):
				print >> sys.stderr, 'skipping line in options.fragmentFile: %s' % (line)
				sys.exit(1)

	# handle last fragment
	if (end > 0):
		interval = Interval(chrom, start, end)
		intersect_tree = intersect_tree.insert(interval, fragmentsCount)
		fragmentsMap[fragmentsCount] = tuple([chrom, end-start])

	return [fragmentsMap, intersect_tree]

def getNext(iterator):
	'''
	get the next read and populate object
	'''

	try:
		return Read(iterator.next())
	except StopIteration:
		return Read("")

def findNextReadPair(samiter):
	'''
	find the next read pair
	'''
	readpair=[getNext(samiter), getNext(samiter)]

	while(readpair[0].qname!= readpair[1].qname):
		if (args.v):
			print "[WARN] File: drop first from unpaired read %s %s" %(readpair[0].qname, readpair[1].qname)
		readpair.pop(0)
		readpair.append(getNext(arr1))

	return readpair

def find(interval, tree):
    ''' Returns a list with the overlapping intervals '''
    out = []
    tree.intersect( interval, lambda x: out.append(x) )
    return [ (x.start, x.end, x.linenum) for x in out ]

def getFragment(read, fragmentList):
	fragmentID = -1
	try:
		# get fragments for both reads
		interval = Interval(pysam.Samfile.getrname(read.tid), read.pos, read.aend)

		fragments = find(interval , intersect_tree)
		if (len(fragments)> 1):
			print >> sys.stderr, '[WARN] number of fragments > 1 : %s (skipping)' % (read)

		#extract fragmentID
		fragmentID = fragments[2] # corresponds to linenum

		if (not fragmentList.has_key(fragmentID)):
			fragmentList[fragmentID] = 0

		fragmentList[fragmentID] += 1

	except:
		if (options.verbose):
			print >> sys.stderr, '[WARN] problems with interval intersection: %s (skipping)' % (read)

	return fragmentID

def countReadsPerFragment(intersect_tree):
	'''
		counts the reads per fragment and generates appropriate output files
	'''
	samfile = pysam.Samfile(args[1], "rb" )

	samiter = samfile.fetch(until_eof=True)

	fragmentList={}
	fragmentPairs = {}

	while(True):
		readpair = findNextReadPair(samiter)
		# if file contains any more reads, exit
		if (readpair[0].qname=="dummy"):
			break

		fragmentID1 = getFragment(readpair[0], fragmentList)
		fragmentID2 = getFragment(readpair[1], fragmentList)
		f_tuple = tuple([min(fragmentID1, fragmentID2), max(fragmentID1, fragmentID2)])
		if (not fragmentPairs.has_key(f_tuple)):
			fragmentPairs[f_tuple] = 0
		fragmentPairs[f_tuple] += 1

	samfile.close()

	return [ fragmentList, fragmentPairs ]

def output(fragmentsMap , fragmentList, fragmentPairs):
	'''
	outputs 2 files, the first containing
	"chr    extraField      fragmentMid     marginalizedContactCount        mappable? (0/1)"

	and the second containing:
	"chr1   fragmentMid1    chr2    fragmentMid2    contactCount"
	'''

	outfile1 = open(options.outputDir+os.basename(args[1])+".fragmentLists","w")
	for fragmentId, contactCounts in fragmentList.iteritems():
		mappable = 0
		if (contactCounts>0):
			mappable = 1
		chrom = fragmentsMap[fragmentId][0]
		midpoint =  fragmentsMap[fragmentId][1]
		outfile1.write("%s\t%d\t%d\t%d\t%d\n" % (chrom, 0, midpoint, contactCounts, mappable))

	outfile1.close()

	outfile2 = open(options.outputDir+os.basename(args[1])+".contactCounts","w")
	for fragmentIds, contactCounts in fragmentPairs.iteritems():
		chrom1 = fragmentsMap[fragmentId[0]][0]
		midpoint1 =  fragmentsMap[fragmentId[0]][1]

		chrom2 = fragmentsMap[fragmentId[1]][0]
		midpoint2 =  fragmentsMap[fragmentId[1]][1]

		outfile2.write("%s\t%d\t%s\t%d\t%d\n" % (chrom1, midpoint1, chrom2, midpoint2, contactCounts))

	outfile2.close()

def process():
	global options
	global args

	[ fragmentsMap, intersect_tree ] = createIntervalTrees()

	[ fragmentList, fragmentPairs ] = countReadsPerFragment(intersect_tree)

	output(fragmentsMap, fragmentList, fragmentPairs)

######################################
# main
######################################
if __name__ == "__main__":
	main()

<|endoftext|>


<commit_before>#!/usr/bin/env python
#
# Long-running server process uses stdin & stdout to communicate JSON
# with a caller
#
from __future__ import print_function
import argparse
import confgen
import json
import kconfiglib
import os
import sys
import tempfile
from confgen import FatalError, __version__

# Min/Max supported protocol versions
MIN_PROTOCOL_VERSION = 1
MAX_PROTOCOL_VERSION = 2

def main():
    parser = argparse.ArgumentParser(description='confserver.py v%s - Config Generation Tool' % __version__, prog=os.path.basename(sys.argv[0]))

    parser.add_argument('--config',
                        help='Project configuration settings',
                        required=True)

    parser.add_argument('--kconfig',
                        help='KConfig file with config item definitions',
                        required=True)

    parser.add_argument('--env', action='append', default=[],
                        help='Environment to set when evaluating the config file', metavar='NAME=VAL')

    parser.add_argument('--version', help='Set protocol version to use on initial status',
                        type=int, default=MAX_PROTOCOL_VERSION)

    args = parser.parse_args()

    if args.version < MIN_PROTOCOL_VERSION:
        print("Version %d is older than minimum supported protocol version %d. Client is much older than ESP-IDF version?" %
              (args.version, MIN_PROTOCOL_VERSION))

    if args.version > MAX_PROTOCOL_VERSION:
        print("Version %d is newer than maximum supported protocol version %d. Client is newer than ESP-IDF version?" %
              (args.version, MAX_PROTOCOL_VERSION))

    try:
        args.env = [(name,value) for (name,value) in (e.split("=",1) for e in args.env)]
    except ValueError:
        print("--env arguments must each contain =. To unset an environment variable, use 'ENV='")
        sys.exit(1)

    for name, value in args.env:
        os.environ[name] = value

    run_server(args.kconfig, args.config)

def run_server(kconfig, sdkconfig, default_version=MAX_PROTOCOL_VERSION):
    config = kconfiglib.Kconfig(kconfig)
    deprecated_options = confgen.DeprecatedOptions(config.config_prefix, path_rename_files=os.environ["IDF_PATH"])
    with tempfile.NamedTemporaryFile(mode='w+b') as f_o:
        with open(sdkconfig, mode='rb') as f_i:
            f_o.write(f_i.read())
        f_o.flush()
        f_o.seek(0)
        deprecated_options.replace(sdkconfig_in=f_o.name, sdkconfig_out=sdkconfig)
    config.load_config(sdkconfig)

    print("Server running, waiting for requests on stdin...", file=sys.stderr)

    config_dict = confgen.get_json_values(config)
    ranges_dict = get_ranges(config)
    visible_dict = get_visible(config)

    if default_version == 1:
        # V1: no 'visibility' key, send value None for any invisible item
        values_dict = dict((k, v if visible_dict[k] else False) for (k,v) in config_dict.items())
        json.dump({"version": 1, "values": values_dict, "ranges": ranges_dict}, sys.stdout)
    else:
        # V2 onwards: separate visibility from version
        json.dump({"version": default_version, "values": config_dict, "ranges": ranges_dict, "visible": visible_dict}, sys.stdout)
    print("\n")

    while True:
        line = sys.stdin.readline()
        if not line:
            break
        try:
            req = json.loads(line)
        except ValueError as e:  # json module throws JSONDecodeError (sublcass of ValueError) on Py3 but ValueError on Py2
            response = {"version": default_version, "error": ["JSON formatting error: %s" % e]}
            json.dump(response, sys.stdout)
            print("\n")
            continue
        before = confgen.get_json_values(config)
        before_ranges = get_ranges(config)
        before_visible = get_visible(config)

        if "load" in req:  # load a new sdkconfig

            if req.get("version", default_version) == 1:
                # for V1 protocol, send all items when loading new sdkconfig.
                # (V2+ will only send changes, same as when setting an item)
                before = {}
                before_ranges = {}
                before_visible = {}

            # if no new filename is supplied, use existing sdkconfig path, otherwise update the path
            if req["load"] is None:
                req["load"] = sdkconfig
            else:
                sdkconfig = req["load"]

        if "save" in req:
            if req["save"] is None:
                req["save"] = sdkconfig
            else:
                sdkconfig = req["save"]

        error = handle_request(deprecated_options, config, req)

        after = confgen.get_json_values(config)
        after_ranges = get_ranges(config)
        after_visible = get_visible(config)

        values_diff = diff(before, after)
        ranges_diff = diff(before_ranges, after_ranges)
        visible_diff = diff(before_visible, after_visible)
        if req["version"] == 1:
            # V1 response, invisible items have value None
            for k in (k for (k,v) in visible_diff.items() if not v):
                values_diff[k] = None
            response = {"version": 1, "values": values_diff, "ranges": ranges_diff}
        else:
            # V2+ response, separate visibility values
            response = {"version": req["version"], "values": values_diff, "ranges": ranges_diff, "visible": visible_diff}
        if error:
            for e in error:
                print("Error: %s" % e, file=sys.stderr)
            response["error"] = error
        json.dump(response, sys.stdout)
        print("\n")

def handle_request(deprecated_options, config, req):
    if "version" not in req:
        return ["All requests must have a 'version'"]

    if req["version"] < MIN_PROTOCOL_VERSION or req["version"] > MAX_PROTOCOL_VERSION:
        return ["Unsupported request version %d. Server supports versions %d-%d" % (
            req["version"],
            MIN_PROTOCOL_VERSION,
            MAX_PROTOCOL_VERSION)]

    error = []

    if "load" in req:
        print("Loading config from %s..." % req["load"], file=sys.stderr)
        try:
            config.load_config(req["load"])
        except Exception as e:
            error += ["Failed to load from %s: %s" % (req["load"], e)]

    if "set" in req:
        handle_set(config, error, req["set"])

    if "save" in req:
        try:
            print("Saving config to %s..." % req["save"], file=sys.stderr)
            confgen.write_config(deprecated_options, config, req["save"])
        except Exception as e:
            error += ["Failed to save to %s: %s" % (req["save"], e)]

    return error

def handle_set(config, error, to_set):
    missing = [k for k in to_set if k not in config.syms]
    if missing:
        error.append("The following config symbol(s) were not found: %s" % (", ".join(missing)))
    # replace name keys with the full config symbol for each key:
    to_set = dict((config.syms[k],v) for (k,v) in to_set.items() if k not in missing)

    # Work through the list of values to set, noting that
    # some may not be immediately applicable (maybe they depend
    # on another value which is being set). Therefore, defer
    # knowing if any value is unsettable until then end

    while len(to_set):
        set_pass = [(k,v) for (k,v) in to_set.items() if k.visibility]
        if not set_pass:
            break  # no visible keys left
        for (sym,val) in set_pass:
            if sym.type in (kconfiglib.BOOL, kconfiglib.TRISTATE):
                if val is True:
                    sym.set_value(2)
                elif val is False:
                    sym.set_value(0)
                else:
                    error.append("Boolean symbol %s only accepts true/false values" % sym.name)
            else:
                sym.set_value(str(val))
            print("Set %s" % sym.name)
            del to_set[sym]

    if len(to_set):
        error.append("The following config symbol(s) were not visible so were not updated: %s" % (", ".join(s.name for s in to_set)))

def diff(before, after):
    """
    Return a dictionary with the difference between 'before' and 'after',
    for items which are present in 'after' dictionary
    """
    diff = dict((k,v) for (k,v) in after.items() if before.get(k, None) != v)
    return diff

def get_ranges(config):
    ranges_dict = {}

    def handle_node(node):
        sym = node.item
        if not isinstance(sym, kconfiglib.Symbol):
            return
        active_range = sym.active_range
        if active_range[0] is not None:
            ranges_dict[sym.name] = active_range

    config.walk_menu(handle_node)
    return ranges_dict

def get_visible(config):
    """
    Return a dict mapping node IDs (config names or menu node IDs) to True/False for their visibility
    """
    result = {}
    menus = []

    # when walking the menu the first time, only
    # record whether the config symbols are visible
    # and make a list of menu nodes (that are not symbols)
    def handle_node(node):
        sym = node.item
        try:
            visible = (sym.visibility != 0)
            result[node] = visible
        except AttributeError:
            menus.append(node)
    config.walk_menu(handle_node)

    # now, figure out visibility for each menu. A menu is visible if any of its children are visible
    for m in reversed(menus):  # reverse to start at leaf nodes
        result[m] = any(v for (n,v) in result.items() if n.parent == m)

    # return a dict mapping the node ID to its visibility.
    result = dict((confgen.get_menu_node_id(n),v) for (n,v) in result.items())

    return result

if __name__ == '__main__':
    try:
        main()
    except FatalError as e:
        print("A fatal error occurred: %s" % e, file=sys.stderr)
        sys.exit(2)
<commit_msg>Stdoutflush: Flushing stdout to avoid issues with 64k char limits<commit_after>#!/usr/bin/env python
#
# Long-running server process uses stdin & stdout to communicate JSON
# with a caller
#
from __future__ import print_function
import argparse
import confgen
import json
import kconfiglib
import os
import sys
import tempfile
from confgen import FatalError, __version__

# Min/Max supported protocol versions
MIN_PROTOCOL_VERSION = 1
MAX_PROTOCOL_VERSION = 2

def main():
    parser = argparse.ArgumentParser(description='confserver.py v%s - Config Generation Tool' % __version__, prog=os.path.basename(sys.argv[0]))

    parser.add_argument('--config',
                        help='Project configuration settings',
                        required=True)

    parser.add_argument('--kconfig',
                        help='KConfig file with config item definitions',
                        required=True)

    parser.add_argument('--env', action='append', default=[],
                        help='Environment to set when evaluating the config file', metavar='NAME=VAL')

    parser.add_argument('--version', help='Set protocol version to use on initial status',
                        type=int, default=MAX_PROTOCOL_VERSION)

    args = parser.parse_args()

    if args.version < MIN_PROTOCOL_VERSION:
        print("Version %d is older than minimum supported protocol version %d. Client is much older than ESP-IDF version?" %
              (args.version, MIN_PROTOCOL_VERSION))

    if args.version > MAX_PROTOCOL_VERSION:
        print("Version %d is newer than maximum supported protocol version %d. Client is newer than ESP-IDF version?" %
              (args.version, MAX_PROTOCOL_VERSION))

    try:
        args.env = [(name,value) for (name,value) in (e.split("=",1) for e in args.env)]
    except ValueError:
        print("--env arguments must each contain =. To unset an environment variable, use 'ENV='")
        sys.exit(1)

    for name, value in args.env:
        os.environ[name] = value

    run_server(args.kconfig, args.config)

def run_server(kconfig, sdkconfig, default_version=MAX_PROTOCOL_VERSION):
    config = kconfiglib.Kconfig(kconfig)
    deprecated_options = confgen.DeprecatedOptions(config.config_prefix, path_rename_files=os.environ["IDF_PATH"])
    with tempfile.NamedTemporaryFile(mode='w+b') as f_o:
        with open(sdkconfig, mode='rb') as f_i:
            f_o.write(f_i.read())
        f_o.flush()
        f_o.seek(0)
        deprecated_options.replace(sdkconfig_in=f_o.name, sdkconfig_out=sdkconfig)
    config.load_config(sdkconfig)

    print("Server running, waiting for requests on stdin...", file=sys.stderr)

    config_dict = confgen.get_json_values(config)
    ranges_dict = get_ranges(config)
    visible_dict = get_visible(config)

    if default_version == 1:
        # V1: no 'visibility' key, send value None for any invisible item
        values_dict = dict((k, v if visible_dict[k] else False) for (k,v) in config_dict.items())
        json.dump({"version": 1, "values": values_dict, "ranges": ranges_dict}, sys.stdout)
        sys.stdout.flush()
    else:
        # V2 onwards: separate visibility from version
        json.dump({"version": default_version, "values": config_dict, "ranges": ranges_dict, "visible": visible_dict}, sys.stdout)
        sys.stdout.flush()
    print("\n")

    while True:
        line = sys.stdin.readline()
        if not line:
            break
        try:
            req = json.loads(line)
        except ValueError as e:  # json module throws JSONDecodeError (sublcass of ValueError) on Py3 but ValueError on Py2
            response = {"version": default_version, "error": ["JSON formatting error: %s" % e]}
            json.dump(response, sys.stdout)
            sys.stdout.flush()
            print("\n")
            continue
        before = confgen.get_json_values(config)
        before_ranges = get_ranges(config)
        before_visible = get_visible(config)

        if "load" in req:  # load a new sdkconfig

            if req.get("version", default_version) == 1:
                # for V1 protocol, send all items when loading new sdkconfig.
                # (V2+ will only send changes, same as when setting an item)
                before = {}
                before_ranges = {}
                before_visible = {}

            # if no new filename is supplied, use existing sdkconfig path, otherwise update the path
            if req["load"] is None:
                req["load"] = sdkconfig
            else:
                sdkconfig = req["load"]

        if "save" in req:
            if req["save"] is None:
                req["save"] = sdkconfig
            else:
                sdkconfig = req["save"]

        error = handle_request(deprecated_options, config, req)

        after = confgen.get_json_values(config)
        after_ranges = get_ranges(config)
        after_visible = get_visible(config)

        values_diff = diff(before, after)
        ranges_diff = diff(before_ranges, after_ranges)
        visible_diff = diff(before_visible, after_visible)
        if req["version"] == 1:
            # V1 response, invisible items have value None
            for k in (k for (k,v) in visible_diff.items() if not v):
                values_diff[k] = None
            response = {"version": 1, "values": values_diff, "ranges": ranges_diff}
        else:
            # V2+ response, separate visibility values
            response = {"version": req["version"], "values": values_diff, "ranges": ranges_diff, "visible": visible_diff}
        if error:
            for e in error:
                print("Error: %s" % e, file=sys.stderr)
            response["error"] = error
        json.dump(response, sys.stdout)
        sys.stdout.flush()
        print("\n")

def handle_request(deprecated_options, config, req):
    if "version" not in req:
        return ["All requests must have a 'version'"]

    if req["version"] < MIN_PROTOCOL_VERSION or req["version"] > MAX_PROTOCOL_VERSION:
        return ["Unsupported request version %d. Server supports versions %d-%d" % (
            req["version"],
            MIN_PROTOCOL_VERSION,
            MAX_PROTOCOL_VERSION)]

    error = []

    if "load" in req:
        print("Loading config from %s..." % req["load"], file=sys.stderr)
        try:
            config.load_config(req["load"])
        except Exception as e:
            error += ["Failed to load from %s: %s" % (req["load"], e)]

    if "set" in req:
        handle_set(config, error, req["set"])

    if "save" in req:
        try:
            print("Saving config to %s..." % req["save"], file=sys.stderr)
            confgen.write_config(deprecated_options, config, req["save"])
        except Exception as e:
            error += ["Failed to save to %s: %s" % (req["save"], e)]

    return error

def handle_set(config, error, to_set):
    missing = [k for k in to_set if k not in config.syms]
    if missing:
        error.append("The following config symbol(s) were not found: %s" % (", ".join(missing)))
    # replace name keys with the full config symbol for each key:
    to_set = dict((config.syms[k],v) for (k,v) in to_set.items() if k not in missing)

    # Work through the list of values to set, noting that
    # some may not be immediately applicable (maybe they depend
    # on another value which is being set). Therefore, defer
    # knowing if any value is unsettable until then end

    while len(to_set):
        set_pass = [(k,v) for (k,v) in to_set.items() if k.visibility]
        if not set_pass:
            break  # no visible keys left
        for (sym,val) in set_pass:
            if sym.type in (kconfiglib.BOOL, kconfiglib.TRISTATE):
                if val is True:
                    sym.set_value(2)
                elif val is False:
                    sym.set_value(0)
                else:
                    error.append("Boolean symbol %s only accepts true/false values" % sym.name)
            else:
                sym.set_value(str(val))
            print("Set %s" % sym.name)
            del to_set[sym]

    if len(to_set):
        error.append("The following config symbol(s) were not visible so were not updated: %s" % (", ".join(s.name for s in to_set)))

def diff(before, after):
    """
    Return a dictionary with the difference between 'before' and 'after',
    for items which are present in 'after' dictionary
    """
    diff = dict((k,v) for (k,v) in after.items() if before.get(k, None) != v)
    return diff

def get_ranges(config):
    ranges_dict = {}

    def handle_node(node):
        sym = node.item
        if not isinstance(sym, kconfiglib.Symbol):
            return
        active_range = sym.active_range
        if active_range[0] is not None:
            ranges_dict[sym.name] = active_range

    config.walk_menu(handle_node)
    return ranges_dict

def get_visible(config):
    """
    Return a dict mapping node IDs (config names or menu node IDs) to True/False for their visibility
    """
    result = {}
    menus = []

    # when walking the menu the first time, only
    # record whether the config symbols are visible
    # and make a list of menu nodes (that are not symbols)
    def handle_node(node):
        sym = node.item
        try:
            visible = (sym.visibility != 0)
            result[node] = visible
        except AttributeError:
            menus.append(node)
    config.walk_menu(handle_node)

    # now, figure out visibility for each menu. A menu is visible if any of its children are visible
    for m in reversed(menus):  # reverse to start at leaf nodes
        result[m] = any(v for (n,v) in result.items() if n.parent == m)

    # return a dict mapping the node ID to its visibility.
    result = dict((confgen.get_menu_node_id(n),v) for (n,v) in result.items())

    return result

if __name__ == '__main__':
    try:
        main()
    except FatalError as e:
        print("A fatal error occurred: %s" % e, file=sys.stderr)
        sys.exit(2)
<|endoftext|>


<commit_before>#!/usr/bin/python

import sys
from psycopg2 import connect
import json

if len(sys.argv) == 1:
    print "usage: ./create.py <file> <output_table>"
    exit(0)

expert_file = sys.argv[1]
out_table = sys.argv[2]

lemmas = {}
tones = {'positive' : 1, 'negative' : -1, 'neutral' : 0}
with open(expert_file, "r") as f:
    for line in f.readlines():
        row = line.split(',')
        lemma = row[2].strip().decode('utf-8')
        tone = row[3].strip()

        if len(lemma.split(' ')) == 1 and tone in tones:
            print lemma, ' ', tones[tone]

            lemmas[lemma] = tones[tone]

# Create connection and save result
print 'Creete connection and save result'
with open(".conn", "r") as connection:
    conn = json.load(connection, encoding='utf8')

# Connect to a database
print 'Connect to a database'
settings = """dbname=%s user=%s password=%s host=%s"""%(
    conn['database'], conn["user"], conn["password"], conn["host"])
connection = connect(settings)
cursor = connection.cursor()

# Create result table
print 'Create result table'
max_term_length = 50
cursor.execute("""CREATE TABLE IF NOT EXISTS {table}(
                        term VARCHAR({max_length}),
                        tone REAL)""".format(table = out_table, max_length =
    max_term_length));
cursor.execute("DELETE FROM {table}".format(table = out_table));

# Save results
print 'Save results'
for pair in lemmas:
    p = (pair, lemmas[pair])
    if (len(p[0]) < max_term_length):
        cursor.execute("""INSERT INTO {table} ({term}, {value})
            VALUES (\'{t}\', {v})""".format(term="term", value='tone',
            table=out_table, t=p[0].encode('utf-8'), v=p[1]));

    connection.commit()
<commit_msg>Исправлен скрипт create.py. Теперь, если встречается слово, которое уже было добавлено, то проверяется совпадение старой тональности с новой. Если этого не происходит, то терм удаляется из лексикона.<commit_after>#!/usr/bin/python

import sys
from psycopg2 import connect
import json

if len(sys.argv) == 1:
    print "usage: ./create.py <file> <output_table>"
    exit(0)

expert_file = sys.argv[1]
out_table = sys.argv[2]

lemmas = {}
tones = {'positive' : 1, 'negative' : -1, 'neutral' : 0}
with open(expert_file, "r") as f:
    for line in f.readlines():
        row = line.split(',')
        lemma = row[2].strip().decode('utf-8')
        tone = row[3].strip()

        if len(lemma.split(' ')) == 1 and tone in tones:
            print lemma, ' ', tones[tone]

            lemmas[lemma] = tones[tone]

# Create connection and save result
print 'Creete connection and save result'
with open(".conn", "r") as connection:
    conn = json.load(connection, encoding='utf8')

# Connect to a database
print 'Connect to a database'
settings = """dbname=%s user=%s password=%s host=%s"""%(
    conn['database'], conn["user"], conn["password"], conn["host"])
connection = connect(settings)
cursor = connection.cursor()

print settings
# Create result table
print 'Create result table'
max_term_length = 50
cursor.execute("""CREATE TABLE IF NOT EXISTS {table}(
                        term VARCHAR({max_length}),
                        tone REAL)""".format(table = out_table, max_length =
    max_term_length));
cursor.execute("DELETE FROM {table}".format(table = out_table));

# Save results
print 'Save results'
all_terms = {}
for pair in lemmas:
    p = (pair, lemmas[pair])
    if (len(p[0]) < max_term_length):
        term_tone = p[1]
        utf_term = p[0].encode('utf-8')

        if not (utf_term in all_terms):
            cursor.execute("""INSERT INTO {table} ({term}, {value})
                VALUES (\'{t}\', {v})""".format(term="term", value='tone',
                table=out_table, t=utf_term, v=term_tone));
            all_terms[utf_term] = term_tone
        elif (term_tone != all_terms[utf_term]):
            cursor.execute("""DELETE FROM {table} WHERE term = '{t}'""".format(
               table=out_table, t=utf_term))
            # set another impossible tone value to prevent an attemt of adding
            # this utf_term in a future
            all_terms[utf_term] = 2;

    connection.commit()
connection.close()
<|endoftext|>


<commit_before>#!/usr/bin/python

# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved
#
# This source code is licensed under the MIT license found in the
# LICENSE file in the root directory of this source tree.

import errno
import os
import time

import fbchisellldbbase as fb
import fbchisellldbobjecthelpers as objectHelpers
import lldb

def lldbcommands():
    return [FBVisualizeCommand()]

def _showImage(commandForImage):
    imageDirectory = "/tmp/xcode_debug_images/"

    imageName = time.strftime("%Y-%m-%d-%H-%M-%S", time.gmtime()) + ".png"
    imagePath = imageDirectory + imageName

    try:
        os.makedirs(imageDirectory)
    except OSError as e:
        if e.errno == errno.EEXIST and os.path.isdir(imageDirectory):
            pass
        else:
            raise

    toPNG = "(id)UIImagePNGRepresentation((id){})".format(commandForImage)
    imageDataAddress = fb.evaluateExpressionValue(toPNG, tryAllThreads=True).GetValue()
    imageBytesStartAddress = fb.evaluateExpression(
        "(void *)[(id)" + imageDataAddress + " bytes]"
    )
    imageBytesLength = fb.evaluateExpression(
        "(NSUInteger)[(id)" + imageDataAddress + " length]"
    )

    address = int(imageBytesStartAddress, 16)
    length = int(imageBytesLength)

    if not (address or length):
        print("Could not get image data.")
        return

    process = lldb.debugger.GetSelectedTarget().GetProcess()
    error = lldb.SBError()
    mem = process.ReadMemory(address, length, error)

    if error is not None and str(error) != "success":
        print(error)
    else:
        with open(imagePath, "wb") as imgFile:
            imgFile.write(mem)
        os.system("open " + imagePath)

def _colorIsCGColorRef(color):
    color = "(CGColorRef)(" + color + ")"

    result = fb.evaluateExpressionValue(
        "(unsigned long)CFGetTypeID({color}) == (unsigned long)CGColorGetTypeID()".format(
            color=color
        )
    )

    if result.GetError() is not None and str(result.GetError()) != "success":
        print("got error: {}".format(result))
        return False
    else:
        isCFColor = result.GetValueAsUnsigned() != 0
        return isCFColor

def _showColor(color):
    color = "(" + color + ")"

    colorToUse = color
    isCF = _colorIsCGColorRef(color)
    if isCF:
        colorToUse = "[[UIColor alloc] initWithCGColor:(CGColorRef){}]".format(color)
    else:
        isCI = objectHelpers.isKindOfClass(color, "CIColor")
        if isCI:
            colorToUse = "[UIColor colorWithCIColor:(CIColor *){}]".format(color)

    imageSize = 58
    fb.evaluateEffect(
        "UIGraphicsBeginImageContextWithOptions((CGSize)CGSizeMake({imageSize}, {imageSize}), NO, 0.0)".format(
            imageSize=imageSize
        )
    )
    fb.evaluateEffect("[(id){} setFill]".format(colorToUse))
    fb.evaluateEffect(
        "UIRectFill((CGRect)CGRectMake(0.0, 0.0, {imageSize}, {imageSize}))".format(
            imageSize=imageSize
        )
    )

    result = fb.evaluateExpressionValue(
        "(UIImage *)UIGraphicsGetImageFromCurrentImageContext()"
    )
    if result.GetError() is not None and str(result.GetError()) != "success":
        print("got error {}".format(result))
        print(result.GetError())
    else:
        image = result.GetValue()
        _showImage(image)

    fb.evaluateEffect("UIGraphicsEndImageContext()")

def _showLayer(layer):
    layer = "(" + layer + ")"
    size = "((CGRect)[(id)" + layer + " bounds]).size"

    width = float(fb.evaluateExpression("(CGFloat)(" + size + ".width)"))
    height = float(fb.evaluateExpression("(CGFloat)(" + size + ".height)"))
    if width == 0.0 or height == 0.0:
        print(
            "Nothing to see here - the size of this element is {} x {}.".format(
                width, height
            )
        )
        return

    fb.evaluateEffect("UIGraphicsBeginImageContextWithOptions(" + size + ", NO, 0.0)")
    fb.evaluateEffect(
        "[(id)" + layer + " renderInContext:(void *)UIGraphicsGetCurrentContext()]"
    )

    result = fb.evaluateExpressionValue(
        "(UIImage *)UIGraphicsGetImageFromCurrentImageContext()"
    )
    if result.GetError() is not None and str(result.GetError()) != "success":
        print(result.GetError())
    else:
        image = result.GetValue()
        _showImage(image)

    fb.evaluateEffect("UIGraphicsEndImageContext()")

def _showPixelBuffer(target):
    fb.evaluateExpression("CGImageRef $imageOut = NULL")
    fb.evaluateExpression(
        "(OSStatus)VTCreateCGImageFromCVPixelBuffer(" + target + ", NULL, &$imageOut)"
    )
    image = fb.evaluateExpression("[UIImage imageWithCGImage:$imageOut]")
    _showImage(image)
    fb.evaluateExpression("CGImageRelease($imageOut)")

def _dataIsImage(data):
    data = "(" + data + ")"

    result = fb.evaluateExpressionValue("(id)[UIImage imageWithData:" + data + "]")

    if result.GetError() is not None and str(result.GetError()) != "success":
        return False
    else:
        isImage = result.GetValueAsUnsigned() != 0
        return isImage

def _dataIsString(data):
    data = "(" + data + ")"

    result = fb.evaluateExpressionValue(
        "(NSString*)[[NSString alloc] initWithData:" + data + " encoding:4]"
    )

    if result.GetError() is not None and str(result.GetError()) != "success":
        return False
    else:
        isString = result.GetValueAsUnsigned() != 0
        return isString

def _visualize(target):
    target = fb.evaluateInputExpression(target)

    if fb.evaluateBooleanExpression(
        "(unsigned long)CFGetTypeID((CFTypeRef)"
        + target
        + ") == (unsigned long)CGImageGetTypeID()"
    ):
        _showImage("(id)[UIImage imageWithCGImage:" + target + "]")
    elif fb.evaluateBooleanExpression(
        "(unsigned long)CFGetTypeID((CFTypeRef)"
        + target
        + ") == (unsigned long)CVPixelBufferGetTypeID()"
    ):
        _showPixelBuffer(target)
    else:
        if objectHelpers.isKindOfClass(target, "UIImage"):
            _showImage(target)
        elif objectHelpers.isKindOfClass(target, "UIView"):
            _showLayer("[(id)" + target + " layer]")
        elif objectHelpers.isKindOfClass(target, "CALayer"):
            _showLayer(target)
        elif (
            objectHelpers.isKindOfClass(target, "UIColor")
            or objectHelpers.isKindOfClass(target, "CIColor")
            or _colorIsCGColorRef(target)
        ):
            _showColor(target)
        elif objectHelpers.isKindOfClass(target, "NSData"):
            if _dataIsImage(target):
                _showImage("(id)[UIImage imageWithData:" + target + "]")
            elif _dataIsString(target):
                print(
                    fb.describeObject(
                        "[[NSString alloc] initWithData:" + target + " encoding:4]"
                    )
                )
            else:
                print("Data isn't an image and isn't a string.")
        else:
            print(
                "{} isn't supported. You can visualize UIImage, CGImageRef, UIView, CALayer, NSData, UIColor, CIColor, or CGColorRef.".format(
                    objectHelpers.className(target)
                )
            )

class FBVisualizeCommand(fb.FBCommand):
    def name(self):
        return "visualize"

    def description(self):
        return "Open a UIImage, CGImageRef, UIView, CALayer, or CVPixelBuffer in Preview.app on your Mac."

    def args(self):
        return [
            fb.FBCommandArgument(
                arg="target", type="(id)", help="The object to visualize."
            )
        ]

    def run(self, arguments, options):
        _visualize(arguments[0])
<commit_msg>Fix visualize CVPixelBuffer command<commit_after>#!/usr/bin/python

# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved
#
# This source code is licensed under the MIT license found in the
# LICENSE file in the root directory of this source tree.

import errno
import os
import time

import fbchisellldbbase as fb
import fbchisellldbobjecthelpers as objectHelpers
import lldb

def lldbcommands():
    return [FBVisualizeCommand()]

def _showImage(commandForImage):
    imageDirectory = "/tmp/xcode_debug_images/"

    imageName = time.strftime("%Y-%m-%d-%H-%M-%S", time.gmtime()) + ".png"
    imagePath = imageDirectory + imageName

    try:
        os.makedirs(imageDirectory)
    except OSError as e:
        if e.errno == errno.EEXIST and os.path.isdir(imageDirectory):
            pass
        else:
            raise

    toPNG = "(id)UIImagePNGRepresentation((id){})".format(commandForImage)
    imageDataAddress = fb.evaluateExpressionValue(toPNG, tryAllThreads=True).GetValue()
    imageBytesStartAddress = fb.evaluateExpression(
        "(void *)[(id)" + imageDataAddress + " bytes]"
    )
    imageBytesLength = fb.evaluateExpression(
        "(NSUInteger)[(id)" + imageDataAddress + " length]"
    )

    address = int(imageBytesStartAddress, 16)
    length = int(imageBytesLength)

    if not (address or length):
        print("Could not get image data.")
        return

    process = lldb.debugger.GetSelectedTarget().GetProcess()
    error = lldb.SBError()
    mem = process.ReadMemory(address, length, error)

    if error is not None and str(error) != "success":
        print(error)
    else:
        with open(imagePath, "wb") as imgFile:
            imgFile.write(mem)
        os.system("open " + imagePath)

def _colorIsCGColorRef(color):
    color = "(CGColorRef)(" + color + ")"

    result = fb.evaluateExpressionValue(
        "(unsigned long)CFGetTypeID({color}) == (unsigned long)CGColorGetTypeID()".format(
            color=color
        )
    )

    if result.GetError() is not None and str(result.GetError()) != "success":
        print("got error: {}".format(result))
        return False
    else:
        isCFColor = result.GetValueAsUnsigned() != 0
        return isCFColor

def _showColor(color):
    color = "(" + color + ")"

    colorToUse = color
    isCF = _colorIsCGColorRef(color)
    if isCF:
        colorToUse = "[[UIColor alloc] initWithCGColor:(CGColorRef){}]".format(color)
    else:
        isCI = objectHelpers.isKindOfClass(color, "CIColor")
        if isCI:
            colorToUse = "[UIColor colorWithCIColor:(CIColor *){}]".format(color)

    imageSize = 58
    fb.evaluateEffect(
        "UIGraphicsBeginImageContextWithOptions((CGSize)CGSizeMake({imageSize}, {imageSize}), NO, 0.0)".format(
            imageSize=imageSize
        )
    )
    fb.evaluateEffect("[(id){} setFill]".format(colorToUse))
    fb.evaluateEffect(
        "UIRectFill((CGRect)CGRectMake(0.0, 0.0, {imageSize}, {imageSize}))".format(
            imageSize=imageSize
        )
    )

    result = fb.evaluateExpressionValue(
        "(UIImage *)UIGraphicsGetImageFromCurrentImageContext()"
    )
    if result.GetError() is not None and str(result.GetError()) != "success":
        print("got error {}".format(result))
        print(result.GetError())
    else:
        image = result.GetValue()
        _showImage(image)

    fb.evaluateEffect("UIGraphicsEndImageContext()")

def _showLayer(layer):
    layer = "(" + layer + ")"
    size = "((CGRect)[(id)" + layer + " bounds]).size"

    width = float(fb.evaluateExpression("(CGFloat)(" + size + ".width)"))
    height = float(fb.evaluateExpression("(CGFloat)(" + size + ".height)"))
    if width == 0.0 or height == 0.0:
        print(
            "Nothing to see here - the size of this element is {} x {}.".format(
                width, height
            )
        )
        return

    fb.evaluateEffect("UIGraphicsBeginImageContextWithOptions(" + size + ", NO, 0.0)")
    fb.evaluateEffect(
        "[(id)" + layer + " renderInContext:(void *)UIGraphicsGetCurrentContext()]"
    )

    result = fb.evaluateExpressionValue(
        "(UIImage *)UIGraphicsGetImageFromCurrentImageContext()"
    )
    if result.GetError() is not None and str(result.GetError()) != "success":
        print(result.GetError())
    else:
        image = result.GetValue()
        _showImage(image)

    fb.evaluateEffect("UIGraphicsEndImageContext()")

def _showPixelBuffer(target):
    imgVar = "$imageOut" + str(round(time.time()))
    fb.evaluateExpression("CGImageRef " + imgVar + " = NULL")
    fb.evaluateExpression(
        "(OSStatus)VTCreateCGImageFromCVPixelBuffer((CVPixelBufferRef)"
        + target
        + ", NULL, &"
        + imgVar
        + ")"
    )
    image = fb.evaluateExpression("[UIImage imageWithCGImage:" + imgVar + "]")
    _showImage(image)
    fb.evaluateExpression("CGImageRelease(" + imgVar + ")")

def _dataIsImage(data):
    data = "(" + data + ")"

    result = fb.evaluateExpressionValue("(id)[UIImage imageWithData:" + data + "]")

    if result.GetError() is not None and str(result.GetError()) != "success":
        return False
    else:
        isImage = result.GetValueAsUnsigned() != 0
        return isImage

def _dataIsString(data):
    data = "(" + data + ")"

    result = fb.evaluateExpressionValue(
        "(NSString*)[[NSString alloc] initWithData:" + data + " encoding:4]"
    )

    if result.GetError() is not None and str(result.GetError()) != "success":
        return False
    else:
        isString = result.GetValueAsUnsigned() != 0
        return isString

def _visualize(target):
    target = fb.evaluateInputExpression(target)

    if fb.evaluateBooleanExpression(
        "(unsigned long)CFGetTypeID((CFTypeRef)"
        + target
        + ") == (unsigned long)CGImageGetTypeID()"
    ):
        _showImage("(id)[UIImage imageWithCGImage:" + target + "]")
    elif fb.evaluateBooleanExpression(
        "(unsigned long)CFGetTypeID((CFTypeRef)"
        + target
        + ") == (unsigned long)CVPixelBufferGetTypeID()"
    ):
        _showPixelBuffer(target)
    else:
        if objectHelpers.isKindOfClass(target, "UIImage"):
            _showImage(target)
        elif objectHelpers.isKindOfClass(target, "UIView"):
            _showLayer("[(id)" + target + " layer]")
        elif objectHelpers.isKindOfClass(target, "CALayer"):
            _showLayer(target)
        elif (
            objectHelpers.isKindOfClass(target, "UIColor")
            or objectHelpers.isKindOfClass(target, "CIColor")
            or _colorIsCGColorRef(target)
        ):
            _showColor(target)
        elif objectHelpers.isKindOfClass(target, "NSData"):
            if _dataIsImage(target):
                _showImage("(id)[UIImage imageWithData:" + target + "]")
            elif _dataIsString(target):
                print(
                    fb.describeObject(
                        "[[NSString alloc] initWithData:" + target + " encoding:4]"
                    )
                )
            else:
                print("Data isn't an image and isn't a string.")
        else:
            print(
                "{} isn't supported. You can visualize UIImage, CGImageRef, UIView, CALayer, NSData, UIColor, CIColor, or CGColorRef.".format(
                    objectHelpers.className(target)
                )
            )

class FBVisualizeCommand(fb.FBCommand):
    def name(self):
        return "visualize"

    def description(self):
        return "Open a UIImage, CGImageRef, UIView, CALayer, or CVPixelBuffer in Preview.app on your Mac."

    def args(self):
        return [
            fb.FBCommandArgument(
                arg="target", type="(id)", help="The object to visualize."
            )
        ]

    def run(self, arguments, options):
        _visualize(arguments[0])
<|endoftext|>


<commit_before>"""Helpers to read settings from setup.cfg or pyproject.toml
"""
import configparser
import importlib
import logging
import os
from collections import UserDict
from functools import wraps
from os import getcwd
from typing import Callable, List

import tomlkit
from tomlkit.exceptions import TOMLKitError

from .errors import ImproperConfigurationError

logger = logging.getLogger(__name__)

def _config():
    cwd = getcwd()
    ini_paths = [
        os.path.join(os.path.dirname(__file__), "defaults.cfg"),
        os.path.join(cwd, "setup.cfg"),
    ]
    ini_config = _config_from_ini(ini_paths)

    toml_path = os.path.join(cwd, "pyproject.toml")
    toml_config = _config_from_pyproject(toml_path)

    # Cast to a UserDict so that we can mock the get() method.
    return UserDict({**ini_config, **toml_config})

def _config_from_ini(paths):
    parser = configparser.ConfigParser()
    parser.read(paths)

    flags = {
        "changelog_capitalize",
        "changelog_scope",
        "check_build_status",
        "commit_version_number",
        "patch_without_tag",
        "major_on_zero",
        "remove_dist",
        "upload_to_pypi",
        "upload_to_release",
    }

    # Iterate through the sections so that default values are applied
    # correctly.  See:
    # https://stackoverflow.com/questions/1773793/convert-configparser-items-to-dictionary
    config = {}
    for key, _ in parser.items("semantic_release"):
        if key in flags:
            config[key] = parser.getboolean("semantic_release", key)
        else:
            config[key] = parser.get("semantic_release", key)

    return config

def _config_from_pyproject(path):
    if os.path.isfile(path):
        try:
            with open(path, "r") as f:
                pyproject = tomlkit.loads(f.read())
            if pyproject:
                return pyproject.get("tool", {}).get("semantic_release", {})
        except TOMLKitError as e:
            logger.debug(f"Could not decode pyproject.toml: {e}")

    return {}

config = _config()

def current_commit_parser() -> Callable:
    """Get the currently-configured commit parser

    :raises ImproperConfigurationError: if ImportError or AttributeError is raised
    :returns: Commit parser
    """

    try:
        # All except the last part is the import path
        parts = config.get("commit_parser").split(".")
        module = ".".join(parts[:-1])
        # The final part is the name of the parse function
        return getattr(importlib.import_module(module), parts[-1])
    except (ImportError, AttributeError) as error:
        raise ImproperConfigurationError(f'Unable to import parser "{error}"')

def current_changelog_components() -> List[Callable]:
    """Get the currently-configured changelog components

    :raises ImproperConfigurationError: if ImportError or AttributeError is raised
    :returns: List of component functions
    """
    component_paths = config.get("changelog_components").split(",")
    components = list()

    for path in component_paths:
        try:
            # All except the last part is the import path
            parts = path.split(".")
            module = ".".join(parts[:-1])
            # The final part is the name of the component function
            components.append(getattr(importlib.import_module(module), parts[-1]))
        except (ImportError, AttributeError) as error:
            raise ImproperConfigurationError(
                f'Unable to import changelog component "{path}"'
            )

    return components

def overload_configuration(func):
    """This decorator gets the content of the "define" array and edits "config"
    according to the pairs of key/value.
    """

    @wraps(func)
    def wrap(*args, **kwargs):
        if "define" in kwargs:
            for defined_param in kwargs["define"]:
                pair = defined_param.split("=", maxsplit=1)
                if len(pair) == 2:
                    config[str(pair[0])] = pair[1]
        return func(*args, **kwargs)

    return wrap
<commit_msg>fix: Change log level of failed toml loading<commit_after>"""Helpers to read settings from setup.cfg or pyproject.toml
"""
import configparser
import importlib
import logging
import os
from collections import UserDict
from functools import wraps
from os import getcwd
from typing import Callable, List

import tomlkit
from tomlkit.exceptions import TOMLKitError

from .errors import ImproperConfigurationError

logger = logging.getLogger(__name__)

def _config():
    cwd = getcwd()
    ini_paths = [
        os.path.join(os.path.dirname(__file__), "defaults.cfg"),
        os.path.join(cwd, "setup.cfg"),
    ]
    ini_config = _config_from_ini(ini_paths)

    toml_path = os.path.join(cwd, "pyproject.toml")
    toml_config = _config_from_pyproject(toml_path)

    # Cast to a UserDict so that we can mock the get() method.
    return UserDict({**ini_config, **toml_config})

def _config_from_ini(paths):
    parser = configparser.ConfigParser()
    parser.read(paths)

    flags = {
        "changelog_capitalize",
        "changelog_scope",
        "check_build_status",
        "commit_version_number",
        "patch_without_tag",
        "major_on_zero",
        "remove_dist",
        "upload_to_pypi",
        "upload_to_release",
    }

    # Iterate through the sections so that default values are applied
    # correctly.  See:
    # https://stackoverflow.com/questions/1773793/convert-configparser-items-to-dictionary
    config = {}
    for key, _ in parser.items("semantic_release"):
        if key in flags:
            config[key] = parser.getboolean("semantic_release", key)
        else:
            config[key] = parser.get("semantic_release", key)

    return config

def _config_from_pyproject(path):
    if os.path.isfile(path):
        try:
            with open(path, "r") as f:
                pyproject = tomlkit.loads(f.read())
            if pyproject:
                return pyproject.get("tool", {}).get("semantic_release", {})
        except TOMLKitError as e:
            logger.warning(f"Could not decode pyproject.toml: {e}")

    return {}

config = _config()

def current_commit_parser() -> Callable:
    """Get the currently-configured commit parser

    :raises ImproperConfigurationError: if ImportError or AttributeError is raised
    :returns: Commit parser
    """

    try:
        # All except the last part is the import path
        parts = config.get("commit_parser").split(".")
        module = ".".join(parts[:-1])
        # The final part is the name of the parse function
        return getattr(importlib.import_module(module), parts[-1])
    except (ImportError, AttributeError) as error:
        raise ImproperConfigurationError(f'Unable to import parser "{error}"')

def current_changelog_components() -> List[Callable]:
    """Get the currently-configured changelog components

    :raises ImproperConfigurationError: if ImportError or AttributeError is raised
    :returns: List of component functions
    """
    component_paths = config.get("changelog_components").split(",")
    components = list()

    for path in component_paths:
        try:
            # All except the last part is the import path
            parts = path.split(".")
            module = ".".join(parts[:-1])
            # The final part is the name of the component function
            components.append(getattr(importlib.import_module(module), parts[-1]))
        except (ImportError, AttributeError) as error:
            raise ImproperConfigurationError(
                f'Unable to import changelog component "{path}"'
            )

    return components

def overload_configuration(func):
    """This decorator gets the content of the "define" array and edits "config"
    according to the pairs of key/value.
    """

    @wraps(func)
    def wrap(*args, **kwargs):
        if "define" in kwargs:
            for defined_param in kwargs["define"]:
                pair = defined_param.split("=", maxsplit=1)
                if len(pair) == 2:
                    config[str(pair[0])] = pair[1]
        return func(*args, **kwargs)

    return wrap
<|endoftext|>


<commit_before>from random import Random
from numpy import zeros

_rand = Random()

def random_sample(size=None):
    if size is None:
        return _rand.random()
    ret = zeros(size)
    for x in xrange(ret.size):
        ret.flat[x] = _rand.random()
    return ret

def rand(*args):
    if len(args) == 0:
        args = None
    return random_sample(args)

def standard_normal(size=None):
    if size is None:
        return _rand.gauss(0., 1.)
    ret = zeros(size)
    for x in xrange(ret.size):
        ret.flat[x] = _rand.gauss(0., 1.)
    return ret

def randn(*args):
    if len(args) == 0:
        args = None
    return standard_normal(args)

def randint(low, high=None, size=None):
    if high is None:
        high = low
        low = 0
    if size is None:
        return _rand.randint(low, high)
    ret = zeros(size, dtype='int')
    for x in xrange(ret.size):
        ret.flat[x] = _rand.randint(low, high)
    return ret

seed = _rand.seed

#from _numpypy.mtrand import *  # doesn't exist yet

for name in '''
beta binomial bytes chisquare exponential f gamma geometric get_state gumbel
hypergeometric laplace logistic lognormal logseries multinomial
multivariate_normal negative_binomial noncentral_chisquare noncentral_f normal
pareto permutation poisson power random_integers rayleigh set_state shuffle
standard_cauchy standard_exponential standard_gamma standard_t triangular
uniform vonmises wald weibull zipf
'''.split():
    if name not in globals():
        globals()[name] = None
<commit_msg>fix random.randint upper bound<commit_after>from random import Random
from numpy import zeros

_rand = Random()

def random_sample(size=None):
    if size is None:
        return _rand.random()
    ret = zeros(size)
    for x in xrange(ret.size):
        ret.flat[x] = _rand.random()
    return ret

def rand(*args):
    if len(args) == 0:
        args = None
    return random_sample(args)

def standard_normal(size=None):
    if size is None:
        return _rand.gauss(0., 1.)
    ret = zeros(size)
    for x in xrange(ret.size):
        ret.flat[x] = _rand.gauss(0., 1.)
    return ret

def randn(*args):
    if len(args) == 0:
        args = None
    return standard_normal(args)

def randint(low, high=None, size=None):
    if high is None:
        high = low
        low = 0
    high -= 1
    if size is None:
        return _rand.randint(low, high)
    ret = zeros(size, dtype='int')
    for x in xrange(ret.size):
        ret.flat[x] = _rand.randint(low, high)
    return ret

seed = _rand.seed

#from _numpypy.mtrand import *  # doesn't exist yet

for name in '''
beta binomial bytes chisquare exponential f gamma geometric get_state gumbel
hypergeometric laplace logistic lognormal logseries multinomial
multivariate_normal negative_binomial noncentral_chisquare noncentral_f normal
pareto permutation poisson power random_integers rayleigh set_state shuffle
standard_cauchy standard_exponential standard_gamma standard_t triangular
uniform vonmises wald weibull zipf
'''.split():
    if name not in globals():
        globals()[name] = None
<|endoftext|>