<commit_before>from chromatica import logger
from chromatica.util import load_external_module

load_external_module(__file__, "")
from clang import cindex

import os
import re

log = logger.logging.getLogger("chromatica.compile_args")

DEFAULT_STD={"c"   : ["-std=c11"], \
             "cpp" : ["-std=c++14"]}

def set_default_std(stds):
    DEFAULT_STD = stds
    return True

class CompileArgsDatabase(object):

    def __init__(self, path, global_args=None):
        if path:
            self.__path = path
        else:
            self.__path = os.getcwd()
        self.compile_args = []
        self.cdb = None
        self.__clang_file = None
        self.__cdb_path = None

        if global_args != None:
            self.compile_args = global_args

        self.__find_clang_file()
        self.__find_cdb_file()

        self.__parse_compile_args()
        self.__try_init_cdb()

    def __find_clang_file(self):
        clang_file_path = self.__path
        while os.path.dirname(clang_file_path) != clang_file_path:
            self.__clang_file = os.path.join(clang_file_path, ".clang")
            if os.path.exists(self.__clang_file):
                return
            clang_file_path = os.path.dirname(clang_file_path)

        self.__clang_file = None

    def __find_cdb_file(self):
        cdb_file_path = self.__path
        while os.path.dirname(cdb_file_path) != cdb_file_path:
            cdb_file = os.path.join(cdb_file_path, "compile_commands.json")
            if os.path.exists(cdb_file):
                self.__cdb_path = cdb_file_path
                return
            cdb_file_path = os.path.dirname(cdb_file_path)

    def __parse_compile_args(self):
        if self.__clang_file == None:
            return
        # read .clang file
        fp = open(self.__clang_file)
        flags = fp.read()
        fp.close()
        m = re.match(r"^flags\s*=\s*", flags)
        if m != None:
            self.compile_args += flags[m.end():].split()

        m = re.match(r"^compilation_database\s*=\s*", flags)
        if m != None:
            cdb_rel_path = flags[m.end():].strip("\"")
            cdb_path = os.path.join(os.path.dirname(self.__clang_file), cdb_rel_path)
            if cdb_path and os.path.isdir(cdb_path):
                self.__cdb_path = cdb_path

    def __try_init_cdb(self):
        if self.__cdb_path != None:
            self.cdb = cindex.CompilationDatabase.fromDirectory(self.__cdb_path)

    def __get_cdb_args(self, filename):
        res = []
        ret = self.cdb.getCompileCommands(filename)
        if ret:
            for cmds in ret:
                cwd = cmds.directory
                skip = 0
                for arg in cmds.arguments:
                    if skip:
                        skip = 0
                        continue
                    if arg == "-o" or arg == "-c":
                        skip = 1
                        continue

                    if arg.startswith('-I'):
                        include_path = arg[2:]
                        if not os.path.isabs(include_path):
                            include_path = os.path.normpath(
                                os.path.join(cwd, include_path))
                        res.append('-I' + include_path)
                    else:
                        res.append(arg)
        else:
            print("Cannot find compile flags for %s in compilation database" % filename)
        return res

    def get_args_filename(self, filename):
        ret = None
        if self.cdb != None:
            ret = self.__get_cdb_args(filename)

        if ret:
            return self.compile_args + ret
        else:
            return self.compile_args

    def get_args_filename_ft(self, filename, filetype):
        if self.cdb != None or filetype not in DEFAULT_STD:
            return self.get_args_filename(filename)

        ret = DEFAULT_STD[filetype]
        return ret + self.compile_args

    @property
    def clang_file(self):
        return self.__clang_file

    @property
    def cdb_file(self):
        if self.__cdb_path:
            return os.path.join(self.__cdb_path, "compile_commands.json")
        else:
            return ""

<commit_msg>add support for multiple flags config in .clang file<commit_after>from chromatica import logger
from chromatica.util import load_external_module

load_external_module(__file__, "")
from clang import cindex

import os
import re

log = logger.logging.getLogger("chromatica.compile_args")

DEFAULT_STD={"c"   : ["-std=c11"], \
             "cpp" : ["-std=c++14"]}

def set_default_std(stds):
    DEFAULT_STD = stds
    return True

class CompileArgsDatabase(object):

    def __init__(self, path, global_args=None):
        if path:
            self.__path = path
        else:
            self.__path = os.getcwd()
        self.compile_args = []
        self.cdb = None
        self.__clang_file = None
        self.__cdb_path = None

        if global_args != None:
            self.compile_args = global_args

        self.__find_clang_file()
        self.__find_cdb_file()

        self.__parse_compile_args()
        self.__try_init_cdb()

    def __find_clang_file(self):
        clang_file_path = self.__path
        while os.path.dirname(clang_file_path) != clang_file_path:
            self.__clang_file = os.path.join(clang_file_path, ".clang")
            if os.path.exists(self.__clang_file):
                return
            clang_file_path = os.path.dirname(clang_file_path)

        self.__clang_file = None

    def __find_cdb_file(self):
        cdb_file_path = self.__path
        while os.path.dirname(cdb_file_path) != cdb_file_path:
            cdb_file = os.path.join(cdb_file_path, "compile_commands.json")
            if os.path.exists(cdb_file):
                self.__cdb_path = cdb_file_path
                return
            cdb_file_path = os.path.dirname(cdb_file_path)

    def __parse_flags_key(self, value):
        self.compile_args = value.split()

    def __parse_flags_append_key(self, value):
        self.compile_args.extend(value.split())

    def __parse_cdb_key(self, value):
        cdb_rel_path = value.strip("\"")
        cdb_path = os.path.join(os.path.dirname(self.__clang_file), cdb_rel_path)
        if cdb_path and os.path.isdir(cdb_path):
            self.__cdb_path = cdb_path

    def __parse_compile_args(self):
        if self.__clang_file == None:
            return
        # read .clang file
        fp = open(self.__clang_file)
        lines = fp.readlines()
        fp.close()

        funcs = {"flags" : self.__parse_flags_key,
                 "flags+" : self.__parse_flags_append_key,
                 "compilation_database" : self.__parse_cdb_key, }

        for line in lines:
            pos = line.find("=")
            key = line[:pos]
            try:
                funcs[key](self, line[pos+1:].strip())
            except:
                log.error("Invalid configuration key: ", key)

    def __try_init_cdb(self):
        if self.__cdb_path != None:
            self.cdb = cindex.CompilationDatabase.fromDirectory(self.__cdb_path)

    def __get_cdb_args(self, filename):
        res = []
        ret = self.cdb.getCompileCommands(filename)
        if ret:
            for cmds in ret:
                cwd = cmds.directory
                skip = 0
                for arg in cmds.arguments:
                    if skip:
                        skip = 0
                        continue
                    if arg == "-o" or arg == "-c":
                        skip = 1
                        continue

                    if arg.startswith('-I'):
                        include_path = arg[2:]
                        if not os.path.isabs(include_path):
                            include_path = os.path.normpath(
                                os.path.join(cwd, include_path))
                        res.append('-I' + include_path)
                    else:
                        res.append(arg)
        else:
            print("Cannot find compile flags for %s in compilation database" % filename)
        return res

    def get_args_filename(self, filename):
        ret = None
        if self.cdb != None:
            ret = self.__get_cdb_args(filename)

        if ret:
            return self.compile_args + ret
        else:
            return self.compile_args

    def get_args_filename_ft(self, filename, filetype):
        if self.cdb != None or filetype not in DEFAULT_STD:
            return self.get_args_filename(filename)

        ret = DEFAULT_STD[filetype]
        return ret + self.compile_args

    @property
    def clang_file(self):
        return self.__clang_file

    @property
    def cdb_file(self):
        if self.__cdb_path:
            return os.path.join(self.__cdb_path, "compile_commands.json")
        else:
            return ""

<|endoftext|>


<commit_before><commit_msg>temporarily disabled the access statements<commit_after><|endoftext|>


<commit_before># Module 'glob' -- filename globbing.

import os
import fnmatch
import regex

def glob(pathname):
	if not has_magic(pathname):
		if os.path.exists(pathname):
			return [pathname]
		else:
			return []
	dirname, basename = os.path.split(pathname)
	if has_magic(dirname):
		list = glob(dirname)
	else:
		list = [dirname]
	if not has_magic(basename):
		result = []
		for dirname in list:
			if basename or os.path.isdir(dirname):
				name = os.path.join(dirname, basename)
				if os.path.exists(name):
					result.append(name)
	else:
		result = []
		for dirname in list:
			sublist = glob1(dirname, basename)
			for name in sublist:
				result.append(os.path.join(dirname, name))
	return result

def glob1(dirname, pattern):
	if not dirname: dirname = os.curdir
	try:
		names = os.listdir(dirname)
	except os.error:
		return []
	result = []
	for name in names:
		if name[0] != '.' or pattern[0] == '.':
			if fnmatch.fnmatch(name, pattern):
				result.append(name)
	return result

magic_check = regex.compile('[*?[]')

def has_magic(s):
	return magic_check.search(s) >= 0
<commit_msg>New doc strings.<commit_after>"""Filename globbing utility."""

import os
import fnmatch
import regex

def glob(pathname):
	"""Return a list of paths matching a pathname pattern.

	The pattern may contain simple shell-style wildcards a la fnmatch.

	"""
	if not has_magic(pathname):
		if os.path.exists(pathname):
			return [pathname]
		else:
			return []
	dirname, basename = os.path.split(pathname)
	if has_magic(dirname):
		list = glob(dirname)
	else:
		list = [dirname]
	if not has_magic(basename):
		result = []
		for dirname in list:
			if basename or os.path.isdir(dirname):
				name = os.path.join(dirname, basename)
				if os.path.exists(name):
					result.append(name)
	else:
		result = []
		for dirname in list:
			sublist = glob1(dirname, basename)
			for name in sublist:
				result.append(os.path.join(dirname, name))
	return result

def glob1(dirname, pattern):
	if not dirname: dirname = os.curdir
	try:
		names = os.listdir(dirname)
	except os.error:
		return []
	result = []
	for name in names:
		if name[0] != '.' or pattern[0] == '.':
			if fnmatch.fnmatch(name, pattern):
				result.append(name)
	return result

magic_check = regex.compile('[*?[]')

def has_magic(s):
	return magic_check.search(s) >= 0
<|endoftext|>


<commit_before>"""Classes to represent arbitrary sets (including sets of sets).

This module implements sets using dictionaries whose values are
ignored.  The usual operations (union, intersection, deletion, etc.)
are provided as both methods and operators.

Important: sets are not sequences!  While they support 'x in s',
'len(s)', and 'for x in s', none of those operations are unique for
sequences; for example, mappings support all three as well.  The
characteristic operation for sequences is subscripting with small
integers: s[i], for i in range(len(s)).  Sets don't support
subscripting at all.  Also, sequences allow multiple occurrences and
their elements have a definite order; sets on the other hand don't
record multiple occurrences and don't remember the order of element
insertion (which is why they don't support s[i]).

The following classes are provided:

BaseSet -- All the operations common to both mutable and immutable
    sets. This is an abstract class, not meant to be directly
    instantiated.

Set -- Mutable sets, subclass of BaseSet; not hashable.

ImmutableSet -- Immutable sets, subclass of BaseSet; hashable.
    An iterable argument is mandatory to create an ImmutableSet.

_TemporarilyImmutableSet -- Not a subclass of BaseSet: just a wrapper
    around a Set, hashable, giving the same hash value as the
    immutable set equivalent would have.  Do not use this class
    directly.

Only hashable objects can be added to a Set. In particular, you cannot
really add a Set as an element to another Set; if you try, what is
actually added is an ImmutableSet built from it (it compares equal to
the one you tried adding).

When you ask if `x in y' where x is a Set and y is a Set or
ImmutableSet, x is wrapped into a _TemporarilyImmutableSet z, and
what's tested is actually `z in y'.

"""

# Code history:
#
# - Greg V. Wilson wrote the first version, using a different approach
#   to the mutable/immutable problem, and inheriting from dict.
#
# - Alex Martelli modified Greg's version to implement the current
#   Set/ImmutableSet approach, and make the data an attribute.
#
# - Guido van Rossum rewrote much of the code, made some API changes,
#   and cleaned up the docstrings.
#
# - Raymond Hettinger added a number of speedups and other
#   improvements.

__all__ = ['BaseSet', 'Set', 'ImmutableSet']

class BaseSet(object):
    """Common base class for mutable and immutable sets."""

    __slots__ = ['_data']

    # Constructor

    def __init__(self):
        """This is an abstract class."""
        # Don't call this from a concrete subclass!
        if self.__class__ is BaseSet:
            raise TypeError, ("BaseSet is an abstract class.  "
                              "Use Set or ImmutableSet.")

    # Standard protocols: __len__, __repr__, __str__, __iter__

    def __len__(self):
        """Return the number of elements of a set."""
        return len(self._data)

    def __repr__(self):
        """Return string representation of a set.

        This looks like 'Set([<list of elements>])'.
        """
        return self._repr()

    # __str__ is the same as __repr__
    __str__ = __repr__

    def _repr(self, sorted=False):
        elements = self._data.keys()
        if sorted:
            elements.sort()
        return '%s(%r)' % (self.__class__.__name__, elements)

    def __iter__(self):
        """Return an iterator over the elements or a set.

        This is the keys iterator for the underlying dict.
        """
        return self._data.iterkeys()

    # Equality comparisons using the underlying dicts

    def __eq__(self, other):
        self._binary_sanity_check(other)
        return self._data == other._data

    def __ne__(self, other):
        self._binary_sanity_check(other)
        return self._data != other._data

    # Copying operations

    def copy(self):
        """Return a shallow copy of a set."""
        result = self.__class__()
        result._data.update(self._data)
        return result

    __copy__ = copy # For the copy module

    def __deepcopy__(self, memo):
        """Return a deep copy of a set; used by copy module."""
        # This pre-creates the result and inserts it in the memo
        # early, in case the deep copy recurses into another reference
        # to this same set.  A set can't be an element of itself, but
        # it can certainly contain an object that has a reference to
        # itself.
        from copy import deepcopy
        result = self.__class__()
        memo[id(self)] = result
        data = result._data
        value = True
        for elt in self:
            data[deepcopy(elt, memo)] = value
        return result

    # Standard set operations: union, intersection, both differences.
    # Each has an operator version (e.g. __or__, invoked with |) and a
    # method version (e.g. union).
    # Subtle:  Each pair requires distinct code so that the outcome is
    # correct when the type of other isn't suitable.  For example, if
    # we did "union = __or__" instead, then Set().union(3) would return
    # NotImplemented instead of raising TypeError (albeit that *why* it
    # raises TypeError as-is is also a bit subtle).

    def __or__(self, other):
        """Return the union of two sets as a new set.

        (I.e. all elements that are in either set.)
        """
        if not isinstance(other, BaseSet):
            return NotImplemented
        result = self.__class__()
        result._data = self._data.copy()
        result._data.update(other._data)
        return result

    def union(self, other):
        """Return the union of two sets as a new set.

        (I.e. all elements that are in either set.)
        """
        return self | other

    def __and__(self, other):
        """Return the intersection of two sets as a new set.

        (I.e. all elements that are in both sets.)
        """
        if not isinstance(other, BaseSet):
            return NotImplemented
        if len(self) <= len(other):
            little, big = self, other
        else:
            little, big = other, self
        common = filter(big._data.has_key, little._data.iterkeys())
        return self.__class__(common)

    def intersection(self, other):
        """Return the intersection of two sets as a new set.

        (I.e. all elements that are in both sets.)
        """
        return self & other

    def __xor__(self, other):
        """Return the symmetric difference of two sets as a new set.

        (I.e. all elements that are in exactly one of the sets.)
        """
        if not isinstance(other, BaseSet):
            return NotImplemented
        result = self.__class__()
        data = result._data
        value = True
        selfdata = self._data
        otherdata = other._data
        for elt in selfdata:
            if elt not in otherdata:
                data[elt] = value
        for elt in otherdata:
            if elt not in selfdata:
                data[elt] = value
        return result

    def symmetric_difference(self, other):
        """Return the symmetric difference of two sets as a new set.

        (I.e. all elements that are in exactly one of the sets.)
        """
        return self ^ other

    def  __sub__(self, other):
        """Return the difference of two sets as a new Set.

        (I.e. all elements that are in this set and not in the other.)
        """
        if not isinstance(other, BaseSet):
            return NotImplemented
        result = self.__class__()
        data = result._data
        otherdata = other._data
        value = True
        for elt in self:
            if elt not in otherdata:
                data[elt] = value
        return result

    def difference(self, other):
        """Return the difference of two sets as a new Set.

        (I.e. all elements that are in this set and not in the other.)
        """
        return self - other

    # Membership test

    def __contains__(self, element):
        """Report whether an element is a member of a set.

        (Called in response to the expression `element in self'.)
        """
        try:
            return element in self._data
        except TypeError:
            transform = getattr(element, "_as_temporarily_immutable", None)
            if transform is None:
                raise # re-raise the TypeError exception we caught
            return transform() in self._data

    # Subset and superset test

    def issubset(self, other):
        """Report whether another set contains this set."""
        self._binary_sanity_check(other)
        if len(self) > len(other):  # Fast check for obvious cases
            return False
        otherdata = other._data
        for elt in self:
            if elt not in otherdata:
                return False
        return True

    def issuperset(self, other):
        """Report whether this set contains another set."""
        self._binary_sanity_check(other)
        if len(self) < len(other):  # Fast check for obvious cases
            return False
        selfdata = self._data
        for elt in other:
            if elt not in selfdata:
                return False
        return True

    # Inequality comparisons using the is-subset relation.
    __le__ = issubset
    __ge__ = issuperset

    def __lt__(self, other):
        self._binary_sanity_check(other)
        return len(self) < len(other) and self.issubset(other)

    def __gt__(self, other):
        self._binary_sanity_check(other)
        return len(self) > len(other) and self.issuperset(other)

    # Assorted helpers

    def _binary_sanity_check(self, other):
        # Check that the other argument to a binary operation is also
        # a set, raising a TypeError otherwise.
        if not isinstance(other, BaseSet):
            raise TypeError, "Binary operation only permitted between sets"

    def _compute_hash(self):
        # Calculate hash code for a set by xor'ing the hash codes of
        # the elements.  This ensures that the hash code does not depend
        # on the order in which elements are added to the set.  This is
        # not called __hash__ because a BaseSet should not be hashable;
        # only an ImmutableSet is hashable.
        result = 0
        for elt in self:
            result ^= hash(elt)
        return result

    def _update(self, iterable):
        # The main loop for update() and the subclass __init__() methods.
        data = self._data
        value = True
        it = iter(iterable)
        while True:
            try:
                for element in it:
                    data[element] = value
                return
            except TypeError:
                transform = getattr(element, "_as_immutable", None)
                if transform is None:
                    raise # re-raise the TypeError exception we caught
                data[transform()] = value

class ImmutableSet(BaseSet):
    """Immutable set class."""

    __slots__ = ['_hashcode']

    # BaseSet + hashing

    def __init__(self, iterable=None):
        """Construct an immutable set from an optional iterable."""
        self._hashcode = None
        self._data = {}
        if iterable is not None:
            self._update(iterable)

    def __hash__(self):
        if self._hashcode is None:
            self._hashcode = self._compute_hash()
        return self._hashcode

class Set(BaseSet):
    """ Mutable set class."""

    __slots__ = []

    # BaseSet + operations requiring mutability; no hashing

    def __init__(self, iterable=None):
        """Construct a set from an optional iterable."""
        self._data = {}
        if iterable is not None:
            self._update(iterable)

    def __hash__(self):
        """A Set cannot be hashed."""
        # We inherit object.__hash__, so we must deny this explicitly
        raise TypeError, "Can't hash a Set, only an ImmutableSet."

    # In-place union, intersection, differences.
    # Subtle:  The xyz_update() functions deliberately return None,
    # as do all mutating operations on built-in container types.
    # The __xyz__ spellings have to return self, though.

    def __ior__(self, other):
        """Update a set with the union of itself and another."""
        self._binary_sanity_check(other)
        self._data.update(other._data)
        return self

    def union_update(self, other):
        """Update a set with the union of itself and another."""
        self |= other

    def __iand__(self, other):
        """Update a set with the intersection of itself and another."""
        self._binary_sanity_check(other)
        for elt in self._data.keys():
            if elt not in other:
                del self._data[elt]
        return self

    def intersection_update(self, other):
        """Update a set with the intersection of itself and another."""
        self &= other

    def __ixor__(self, other):
        """Update a set with the symmetric difference of itself and another."""
        self._binary_sanity_check(other)
        data = self._data
        value = True
        for elt in other:
            if elt in data:
                del data[elt]
            else:
                data[elt] = value
        return self

    def symmetric_difference_update(self, other):
        """Update a set with the symmetric difference of itself and another."""
        self ^= other

    def __isub__(self, other):
        """Remove all elements of another set from this set."""
        self._binary_sanity_check(other)
        data = self._data
        for elt in other:
            if elt in data:
                del data[elt]
        return self

    def difference_update(self, other):
        """Remove all elements of another set from this set."""
        self -= other

    # Python dict-like mass mutations: update, clear

    def update(self, iterable):
        """Add all values from an iterable (such as a list or file)."""
        self._update(iterable)

    def clear(self):
        """Remove all elements from this set."""
        self._data.clear()

    # Single-element mutations: add, remove, discard

    def add(self, element):
        """Add an element to a set.

        This has no effect if the element is already present.
        """
        try:
            self._data[element] = True
        except TypeError:
            transform = getattr(element, "_as_immutable", None)
            if transform is None:
                raise # re-raise the TypeError exception we caught
            self._data[transform()] = True

    def remove(self, element):
        """Remove an element from a set; it must be a member.

        If the element is not a member, raise a KeyError.
        """
        try:
            del self._data[element]
        except TypeError:
            transform = getattr(element, "_as_temporarily_immutable", None)
            if transform is None:
                raise # re-raise the TypeError exception we caught
            del self._data[transform()]

    def discard(self, element):
        """Remove an element from a set if it is a member.

        If the element is not a member, do nothing.
        """
        try:
            self.remove(element)
        except KeyError:
            pass

    def pop(self):
        """Remove and return an arbitrary set element."""
        return self._data.popitem()[0]

    def _as_immutable(self):
        # Return a copy of self as an immutable set
        return ImmutableSet(self)

    def _as_temporarily_immutable(self):
        # Return self wrapped in a temporarily immutable set
        return _TemporarilyImmutableSet(self)

class _TemporarilyImmutableSet(BaseSet):
    # Wrap a mutable set as if it was temporarily immutable.
    # This only supplies hashing and equality comparisons.

    def __init__(self, set):
        self._set = set
        self._data = set._data  # Needed by ImmutableSet.__eq__()

    def __hash__(self):
        return self._set._compute_hash()
<commit_msg>Gave intersection_update a speed boost.<commit_after>"""Classes to represent arbitrary sets (including sets of sets).

This module implements sets using dictionaries whose values are
ignored.  The usual operations (union, intersection, deletion, etc.)
are provided as both methods and operators.

Important: sets are not sequences!  While they support 'x in s',
'len(s)', and 'for x in s', none of those operations are unique for
sequences; for example, mappings support all three as well.  The
characteristic operation for sequences is subscripting with small
integers: s[i], for i in range(len(s)).  Sets don't support
subscripting at all.  Also, sequences allow multiple occurrences and
their elements have a definite order; sets on the other hand don't
record multiple occurrences and don't remember the order of element
insertion (which is why they don't support s[i]).

The following classes are provided:

BaseSet -- All the operations common to both mutable and immutable
    sets. This is an abstract class, not meant to be directly
    instantiated.

Set -- Mutable sets, subclass of BaseSet; not hashable.

ImmutableSet -- Immutable sets, subclass of BaseSet; hashable.
    An iterable argument is mandatory to create an ImmutableSet.

_TemporarilyImmutableSet -- Not a subclass of BaseSet: just a wrapper
    around a Set, hashable, giving the same hash value as the
    immutable set equivalent would have.  Do not use this class
    directly.

Only hashable objects can be added to a Set. In particular, you cannot
really add a Set as an element to another Set; if you try, what is
actually added is an ImmutableSet built from it (it compares equal to
the one you tried adding).

When you ask if `x in y' where x is a Set and y is a Set or
ImmutableSet, x is wrapped into a _TemporarilyImmutableSet z, and
what's tested is actually `z in y'.

"""

# Code history:
#
# - Greg V. Wilson wrote the first version, using a different approach
#   to the mutable/immutable problem, and inheriting from dict.
#
# - Alex Martelli modified Greg's version to implement the current
#   Set/ImmutableSet approach, and make the data an attribute.
#
# - Guido van Rossum rewrote much of the code, made some API changes,
#   and cleaned up the docstrings.
#
# - Raymond Hettinger added a number of speedups and other
#   improvements.

__all__ = ['BaseSet', 'Set', 'ImmutableSet']

class BaseSet(object):
    """Common base class for mutable and immutable sets."""

    __slots__ = ['_data']

    # Constructor

    def __init__(self):
        """This is an abstract class."""
        # Don't call this from a concrete subclass!
        if self.__class__ is BaseSet:
            raise TypeError, ("BaseSet is an abstract class.  "
                              "Use Set or ImmutableSet.")

    # Standard protocols: __len__, __repr__, __str__, __iter__

    def __len__(self):
        """Return the number of elements of a set."""
        return len(self._data)

    def __repr__(self):
        """Return string representation of a set.

        This looks like 'Set([<list of elements>])'.
        """
        return self._repr()

    # __str__ is the same as __repr__
    __str__ = __repr__

    def _repr(self, sorted=False):
        elements = self._data.keys()
        if sorted:
            elements.sort()
        return '%s(%r)' % (self.__class__.__name__, elements)

    def __iter__(self):
        """Return an iterator over the elements or a set.

        This is the keys iterator for the underlying dict.
        """
        return self._data.iterkeys()

    # Equality comparisons using the underlying dicts

    def __eq__(self, other):
        self._binary_sanity_check(other)
        return self._data == other._data

    def __ne__(self, other):
        self._binary_sanity_check(other)
        return self._data != other._data

    # Copying operations

    def copy(self):
        """Return a shallow copy of a set."""
        result = self.__class__()
        result._data.update(self._data)
        return result

    __copy__ = copy # For the copy module

    def __deepcopy__(self, memo):
        """Return a deep copy of a set; used by copy module."""
        # This pre-creates the result and inserts it in the memo
        # early, in case the deep copy recurses into another reference
        # to this same set.  A set can't be an element of itself, but
        # it can certainly contain an object that has a reference to
        # itself.
        from copy import deepcopy
        result = self.__class__()
        memo[id(self)] = result
        data = result._data
        value = True
        for elt in self:
            data[deepcopy(elt, memo)] = value
        return result

    # Standard set operations: union, intersection, both differences.
    # Each has an operator version (e.g. __or__, invoked with |) and a
    # method version (e.g. union).
    # Subtle:  Each pair requires distinct code so that the outcome is
    # correct when the type of other isn't suitable.  For example, if
    # we did "union = __or__" instead, then Set().union(3) would return
    # NotImplemented instead of raising TypeError (albeit that *why* it
    # raises TypeError as-is is also a bit subtle).

    def __or__(self, other):
        """Return the union of two sets as a new set.

        (I.e. all elements that are in either set.)
        """
        if not isinstance(other, BaseSet):
            return NotImplemented
        result = self.__class__()
        result._data = self._data.copy()
        result._data.update(other._data)
        return result

    def union(self, other):
        """Return the union of two sets as a new set.

        (I.e. all elements that are in either set.)
        """
        return self | other

    def __and__(self, other):
        """Return the intersection of two sets as a new set.

        (I.e. all elements that are in both sets.)
        """
        if not isinstance(other, BaseSet):
            return NotImplemented
        if len(self) <= len(other):
            little, big = self, other
        else:
            little, big = other, self
        common = filter(big._data.has_key, little._data.iterkeys())
        return self.__class__(common)

    def intersection(self, other):
        """Return the intersection of two sets as a new set.

        (I.e. all elements that are in both sets.)
        """
        return self & other

    def __xor__(self, other):
        """Return the symmetric difference of two sets as a new set.

        (I.e. all elements that are in exactly one of the sets.)
        """
        if not isinstance(other, BaseSet):
            return NotImplemented
        result = self.__class__()
        data = result._data
        value = True
        selfdata = self._data
        otherdata = other._data
        for elt in selfdata:
            if elt not in otherdata:
                data[elt] = value
        for elt in otherdata:
            if elt not in selfdata:
                data[elt] = value
        return result

    def symmetric_difference(self, other):
        """Return the symmetric difference of two sets as a new set.

        (I.e. all elements that are in exactly one of the sets.)
        """
        return self ^ other

    def  __sub__(self, other):
        """Return the difference of two sets as a new Set.

        (I.e. all elements that are in this set and not in the other.)
        """
        if not isinstance(other, BaseSet):
            return NotImplemented
        result = self.__class__()
        data = result._data
        otherdata = other._data
        value = True
        for elt in self:
            if elt not in otherdata:
                data[elt] = value
        return result

    def difference(self, other):
        """Return the difference of two sets as a new Set.

        (I.e. all elements that are in this set and not in the other.)
        """
        return self - other

    # Membership test

    def __contains__(self, element):
        """Report whether an element is a member of a set.

        (Called in response to the expression `element in self'.)
        """
        try:
            return element in self._data
        except TypeError:
            transform = getattr(element, "_as_temporarily_immutable", None)
            if transform is None:
                raise # re-raise the TypeError exception we caught
            return transform() in self._data

    # Subset and superset test

    def issubset(self, other):
        """Report whether another set contains this set."""
        self._binary_sanity_check(other)
        if len(self) > len(other):  # Fast check for obvious cases
            return False
        otherdata = other._data
        for elt in self:
            if elt not in otherdata:
                return False
        return True

    def issuperset(self, other):
        """Report whether this set contains another set."""
        self._binary_sanity_check(other)
        if len(self) < len(other):  # Fast check for obvious cases
            return False
        selfdata = self._data
        for elt in other:
            if elt not in selfdata:
                return False
        return True

    # Inequality comparisons using the is-subset relation.
    __le__ = issubset
    __ge__ = issuperset

    def __lt__(self, other):
        self._binary_sanity_check(other)
        return len(self) < len(other) and self.issubset(other)

    def __gt__(self, other):
        self._binary_sanity_check(other)
        return len(self) > len(other) and self.issuperset(other)

    # Assorted helpers

    def _binary_sanity_check(self, other):
        # Check that the other argument to a binary operation is also
        # a set, raising a TypeError otherwise.
        if not isinstance(other, BaseSet):
            raise TypeError, "Binary operation only permitted between sets"

    def _compute_hash(self):
        # Calculate hash code for a set by xor'ing the hash codes of
        # the elements.  This ensures that the hash code does not depend
        # on the order in which elements are added to the set.  This is
        # not called __hash__ because a BaseSet should not be hashable;
        # only an ImmutableSet is hashable.
        result = 0
        for elt in self:
            result ^= hash(elt)
        return result

    def _update(self, iterable):
        # The main loop for update() and the subclass __init__() methods.
        data = self._data
        value = True
        it = iter(iterable)
        while True:
            try:
                for element in it:
                    data[element] = value
                return
            except TypeError:
                transform = getattr(element, "_as_immutable", None)
                if transform is None:
                    raise # re-raise the TypeError exception we caught
                data[transform()] = value

class ImmutableSet(BaseSet):
    """Immutable set class."""

    __slots__ = ['_hashcode']

    # BaseSet + hashing

    def __init__(self, iterable=None):
        """Construct an immutable set from an optional iterable."""
        self._hashcode = None
        self._data = {}
        if iterable is not None:
            self._update(iterable)

    def __hash__(self):
        if self._hashcode is None:
            self._hashcode = self._compute_hash()
        return self._hashcode

class Set(BaseSet):
    """ Mutable set class."""

    __slots__ = []

    # BaseSet + operations requiring mutability; no hashing

    def __init__(self, iterable=None):
        """Construct a set from an optional iterable."""
        self._data = {}
        if iterable is not None:
            self._update(iterable)

    def __hash__(self):
        """A Set cannot be hashed."""
        # We inherit object.__hash__, so we must deny this explicitly
        raise TypeError, "Can't hash a Set, only an ImmutableSet."

    # In-place union, intersection, differences.
    # Subtle:  The xyz_update() functions deliberately return None,
    # as do all mutating operations on built-in container types.
    # The __xyz__ spellings have to return self, though.

    def __ior__(self, other):
        """Update a set with the union of itself and another."""
        self._binary_sanity_check(other)
        self._data.update(other._data)
        return self

    def union_update(self, other):
        """Update a set with the union of itself and another."""
        self |= other

    def __iand__(self, other):
        """Update a set with the intersection of itself and another."""
        self._binary_sanity_check(other)
        self._data = (self & other)._data
        return self

    def intersection_update(self, other):
        """Update a set with the intersection of itself and another."""
        self &= other

    def __ixor__(self, other):
        """Update a set with the symmetric difference of itself and another."""
        self._binary_sanity_check(other)
        data = self._data
        value = True
        for elt in other:
            if elt in data:
                del data[elt]
            else:
                data[elt] = value
        return self

    def symmetric_difference_update(self, other):
        """Update a set with the symmetric difference of itself and another."""
        self ^= other

    def __isub__(self, other):
        """Remove all elements of another set from this set."""
        self._binary_sanity_check(other)
        data = self._data
        for elt in other:
            if elt in data:
                del data[elt]
        return self

    def difference_update(self, other):
        """Remove all elements of another set from this set."""
        self -= other

    # Python dict-like mass mutations: update, clear

    def update(self, iterable):
        """Add all values from an iterable (such as a list or file)."""
        self._update(iterable)

    def clear(self):
        """Remove all elements from this set."""
        self._data.clear()

    # Single-element mutations: add, remove, discard

    def add(self, element):
        """Add an element to a set.

        This has no effect if the element is already present.
        """
        try:
            self._data[element] = True
        except TypeError:
            transform = getattr(element, "_as_immutable", None)
            if transform is None:
                raise # re-raise the TypeError exception we caught
            self._data[transform()] = True

    def remove(self, element):
        """Remove an element from a set; it must be a member.

        If the element is not a member, raise a KeyError.
        """
        try:
            del self._data[element]
        except TypeError:
            transform = getattr(element, "_as_temporarily_immutable", None)
            if transform is None:
                raise # re-raise the TypeError exception we caught
            del self._data[transform()]

    def discard(self, element):
        """Remove an element from a set if it is a member.

        If the element is not a member, do nothing.
        """
        try:
            self.remove(element)
        except KeyError:
            pass

    def pop(self):
        """Remove and return an arbitrary set element."""
        return self._data.popitem()[0]

    def _as_immutable(self):
        # Return a copy of self as an immutable set
        return ImmutableSet(self)

    def _as_temporarily_immutable(self):
        # Return self wrapped in a temporarily immutable set
        return _TemporarilyImmutableSet(self)

class _TemporarilyImmutableSet(BaseSet):
    # Wrap a mutable set as if it was temporarily immutable.
    # This only supplies hashing and equality comparisons.

    def __init__(self, set):
        self._set = set
        self._data = set._data  # Needed by ImmutableSet.__eq__()

    def __hash__(self):
        return self._set._compute_hash()
<|endoftext|>


<commit_before>import json
from node import Node
from link import Link

from bs4 import BeautifulSoup

# Python2/3 compatiblity hacks
try:
	from urllib.request import urlopen
except ImportError:
	from urllib import urlopen

try:
	basestring
except NameError:
	basestring = str

class NodeDB:
  def __init__(self):
    self._nodes = []
    self._links = set()

  # fetch list of links
  def get_links(self):
    return [self.map_link(x) for x in self._links]

  # fetch list of nodes
  def get_nodes(self):
    return self._nodes

  def maybe_node_by_mac(self, macs):
    for node in self._nodes:
      for mac in macs:
        if mac.lower() in node.macs:
          return node

    raise

  # import_batman(list(fileinput.input(options['batmanjson'])))
  def import_batman(self, lines):

    for line in lines:
      x = json.loads(line)

      if 'of' in x:
        try:
          node = self.maybe_node_by_mac((x['of'], x['secondary']))
        except:
          node = Node()
          node.online = True
          self._nodes.append(node)

        node.add_mac(x['of'])
        node.add_mac(x['secondary'])

    for line in lines:
      x = json.loads(line)

      if 'router' in x:
        try:
          node = self.maybe_node_by_mac((x['router'], ))
        except:
          node = Node()
          node.online = True
          node.add_mac(x['router'])
          self._nodes.append(node)

        try:
          if 'gateway' in x:
            x['neighbor'] = x['gateway']

          node = self.maybe_node_by_mac((x['neighbor'], ))
        except:
          node = Node()
          node.online = True
          if x['label'] == 'TT':
            node.group = 3

          node.add_mac(x['neighbor'])
          self._nodes.append(node)

    for line in lines:
      x = json.loads(line)

      if 'router' in x:
        try:
          if 'gateway' in x:
            x['neighbor'] = x['gateway']

          router = self.maybe_node_by_mac((x['router'], ))
          neighbor = self.maybe_node_by_mac((x['neighbor'], ))
        except:
          continue

        a = self._nodes.index(router)
        b = self._nodes.index(neighbor)

        self._links.add(tuple(sorted((a,b))))

    for line in lines:
      x = json.loads(line)

      if 'primary' in x:
        try:
          node = self.maybe_node_by_mac((x['primary'], ))
        except:
          continue

        node.id = x['primary']

  def import_aliases(self, aliases):
    for mac, alias in aliases.items():
      try:
        node = self.maybe_node_by_mac((mac, ))
      except:
        continue

      node.name = alias['name']
      if 'group' in alias:
        node.group = alias['group']

  # list of macs
  # if options['gateway']:
  #   mark_gateways(options['gateway'])
  def mark_gateways(self, gateways):
    for gateway in gateways:
      try:
        node = self.maybe_node_by_mac((gateway, ))
      except:
        continue

      node.group = 2

  def map_link(self, pair):
    distance = 80
    strength = 0.2
    if any(filter(lambda x: self._nodes[x].group == 3, pair)):
      distance = 10
      strength = 1

    link = Link()
    link.pair = pair
    link.distance = distance
    link.strength = strength
    return link

  def import_wikigps(self, url):
    def fetch_wikitable(url):
      f = urlopen(url)

      soup = BeautifulSoup(f)

      table = soup.find_all("table")[0]

      rows = table.find_all("tr")

      headers = []

      data = []

      def maybe_strip(x):
        if isinstance(x.string, basestring):
          return x.string.strip()
        else:
          return ""

      for row in rows:
        tds = list([maybe_strip(x) for x in row.find_all("td")])
        ths = list([maybe_strip(x) for x in row.find_all("th")])

        if any(tds):
          data.append(tds)

        if any(ths):
          headers = ths

      nodes = []

      for d in data:
        nodes.append(dict(zip(headers, d)))

      return nodes

    nodes = fetch_wikitable(url)

    for node in nodes:
      if not ('MAC' in node and 'GPS' in node):
        continue

      macs = [s for s in [s.strip() for s in node['MAC'].split(',')] if s]
      gps = [s for s in [s.strip() for s in node['GPS'].split(',')] if s]
      zipped = zip(macs, gps)

      if 'Nick' in node:
        names = [s for s in [s.strip() for s in node['Nick'].split(',')] if s]
        if names:
          zipped = zip(macs, gps, names)

      for pair in zipped:
        try:
          node = self.maybe_node_by_mac((pair[0], ))
        except:
          node = Node()
          node.add_mac(pair[0])
          self._nodes.append(node)

        if '2' in pair:
          if pair[2]:
            node.name = pair[2]

        node.gps = pair[1]

  def find_link(self, i):
    for link in self._links:
      if i in link:
        return link

  def wilder_scheiss(self):
    for node in self._nodes:
      if node.group == 3 and node.gps:
        i = self._nodes.index(node)
        link = self.find_link(i)
        if link:
          linklist = list(link)
          linklist.remove(i)
          j = linklist[0]

          for mac in node.macs:
            self._nodes[j].add_mac(mac)

          self._nodes[j].gps = node.gps

          node.gps = None

<commit_msg>bugfix<commit_after>import json
from node import Node
from link import Link

from bs4 import BeautifulSoup

# Python2/3 compatiblity hacks
try:
	from urllib.request import urlopen
except ImportError:
	from urllib import urlopen

try:
	basestring
except NameError:
	basestring = str

class NodeDB:
  def __init__(self):
    self._nodes = []
    self._links = set()

  # fetch list of links
  def get_links(self):
    return [self.map_link(x) for x in self._links]

  # fetch list of nodes
  def get_nodes(self):
    return self._nodes

  def maybe_node_by_mac(self, macs):
    for node in self._nodes:
      for mac in macs:
        if mac.lower() in node.macs:
          return node

    raise

  # import_batman(list(fileinput.input(options['batmanjson'])))
  def import_batman(self, lines):

    for line in lines:
      x = json.loads(line)

      if 'of' in x:
        try:
          node = self.maybe_node_by_mac((x['of'], x['secondary']))
        except:
          node = Node()
          node.online = True
          self._nodes.append(node)

        node.add_mac(x['of'])
        node.add_mac(x['secondary'])

    for line in lines:
      x = json.loads(line)

      if 'router' in x:
        try:
          node = self.maybe_node_by_mac((x['router'], ))
        except:
          node = Node()
          node.online = True
          node.add_mac(x['router'])
          self._nodes.append(node)

        try:
          if 'gateway' in x:
            x['neighbor'] = x['gateway']

          node = self.maybe_node_by_mac((x['neighbor'], ))
        except:
          node = Node()
          node.online = True
          if x['label'] == 'TT':
            node.group = 3

          node.add_mac(x['neighbor'])
          self._nodes.append(node)

    for line in lines:
      x = json.loads(line)

      if 'router' in x:
        try:
          if 'gateway' in x:
            x['neighbor'] = x['gateway']

          router = self.maybe_node_by_mac((x['router'], ))
          neighbor = self.maybe_node_by_mac((x['neighbor'], ))
        except:
          continue

        a = self._nodes.index(router)
        b = self._nodes.index(neighbor)

        self._links.add(tuple(sorted((a,b))))

    for line in lines:
      x = json.loads(line)

      if 'primary' in x:
        try:
          node = self.maybe_node_by_mac((x['primary'], ))
        except:
          continue

        node.id = x['primary']

  def import_aliases(self, aliases):
    for mac, alias in aliases.items():
      try:
        node = self.maybe_node_by_mac((mac, ))
      except:
        continue

      node.name = alias['name']
      if 'group' in alias:
        node.group = alias['group']

  # list of macs
  # if options['gateway']:
  #   mark_gateways(options['gateway'])
  def mark_gateways(self, gateways):
    for gateway in gateways:
      try:
        node = self.maybe_node_by_mac((gateway, ))
      except:
        continue

      node.group = 2

  def map_link(self, pair):
    distance = 80
    strength = 0.2
    if any(filter(lambda x: self._nodes[x].group == 3, pair)):
      distance = 10
      strength = 1

    link = Link()
    link.pair = pair
    link.distance = distance
    link.strength = strength
    return link

  def import_wikigps(self, url):
    def fetch_wikitable(url):
      f = urlopen(url)

      soup = BeautifulSoup(f)

      table = soup.find_all("table")[0]

      rows = table.find_all("tr")

      headers = []

      data = []

      def maybe_strip(x):
        if isinstance(x.string, basestring):
          return x.string.strip()
        else:
          return ""

      for row in rows:
        tds = list([maybe_strip(x) for x in row.find_all("td")])
        ths = list([maybe_strip(x) for x in row.find_all("th")])

        if any(tds):
          data.append(tds)

        if any(ths):
          headers = ths

      nodes = []

      for d in data:
        nodes.append(dict(zip(headers, d)))

      return nodes

    nodes = fetch_wikitable(url)

    for node in nodes:
      if not ('MAC' in node and 'GPS' in node):
        continue

      macs = [s for s in [s.strip() for s in node['MAC'].split(',')] if s]
      gps = [s for s in [s.strip() for s in node['GPS'].split(',')] if s]
      zipped = zip(macs, gps)

      if 'Nick' in node:
        names = [s for s in [s.strip() for s in node['Nick'].split(',')] if s]
        if names:
          zipped = zip(macs, gps, names)

      for pair in zipped:
        try:
          node = self.maybe_node_by_mac((pair[0], ))
        except:
          node = Node()
          node.add_mac(pair[0])
          self._nodes.append(node)

        if len(pair) > 2:
          if pair[2]:
            node.name = pair[2]

        node.gps = pair[1]

  def find_link(self, i):
    for link in self._links:
      if i in link:
        return link

  def wilder_scheiss(self):
    for node in self._nodes:
      if node.group == 3 and node.gps:
        i = self._nodes.index(node)
        link = self.find_link(i)
        if link:
          linklist = list(link)
          linklist.remove(i)
          j = linklist[0]

          for mac in node.macs:
            self._nodes[j].add_mac(mac)

          self._nodes[j].gps = node.gps

          node.gps = None

<|endoftext|>


<commit_before>"""
A collection of classes that write out the demand results files. The `cea.globalvar.GlobalVariables.demand_writer`
variable references the `DemandWriter` to use. The default is `HourlyDemandWriter`. A `MonthlyDemandWriter` is provided
that sums the values up monthly. See the `cea.analysis.sensitivity.sensitivity_demand` module for an example of using
the `MonthlyDemandWriter`.
"""

import pandas as pd
import h5py
import numpy as np

# index into the `vars_to_print` structure, that corresponds to `gv.demand_building_csv_columns`
FLOAT_FORMAT = '%.3f'

class DemandWriter(object):
    """
    This is meant to be an abstract base class: Use the subclasses of this class instead.

    Subclasses are expected to:

    - set the `gv` field to a `cea.globalvar.GlobalVariables` instance in the constructor
    - set the `vars_to_print` field in the constructor (FIXME: describe the `vars_to_print` structure.
    - implement the `write_to_csv` method
    """

    def __init__(self, loads, massflows, temperatures):
        if not loads:
            self.load_vars = ['QEf', 'QHf', 'QCf', 'Ef', 'Egenf_cs', 'Qhs_sen_shu', 'Qhs_sen_ahu', 'Qhs_lat_ahu',
                              'Qhs_sen_aru', 'Qhs_lat_aru', 'Qhs_sen_sys', 'Qhs_lat_sys', 'Qhs_em_ls', 'Qhs_dis_ls',
                              'Qhs', 'Qhsf', 'Qhsf_lat', 'Qwwf', 'Qww', 'Qhsf_shu', 'Qhsf_ahu', 'Qhsf_aru',
                              'Qcs', 'Qcdataf', 'Qcref', 'Qcs_sen_scu', 'Qcs_sen_ahu',
                              'Qcsf_scu', 'Qcsf_ahu', 'Qcsf_aru',
                              'Qcs_lat_ahu', 'Qcs_sen_aru', 'Qcs_lat_aru', 'Qcs_sen_sys', 'Qcs_lat_sys', 'Qcs_em_ls',
                              'Qcs_dis_ls', 'Qcsf', 'Qcs', 'Qcsf_lat', 'Qhprof', 'Edataf', 'Ealf', 'Eaf', 'Elf',
                              'Eref', 'Eauxf', 'Eauxf_ve', 'Eauxf_hs', 'Eauxf_cs', 'Eauxf_ww', 'Eauxf_fw',
                              'Eprof', 'Ecaf', 'Egenf_cs', 'Q_gain_sen_app', 'Q_gain_sen_light', 'Q_gain_sen_peop',
                              'Q_gain_sen_data',
                              'Q_loss_sen_ref', 'Q_gain_sen_env', 'Q_gain_sen_vent',
                              'Q_gain_sen_wind',
                              'I_sol', 'I_rad', 'Q_gain_lat_peop']
        else:
            self.load_vars = loads

        if not massflows:
            self.mass_flow_vars = ['mcpwwf', 'mcpdataf', 'mcpref', 'mcptw',
                                   'mcpcsf_ahu', 'mcpcsf_aru', 'mcpcsf_scu',
                                   'mcphsf_ahu', 'mcphsf_aru', 'mcphsf_shu']
        else:
            self.mass_flow_vars = massflows

        if not temperatures:
            self.temperature_vars = ['Twwf_sup', 'T_int', 'T_ext',
                                     'Twwf_re',
                                     'Tcdataf_re', 'Tcdataf_sup', 'Tcref_re',
                                     'Tcref_sup', 'Tcsf_re_ahu', 'Tcsf_re_aru', 'Tcsf_re_scu', 'Tcsf_sup_ahu',
                                     'Tcsf_sup_aru', 'Tcsf_sup_scu',
                                     'Thsf_re_ahu', 'Thsf_re_aru', 'Thsf_re_shu', 'Thsf_sup_ahu', 'Thsf_sup_aru',
                                     'Thsf_sup_shu', 'Thsf_sup', 'Tcsf_sup', 'Thsf_re', 'Tcsf_re']
        else:
            self.temperature_vars = temperatures

        self.OTHER_VARS = ['Name', 'Af_m2', 'Aroof_m2', 'GFA_m2', 'people0']

    def results_to_hdf5(self, tsd, bpr, locator, date, building_name):
        columns, hourly_data = self.calc_hourly_dataframe(building_name, date, tsd)
        self.write_to_hdf5(building_name, columns, hourly_data, locator)

        # save total for the year
        columns, data = self.calc_yearly_dataframe(bpr, building_name, tsd)
        # save to disc
        partial_total_data = pd.DataFrame(data, index=[0])
        partial_total_data.drop('Name', inplace=True, axis=1)
        partial_total_data.to_hdf(
            locator.get_temporary_file('%(building_name)sT.hdf' % locals()),
            key='dataset')

    def results_to_csv(self, tsd, bpr, locator, date, building_name):
        # save hourly data
        columns, hourly_data = self.calc_hourly_dataframe(building_name, date, tsd)
        self.write_to_csv(building_name, columns, hourly_data, locator)

        # save total for the year
        columns, data = self.calc_yearly_dataframe(bpr, building_name, tsd)
        # save to disc
        pd.DataFrame(data, index=[0]).to_csv(
            locator.get_temporary_file('%(building_name)sT.csv' % locals()),
            index=False, columns=columns, float_format='%.3f')

    def calc_yearly_dataframe(self, bpr, building_name, tsd):
        # if printing total values is necessary
        # treating timeseries data from W to MWh
        data = dict((x + '_MWhyr', tsd[x].sum() / 1000000) for x in self.load_vars)
        data.update(dict((x + '0_kW', tsd[x].max() / 1000) for x in self.load_vars))
        # get order of columns
        keys = data.keys()
        columns = self.OTHER_VARS
        columns.extend(keys)
        # add other default elements
        data.update({'Name': building_name, 'Af_m2': bpr.rc_model['Af'], 'Aroof_m2': bpr.rc_model['Aroof'],
                     'GFA_m2': bpr.rc_model['GFA_m2'], 'people0': tsd['people'].max()})
        return columns, data

    def calc_hourly_dataframe(self, building_name, date, tsd):
        # treating time series data of loads from W to kW
        data = dict((x + '_kWh', np.nan_to_num(tsd[x]) / 1000) for x in self.load_vars)  # TODO: convert nan to num at the very end.
        # treating time series data of mass_flows from W/C to kW/C
        data.update(dict((x + '_kWperC', np.nan_to_num(tsd[x]) / 1000) for x in self.mass_flow_vars))  # TODO: convert nan to num at the very end.
        # treating time series data of temperatures from W/C to kW/C
        data.update(dict((x + '_C', np.nan_to_num(tsd[x])) for x in self.temperature_vars))  # TODO: convert nan to num at the very end.
        # get order of columns
        columns = ['Name', 'people']
        columns.extend([x + '_kWh' for x in self.load_vars])
        columns.extend([x + '_kWperC' for x in self.mass_flow_vars])
        columns.extend([x + '_C' for x in self.temperature_vars])
        # add other default elements
        data.update({'DATE': date, 'Name': building_name, 'people': tsd['people']})
        # create dataframe with hourly values of selected data
        hourly_data = pd.DataFrame(data).set_index('DATE')
        return columns, hourly_data

class HourlyDemandWriter(DemandWriter):
    """Write out the hourly demand results"""

    def __init__(self, loads, massflows, temperatures):
        super(HourlyDemandWriter, self).__init__(loads, massflows, temperatures)

    def write_to_csv(self, building_name, columns, hourly_data, locator):
        hourly_data.to_csv(locator.get_demand_results_file(building_name, 'csv'), columns=columns,
                           float_format=FLOAT_FORMAT)

    def write_to_hdf5(self, building_name, columns, hourly_data, locator):
        # fixing columns with strings
        hourly_data.drop('Name', inplace=True, axis=1)
        hourly_data.to_hdf(locator.get_demand_results_file(building_name, 'hdf'), key='dataset')

class MonthlyDemandWriter(DemandWriter):
    """Write out the monthly demand results"""

    def __init__(self, loads, massflows, temperatures):
        super(MonthlyDemandWriter, self).__init__(loads, massflows, temperatures)
        self.MONTHS = ['january', 'february', 'march', 'april', 'may', 'june', 'july', 'august', 'september',
                       'october', 'november', 'december']

    def write_to_csv(self, building_name, columns, hourly_data, locator):
        # get monthly totals and rename to MWhyr
        monthly_data_new = self.calc_monthly_dataframe(building_name, hourly_data)
        monthly_data_new.to_csv(locator.get_demand_results_file(building_name, 'csv'), index=False,
                                float_format=FLOAT_FORMAT)

    def write_to_hdf5(self, building_name, columns, hourly_data, locator):
        # get monthly totals and rename to MWhyr
        monthly_data_new = self.calc_monthly_dataframe(building_name, hourly_data)
        monthly_data_new.to_hdf(locator.get_demand_results_file(building_name, 'hdf'), key=building_name)

    def calc_monthly_dataframe(self, building_name, hourly_data):
        monthly_data = hourly_data[[x + '_kWh' for x in self.load_vars]].groupby(
            by=[hourly_data.index.month]).sum() / 1000

        monthly_data = monthly_data.rename(
            columns=dict((x + '_kWh', x + '_MWhyr') for x in self.load_vars))

        peaks = hourly_data[[x + '_kWh' for x in self.load_vars]].groupby(
            by=[hourly_data.index.month]).max()

        peaks = peaks.rename(
            columns=dict((x + '_kWh', x + '0_kW') for x in self.load_vars))
        monthly_data_new = monthly_data.merge(peaks, left_index=True, right_index=True)
        monthly_data_new['Name'] = building_name
        monthly_data_new['Month'] = self.MONTHS

        return monthly_data_new

class YearlyDemandWriter(DemandWriter):
    """Write out the hourly demand results"""

    def __init__(self, loads, massflows, temperatures):
        super(YearlyDemandWriter, self).__init__(loads, massflows, temperatures)

    def write_to_csv(self, list_buildings, locator):
        """read in the temporary results files and append them to the Totals.csv file."""
        df = None
        for name in list_buildings:
            temporary_file = locator.get_temporary_file('%(name)sT.csv' % locals())
            if df is None:
                df = pd.read_csv(temporary_file)
            else:
                df = df.append(pd.read_csv(temporary_file), ignore_index=True)
        df.to_csv(locator.get_total_demand('csv'), index=False, float_format='%.3f')

        """read saved data of monthly values and return as totals"""
        monthly_data_buildings = [pd.read_csv(locator.get_demand_results_file(building_name, 'csv')) for building_name
                                  in
                                  list_buildings]
        return df, monthly_data_buildings

    def write_to_hdf5(self, list_buildings, locator):
        """read in the temporary results files and append them to the Totals.csv file."""
        df = None
        for name in list_buildings:
            temporary_file = locator.get_temporary_file('%(name)sT.hdf' % locals())
            if df is None:
                df = pd.read_hdf(temporary_file, key='dataset')
            else:
                df = df.append(pd.read_hdf(temporary_file, key='dataset'))
        df.to_hdf(locator.get_total_demand('hdf'), key='dataset')

        """read saved data of monthly values and return as totals"""
        monthly_data_buildings = [pd.read_hdf(locator.get_demand_results_file(building_name, 'hdf'), key=building_name)
                                  for building_name in
                                  list_buildings]
        return df, monthly_data_buildings
<commit_msg>reintroduce mass flows<commit_after>"""
A collection of classes that write out the demand results files. The `cea.globalvar.GlobalVariables.demand_writer`
variable references the `DemandWriter` to use. The default is `HourlyDemandWriter`. A `MonthlyDemandWriter` is provided
that sums the values up monthly. See the `cea.analysis.sensitivity.sensitivity_demand` module for an example of using
the `MonthlyDemandWriter`.
"""

import pandas as pd
import h5py
import numpy as np

# index into the `vars_to_print` structure, that corresponds to `gv.demand_building_csv_columns`
FLOAT_FORMAT = '%.3f'

class DemandWriter(object):
    """
    This is meant to be an abstract base class: Use the subclasses of this class instead.

    Subclasses are expected to:

    - set the `gv` field to a `cea.globalvar.GlobalVariables` instance in the constructor
    - set the `vars_to_print` field in the constructor (FIXME: describe the `vars_to_print` structure.
    - implement the `write_to_csv` method
    """

    def __init__(self, loads, massflows, temperatures):
        if not loads:
            self.load_vars = ['QEf', 'QHf', 'QCf', 'Ef', 'Egenf_cs', 'Qhs_sen_shu', 'Qhs_sen_ahu', 'Qhs_lat_ahu',
                              'Qhs_sen_aru', 'Qhs_lat_aru', 'Qhs_sen_sys', 'Qhs_lat_sys', 'Qhs_em_ls', 'Qhs_dis_ls',
                              'Qhs', 'Qhsf', 'Qhsf_lat', 'Qwwf', 'Qww', 'Qhsf_shu', 'Qhsf_ahu', 'Qhsf_aru',
                              'Qcs', 'Qcdataf', 'Qcref', 'Qcs_sen_scu', 'Qcs_sen_ahu',
                              'Qcsf_scu', 'Qcsf_ahu', 'Qcsf_aru',
                              'Qcs_lat_ahu', 'Qcs_sen_aru', 'Qcs_lat_aru', 'Qcs_sen_sys', 'Qcs_lat_sys', 'Qcs_em_ls',
                              'Qcs_dis_ls', 'Qcsf', 'Qcs', 'Qcsf_lat', 'Qhprof', 'Edataf', 'Ealf', 'Eaf', 'Elf',
                              'Eref', 'Eauxf', 'Eauxf_ve', 'Eauxf_hs', 'Eauxf_cs', 'Eauxf_ww', 'Eauxf_fw',
                              'Eprof', 'Ecaf', 'Egenf_cs', 'Q_gain_sen_app', 'Q_gain_sen_light', 'Q_gain_sen_peop',
                              'Q_gain_sen_data',
                              'Q_loss_sen_ref', 'Q_gain_sen_env', 'Q_gain_sen_vent',
                              'Q_gain_sen_wind',
                              'I_sol', 'I_rad', 'Q_gain_lat_peop']
        else:
            self.load_vars = loads

        if not massflows:
            self.mass_flow_vars = ['mcpwwf', 'mcpdataf', 'mcpref', 'mcptw',
                                   'mcpcsf_ahu', 'mcpcsf_aru', 'mcpcsf_scu',
                                   'mcphsf_ahu', 'mcphsf_aru', 'mcphsf_shu',
                                   'mcpcsf', 'mcphsf']
        else:
            self.mass_flow_vars = massflows

        if not temperatures:
            self.temperature_vars = ['Twwf_sup', 'T_int', 'T_ext',
                                     'Twwf_re',
                                     'Tcdataf_re', 'Tcdataf_sup', 'Tcref_re',
                                     'Tcref_sup', 'Tcsf_re_ahu', 'Tcsf_re_aru', 'Tcsf_re_scu', 'Tcsf_sup_ahu',
                                     'Tcsf_sup_aru', 'Tcsf_sup_scu',
                                     'Thsf_re_ahu', 'Thsf_re_aru', 'Thsf_re_shu', 'Thsf_sup_ahu', 'Thsf_sup_aru',
                                     'Thsf_sup_shu', 'Thsf_sup', 'Tcsf_sup', 'Thsf_re', 'Tcsf_re']
        else:
            self.temperature_vars = temperatures

        self.OTHER_VARS = ['Name', 'Af_m2', 'Aroof_m2', 'GFA_m2', 'people0']

    def results_to_hdf5(self, tsd, bpr, locator, date, building_name):
        columns, hourly_data = self.calc_hourly_dataframe(building_name, date, tsd)
        self.write_to_hdf5(building_name, columns, hourly_data, locator)

        # save total for the year
        columns, data = self.calc_yearly_dataframe(bpr, building_name, tsd)
        # save to disc
        partial_total_data = pd.DataFrame(data, index=[0])
        partial_total_data.drop('Name', inplace=True, axis=1)
        partial_total_data.to_hdf(
            locator.get_temporary_file('%(building_name)sT.hdf' % locals()),
            key='dataset')

    def results_to_csv(self, tsd, bpr, locator, date, building_name):
        # save hourly data
        columns, hourly_data = self.calc_hourly_dataframe(building_name, date, tsd)
        self.write_to_csv(building_name, columns, hourly_data, locator)

        # save total for the year
        columns, data = self.calc_yearly_dataframe(bpr, building_name, tsd)
        # save to disc
        pd.DataFrame(data, index=[0]).to_csv(
            locator.get_temporary_file('%(building_name)sT.csv' % locals()),
            index=False, columns=columns, float_format='%.3f')

    def calc_yearly_dataframe(self, bpr, building_name, tsd):
        # if printing total values is necessary
        # treating timeseries data from W to MWh
        data = dict((x + '_MWhyr', tsd[x].sum() / 1000000) for x in self.load_vars)
        data.update(dict((x + '0_kW', tsd[x].max() / 1000) for x in self.load_vars))
        # get order of columns
        keys = data.keys()
        columns = self.OTHER_VARS
        columns.extend(keys)
        # add other default elements
        data.update({'Name': building_name, 'Af_m2': bpr.rc_model['Af'], 'Aroof_m2': bpr.rc_model['Aroof'],
                     'GFA_m2': bpr.rc_model['GFA_m2'], 'people0': tsd['people'].max()})
        return columns, data

    def calc_hourly_dataframe(self, building_name, date, tsd):
        # treating time series data of loads from W to kW
        data = dict((x + '_kWh', np.nan_to_num(tsd[x]) / 1000) for x in self.load_vars)  # TODO: convert nan to num at the very end.
        # treating time series data of mass_flows from W/C to kW/C
        data.update(dict((x + '_kWperC', np.nan_to_num(tsd[x]) / 1000) for x in self.mass_flow_vars))  # TODO: convert nan to num at the very end.
        # treating time series data of temperatures from W/C to kW/C
        data.update(dict((x + '_C', np.nan_to_num(tsd[x])) for x in self.temperature_vars))  # TODO: convert nan to num at the very end.
        # get order of columns
        columns = ['Name', 'people']
        columns.extend([x + '_kWh' for x in self.load_vars])
        columns.extend([x + '_kWperC' for x in self.mass_flow_vars])
        columns.extend([x + '_C' for x in self.temperature_vars])
        # add other default elements
        data.update({'DATE': date, 'Name': building_name, 'people': tsd['people']})
        # create dataframe with hourly values of selected data
        hourly_data = pd.DataFrame(data).set_index('DATE')
        return columns, hourly_data

class HourlyDemandWriter(DemandWriter):
    """Write out the hourly demand results"""

    def __init__(self, loads, massflows, temperatures):
        super(HourlyDemandWriter, self).__init__(loads, massflows, temperatures)

    def write_to_csv(self, building_name, columns, hourly_data, locator):
        hourly_data.to_csv(locator.get_demand_results_file(building_name, 'csv'), columns=columns,
                           float_format=FLOAT_FORMAT)

    def write_to_hdf5(self, building_name, columns, hourly_data, locator):
        # fixing columns with strings
        hourly_data.drop('Name', inplace=True, axis=1)
        hourly_data.to_hdf(locator.get_demand_results_file(building_name, 'hdf'), key='dataset')

class MonthlyDemandWriter(DemandWriter):
    """Write out the monthly demand results"""

    def __init__(self, loads, massflows, temperatures):
        super(MonthlyDemandWriter, self).__init__(loads, massflows, temperatures)
        self.MONTHS = ['january', 'february', 'march', 'april', 'may', 'june', 'july', 'august', 'september',
                       'october', 'november', 'december']

    def write_to_csv(self, building_name, columns, hourly_data, locator):
        # get monthly totals and rename to MWhyr
        monthly_data_new = self.calc_monthly_dataframe(building_name, hourly_data)
        monthly_data_new.to_csv(locator.get_demand_results_file(building_name, 'csv'), index=False,
                                float_format=FLOAT_FORMAT)

    def write_to_hdf5(self, building_name, columns, hourly_data, locator):
        # get monthly totals and rename to MWhyr
        monthly_data_new = self.calc_monthly_dataframe(building_name, hourly_data)
        monthly_data_new.to_hdf(locator.get_demand_results_file(building_name, 'hdf'), key=building_name)

    def calc_monthly_dataframe(self, building_name, hourly_data):
        monthly_data = hourly_data[[x + '_kWh' for x in self.load_vars]].groupby(
            by=[hourly_data.index.month]).sum() / 1000

        monthly_data = monthly_data.rename(
            columns=dict((x + '_kWh', x + '_MWhyr') for x in self.load_vars))

        peaks = hourly_data[[x + '_kWh' for x in self.load_vars]].groupby(
            by=[hourly_data.index.month]).max()

        peaks = peaks.rename(
            columns=dict((x + '_kWh', x + '0_kW') for x in self.load_vars))
        monthly_data_new = monthly_data.merge(peaks, left_index=True, right_index=True)
        monthly_data_new['Name'] = building_name
        monthly_data_new['Month'] = self.MONTHS

        return monthly_data_new

class YearlyDemandWriter(DemandWriter):
    """Write out the hourly demand results"""

    def __init__(self, loads, massflows, temperatures):
        super(YearlyDemandWriter, self).__init__(loads, massflows, temperatures)

    def write_to_csv(self, list_buildings, locator):
        """read in the temporary results files and append them to the Totals.csv file."""
        df = None
        for name in list_buildings:
            temporary_file = locator.get_temporary_file('%(name)sT.csv' % locals())
            if df is None:
                df = pd.read_csv(temporary_file)
            else:
                df = df.append(pd.read_csv(temporary_file), ignore_index=True)
        df.to_csv(locator.get_total_demand('csv'), index=False, float_format='%.3f')

        """read saved data of monthly values and return as totals"""
        monthly_data_buildings = [pd.read_csv(locator.get_demand_results_file(building_name, 'csv')) for building_name
                                  in
                                  list_buildings]
        return df, monthly_data_buildings

    def write_to_hdf5(self, list_buildings, locator):
        """read in the temporary results files and append them to the Totals.csv file."""
        df = None
        for name in list_buildings:
            temporary_file = locator.get_temporary_file('%(name)sT.hdf' % locals())
            if df is None:
                df = pd.read_hdf(temporary_file, key='dataset')
            else:
                df = df.append(pd.read_hdf(temporary_file, key='dataset'))
        df.to_hdf(locator.get_total_demand('hdf'), key='dataset')

        """read saved data of monthly values and return as totals"""
        monthly_data_buildings = [pd.read_hdf(locator.get_demand_results_file(building_name, 'hdf'), key=building_name)
                                  for building_name in
                                  list_buildings]
        return df, monthly_data_buildings
<|endoftext|>


<commit_before>import errno
import socket
import os
import tempfile
import platform

def platform_information():
    """ detect platform information from remote host """
    distro, release, codename = platform.linux_distribution()
    return (
        str(distro).rstrip(),
        str(release).rstrip(),
        str(codename).rstrip()
    )

def machine_type():
    """ detect machine type """
    return platform.machine()

def write_sources_list(url, codename):
    """add ceph deb repo to sources.list"""
    with file('/etc/apt/sources.list.d/ceph.list', 'w') as f:
        f.write('deb {url} {codename} main\n'.format(
                url=url,
                codename=codename,
                ))

def write_conf(cluster, conf, overwrite):
    """ write cluster configuration to /etc/ceph/{cluster}.conf """
    path = '/etc/ceph/{cluster}.conf'.format(cluster=cluster)
    tmp_file = tempfile.NamedTemporaryFile()
    err_msg = 'config file %s exists with different content; use --overwrite-conf to overwrite' % path

    if os.path.exists(path):
        with file(path, 'rb') as f:
            old = f.read()
            if old != conf and not overwrite:
                raise RuntimeError(err_msg)
    tmp_file.write(conf)
    os.rename(tmp_file.name, path)

def create_mon_path(path):
    """create the mon path if it does not exist"""
    if not os.path.exists(path):
        os.makedirs(path)

def create_done_path(done_path):
    """create a done file to avoid re-doing the mon deployment"""
    with file(done_path, 'w'):
        pass

def create_init_path(init_path):
    """create the init path if it does not exist"""
    if not os.path.exists(init_path):
        with file(init_path, 'w'):
            pass

def path_exists(path):
    return os.path.exists(path)

def makedir(path):
    os.makedirs(path)

def unlink(_file):
    os.unlink(_file)

def write_monitor_keyring(keyring, monitor_keyring):
    """create the monitor keyring file"""
    with file(keyring, 'w') as f:
        f.write(monitor_keyring)

def shortname():
    """get remote short hostname"""
    return socket.gethostname().split('.', 1)[0]

def which_service():
    """ locating the `service` executable... """
    locations = ['/sbin/service', '/usr/sbin/service']
    for location in locations:
        if os.path.exists(location):
            return location

def make_mon_removed_dir(path, file_name):
    """ move old monitor data """
    try:
        os.makedirs('/var/lib/ceph/mon-removed')
    except OSError, e:
        if e.errno != errno.EEXIST:
            raise
    os.rename(path, os.path.join('/var/lib/ceph/mon-removed/', file_name))

# remoto magic, needed to execute these functions remotely
if __name__ == '__channelexec__':
    for item in channel:  # noqa
        channel.send(eval(item))  # noqa
<commit_msg>fix the early deletion of temp files<commit_after>import errno
import socket
import os
import tempfile
import platform

def platform_information():
    """ detect platform information from remote host """
    distro, release, codename = platform.linux_distribution()
    return (
        str(distro).rstrip(),
        str(release).rstrip(),
        str(codename).rstrip()
    )

def machine_type():
    """ detect machine type """
    return platform.machine()

def write_sources_list(url, codename):
    """add ceph deb repo to sources.list"""
    with file('/etc/apt/sources.list.d/ceph.list', 'w') as f:
        f.write('deb {url} {codename} main\n'.format(
                url=url,
                codename=codename,
                ))

def write_conf(cluster, conf, overwrite):
    """ write cluster configuration to /etc/ceph/{cluster}.conf """
    path = '/etc/ceph/{cluster}.conf'.format(cluster=cluster)
    tmp_file = tempfile.NamedTemporaryFile(delete=False)
    err_msg = 'config file %s exists with different content; use --overwrite-conf to overwrite' % path

    if os.path.exists(path):
        with file(path, 'rb') as f:
            old = f.read()
            if old != conf and not overwrite:
                raise RuntimeError(err_msg)
    tmp_file.write(conf)
    os.rename(tmp_file.name, path)

def create_mon_path(path):
    """create the mon path if it does not exist"""
    if not os.path.exists(path):
        os.makedirs(path)

def create_done_path(done_path):
    """create a done file to avoid re-doing the mon deployment"""
    with file(done_path, 'w'):
        pass

def create_init_path(init_path):
    """create the init path if it does not exist"""
    if not os.path.exists(init_path):
        with file(init_path, 'w'):
            pass

def path_exists(path):
    return os.path.exists(path)

def makedir(path):
    os.makedirs(path)

def unlink(_file):
    os.unlink(_file)

def write_monitor_keyring(keyring, monitor_keyring):
    """create the monitor keyring file"""
    with file(keyring, 'w') as f:
        f.write(monitor_keyring)

def shortname():
    """get remote short hostname"""
    return socket.gethostname().split('.', 1)[0]

def which_service():
    """ locating the `service` executable... """
    locations = ['/sbin/service', '/usr/sbin/service']
    for location in locations:
        if os.path.exists(location):
            return location

def make_mon_removed_dir(path, file_name):
    """ move old monitor data """
    try:
        os.makedirs('/var/lib/ceph/mon-removed')
    except OSError, e:
        if e.errno != errno.EEXIST:
            raise
    os.rename(path, os.path.join('/var/lib/ceph/mon-removed/', file_name))

# remoto magic, needed to execute these functions remotely
if __name__ == '__channelexec__':
    for item in channel:  # noqa
        channel.send(eval(item))  # noqa
<|endoftext|>