<commit_before>##############################################################################
# Copyright (c) 2016, Lawrence Livermore National Security, LLC.
# Produced at the Lawrence Livermore National Laboratory.
#
# This file is part of Spack.
# Written by Elizabeth Fischer
# LLNL-CODE-647188
#
# For details, see https://github.com/llnl/spack
# Please also see the LICENSE file for our notice and the LGPL.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License (as published by
# the Free Software Foundation) version 2.1 dated February 1999.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and
# conditions of the GNU General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with this program; if not, write to the Free Software Foundation,
# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
##############################################################################
import argparse
import os
import string
import sys

import llnl.util.tty as tty
import spack
import spack.cmd
from spack import which
from spack.cmd.edit import edit_package
from spack.stage import DIYStage
from llnl.util.filesystem import set_executable

description = "Create a configuration script and module, but don't build."

def setup_parser(subparser):
    subparser.add_argument(
        '-i', '--ignore-dependencies', action='store_true', dest='ignore_deps',
        help="Do not try to install dependencies of requested packages.")
    subparser.add_argument(
        '-v', '--verbose', action='store_true', dest='verbose',
        help="Display verbose build output while installing.")
    subparser.add_argument(
        'spec', nargs=argparse.REMAINDER,
        help="specs to use for install.  Must contain package AND version.")
    subparser.add_argument(
        '--dirty', action='store_true', dest='dirty',
        help="Install a package *without* cleaning the environment.")

def spack_transitive_include_path():
    return ';'.join(
        os.path.join(dep, 'include')
        for dep in os.environ['SPACK_DEPENDENCIES'].split(os.pathsep)
    )

def write_spconfig(package):
    # Set-up the environment
    spack.build_environment.setup_package(package)

    cmd = [str(which('cmake'))] + package.std_cmake_args + package.cmake_args()

    env = dict()

    paths = os.environ['PATH'].split(':')
    paths = [item for item in paths if 'spack/env' not in item]
    env['PATH'] = ':'.join(paths)
    env['SPACK_TRANSITIVE_INCLUDE_PATH'] = spack_transitive_include_path()
    env['CMAKE_PREFIX_PATH'] = os.environ['CMAKE_PREFIX_PATH']
    env['CC'] = os.environ['SPACK_CC']
    env['CXX'] = os.environ['SPACK_CXX']
    env['FC'] = os.environ['SPACK_FC']

    setup_fname = 'spconfig.py'
    with open(setup_fname, 'w') as fout:
        fout.write(
            r"""#!%s
#

import sys
import os
import subprocess

def cmdlist(str):
    return list(x.strip().replace("'",'') for x in str.split('\n') if x)
env = dict(os.environ)
""" % sys.executable)

        env_vars = sorted(list(env.keys()))
        for name in env_vars:
            val = env[name]
            if string.find(name, 'PATH') < 0:
                fout.write('env[%s] = %s\n' % (repr(name), repr(val)))
            else:
                if name == 'SPACK_TRANSITIVE_INCLUDE_PATH':
                    sep = ';'
                else:
                    sep = ':'

                fout.write(
                    'env[%s] = "%s".join(cmdlist("""\n' % (repr(name), sep))
                for part in string.split(val, sep):
                    fout.write('    %s\n' % part)
                fout.write('"""))\n')

        fout.write("env['CMAKE_TRANSITIVE_INCLUDE_PATH'] = env['SPACK_TRANSITIVE_INCLUDE_PATH']   # Deprecated\n")  # NOQA: ignore=E501
        fout.write('\ncmd = cmdlist("""\n')
        fout.write('%s\n' % cmd[0])
        for arg in cmd[1:]:
            fout.write('    %s\n' % arg)
        fout.write('""") + sys.argv[1:]\n')
        fout.write('\nproc = subprocess.Popen(cmd, env=env)\nproc.wait()\n')
        set_executable(setup_fname)

def setup(self, args):
    if not args.spec:
        tty.die("spack setup requires a package spec argument.")

    specs = spack.cmd.parse_specs(args.spec)
    if len(specs) > 1:
        tty.die("spack setup only takes one spec.")

    # Take a write lock before checking for existence.
    with spack.installed_db.write_transaction():
        spec = specs[0]
        if not spack.repo.exists(spec.name):
            tty.warn("No such package: %s" % spec.name)
            create = tty.get_yes_or_no("Create this package?", default=False)
            if not create:
                tty.msg("Exiting without creating.")
                sys.exit(1)
            else:
                tty.msg("Running 'spack edit -f %s'" % spec.name)
                edit_package(spec.name, spack.repo.first_repo(), None, True)
                return

        if not spec.versions.concrete:
            tty.die(
                "spack setup spec must have a single, concrete version. "
                "Did you forget a package version number?")

        spec.concretize()
        package = spack.repo.get(spec)

        # It's OK if the package is already installed.

        # Forces the build to run out of the current directory.
        package.stage = DIYStage(os.getcwd())

        # TODO: make this an argument, not a global.
        spack.do_checksum = False

        if not isinstance(package, spack.CMakePackage):
            raise RuntimeError(
                'Support for {0} not yet implemented'.format(type(package)))

        write_spconfig(package)
<commit_msg>spack setup : improved error message<commit_after>##############################################################################
# Copyright (c) 2016, Lawrence Livermore National Security, LLC.
# Produced at the Lawrence Livermore National Laboratory.
#
# This file is part of Spack.
# Written by Elizabeth Fischer
# LLNL-CODE-647188
#
# For details, see https://github.com/llnl/spack
# Please also see the LICENSE file for our notice and the LGPL.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License (as published by
# the Free Software Foundation) version 2.1 dated February 1999.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and
# conditions of the GNU General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with this program; if not, write to the Free Software Foundation,
# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
##############################################################################
import argparse
import os
import string
import sys

import llnl.util.tty as tty
import spack
import spack.cmd
from spack import which
from spack.cmd.edit import edit_package
from spack.stage import DIYStage
from llnl.util.filesystem import set_executable

description = "Create a configuration script and module, but don't build."

def setup_parser(subparser):
    subparser.add_argument(
        '-i', '--ignore-dependencies', action='store_true', dest='ignore_deps',
        help="Do not try to install dependencies of requested packages.")
    subparser.add_argument(
        '-v', '--verbose', action='store_true', dest='verbose',
        help="Display verbose build output while installing.")
    subparser.add_argument(
        'spec', nargs=argparse.REMAINDER,
        help="specs to use for install.  Must contain package AND version.")
    subparser.add_argument(
        '--dirty', action='store_true', dest='dirty',
        help="Install a package *without* cleaning the environment.")

def spack_transitive_include_path():
    return ';'.join(
        os.path.join(dep, 'include')
        for dep in os.environ['SPACK_DEPENDENCIES'].split(os.pathsep)
    )

def write_spconfig(package):
    # Set-up the environment
    spack.build_environment.setup_package(package)

    cmd = [str(which('cmake'))] + package.std_cmake_args + package.cmake_args()

    env = dict()

    paths = os.environ['PATH'].split(':')
    paths = [item for item in paths if 'spack/env' not in item]
    env['PATH'] = ':'.join(paths)
    env['SPACK_TRANSITIVE_INCLUDE_PATH'] = spack_transitive_include_path()
    env['CMAKE_PREFIX_PATH'] = os.environ['CMAKE_PREFIX_PATH']
    env['CC'] = os.environ['SPACK_CC']
    env['CXX'] = os.environ['SPACK_CXX']
    env['FC'] = os.environ['SPACK_FC']

    setup_fname = 'spconfig.py'
    with open(setup_fname, 'w') as fout:
        fout.write(
            r"""#!%s
#

import sys
import os
import subprocess

def cmdlist(str):
    return list(x.strip().replace("'",'') for x in str.split('\n') if x)
env = dict(os.environ)
""" % sys.executable)

        env_vars = sorted(list(env.keys()))
        for name in env_vars:
            val = env[name]
            if string.find(name, 'PATH') < 0:
                fout.write('env[%s] = %s\n' % (repr(name), repr(val)))
            else:
                if name == 'SPACK_TRANSITIVE_INCLUDE_PATH':
                    sep = ';'
                else:
                    sep = ':'

                fout.write(
                    'env[%s] = "%s".join(cmdlist("""\n' % (repr(name), sep))
                for part in string.split(val, sep):
                    fout.write('    %s\n' % part)
                fout.write('"""))\n')

        fout.write("env['CMAKE_TRANSITIVE_INCLUDE_PATH'] = env['SPACK_TRANSITIVE_INCLUDE_PATH']   # Deprecated\n")  # NOQA: ignore=E501
        fout.write('\ncmd = cmdlist("""\n')
        fout.write('%s\n' % cmd[0])
        for arg in cmd[1:]:
            fout.write('    %s\n' % arg)
        fout.write('""") + sys.argv[1:]\n')
        fout.write('\nproc = subprocess.Popen(cmd, env=env)\nproc.wait()\n')
        set_executable(setup_fname)

def setup(self, args):
    if not args.spec:
        tty.die("spack setup requires a package spec argument.")

    specs = spack.cmd.parse_specs(args.spec)
    if len(specs) > 1:
        tty.die("spack setup only takes one spec.")

    # Take a write lock before checking for existence.
    with spack.installed_db.write_transaction():
        spec = specs[0]
        if not spack.repo.exists(spec.name):
            tty.warn("No such package: %s" % spec.name)
            create = tty.get_yes_or_no("Create this package?", default=False)
            if not create:
                tty.msg("Exiting without creating.")
                sys.exit(1)
            else:
                tty.msg("Running 'spack edit -f %s'" % spec.name)
                edit_package(spec.name, spack.repo.first_repo(), None, True)
                return

        if not spec.versions.concrete:
            tty.die(
                "spack setup spec must have a single, concrete version. "
                "Did you forget a package version number?")

        spec.concretize()
        package = spack.repo.get(spec)
        if not isinstance(package, spack.CMakePackage):
            tty.die(
                'Support for {0} derived packages not yet implemented'.format(
                    package.build_system_class
                )
            )

        # It's OK if the package is already installed.

        # Forces the build to run out of the current directory.
        package.stage = DIYStage(os.getcwd())

        # TODO: make this an argument, not a global.
        spack.do_checksum = False

        write_spconfig(package)
<|endoftext|>


<commit_before>
from __future__ import (
    unicode_literals,
    absolute_import,
    print_function,
    division,
    )
from io import BytesIO
import io
import traceback
import os
from uuid import UUID, uuid4
import sys
import datetime

from .utils import (
    read_u8,
    read_u16le,
    read_u32le,
    write_u8,
    write_u16le,
    write_u32le,
    )

from .cfb import (CompoundFileBinary, DirEntry)
from .core import AAFObject
from .properties import StrongRefProperty, StrongRefArrayProperty, StrongRefSetProperty
from .metadict import MetaDictionary

class AAFFactory(object):

    def __init__(self, root):
        self.root = root
        self.metadict = root.metadict
        self.class_name = None

    def __getattr__(self, name):
        self.class_name = name
        return self.create_instance

    def from_name(self, name, *args, **kwargs):

        classdef = self.metadict.lookup_classdef(name)
        if classdef is None:
            raise ValueError("no class found with name: %s" % name)

        classobj = self.metadict.lookup_class(name)

        obj = classobj(self.root, *args, **kwargs)
        if isinstance(obj, AAFObject):
            obj.class_id = classdef.uuid

        return obj

    def create_instance(self, *args, **kwargs):
        return self.from_name(self.class_name, *args, **kwargs)

class AAFFile(object):

    def __init__(self, path=None, mode='r'):

        if mode in ('r', 'rb'):
            mode = 'rb'
        elif mode in ('r+', 'rb+'):
            mode = 'rb+'
        elif mode in ('w', 'w+', 'wb+'):
            mode = 'wb+'
        else:
            raise ValueError("invalid mode: %s" % mode)
        self.mode = mode
        if path is None:
            self.mode = 'wb+'
            self.f = BytesIO()
        else:
            self.f = io.open(path, mode)

        self.cfb = CompoundFileBinary(self.f, self.mode)
        self.weakref_table = []
        self.path_cache = {}
        self.metadict = MetaDictionary(self)
        self.create = AAFFactory(self)

        if self.mode in ("rb", "rb+"):
            self.metadict.dir = self.cfb.find('/MetaDictionary-1')
            self.path_cache['/MetaDictionary-1'] = self.metadict
            self.root = self.read_object("/")
            self.read_reference_properties()
            self.metadict.read_properties()
            header_pid = 0x02
            self.header = self.root.property_entries[header_pid].value
            self.storage = self.header['Content'].value

        elif self.mode in ("wb+",):
            self.setup_empty()

    def setup_empty(self):
        now = datetime.datetime.now()
        self.root = self.create.Root()
        self.root.attach(self.cfb.find("/"))
        self.root['MetaDictionary'].value = self.metadict
        self.metadict.setup_defaults()

        self.header = self.create.Header()
        self.root['Header'].value = self.header
        self.storage = self.create.ContentStorage()
        d = self.create.Dictionary()
        self.header['Dictionary'].value = d
        d.setup_defaults()
        self.header['Content'].value = self.storage
        self.header['OperationalPattern'].value = UUID("0d011201-0100-0000-060e-2b3404010105")
        self.header['ObjectModelVersion'].value = 1
        self.header['Version'].value =  {u'major': 1, u'minor': 1}

        i = self.create.Identification()
        i['ProductName'].value = "PyAAF"
        i['CompanyName'].value = "CompanyName"
        i['ProductVersionString'].value = '2.0.0'
        i['ProductID'].value = UUID("97e04c67-dbe6-4d11-bcd7-3a3a4253a2ef")
        i['Date'].value = now
        i['Platform'].value = sys.platform
        i['GenerationAUID'].value = uuid4()

        self.header['IdentificationList'].value = [i]
        self.header['LastModified'].value = now
        self.header['ByteOrder'].value = 0x4949

        self.storage['Mobs'].value = []

    def read_object(self, path):
        if isinstance(path, DirEntry):
            dir_entry = path
            path = dir_entry.path()
            if path in self.path_cache:
                return self.path_cache[path]
        else:
            if path in self.path_cache:
                return self.path_cache[path]

            dir_entry = self.cfb.find(path)

        if dir_entry is None:
            raise ValueError("cannot find path: %s" % path)

        obj_class = self.metadict.lookup_class(dir_entry.class_id)

        obj = obj_class(self)
        obj.dir = dir_entry
        if obj_class is AAFObject:
            obj.class_id = dir_entry.class_id
        obj.read_properties()

        self.path_cache[path] = obj
        # obj.dump()
        return obj

    def resovle_weakref(self, index, ref_pid, ref):
        # print(index, ref_pid, ref)
        parent, p = self.weakref_prop(index)
        v = p.value.get(ref)
        return v

    def weakref_prop(self, index):
        path = self.weakref_table[index]
        root = self.root

        for pid in path[:-1]:
            p = root.property_entries[pid]
            root = p.value

        p = root.property_entries[path[-1]]
        return root, p

    def create_weakref(self, obj, pid_path):

        if pid_path in self.weakref_table:
            index = self.weakref_table.index(pid_path)
        else:
            index = len(self.weakref_table)
            self.weakref_table.append(pid_path)

        parent_obj, p = self.weakref_prop(index)
        assert isinstance(p, StrongRefSetProperty)
        weakref_pid, key_size = self.metadict.weakref_pid(parent_obj.classdef, p.propertydef)

        ref_key = obj.unique_key

        return index, weakref_pid, ref_key

    def read_reference_properties(self):
        f = self.cfb.open("/referenced properties")

        byte_order = read_u8(f)
        if byte_order != 0x4c:
            raise NotImplementedError("be byteorder")

        path_count = read_u16le(f)
        pid_count = read_u32le(f)

        self.weakref_table = []
        path = []
        for i in range(pid_count):
            pid = read_u16le(f)
            if pid != 0:
                path.append(pid)
            else:
                self.weakref_table.append(path)
                path = []
        assert len(self.weakref_table) == path_count

    def write_reference_properties(self):
        f = self.cfb.open("/referenced properties", 'w')
        byte_order = 0x4c
        path_count = len(self.weakref_table)
        pid_count = 0
        for path in self.weakref_table:
            pid_count += len(path)
            pid_count += 1 # null byte

        write_u8(f, byte_order)
        write_u16le(f, path_count)
        write_u32le(f, pid_count)
        for path in self.weakref_table:
            for pid in path:
                write_u16le(f, pid)
            write_u16le(f, 0) # null terminated

    def __exit__(self, exc_type, exc_value, traceback):
        self.save()

    def __enter__(self):
        return self

    def dump(self):
        self.root.dump()

    def save(self, path = None):

        if path:
            mode = 'wb+'
            f = io.open(path, mode)
            cfb = CompoundFileBinary(f, mode)

            self.root = self.root.copy(cfb.root)
            self.cfb = cfb
            self.f = f
            self.mode = mode
            self.path_cache = {}

            metadict_pid = 0x01
            metadict = self.metadict
            metadict.detach()
            self.root.property_entries[metadict_pid].value = metadict
            self.path_cache['/MetaDictionary-1'] = self.metadict

            header_pid = 0x02
            self.header = self.root.property_entries[header_pid].value
            self.storage = self.header['Content'].value

            for item, streams in self.root.walk_references():
                self.path_cache[item.dir.path()] = item

        if self.mode in ("wb+", 'rb+'):
            self.write_reference_properties()
            for path, obj in self.path_cache.items():
                obj.write_properties()

        self.cfb.close()
<commit_msg>replace header, content and dictionary with property<commit_after>
from __future__ import (
    unicode_literals,
    absolute_import,
    print_function,
    division,
    )
from io import BytesIO
import io
import traceback
import os
from uuid import UUID, uuid4
import sys
import datetime

from .utils import (
    read_u8,
    read_u16le,
    read_u32le,
    write_u8,
    write_u16le,
    write_u32le,
    )

from .cfb import (CompoundFileBinary, DirEntry)
from .core import AAFObject
from .properties import StrongRefProperty, StrongRefArrayProperty, StrongRefSetProperty
from .metadict import MetaDictionary

class AAFFactory(object):

    def __init__(self, root):
        self.root = root
        self.metadict = root.metadict
        self.class_name = None

    def __getattr__(self, name):
        self.class_name = name
        return self.create_instance

    def from_name(self, name, *args, **kwargs):

        classdef = self.metadict.lookup_classdef(name)
        if classdef is None:
            raise ValueError("no class found with name: %s" % name)

        classobj = self.metadict.lookup_class(name)

        obj = classobj(self.root, *args, **kwargs)
        if isinstance(obj, AAFObject):
            obj.class_id = classdef.uuid

        return obj

    def create_instance(self, *args, **kwargs):
        return self.from_name(self.class_name, *args, **kwargs)

class AAFFile(object):

    def __init__(self, path=None, mode='r'):

        if mode in ('r', 'rb'):
            mode = 'rb'
        elif mode in ('r+', 'rb+'):
            mode = 'rb+'
        elif mode in ('w', 'w+', 'wb+'):
            mode = 'wb+'
        else:
            raise ValueError("invalid mode: %s" % mode)
        self.mode = mode
        if path is None:
            self.mode = 'wb+'
            self.f = BytesIO()
        else:
            self.f = io.open(path, mode)

        self.cfb = CompoundFileBinary(self.f, self.mode)
        self.weakref_table = []
        self.path_cache = {}
        self.metadict = MetaDictionary(self)
        self.create = AAFFactory(self)

        if self.mode in ("rb", "rb+"):
            self.metadict.dir = self.cfb.find('/MetaDictionary-1')
            self.path_cache['/MetaDictionary-1'] = self.metadict
            self.root = self.read_object("/")
            self.read_reference_properties()
            self.metadict.read_properties()

        elif self.mode in ("wb+",):
            self.setup_empty()

    @property
    def header(self):
        header_pid = 0x02
        return self.root.property_entries[header_pid].value

    @property
    def content(self):
        return self.header['Content'].value

    @property
    def dictionary(self):
        return self.header['Dictionary'].value

    def setup_empty(self):
        now = datetime.datetime.now()
        self.root = self.create.Root()
        self.root.attach(self.cfb.find("/"))
        self.root['MetaDictionary'].value = self.metadict
        self.metadict.setup_defaults()
        self.root['Header'].value = self.create.Header()

        self.header['Dictionary'].value = self.create.Dictionary()
        self.dictionary.setup_defaults()

        self.header['Content'].value = self.create.ContentStorage()
        self.header['OperationalPattern'].value = UUID("0d011201-0100-0000-060e-2b3404010105")
        self.header['ObjectModelVersion'].value = 1
        self.header['Version'].value =  {u'major': 1, u'minor': 1}

        i = self.create.Identification()
        i['ProductName'].value = "PyAAF"
        i['CompanyName'].value = "CompanyName"
        i['ProductVersionString'].value = '2.0.0'
        i['ProductID'].value = UUID("97e04c67-dbe6-4d11-bcd7-3a3a4253a2ef")
        i['Date'].value = now
        i['Platform'].value = sys.platform
        i['GenerationAUID'].value = uuid4()

        self.header['IdentificationList'].value = [i]
        self.header['LastModified'].value = now
        self.header['ByteOrder'].value = 0x4949

        self.storage['Mobs'].value = []

    def read_object(self, path):
        if isinstance(path, DirEntry):
            dir_entry = path
            path = dir_entry.path()
            if path in self.path_cache:
                return self.path_cache[path]
        else:
            if path in self.path_cache:
                return self.path_cache[path]

            dir_entry = self.cfb.find(path)

        if dir_entry is None:
            raise ValueError("cannot find path: %s" % path)

        obj_class = self.metadict.lookup_class(dir_entry.class_id)

        obj = obj_class(self)
        obj.dir = dir_entry
        if obj_class is AAFObject:
            obj.class_id = dir_entry.class_id
        obj.read_properties()

        self.path_cache[path] = obj
        # obj.dump()
        return obj

    def resovle_weakref(self, index, ref_pid, ref):
        # print(index, ref_pid, ref)
        parent, p = self.weakref_prop(index)
        v = p.value.get(ref)
        return v

    def weakref_prop(self, index):
        path = self.weakref_table[index]
        root = self.root

        for pid in path[:-1]:
            p = root.property_entries[pid]
            root = p.value

        p = root.property_entries[path[-1]]
        return root, p

    def create_weakref(self, obj, pid_path):

        if pid_path in self.weakref_table:
            index = self.weakref_table.index(pid_path)
        else:
            index = len(self.weakref_table)
            self.weakref_table.append(pid_path)

        parent_obj, p = self.weakref_prop(index)
        assert isinstance(p, StrongRefSetProperty)
        weakref_pid, key_size = self.metadict.weakref_pid(parent_obj.classdef, p.propertydef)

        ref_key = obj.unique_key

        return index, weakref_pid, ref_key

    def read_reference_properties(self):
        f = self.cfb.open("/referenced properties")

        byte_order = read_u8(f)
        if byte_order != 0x4c:
            raise NotImplementedError("be byteorder")

        path_count = read_u16le(f)
        pid_count = read_u32le(f)

        self.weakref_table = []
        path = []
        for i in range(pid_count):
            pid = read_u16le(f)
            if pid != 0:
                path.append(pid)
            else:
                self.weakref_table.append(path)
                path = []
        assert len(self.weakref_table) == path_count

    def write_reference_properties(self):
        f = self.cfb.open("/referenced properties", 'w')
        byte_order = 0x4c
        path_count = len(self.weakref_table)
        pid_count = 0
        for path in self.weakref_table:
            pid_count += len(path)
            pid_count += 1 # null byte

        write_u8(f, byte_order)
        write_u16le(f, path_count)
        write_u32le(f, pid_count)
        for path in self.weakref_table:
            for pid in path:
                write_u16le(f, pid)
            write_u16le(f, 0) # null terminated

    def __exit__(self, exc_type, exc_value, traceback):
        self.save()

    def __enter__(self):
        return self

    def dump(self):
        self.root.dump()

    def save(self, path = None):

        if path:
            mode = 'wb+'
            f = io.open(path, mode)
            cfb = CompoundFileBinary(f, mode)

            self.root = self.root.copy(cfb.root)
            self.cfb = cfb
            self.f = f
            self.mode = mode
            self.path_cache = {}

            metadict_pid = 0x01
            metadict = self.metadict
            metadict.detach()
            self.root.property_entries[metadict_pid].value = metadict
            self.path_cache['/MetaDictionary-1'] = self.metadict

            for item, streams in self.root.walk_references():
                self.path_cache[item.dir.path()] = item

        if self.mode in ("wb+", 'rb+'):
            self.write_reference_properties()
            for path, obj in self.path_cache.items():
                obj.write_properties()

        self.cfb.close()
<|endoftext|>


<commit_before>from unittest import TestCase

from mock import patch

from unicore.google.tasks import pageview

class TestTasks(TestTasks):

    @patch('urllib2.urlopen')
    def test_pageview(self, mock_request):
        mock_request.return_value = True
        pageview('profile_id', 'client_id', {
            'path': 'foo',
            'user_agent': 'user agent',
            'uip': 'the-ip',
            'dr': 'referrer',
            'dh': 'domain',
        })
        mock_request.assert_called_with('foo')
<commit_msg>fix typo<commit_after>from unittest import TestCase

from mock import patch

from unicore.google.tasks import pageview

class TestTasks(TestCase):

    @patch('urllib2.urlopen')
    def test_pageview(self, mock_request):
        mock_request.return_value = True
        pageview('profile_id', 'client_id', {
            'path': 'foo',
            'user_agent': 'user agent',
            'uip': 'the-ip',
            'dr': 'referrer',
            'dh': 'domain',
        })
        mock_request.assert_called_with('foo')
<|endoftext|>


<commit_before># coding: utf8
"""
Allows setting/changing/removing of chosen url query string parameters,
while maintaining any existing others.

Expects the current request to be available in the context as ``request``.

Examples:

    {% set_url_param page=next_page %}
    {% set_url_param page="" %}
    {% set_url_param filter="books" page=1 %}
"""

import urllib
import tokenize
import StringIO
from django import template

import six

register = template.Library()

class SetUrlParamNode(template.Node):
    def __init__(self, changes):
        self.changes = changes

    def render(self, context):
        request = context.get('request', None)
        if not request:
            return ""

        # Note that we want params to **not** be a ``QueryDict`` (thus we
        # don't use it's ``copy()`` method), as it would force all values
        # to be unicode, and ``urllib.urlencode`` can't handle that.
        params = dict(request.GET)
        for key, newvalue in self.changes.items():
            newvalue = newvalue.resolve(context)
            if newvalue == '' or newvalue is None:
                params.pop(key, False)
            else:
                params[key] = unicode(newvalue)
        # ``urlencode`` chokes on unicode input, so convert everything to
        # utf8. Note that if some query arguments passed to the site have
        # their non-ascii characters screwed up when passed though this,
        # it's most likely not our fault. Django (the ``QueryDict`` class
        # to be exact) uses your projects DEFAULT_CHARSET to decode incoming
        # query strings, whereas your browser might encode the url
        # differently. For example, typing "ä" in my German Firefox's (v2)
        # address bar results in "%E4" being passed to the server (in
        # iso-8859-1), but Django might expect utf-8, where ä would be
        # "%C3%A4"

        def mkstr(s):
            if isinstance(s, list):
                return map(mkstr, s)
            else:
                return (isinstance(s, unicode) and [s.encode('utf-8')] or [s])[0]  # noqa
        params = dict([(mkstr(k), mkstr(v)) for k, v in params.items()])
        # done, return (string is already safe)
        return '?'+urllib.urlencode(params, doseq=True)

def do_seturlparam(parser, token):
    bits = token.contents.split()
    qschanges = {}
    for i in bits[1:]:
        try:
            a, b = i.split('=', 1)
            a = a.strip()
            b = b.strip()
            keys = list(
                tokenize.generate_tokens(StringIO.StringIO(a).readline),
            )
            if keys[0][0] == tokenize.NAME:
                if b == '""':
                    b = template.Variable('""')  # workaround bug #5270
                else:
                    b = parser.compile_filter(b)
                qschanges[str(a)] = b
            else:
                raise ValueError
        except ValueError:
            six.raise_from(
                template.TemplateSyntaxError,
                "Argument syntax wrong: should be key=value",
            )
    return SetUrlParamNode(qschanges)

register.tag('set_url_param', do_seturlparam)
<commit_msg>refs #4: StringIO is gone in python3<commit_after># coding: utf8
"""
Allows setting/changing/removing of chosen url query string parameters,
while maintaining any existing others.

Expects the current request to be available in the context as ``request``.

Examples:

    {% set_url_param page=next_page %}
    {% set_url_param page="" %}
    {% set_url_param filter="books" page=1 %}
"""

import urllib
import tokenize
from django import template

import six

register = template.Library()

class SetUrlParamNode(template.Node):
    def __init__(self, changes):
        self.changes = changes

    def render(self, context):
        request = context.get('request', None)
        if not request:
            return ""

        # Note that we want params to **not** be a ``QueryDict`` (thus we
        # don't use it's ``copy()`` method), as it would force all values
        # to be unicode, and ``urllib.urlencode`` can't handle that.
        params = dict(request.GET)
        for key, newvalue in self.changes.items():
            newvalue = newvalue.resolve(context)
            if newvalue == '' or newvalue is None:
                params.pop(key, False)
            else:
                params[key] = unicode(newvalue)
        # ``urlencode`` chokes on unicode input, so convert everything to
        # utf8. Note that if some query arguments passed to the site have
        # their non-ascii characters screwed up when passed though this,
        # it's most likely not our fault. Django (the ``QueryDict`` class
        # to be exact) uses your projects DEFAULT_CHARSET to decode incoming
        # query strings, whereas your browser might encode the url
        # differently. For example, typing "ä" in my German Firefox's (v2)
        # address bar results in "%E4" being passed to the server (in
        # iso-8859-1), but Django might expect utf-8, where ä would be
        # "%C3%A4"

        def mkstr(s):
            if isinstance(s, list):
                return map(mkstr, s)
            else:
                return (isinstance(s, unicode) and [s.encode('utf-8')] or [s])[0]  # noqa
        params = dict([(mkstr(k), mkstr(v)) for k, v in params.items()])
        # done, return (string is already safe)
        return '?'+urllib.urlencode(params, doseq=True)

def do_seturlparam(parser, token):
    bits = token.contents.split()
    qschanges = {}
    for i in bits[1:]:
        try:
            a, b = i.split('=', 1)
            a = a.strip()
            b = b.strip()
            keys = list(
                tokenize.generate_tokens(six.StringIO(a).readline),
            )
            if keys[0][0] == tokenize.NAME:
                if b == '""':
                    b = template.Variable('""')  # workaround bug #5270
                else:
                    b = parser.compile_filter(b)
                qschanges[str(a)] = b
            else:
                raise ValueError
        except ValueError:
            six.raise_from(
                template.TemplateSyntaxError,
                "Argument syntax wrong: should be key=value",
            )
    return SetUrlParamNode(qschanges)

register.tag('set_url_param', do_seturlparam)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from rest_framework.compat import six
from rest_framework.exceptions import ParseError
from rest_framework.parsers import JSONParser
from django.conf import settings
import re
import json

first_cap_re = re.compile('(.)([A-Z][a-z]+)')
all_cap_re = re.compile('([a-z0-9])([A-Z])')

def camel_to_underscore(name):
    s1 = first_cap_re.sub(r'\1_\2', name)
    return all_cap_re.sub(r'\1_\2', s1).lower()

def underscoreize(data):
    if isinstance(data, dict):
        new_dict = {}
        for key, value in data.items():
            new_key = camel_to_underscore(key)
            new_dict[new_key] = underscoreize(value)
        return new_dict
    if isinstance(data, (list, tuple)):
        for i in range(len(data)):
            data[i] = underscoreize(data[i])
        return data
    return data

class CamelCaseJSONParser(JSONParser):
    def parse(self, stream, media_type=None, parser_context=None):
        parser_context = parser_context or {}
        encoding = parser_context.get('encoding', settings.DEFAULT_CHARSET)

        try:
            data = stream.read().decode(encoding)
            return underscoreize(json.loads(data))
        except ValueError as exc:
            raise ParseError('JSON parse error - %s' % six.text_type(exc))
<commit_msg>BUGFIX: Added missing imports<commit_after># -*- coding: utf-8 -*-
from rest_framework.parsers import JSONParser, ParseError, six
from django.conf import settings
import re
import json

first_cap_re = re.compile('(.)([A-Z][a-z]+)')
all_cap_re = re.compile('([a-z0-9])([A-Z])')

def camel_to_underscore(name):
    s1 = first_cap_re.sub(r'\1_\2', name)
    return all_cap_re.sub(r'\1_\2', s1).lower()

def underscoreize(data):
    if isinstance(data, dict):
        new_dict = {}
        for key, value in data.items():
            new_key = camel_to_underscore(key)
            new_dict[new_key] = underscoreize(value)
        return new_dict
    if isinstance(data, (list, tuple)):
        for i in range(len(data)):
            data[i] = underscoreize(data[i])
        return data
    return data

class CamelCaseJSONParser(JSONParser):

    def parse(self, stream, media_type=None, parser_context=None):
        parser_context = parser_context or {}
        encoding = parser_context.get('encoding', settings.DEFAULT_CHARSET)

        try:
            data = stream.read().decode(encoding)
            return underscoreize(json.loads(data))
        except ValueError as exc:
            raise ParseError('JSON parse error - %s' % six.text_type(exc))
<|endoftext|>


<commit_before>"""
Create a graph of function calls as an overview of the CEA software.

In a first step, this only runs on the demand script and just logs which functions call which other functions.

The strategy used is to provide a handler for `sys.settrace` and log the "call" events as a tuple in a set:

    (caller, callee)

This tuple is then used to produce a GraphViz digraph that is saved to the filie "%TEMP%\demand_function_graph.gv"
"""
import inspect
import os
import pickle
import shutil
import sys
import tempfile
import zipfile

import requests

FUNCTIONS = ["benchmark_graphs", "data_helper", "demand_graphs", "demand", "embodied_energy", "emissions", "heatmaps",
          "mobility", "radiation", "scenario_plots"]

def download_radiation(locator):
    """Download radiation and surface properties for running the demand on the nincubes samples"""
    data = {'properties_surfaces': 'https://shared.ethz.ch/owncloud/s/NALgN4Tlhho6QEC/download',
            'radiation': 'https://shared.ethz.ch/owncloud/s/uF6f4EWhPF31ko4/download'}
    r = requests.get(data['properties_surfaces'])
    assert r.ok, 'could not download the properties_surfaces.csv file'
    with open(locator.get_surface_properties(), 'w') as f:
        f.write(r.content)
    r = requests.get(data['radiation'])
    assert r.ok, 'could not download the radiation.csv file'
    with open(locator.get_radiation(), 'w') as f:
        f.write(r.content)

class TraceDataInfo(object):
    """Collects data about a trace event's frame"""
    def __init__(self, frame):
        code = frame.f_code
        self.path = code.co_filename
        self.name = code.co_name
        self.module = ''

        # figure out fully qualified name (fqname)
        fqname_parts = []
        module = inspect.getmodule(code)
        if module:
            fqname_parts.append(module.__name__)
            self.module = module.__name__

        try:
            fqname_parts.append(frame.f_locals['self'].__class__.__name__)
        except (KeyError, AttributeError):
            pass

        fqname_parts.append(self.name)
        self.fqname = '.'.join(fqname_parts)

    def is_cea(self):
        return self.fqname.startswith('cea.')

    def __repr__(self):
        return self.fqname

def trace_function(function_to_trace):
    """Extract the ninecubes.zip reference-case to a temporary directory and run the script 'function_to_trace' on it
    with a default weather file, collecting trace information of each function called."""
    trace_data = []
    uniques = set()  # of (src.fqname, dst.fqname)

    def trace_calls(frame, event, arg):
        """handle trace events and update the trace_data with information of the source and destination of the call"""
        if event != 'call':
            return

        call_dst = TraceDataInfo(frame)
        if not call_dst.is_cea():
            return
        #print('call_dst: %s' % call_dst)

        call_src = find_last_cea(frame)
        #print ('call_src: %s -- %s' % (call_src, trace_data))
        if call_src and not (call_src.fqname, call_dst.fqname) in uniques:
            trace_data.append((call_src, call_dst))
            uniques.add((call_src.fqname, call_dst.fqname))
            print(call_src, call_dst)
        return

    try:
        import cea.inputlocator
        import cea.globalvar

        scenario_path = extract_ninecubes()
        locator = cea.inputlocator.InputLocator(scenario_path=scenario_path)
        gv = cea.globalvar.GlobalVariables()
        gv.multiprocessing = False
        download_radiation(locator)
        weather_path = locator.get_default_weather()

        sys.settrace(trace_calls)
        function_to_trace(gv, locator, weather_path)
        sys.settrace(None)

    finally:
        # make sure we clean up after ourselves...
        remove_ninecubes()
        pass
    return trace_data

def run_demand(gv, locator, weather_path):
    import cea.demand.demand_main
    cea.demand.demand_main.demand_calculation(locator=locator, weather_path=weather_path, gv=gv)

def find_last_cea(frame):
    """"Search through the stack frame for the last origination of a call from cea code"""
    if not hasattr(frame, 'f_back'):
        #print('frame has no attr f_back')
        return None
    call_src = TraceDataInfo(frame.f_back)
    if call_src.name == 'trace_demand':
        #print('call_src.name == trace_demand')
        return None

    if call_src.is_cea():
        return call_src
    else:
        #print('call_src is not cea: %s' % call_src)
        return find_last_cea(frame.f_back)

def extract_ninecubes():
    ninecubes_src = os.path.join(os.path.dirname(__file__), '..', 'examples', 'ninecubes.zip')
    ninecubes_dst = os.path.join(tempfile.gettempdir(), 'ninecubes.zip')
    shutil.copyfile(ninecubes_src, ninecubes_dst)
    archive = zipfile.ZipFile(ninecubes_dst)
    extraction_location = os.path.join(tempfile.gettempdir(), 'ninecubes')
    archive.extractall(extraction_location)
    scenario_path = os.path.join(extraction_location, 'nine cubes', 'baseline')
    return scenario_path

def remove_ninecubes():
    """delete the ninecubes stuff from the temp directory"""
    ninecubes_zip = os.path.join(tempfile.gettempdir(), 'ninecubes.zip')
    if os.path.exists(ninecubes_zip):
        os.remove(ninecubes_zip)

    ninecubes_dir = os.path.join(tempfile.gettempdir(), 'ninecubes')
    if os.path.exists(ninecubes_dir):
        shutil.rmtree(ninecubes_dir)

def package_names(trace_data):
    """Extract the names of the packages from the (filtered?) trace_data."""
    return ('a', 'b')

def edge_names(trace_data_item):
    return ('a', 'b')

def extract_namespace(fqname):
    return fqname.rsplit('.', 1)[0]

def extract_function_name(fqname):
    return fqname.rsplit('.', 1)[1]

def filter_trace_data(trace_data):
    """remove calls to (and from) the namespaces in the stoplist.
    This should make the output easier to read.
    """
    stoplist = ['cea.inputlocator.InputLocator',
                'cea.utilities.helpers',
                'cea.demand.thermal_loads.BuildingProperties',
                'cea.globalvar.GlobalVariables',
                'cea.demand.demand_writers.HourlyDemandWriter',
                'cea.demand.thermal_loads.BuildingPropertiesRow']
    for tdi_src, tdi_dst in trace_data:
        if extract_namespace(tdi_src.fqname) in stoplist:
            continue
        elif extract_namespace(tdi_dst.fqname) in stoplist:
            continue
        yield (tdi_src, tdi_dst)

def print_digraph(trace_data, f):
    trace_data = list(filter_trace_data(trace_data))
    all_nodes = set(tdi.fqname for td in trace_data for tdi in td)
    namespaces = set(extract_namespace(node) for node in all_nodes)

    f.writelines([
        "digraph demand_function_graph {\n",
        "  rankdir=LR;\n",
        "  ratio=0.7072135785007072;\n",
        "  edge[weight=1.2];\n",
        "  node [shape=plaintext, ranksep=0.7, nodesep=0.7, fontsize=10, fontname=monospace, color=none];\n",
    ])

    for namespace in namespaces:
        node_names = '\n'.join('    "%s";' % extract_function_name(node) for node in all_nodes if
                               extract_namespace(node) == namespace)
        f.writelines([
            '  subgraph "cluster_%(namespace)s" {\n' % locals(),
            node_names,
            '\n',
            '    label="%(namespace)s";\n' % locals(),
            '  }\n',
        ])

    for src, dst in trace_data:
        f.write('  "%s" -> "%s";\n' % (extract_function_name(src.fqname), extract_function_name(dst.fqname)))

    f.write('}\n')

def create_module_overview(trace_data):
    """Compress the list of connections in the trace_data to only the modules. So instead of tracing function
    calls, trace module-to-module calls. This can give more overview of the code as it is higher level.
    """
    class OverviewTraceDataInfo(object):
        """Imitate a TDI, but only sets the fqname attribute"""
        def __init__(self, fqname):
            self.fqname = fqname

    module_calls = set()  # list of (src_module, dst_module)
    for tdi_src, tdi_dst in filter_trace_data(trace_data):
        src_module = extract_namespace(tdi_src.fqname)
        dst_module = extract_namespace(tdi_dst.fqname)
        if src_module == dst_module:
            # skip self-references
            continue
        module_calls.add((src_module, dst_module))
    return [(OverviewTraceDataInfo(src_module), OverviewTraceDataInfo(dst_module))
            for src_module, dst_module in module_calls]

def get_function_to_trace(function):
    functions = {
        "benchmark_graphs": None,
        "data_helper": None,
        "demand_graphs": None,
        "demand": run_demand,
        "embodied_energy": None,
        "emissions": None,
        "heatmaps": None,
        "mobility": None,
        "radiation": None,
        "scenario_plots": None}
    return functions[function]

def create_function_graph(input=None, output=None, save_trace_data=None, module_overview=False, function_name='demand'):
    if input:
        # create a graph using trace data that was previously saved to a file
        with open(input, 'r') as f:
            trace_data = pickle.load(f)
    else:
        # run function to create trace data
        trace_data = trace_function(get_function_to_trace(function_name))

    if save_trace_data:
        # save a copy of the trace data to a file for later retrieval
        with open(save_trace_data, 'w') as f:
            pickle.dump(trace_data, f)

    if output:
        # write out a GraphViz graph to this file
        if module_overview:
            trace_data = create_module_overview(trace_data)

        print_digraph(trace_data, f)

def parse_arguments(argv):
    import argparse
    parser = argparse.ArgumentParser()
    parser.add_argument('-s', '--save-trace-data', help='Save trace data to file', default=None)
    parser.add_argument('-i', '--input', help='Load trace data from file', default=None)
    parser.add_argument('-o', '--output', help='Save graphviz output to this file', default=None)
    parser.add_argument('-m', '--module-overview', help='Create a module overview', action='store_true')
    parser.add_argument('-f', '--function', help='Function to run', default="demand", choices=FUNCTIONS)
    args = parser.parse_args(argv)
    return args

if __name__ == '__main__':
    args = parse_arguments(sys.argv[1:])
    create_function_graph(input=args.input, output=args.output, save_trace_data=args.save_trace_data,
                          module_overview=args.module_overview, function_name=args.function)
<commit_msg>fixed bug with writing out the graph<commit_after>"""
Create a graph of function calls as an overview of the CEA software.

In a first step, this only runs on the demand script and just logs which functions call which other functions.

The strategy used is to provide a handler for `sys.settrace` and log the "call" events as a tuple in a set:

    (caller, callee)

This tuple is then used to produce a GraphViz digraph that is saved to the filie "%TEMP%\demand_function_graph.gv"
"""
import inspect
import os
import pickle
import shutil
import sys
import tempfile
import zipfile

import requests

FUNCTIONS = ["benchmark_graphs", "data_helper", "demand_graphs", "demand", "embodied_energy", "emissions", "heatmaps",
          "mobility", "radiation", "scenario_plots"]

def download_radiation(locator):
    """Download radiation and surface properties for running the demand on the nincubes samples"""
    data = {'properties_surfaces': 'https://shared.ethz.ch/owncloud/s/NALgN4Tlhho6QEC/download',
            'radiation': 'https://shared.ethz.ch/owncloud/s/uF6f4EWhPF31ko4/download'}
    r = requests.get(data['properties_surfaces'])
    assert r.ok, 'could not download the properties_surfaces.csv file'
    with open(locator.get_surface_properties(), 'w') as f:
        f.write(r.content)
    r = requests.get(data['radiation'])
    assert r.ok, 'could not download the radiation.csv file'
    with open(locator.get_radiation(), 'w') as f:
        f.write(r.content)

class TraceDataInfo(object):
    """Collects data about a trace event's frame"""
    def __init__(self, frame):
        code = frame.f_code
        self.path = code.co_filename
        self.name = code.co_name
        self.module = ''

        # figure out fully qualified name (fqname)
        fqname_parts = []
        module = inspect.getmodule(code)
        if module:
            fqname_parts.append(module.__name__)
            self.module = module.__name__

        try:
            fqname_parts.append(frame.f_locals['self'].__class__.__name__)
        except (KeyError, AttributeError):
            pass

        fqname_parts.append(self.name)
        self.fqname = '.'.join(fqname_parts)

    def is_cea(self):
        return self.fqname.startswith('cea.')

    def __repr__(self):
        return self.fqname

def trace_function(function_to_trace):
    """Extract the ninecubes.zip reference-case to a temporary directory and run the script 'function_to_trace' on it
    with a default weather file, collecting trace information of each function called."""
    trace_data = []
    uniques = set()  # of (src.fqname, dst.fqname)

    def trace_calls(frame, event, arg):
        """handle trace events and update the trace_data with information of the source and destination of the call"""
        if event != 'call':
            return

        call_dst = TraceDataInfo(frame)
        if not call_dst.is_cea():
            return
        #print('call_dst: %s' % call_dst)

        call_src = find_last_cea(frame)
        #print ('call_src: %s -- %s' % (call_src, trace_data))
        if call_src and not (call_src.fqname, call_dst.fqname) in uniques:
            trace_data.append((call_src, call_dst))
            uniques.add((call_src.fqname, call_dst.fqname))
            print(call_src, call_dst)
        return

    try:
        import cea.inputlocator
        import cea.globalvar

        scenario_path = extract_ninecubes()
        locator = cea.inputlocator.InputLocator(scenario_path=scenario_path)
        gv = cea.globalvar.GlobalVariables()
        gv.multiprocessing = False
        download_radiation(locator)
        weather_path = locator.get_default_weather()

        sys.settrace(trace_calls)
        function_to_trace(gv, locator, weather_path)
        sys.settrace(None)

    finally:
        # make sure we clean up after ourselves...
        remove_ninecubes()
        pass
    return trace_data

def run_demand(gv, locator, weather_path):
    import cea.demand.demand_main
    cea.demand.demand_main.demand_calculation(locator=locator, weather_path=weather_path, gv=gv)

def find_last_cea(frame):
    """"Search through the stack frame for the last origination of a call from cea code"""
    if not hasattr(frame, 'f_back'):
        #print('frame has no attr f_back')
        return None
    call_src = TraceDataInfo(frame.f_back)
    if call_src.name == 'trace_demand':
        #print('call_src.name == trace_demand')
        return None

    if call_src.is_cea():
        return call_src
    else:
        #print('call_src is not cea: %s' % call_src)
        return find_last_cea(frame.f_back)

def extract_ninecubes():
    ninecubes_src = os.path.join(os.path.dirname(__file__), '..', 'examples', 'ninecubes.zip')
    ninecubes_dst = os.path.join(tempfile.gettempdir(), 'ninecubes.zip')
    shutil.copyfile(ninecubes_src, ninecubes_dst)
    archive = zipfile.ZipFile(ninecubes_dst)
    extraction_location = os.path.join(tempfile.gettempdir(), 'ninecubes')
    archive.extractall(extraction_location)
    scenario_path = os.path.join(extraction_location, 'nine cubes', 'baseline')
    return scenario_path

def remove_ninecubes():
    """delete the ninecubes stuff from the temp directory"""
    ninecubes_zip = os.path.join(tempfile.gettempdir(), 'ninecubes.zip')
    if os.path.exists(ninecubes_zip):
        os.remove(ninecubes_zip)

    ninecubes_dir = os.path.join(tempfile.gettempdir(), 'ninecubes')
    if os.path.exists(ninecubes_dir):
        shutil.rmtree(ninecubes_dir)

def package_names(trace_data):
    """Extract the names of the packages from the (filtered?) trace_data."""
    return ('a', 'b')

def edge_names(trace_data_item):
    return ('a', 'b')

def extract_namespace(fqname):
    return fqname.rsplit('.', 1)[0]

def extract_function_name(fqname):
    return fqname.rsplit('.', 1)[1]

def filter_trace_data(trace_data):
    """remove calls to (and from) the namespaces in the stoplist.
    This should make the output easier to read.
    """
    stoplist = ['cea.inputlocator.InputLocator',
                'cea.utilities.helpers',
                'cea.demand.thermal_loads.BuildingProperties',
                'cea.globalvar.GlobalVariables',
                'cea.demand.demand_writers.HourlyDemandWriter',
                'cea.demand.thermal_loads.BuildingPropertiesRow']
    for tdi_src, tdi_dst in trace_data:
        if extract_namespace(tdi_src.fqname) in stoplist:
            continue
        elif extract_namespace(tdi_dst.fqname) in stoplist:
            continue
        yield (tdi_src, tdi_dst)

def print_digraph(trace_data, f):
    trace_data = list(filter_trace_data(trace_data))
    all_nodes = set(tdi.fqname for td in trace_data for tdi in td)
    namespaces = set(extract_namespace(node) for node in all_nodes)

    f.writelines([
        "digraph demand_function_graph {\n",
        "  rankdir=LR;\n",
        "  ratio=0.7072135785007072;\n",
        "  edge[weight=1.2];\n",
        "  node [shape=plaintext, ranksep=0.7, nodesep=0.7, fontsize=10, fontname=monospace, color=none];\n",
    ])

    for namespace in namespaces:
        node_names = '\n'.join('    "%s";' % extract_function_name(node) for node in all_nodes if
                               extract_namespace(node) == namespace)
        f.writelines([
            '  subgraph "cluster_%(namespace)s" {\n' % locals(),
            node_names,
            '\n',
            '    label="%(namespace)s";\n' % locals(),
            '  }\n',
        ])

    for src, dst in trace_data:
        f.write('  "%s" -> "%s";\n' % (extract_function_name(src.fqname), extract_function_name(dst.fqname)))

    f.write('}\n')

def create_module_overview(trace_data):
    """Compress the list of connections in the trace_data to only the modules. So instead of tracing function
    calls, trace module-to-module calls. This can give more overview of the code as it is higher level.
    """
    class OverviewTraceDataInfo(object):
        """Imitate a TDI, but only sets the fqname attribute"""
        def __init__(self, fqname):
            self.fqname = fqname

    module_calls = set()  # list of (src_module, dst_module)
    for tdi_src, tdi_dst in filter_trace_data(trace_data):
        src_module = extract_namespace(tdi_src.fqname)
        dst_module = extract_namespace(tdi_dst.fqname)
        if src_module == dst_module:
            # skip self-references
            continue
        module_calls.add((src_module, dst_module))
    return [(OverviewTraceDataInfo(src_module), OverviewTraceDataInfo(dst_module))
            for src_module, dst_module in module_calls]

def get_function_to_trace(function):
    functions = {
        "benchmark_graphs": None,
        "data_helper": None,
        "demand_graphs": None,
        "demand": run_demand,
        "embodied_energy": None,
        "emissions": None,
        "heatmaps": None,
        "mobility": None,
        "radiation": None,
        "scenario_plots": None}
    return functions[function]

def create_function_graph(input=None, output=None, save_trace_data=None, module_overview=False, function_name='demand'):
    if input:
        # create a graph using trace data that was previously saved to a file
        with open(input, 'r') as f:
            trace_data = pickle.load(f)
    else:
        # run function to create trace data
        trace_data = trace_function(get_function_to_trace(function_name))

    if save_trace_data:
        # save a copy of the trace data to a file for later retrieval
        with open(save_trace_data, 'w') as f:
            pickle.dump(trace_data, f)

    if output:
        # write out a GraphViz graph to this file
        if module_overview:
            trace_data = create_module_overview(trace_data)
        with open(output, 'w') as f:
            print_digraph(trace_data, f)

def parse_arguments(argv):
    import argparse
    parser = argparse.ArgumentParser()
    parser.add_argument('-s', '--save-trace-data', help='Save trace data to file', default=None)
    parser.add_argument('-i', '--input', help='Load trace data from file', default=None)
    parser.add_argument('-o', '--output', help='Save graphviz output to this file', default=None)
    parser.add_argument('-m', '--module-overview', help='Create a module overview', action='store_true')
    parser.add_argument('-f', '--function', help='Function to run', default="demand", choices=FUNCTIONS)
    args = parser.parse_args(argv)
    return args

if __name__ == '__main__':
    args = parse_arguments(sys.argv[1:])
    create_function_graph(input=args.input, output=args.output, save_trace_data=args.save_trace_data,
                          module_overview=args.module_overview, function_name=args.function)
<|endoftext|>


<commit_before># Copyright 2021 The Meson development team

# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at

#     http://www.apache.org/licenses/LICENSE-2.0

# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import os
import tempfile
from unittest import skipIf

from .baseplatformtests import BasePlatformTests
from .helpers import is_ci
from mesonbuild.mesonlib import is_linux
from mesonbuild.optinterpreter import OptionInterpreter, OptionException

@skipIf(is_ci() and not is_linux(), "Run only on fast platforms")
class PlatformAgnosticTests(BasePlatformTests):
    '''
    Tests that does not need to run on all platforms during CI
    '''

    def test_relative_find_program(self):
        '''
        Tests that find_program() with a relative path does not find the program
        in current workdir.
        '''
        testdir = os.path.join(self.unit_test_dir, '100 relative find program')
        self.init(testdir, workdir=testdir)

    def test_invalid_option_names(self):
        interp = OptionInterpreter('')

        def write_file(code: str):
            with tempfile.NamedTemporaryFile('w', dir=self.builddir, encoding='utf-8', delete=False) as f:
                f.write(code)
                return f.name

        fname = write_file("option('default_library', type: 'string')")
        self.assertRaisesRegex(OptionException, 'Option name default_library is reserved.',
                               interp.process, fname)

        fname = write_file("option('c_anything', type: 'string')")
        self.assertRaisesRegex(OptionException, 'Option name c_anything is reserved.',
                               interp.process, fname)

        fname = write_file("option('b_anything', type: 'string')")
        self.assertRaisesRegex(OptionException, 'Option name b_anything is reserved.',
                               interp.process, fname)

        fname = write_file("option('backend_anything', type: 'string')")
        self.assertRaisesRegex(OptionException, 'Option name backend_anything is reserved.',
                               interp.process, fname)

        fname = write_file("option('foo.bar', type: 'string')")
        self.assertRaisesRegex(OptionException, 'Option names can only contain letters, numbers or dashes.',
                               interp.process, fname)

        # platlib is allowed, only python.platlib is reserved.
        fname = write_file("option('platlib', type: 'string')")
        interp.process(fname)

    def test_python_dependency_without_pkgconfig(self):
        testdir = os.path.join(self.unit_test_dir, '102 python without pkgconfig')
        self.init(testdir, override_envvars={'PKG_CONFIG': 'notfound'})

    def test_debug_function_outputs_to_meson_log(self):
        testdir = os.path.join(self.unit_test_dir, '104 debug function')
        log_msg = 'This is an example debug output, should only end up in debug log'
        output = self.init(testdir)

        # Check if message is not printed to stdout while configuring
        assert(log_msg not in output)

        # Check if message is written to the meson log
        mesonlog = os.path.join(self.builddir, 'meson-logs/meson-log.txt')
        with open(mesonlog, mode='r', encoding='utf-8') as file:
            assert(log_msg in file.read())
<commit_msg>unittests: correctly use test asserts instead of regular ones<commit_after># Copyright 2021 The Meson development team

# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at

#     http://www.apache.org/licenses/LICENSE-2.0

# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import os
import tempfile
from unittest import skipIf

from .baseplatformtests import BasePlatformTests
from .helpers import is_ci
from mesonbuild.mesonlib import is_linux
from mesonbuild.optinterpreter import OptionInterpreter, OptionException

@skipIf(is_ci() and not is_linux(), "Run only on fast platforms")
class PlatformAgnosticTests(BasePlatformTests):
    '''
    Tests that does not need to run on all platforms during CI
    '''

    def test_relative_find_program(self):
        '''
        Tests that find_program() with a relative path does not find the program
        in current workdir.
        '''
        testdir = os.path.join(self.unit_test_dir, '100 relative find program')
        self.init(testdir, workdir=testdir)

    def test_invalid_option_names(self):
        interp = OptionInterpreter('')

        def write_file(code: str):
            with tempfile.NamedTemporaryFile('w', dir=self.builddir, encoding='utf-8', delete=False) as f:
                f.write(code)
                return f.name

        fname = write_file("option('default_library', type: 'string')")
        self.assertRaisesRegex(OptionException, 'Option name default_library is reserved.',
                               interp.process, fname)

        fname = write_file("option('c_anything', type: 'string')")
        self.assertRaisesRegex(OptionException, 'Option name c_anything is reserved.',
                               interp.process, fname)

        fname = write_file("option('b_anything', type: 'string')")
        self.assertRaisesRegex(OptionException, 'Option name b_anything is reserved.',
                               interp.process, fname)

        fname = write_file("option('backend_anything', type: 'string')")
        self.assertRaisesRegex(OptionException, 'Option name backend_anything is reserved.',
                               interp.process, fname)

        fname = write_file("option('foo.bar', type: 'string')")
        self.assertRaisesRegex(OptionException, 'Option names can only contain letters, numbers or dashes.',
                               interp.process, fname)

        # platlib is allowed, only python.platlib is reserved.
        fname = write_file("option('platlib', type: 'string')")
        interp.process(fname)

    def test_python_dependency_without_pkgconfig(self):
        testdir = os.path.join(self.unit_test_dir, '102 python without pkgconfig')
        self.init(testdir, override_envvars={'PKG_CONFIG': 'notfound'})

    def test_debug_function_outputs_to_meson_log(self):
        testdir = os.path.join(self.unit_test_dir, '104 debug function')
        log_msg = 'This is an example debug output, should only end up in debug log'
        output = self.init(testdir)

        # Check if message is not printed to stdout while configuring
        self.assertNotIn(log_msg, output)

        # Check if message is written to the meson log
        mesonlog = os.path.join(self.builddir, 'meson-logs/meson-log.txt')
        with open(mesonlog, mode='r', encoding='utf-8') as file:
            self.assertIn(log_msg, file.read())
<|endoftext|>


<commit_before>import unittest
from gimmemotifs.db import MotifDb

class TestMotifDatabases(unittest.TestCase):
    """ A test class for motif database functionality """

    def setUp(self):
        pass

    def test_list(self):
        """Test motif db list"""

        l = sorted(MotifDb.list_databases())
        expected = ['encode', 'factorbook', 'hocomoco', 'homer', 'image', 'jaspar', 'swissregulon']
        self.assertEqual(l, expected)

    def tearDown(self):
        pass

if __name__ == '__main__':
    unittest.main()
<commit_msg>update test<commit_after>import unittest
from gimmemotifs.db import MotifDb

class TestMotifDatabases(unittest.TestCase):
    """ A test class for motif database functionality """

    def setUp(self):
        pass

    def test_list(self):
        """Test motif db list"""

        l = sorted(MotifDb.list_databases())
        expected = ['cis-bp', 'encode', 'factorbook', 'hocomoco', 'homer', 'image', 'jaspar', 'swissregulon']
        self.assertEqual(l, expected)

    def tearDown(self):
        pass

if __name__ == '__main__':
    unittest.main()
<|endoftext|>


<commit_before># __BEGIN_LICENSE__
# Copyright (c) 2015, United States Government, as represented by the
# Administrator of the National Aeronautics and Space Administration.
# All rights reserved.
#
# The xGDS platform is licensed under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
# http://www.apache.org/licenses/LICENSE-2.0.
#
# Unless required by applicable law or agreed to in writing, software distributed
# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
# CONDITIONS OF ANY KIND, either express or implied. See the License for the
# specific language governing permissions and limitations under the License.
# __END_LICENSE__
# pylint: disable=W0702

import pytz
from uuid import uuid4
from django.conf import settings
from django.core.exceptions import ObjectDoesNotExist
from geocamUtil.loader import LazyGetModelByName

ACTIVE_FLIGHT_MODEL = LazyGetModelByName(settings.XGDS_CORE_ACTIVE_FLIGHT_MODEL)
FLIGHT_MODEL = LazyGetModelByName(settings.XGDS_CORE_FLIGHT_MODEL)
GROUP_FLIGHT_MODEL = LazyGetModelByName(settings.XGDS_CORE_GROUP_FLIGHT_MODEL)
VEHICLE_MODEL = LazyGetModelByName(settings.XGDS_CORE_VEHICLE_MODEL)

def getFlight(event_time, vehicle=None):
    """
     Returns the flight that contains that event_time
    If we are in live mode it first looks in the active table.
    :param event_time: the event time to find a flight for
    :param vehicle: the vehicle, will default to the default vehicle
    :return: the flight that matches the time for the vehicle, or None
    """

    if not event_time:
        return None
    if settings.GEOCAM_UTIL_LIVE_MODE:
        if not vehicle:
            vehicle = get_default_vehicle()

        # check active flights first if we are in live mode
        active_flights = ACTIVE_FLIGHT_MODEL.get().objects.filter(flight__vehicle=vehicle)

        if active_flights:
            active_flights.filter(flight__start_time__gte=event_time)

        if active_flights:
            return active_flights.last().flight

    # if we are not in live mode then it should be from a completed flight
    if vehicle:
        found_flights = FLIGHT_MODEL.get().objects.exclude(end_time__isnull=True).filter(vehicle=vehicle,
                                                                                start_time__lte=event_time,
                                                                                end_time__gte=event_time)
    else:
        found_flights = FLIGHT_MODEL.get().objects.exclude(end_time__isnull=True).filter(start_time__lte=event_time,
                                                                                end_time__gte=event_time)

    # if found_flights.count() == 0:
    #     found_active_flight = getActiveFlight(vehicle)
    #     if found_active_flight and found_active_flight.start_time:
    #         if event_time >= found_active_flight.start_time:
    #             return found_active_flight;
    #     return None
    # else:
    if found_flights.count() > 1:
        filtered_flights = found_flights.filter(vehicle_id=settings.XGDS_CORE_DEFAULT_VEHICLE_PK)
        if filtered_flights:
            return filtered_flights[0]
    return found_flights[0]

def getNextAlphabet(character):
    """
    For getting the next letter of the alphabet for prefix.
    """
    # a is '97' and z is '122'. There are 26 letters total
    nextChar = ord(character) + 1
    if nextChar > 122:
        nextChar = (nextChar - 97) % 26 + 97
    return chr(nextChar)

def get_next_available_group_flight_name(prefix):
    character = 'A'
    gModel = GROUP_FLIGHT_MODEL.get()
    while True:
        try:
            gModel.objects.get(name=prefix + character)
            character = getNextAlphabet(character)
        except:
            return prefix + character

def getActiveFlight(vehicle=None):
    if vehicle:
        found_active_flights = ACTIVE_FLIGHT_MODEL.get().objects.filter(flight__vehicle=vehicle)
    else:
        found_active_flights = ACTIVE_FLIGHT_MODEL.get().objects.filter()

    if found_active_flights:
        return found_active_flights.last().flight
    return None

def get_default_vehicle():
    """
    Gets the default vehicle.
    :return:
    """
    return VEHICLE_MODEL.get().objects.get(pk=settings.XGDS_CORE_DEFAULT_VEHICLE_PK)

def get_vehicle(vehicle_name=None):
    """
    Gets the vehicle.
    :param vehicle_name: the name of the vehicle
    :return:
    """
    if not vehicle_name:
        return get_default_vehicle()
    return VEHICLE_MODEL.get().objects.get(name=vehicle_name)

def create_group_flight(group_flight_name, notes=None, vehicles=None, active=False, start_time=None):
    """
    Create a new group flight
    :param group_flight_name:
    :param notes:
    :param vehicles:
    :param active: true to make the flights active
    :param start_time: the start time for the group flight and flights
    :return: the group flight
    """
    group_flight = GROUP_FLIGHT_MODEL.get()(name=group_flight_name, notes=notes)
    group_flight.save()

    if not vehicles:
        # TODO deal with generic vehicle better?
        vehicles = VEHICLE_MODEL.get().objects.filter(primary=True).exclude(name='generic')
    for vehicle in vehicles:
        new_flight = FLIGHT_MODEL.get()()
        new_flight.group = group_flight
        new_flight.vehicle = vehicle
        new_flight.name = group_flight_name + "_" + vehicle.name
        new_flight.uuid = uuid4()
        new_flight.start_time = start_time
        new_flight.save()

        if active:
            af = ACTIVE_FLIGHT_MODEL.get()(flight=new_flight)
            af.save()

    return group_flight

def end_group_flight(group_flight_name, end_time=None):
    """
    Remove all the flights from active if they are, and mark their end times
    :param group_flight_name:
    :param end_time:
    :return:
    """
    group_flight = GROUP_FLIGHT_MODEL.get().objects.get(name=group_flight_name)

    for flight in group_flight.flights:
        if end_time:
            flight.end_time = end_time
            flight.save()

        try:
            active_flight = ACTIVE_FLIGHT_MODEL.get().objects.get(flight=flight)
            active_flight.delete()
        except:
            pass

def lookup_vehicle(vehicle_name):
    """ Look up the vehicle by name or get the default
    :param vehicle_name: The name of the vehicle
    :return: the found vehicle, or None
    """
    vehicle = None
    try:
        if vehicle_name:
            vehicle = VEHICLE_MODEL.get().objects.get(name=vehicle_name)
    except:
        pass
    if not vehicle:
        vehicle = get_default_vehicle()
    return vehicle

def lookup_flight(flight_name):
    """ Look up the flight by name
    :param flight_name: The name of the flight
    :return: the found flight, or None
    """
    flight = None
    if flight_name:
        try:
            flight = FLIGHT_MODEL.get().objects.get(name=flight_name)
        except:
            pass
    return flight

def get_or_create_flight(start_time, vehicle=None, check_flight_exists=True, end_time=None, name=None):
    """
    Get or create a flight; will create the group flight and the flights, and set the start time.
    :param start_time: the start time of the flight
    :param vehicle: the vehicle
    :param check_flight_exists: flag to check if the flight exists
    :param end_time: Not used for search, but if you want to set the end time of the newly created flight pass it here
    :param name: The name for the group flight
    :return: the flight
    """
    if not vehicle:
        vehicle = get_default_vehicle()
    flight = None
    if check_flight_exists:
        flight = getFlight(start_time, vehicle)
    if not flight:
        # There was not a valid flight, so let's make a new one.  We will make a new group flight.

        # localize the time
        if not name:
            local_start_time = start_time.astimezone(pytz.timezone(settings.TIME_ZONE))
            name = get_next_available_group_flight_name(local_start_time.strftime('%Y%m%d'))
        group_flight = create_group_flight(name)
        if group_flight:
            flights = group_flight.flights.filter(vehicle=vehicle)
            flight = flights[0]  # there should only be one

            # set its start time
            flight.start_time = start_time
            flight.end_time = end_time
            flight.save()
    else:
        flight.update_start_time(start_time)
    return flight

def get_or_create_flight_with_source_root(source_root, timestamp, end_time=None, name=None):
    """
    See if there is already a flight for this directory, or create it.
    It is on us to guarantee uniqueness (db has limitation of 255 characters for unique)
    #TODO have flexible flight naming convention and make sure that works
    :param source_root: the path that should correspond to a flight
    :param timestamp: the timestamp to be used when building flight name
    :param end_time: the end time for the flight
    :param name: the custom name for the flight (and group flight)
    :return: the flight
    """
    try:
        flight = FLIGHT_MODEL.get().objects.get(source_root__icontains=source_root)
    except ObjectDoesNotExist:
        # create it
        flight = get_or_create_flight(timestamp, check_flight_exists=False, end_time=end_time, name=name)
        flight.source_root = source_root
        flight.save()
    return flight
<commit_msg>use default vehicle for getActiveFlight if none specified<commit_after># __BEGIN_LICENSE__
# Copyright (c) 2015, United States Government, as represented by the
# Administrator of the National Aeronautics and Space Administration.
# All rights reserved.
#
# The xGDS platform is licensed under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
# http://www.apache.org/licenses/LICENSE-2.0.
#
# Unless required by applicable law or agreed to in writing, software distributed
# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
# CONDITIONS OF ANY KIND, either express or implied. See the License for the
# specific language governing permissions and limitations under the License.
# __END_LICENSE__
# pylint: disable=W0702

import pytz
from uuid import uuid4
from django.conf import settings
from django.core.exceptions import ObjectDoesNotExist
from geocamUtil.loader import LazyGetModelByName

ACTIVE_FLIGHT_MODEL = LazyGetModelByName(settings.XGDS_CORE_ACTIVE_FLIGHT_MODEL)
FLIGHT_MODEL = LazyGetModelByName(settings.XGDS_CORE_FLIGHT_MODEL)
GROUP_FLIGHT_MODEL = LazyGetModelByName(settings.XGDS_CORE_GROUP_FLIGHT_MODEL)
VEHICLE_MODEL = LazyGetModelByName(settings.XGDS_CORE_VEHICLE_MODEL)

def getFlight(event_time, vehicle=None):
    """
     Returns the flight that contains that event_time
    If we are in live mode it first looks in the active table.
    :param event_time: the event time to find a flight for
    :param vehicle: the vehicle, will default to the default vehicle
    :return: the flight that matches the time for the vehicle, or None
    """

    if not event_time:
        return None
    if settings.GEOCAM_UTIL_LIVE_MODE:
        if not vehicle:
            vehicle = get_default_vehicle()

        # check active flights first if we are in live mode
        active_flights = ACTIVE_FLIGHT_MODEL.get().objects.filter(flight__vehicle=vehicle)

        if active_flights:
            active_flights.filter(flight__start_time__gte=event_time)

        if active_flights:
            return active_flights.last().flight

    # if we are not in live mode then it should be from a completed flight
    if vehicle:
        found_flights = FLIGHT_MODEL.get().objects.exclude(end_time__isnull=True).filter(vehicle=vehicle,
                                                                                start_time__lte=event_time,
                                                                                end_time__gte=event_time)
    else:
        found_flights = FLIGHT_MODEL.get().objects.exclude(end_time__isnull=True).filter(start_time__lte=event_time,
                                                                                end_time__gte=event_time)

    # if found_flights.count() == 0:
    #     found_active_flight = getActiveFlight(vehicle)
    #     if found_active_flight and found_active_flight.start_time:
    #         if event_time >= found_active_flight.start_time:
    #             return found_active_flight;
    #     return None
    # else:
    if found_flights.count() > 1:
        filtered_flights = found_flights.filter(vehicle_id=settings.XGDS_CORE_DEFAULT_VEHICLE_PK)
        if filtered_flights:
            return filtered_flights[0]
    return found_flights[0]

def getNextAlphabet(character):
    """
    For getting the next letter of the alphabet for prefix.
    """
    # a is '97' and z is '122'. There are 26 letters total
    nextChar = ord(character) + 1
    if nextChar > 122:
        nextChar = (nextChar - 97) % 26 + 97
    return chr(nextChar)

def get_next_available_group_flight_name(prefix):
    character = 'A'
    gModel = GROUP_FLIGHT_MODEL.get()
    while True:
        try:
            gModel.objects.get(name=prefix + character)
            character = getNextAlphabet(character)
        except:
            return prefix + character

def getActiveFlight(vehicle=None):
    if vehicle is not None:
        found_active_flights = ACTIVE_FLIGHT_MODEL.get().objects.filter(flight__vehicle=vehicle)
    else:
        vehicle = get_default_vehicle()
        found_active_flights = ACTIVE_FLIGHT_MODEL.get().objects.filter(flight__vehicle=vehicle)
        if found_active_flights.count()<1:
            found_active_flights = ACTIVE_FLIGHT_MODEL.get().objects.filter()

    if found_active_flights:
        return found_active_flights.last().flight
    return None

def get_default_vehicle():
    """
    Gets the default vehicle.
    :return:
    """
    return VEHICLE_MODEL.get().objects.get(pk=settings.XGDS_CORE_DEFAULT_VEHICLE_PK)

def get_vehicle(vehicle_name=None):
    """
    Gets the vehicle.
    :param vehicle_name: the name of the vehicle
    :return:
    """
    if not vehicle_name:
        return get_default_vehicle()
    return VEHICLE_MODEL.get().objects.get(name=vehicle_name)

def create_group_flight(group_flight_name, notes=None, vehicles=None, active=False, start_time=None):
    """
    Create a new group flight
    :param group_flight_name:
    :param notes:
    :param vehicles:
    :param active: true to make the flights active
    :param start_time: the start time for the group flight and flights
    :return: the group flight
    """
    group_flight = GROUP_FLIGHT_MODEL.get()(name=group_flight_name, notes=notes)
    group_flight.save()

    if not vehicles:
        # TODO deal with generic vehicle better?
        vehicles = VEHICLE_MODEL.get().objects.filter(primary=True).exclude(name='generic')
    for vehicle in vehicles:
        new_flight = FLIGHT_MODEL.get()()
        new_flight.group = group_flight
        new_flight.vehicle = vehicle
        new_flight.name = group_flight_name + "_" + vehicle.name
        new_flight.uuid = uuid4()
        new_flight.start_time = start_time
        new_flight.save()

        if active:
            af = ACTIVE_FLIGHT_MODEL.get()(flight=new_flight)
            af.save()

    return group_flight

def end_group_flight(group_flight_name, end_time=None):
    """
    Remove all the flights from active if they are, and mark their end times
    :param group_flight_name:
    :param end_time:
    :return:
    """
    group_flight = GROUP_FLIGHT_MODEL.get().objects.get(name=group_flight_name)

    for flight in group_flight.flights:
        if end_time:
            flight.end_time = end_time
            flight.save()

        try:
            active_flight = ACTIVE_FLIGHT_MODEL.get().objects.get(flight=flight)
            active_flight.delete()
        except:
            pass

def lookup_vehicle(vehicle_name):
    """ Look up the vehicle by name or get the default
    :param vehicle_name: The name of the vehicle
    :return: the found vehicle, or None
    """
    vehicle = None
    try:
        if vehicle_name:
            vehicle = VEHICLE_MODEL.get().objects.get(name=vehicle_name)
    except:
        pass
    if not vehicle:
        vehicle = get_default_vehicle()
    return vehicle

def lookup_flight(flight_name):
    """ Look up the flight by name
    :param flight_name: The name of the flight
    :return: the found flight, or None
    """
    flight = None
    if flight_name:
        try:
            flight = FLIGHT_MODEL.get().objects.get(name=flight_name)
        except:
            pass
    return flight

def get_or_create_flight(start_time, vehicle=None, check_flight_exists=True, end_time=None, name=None):
    """
    Get or create a flight; will create the group flight and the flights, and set the start time.
    :param start_time: the start time of the flight
    :param vehicle: the vehicle
    :param check_flight_exists: flag to check if the flight exists
    :param end_time: Not used for search, but if you want to set the end time of the newly created flight pass it here
    :param name: The name for the group flight
    :return: the flight
    """
    if not vehicle:
        vehicle = get_default_vehicle()
    flight = None
    if check_flight_exists:
        flight = getFlight(start_time, vehicle)
    if not flight:
        # There was not a valid flight, so let's make a new one.  We will make a new group flight.

        # localize the time
        if not name:
            local_start_time = start_time.astimezone(pytz.timezone(settings.TIME_ZONE))
            name = get_next_available_group_flight_name(local_start_time.strftime('%Y%m%d'))
        group_flight = create_group_flight(name)
        if group_flight:
            flights = group_flight.flights.filter(vehicle=vehicle)
            flight = flights[0]  # there should only be one

            # set its start time
            flight.start_time = start_time
            flight.end_time = end_time
            flight.save()
    else:
        flight.update_start_time(start_time)
    return flight

def get_or_create_flight_with_source_root(source_root, timestamp, end_time=None, name=None):
    """
    See if there is already a flight for this directory, or create it.
    It is on us to guarantee uniqueness (db has limitation of 255 characters for unique)
    #TODO have flexible flight naming convention and make sure that works
    :param source_root: the path that should correspond to a flight
    :param timestamp: the timestamp to be used when building flight name
    :param end_time: the end time for the flight
    :param name: the custom name for the flight (and group flight)
    :return: the flight
    """
    try:
        flight = FLIGHT_MODEL.get().objects.get(source_root__icontains=source_root)
    except ObjectDoesNotExist:
        # create it
        flight = get_or_create_flight(timestamp, check_flight_exists=False, end_time=end_time, name=name)
        flight.source_root = source_root
        flight.save()
    return flight
<|endoftext|>


<commit_before>from __future__ import division, print_function

from os.path import join, split, dirname
import os
import sys
from distutils.dep_util import newer
from distutils.msvccompiler import get_build_version as get_msvc_build_version

def needs_mingw_ftime_workaround():
    # We need the mingw workaround for _ftime if the msvc runtime version is
    # 7.1 or above and we build with mingw ...
    # ... but we can't easily detect compiler version outside distutils command
    # context, so we will need to detect in randomkit whether we build with gcc
    msver = get_msvc_build_version()
    if msver and msver >= 8:
        return True

    return False

def configuration(parent_package='',top_path=None):
    from numpy.distutils.misc_util import Configuration, get_mathlibs
    config = Configuration('random', parent_package, top_path)

    def generate_libraries(ext, build_dir):
        config_cmd = config.get_config_cmd()
        libs = get_mathlibs()
        tc = testcode_wincrypt()
        if config_cmd.try_run(tc):
            libs.append('Advapi32')
        ext.libraries.extend(libs)
        return None

    # enable unix large file support on 32 bit systems
    # (64 bit off_t, lseek -> lseek64 etc.)
    defs = [('_FILE_OFFSET_BITS', '64'),
            ('_LARGEFILE_SOURCE', '1'),
            ('_LARGEFILE64_SOURCE', '1')]
    if needs_mingw_ftime_workaround():
        defs.append(("NPY_NEEDS_MINGW_TIME_WORKAROUND", None))

    libs = []
    # Configure mtrand
    config.add_extension('mtrand',
                         sources=[join('mtrand', x) for x in
                                  ['mtrand.c', 'randomkit.c', 'initarray.c',
                                   'distributions.c']]+[generate_libraries],
                         libraries=libs,
                         depends=[join('mtrand', '*.h'),
                                  join('mtrand', '*.pyx'),
                                  join('mtrand', '*.pxi'),],
                         define_macros=defs,
                         )

    config.add_data_files(('.', join('mtrand', 'randomkit.h')))
    config.add_data_dir('tests')

    return config

def testcode_wincrypt():
    return """\
/* check to see if _WIN32 is defined */
int main(int argc, char *argv[])
{
#ifdef _WIN32
    return 0;
#else
    return 1;
#endif
}
"""

if __name__ == '__main__':
    from numpy.distutils.core import setup
    setup(configuration=configuration)
<commit_msg>BLD: try_run fails on Python 3.5b4 for Windows<commit_after>from __future__ import division, print_function

from os.path import join, split, dirname
import os
import sys
from distutils.dep_util import newer
from distutils.msvccompiler import get_build_version as get_msvc_build_version

def needs_mingw_ftime_workaround():
    # We need the mingw workaround for _ftime if the msvc runtime version is
    # 7.1 or above and we build with mingw ...
    # ... but we can't easily detect compiler version outside distutils command
    # context, so we will need to detect in randomkit whether we build with gcc
    msver = get_msvc_build_version()
    if msver and msver >= 8:
        return True

    return False

def configuration(parent_package='',top_path=None):
    from numpy.distutils.misc_util import Configuration, get_mathlibs
    config = Configuration('random', parent_package, top_path)

    def generate_libraries(ext, build_dir):
        config_cmd = config.get_config_cmd()
        libs = get_mathlibs()
        if sys.platform == 'win32':
            libs.append('Advapi32')
        ext.libraries.extend(libs)
        return None

    # enable unix large file support on 32 bit systems
    # (64 bit off_t, lseek -> lseek64 etc.)
    defs = [('_FILE_OFFSET_BITS', '64'),
            ('_LARGEFILE_SOURCE', '1'),
            ('_LARGEFILE64_SOURCE', '1')]
    if needs_mingw_ftime_workaround():
        defs.append(("NPY_NEEDS_MINGW_TIME_WORKAROUND", None))

    libs = []
    # Configure mtrand
    config.add_extension('mtrand',
                         sources=[join('mtrand', x) for x in
                                  ['mtrand.c', 'randomkit.c', 'initarray.c',
                                   'distributions.c']]+[generate_libraries],
                         libraries=libs,
                         depends=[join('mtrand', '*.h'),
                                  join('mtrand', '*.pyx'),
                                  join('mtrand', '*.pxi'),],
                         define_macros=defs,
                         )

    config.add_data_files(('.', join('mtrand', 'randomkit.h')))
    config.add_data_dir('tests')

    return config

if __name__ == '__main__':
    from numpy.distutils.core import setup
    setup(configuration=configuration)
<|endoftext|>