<commit_before>#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

# This file is ported from spark/util/StatCounter.scala
# Modified from pyspark's statcounter.py.

import copy
import math
from itertools import chain

try:
    from numpy import maximum, minimum, sqrt
except ImportError:
    maximum = max
    minimum = min
    sqrt = math.sqrt

class StatCounter(object):

    REQUIRED_FOR = {
        'mean': ('mu',),
        'sum': ('mu',),
        'min': ('minValue',),
        'max': ('maxValue',),
        'variance': ('mu', 'm2'),
        'sampleVariance': ('mu', 'm2'),
        'stdev': ('mu', 'm2'),
        'sampleStdev': ('mu', 'm2'),
        'all': ('mu', 'm2', 'minValue', 'maxValue')
    }

    def __init__(self, values=(), stats='all'):
        self.n = 0L    # Running count of our values
        self.mu = 0.0  # Running mean of our values
        self.m2 = 0.0  # Running variance numerator (sum of (x - mean)^2)
        self.maxValue = float("-inf")
        self.minValue = float("inf")

        if isinstance(stats, basestring):
            stats = [stats]
        self.requiredAttrs = frozenset(chain().from_iterable([StatCounter.REQUIRED_FOR[stat] for stat in stats]))

        for v in values:
            self.merge(v)

    # Add a value into this StatCounter, updating the internal statistics.
    def merge(self, value):
        self.n += 1
        if self.__requires('mu'):
            delta = value - self.mu
            self.mu += delta / self.n
            if self.__requires('m2'):
                self.m2 += delta * (value - self.mu)
        if self.__requires('maxValue'):
            self.maxValue = maximum(self.maxValue, value) if not self.maxValue is None else value
        if self.__requires('minValue'):
            self.minValue = minimum(self.minValue, value)

        return self

    # checks whether the passed attribute name is required to be updated in order to support the
    # statistics requested in self.requestedStats.
    def __requires(self, attrName):
        return attrName in self.requiredAttrs

    # Merge another StatCounter into this one, adding up the internal statistics.
    def mergeStats(self, other):
        if not isinstance(other, StatCounter):
            raise Exception("Can only merge Statcounters!")

        if other is self:  # reference equality holds
            self.merge(copy.deepcopy(other))  # Avoid overwriting fields in a weird order
        else:
            # accumulator should only be updated if it's valid in both statcounters:
            self.requiredAttrs = set(self.requiredAttrs).intersection(set(other.requiredAttrs))

            if self.n == 0:
                self.n = other.n
                for attrname in ('mu', 'm2', 'maxValue', 'minValue'):
                    if self.__requires(attrname):
                        setattr(self, attrname, getattr(other, attrname))

            elif other.n != 0:
                if self.__requires('mu'):
                    delta = other.mu - self.mu
                    if other.n * 10 < self.n:
                        self.mu = self.mu + (delta * other.n) / (self.n + other.n)
                    elif self.n * 10 < other.n:
                        self.mu = other.mu - (delta * self.n) / (self.n + other.n)
                    else:
                        self.mu = (self.mu * self.n + other.mu * other.n) / (self.n + other.n)

                    if self.__requires('m2'):
                        self.m2 += other.m2 + (delta * delta * self.n * other.n) / (self.n + other.n)

                if self.__requires('maxValue'):
                    self.maxValue = maximum(self.maxValue, other.maxValue)
                if self.__requires('minValue'):
                    self.minValue = minimum(self.minValue, other.minValue)

                self.n += other.n
        return self

    # Clone this StatCounter
    def copy(self):
        return copy.deepcopy(self)

    def count(self):
        return self.n

    def __checkAvail(self, statName):
        if not all(attr in self.requiredAttrs for attr in StatCounter.REQUIRED_FOR[statName]):
            raise ValueError("'%s' stat not available, must be requested at StatCounter instantiation" % statName)

    def mean(self):
        self.__checkAvail('mean')
        return self.mu

    def sum(self):
        self.__checkAvail('sum')
        return self.n * self.mu

    def min(self):
        self.__checkAvail('min')
        return self.minValue

    def max(self):
        self.__checkAvail('max')
        return self.maxValue

    # Return the variance of the values.
    def variance(self):
        self.__checkAvail('variance')
        if self.n == 0:
            return float('nan')
        else:
            return self.m2 / self.n

    #
    # Return the sample variance, which corrects for bias in estimating the variance by dividing
    # by N-1 instead of N.
    #
    def sampleVariance(self):
        self.__checkAvail('sampleVariance')
        if self.n <= 1:
            return float('nan')
        else:
            return self.m2 / (self.n - 1)

    # Return the standard deviation of the values.
    def stdev(self):
        self.__checkAvail('stdev')
        return sqrt(self.variance())

    #
    # Return the sample standard deviation of the values, which corrects for bias in estimating the
    # variance by dividing by N-1 instead of N.
    #
    def sampleStdev(self):
        self.__checkAvail('sampleStdev')
        return sqrt(self.sampleVariance())

    def __repr__(self):
        return ("(count: %s, mean: %s, stdev: %s, max: %s, min: %s, required: %s)" %
                (self.count(), self.mean(), self.stdev(), self.max(), self.min(), str(tuple(self.requiredAttrs))))
<commit_msg>fix for implicit casting to float in stats max/min methods<commit_after>#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

# This file is ported from spark/util/StatCounter.scala
# Modified from pyspark's statcounter.py.

import copy
import math
from itertools import chain

try:
    from numpy import maximum, minimum, sqrt
except ImportError:
    maximum = max
    minimum = min
    sqrt = math.sqrt

class StatCounter(object):

    REQUIRED_FOR = {
        'mean': ('mu',),
        'sum': ('mu',),
        'min': ('minValue',),
        'max': ('maxValue',),
        'variance': ('mu', 'm2'),
        'sampleVariance': ('mu', 'm2'),
        'stdev': ('mu', 'm2'),
        'sampleStdev': ('mu', 'm2'),
        'all': ('mu', 'm2', 'minValue', 'maxValue')
    }

    def __init__(self, values=(), stats='all'):
        self.n = 0L    # Running count of our values
        self.mu = 0.0  # Running mean of our values
        self.m2 = 0.0  # Running variance numerator (sum of (x - mean)^2)
        self.maxValue = None
        self.minValue = None

        if isinstance(stats, basestring):
            stats = [stats]
        self.requiredAttrs = frozenset(chain().from_iterable([StatCounter.REQUIRED_FOR[stat] for stat in stats]))

        for v in values:
            self.merge(v)

    # Add a value into this StatCounter, updating the internal statistics.
    def merge(self, value):
        self.n += 1
        if self.__requires('mu'):
            delta = value - self.mu
            self.mu += delta / self.n
            if self.__requires('m2'):
                self.m2 += delta * (value - self.mu)
        if self.__requires('maxValue'):
            self.maxValue = maximum(self.maxValue, value) if not self.maxValue is None else value
        if self.__requires('minValue'):
            self.minValue = minimum(self.minValue, value) if not self.minValue is None else value

        return self

    # checks whether the passed attribute name is required to be updated in order to support the
    # statistics requested in self.requestedStats.
    def __requires(self, attrName):
        return attrName in self.requiredAttrs

    # Merge another StatCounter into this one, adding up the internal statistics.
    def mergeStats(self, other):
        if not isinstance(other, StatCounter):
            raise Exception("Can only merge Statcounters!")

        if other is self:  # reference equality holds
            self.merge(copy.deepcopy(other))  # Avoid overwriting fields in a weird order
        else:
            # accumulator should only be updated if it's valid in both statcounters:
            self.requiredAttrs = set(self.requiredAttrs).intersection(set(other.requiredAttrs))

            if self.n == 0:
                self.n = other.n
                for attrname in ('mu', 'm2', 'maxValue', 'minValue'):
                    if self.__requires(attrname):
                        setattr(self, attrname, getattr(other, attrname))

            elif other.n != 0:
                if self.__requires('mu'):
                    delta = other.mu - self.mu
                    if other.n * 10 < self.n:
                        self.mu = self.mu + (delta * other.n) / (self.n + other.n)
                    elif self.n * 10 < other.n:
                        self.mu = other.mu - (delta * self.n) / (self.n + other.n)
                    else:
                        self.mu = (self.mu * self.n + other.mu * other.n) / (self.n + other.n)

                    if self.__requires('m2'):
                        self.m2 += other.m2 + (delta * delta * self.n * other.n) / (self.n + other.n)

                if self.__requires('maxValue'):
                    self.maxValue = maximum(self.maxValue, other.maxValue)
                if self.__requires('minValue'):
                    self.minValue = minimum(self.minValue, other.minValue)

                self.n += other.n
        return self

    # Clone this StatCounter
    def copy(self):
        return copy.deepcopy(self)

    def count(self):
        return self.n

    def __checkAvail(self, statName):
        if not all(attr in self.requiredAttrs for attr in StatCounter.REQUIRED_FOR[statName]):
            raise ValueError("'%s' stat not available, must be requested at StatCounter instantiation" % statName)

    def mean(self):
        self.__checkAvail('mean')
        return self.mu

    def sum(self):
        self.__checkAvail('sum')
        return self.n * self.mu

    def min(self):
        self.__checkAvail('min')
        return self.minValue

    def max(self):
        self.__checkAvail('max')
        return self.maxValue

    # Return the variance of the values.
    def variance(self):
        self.__checkAvail('variance')
        if self.n == 0:
            return float('nan')
        else:
            return self.m2 / self.n

    #
    # Return the sample variance, which corrects for bias in estimating the variance by dividing
    # by N-1 instead of N.
    #
    def sampleVariance(self):
        self.__checkAvail('sampleVariance')
        if self.n <= 1:
            return float('nan')
        else:
            return self.m2 / (self.n - 1)

    # Return the standard deviation of the values.
    def stdev(self):
        self.__checkAvail('stdev')
        return sqrt(self.variance())

    #
    # Return the sample standard deviation of the values, which corrects for bias in estimating the
    # variance by dividing by N-1 instead of N.
    #
    def sampleStdev(self):
        self.__checkAvail('sampleStdev')
        return sqrt(self.sampleVariance())

    def __repr__(self):
        return ("(count: %s, mean: %s, stdev: %s, max: %s, min: %s, required: %s)" %
                (self.count(), self.mean(), self.stdev(), self.max(), self.min(), str(tuple(self.requiredAttrs))))
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

#
# Based on AboutHashes in the Ruby Koans
#

from runner.koan import *

class AboutDictionaries(Koan):
    def test_creating_dictionaries(self):
        empty_dict = dict()
        self.assertEqual(dict, type(empty_dict))
        self.assertEqual(dict(), empty_dict)
        self.assertEqual(__, len(empty_dict))

    def test_dictionary_literals(self):
        empty_dict = {}
        self.assertEqual(dict, type(empty_dict))
        babel_fish = {'one': 'uno', 'two': 'dos'}
        self.assertEqual(__, len(babel_fish))

    def test_accessing_dictionaries(self):
        babel_fish = {'one': 'uno', 'two': 'dos'}
        self.assertEqual(__, babel_fish['one'])
        self.assertEqual(__, babel_fish['two'])

    def test_changing_dictionaries(self):
        babel_fish = {'one': 'uno', 'two': 'dos'}
        babel_fish['one'] = 'eins'

        expected = {'two': 'dos', 'one': __}
        self.assertEqual(expected, babel_fish)

    def test_dictionary_is_unordered(self):
        dict1 = {'one': 'uno', 'two': 'dos'}
        dict2 = {'two': 'dos', 'one': 'uno'}

        self.assertEqual(____, dict1 == dict2)

    def test_dictionary_keys_and_values(self):
        babel_fish = {'one': 'uno', 'two': 'dos'}
        self.assertEqual(__, len(babel_fish.keys()))
        self.assertEqual(__, len(babel_fish.values()))
        self.assertEqual(__, 'one' in babel_fish.keys())
        self.assertEqual(__, 'two' in babel_fish.values())
        self.assertEqual(__, 'uno' in babel_fish.keys())
        self.assertEqual(__, 'dos' in babel_fish.values())

    def test_making_a_dictionary_from_a_sequence_of_keys(self):
        cards = {}.fromkeys(
            ('red warrior', 'green elf', 'blue valkyrie', 'yellow dwarf',
             'confused looking zebra'),
            42)

        self.assertEqual(__, len(cards))
        self.assertEqual(__, cards['green elf'])
        self.assertEqual(__, cards['yellow dwarf'])
<commit_msg>AboutDictionaries Lesson<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

#
# Based on AboutHashes in the Ruby Koans
#

from runner.koan import *

class AboutDictionaries(Koan):
    def test_creating_dictionaries(self):
        empty_dict = dict()
        self.assertEqual(dict, type(empty_dict))
        self.assertEqual(dict(), empty_dict)
        self.assertEqual(0, len(empty_dict))

    def test_dictionary_literals(self):
        empty_dict = {}
        self.assertEqual(dict, type(empty_dict))
        babel_fish = {'one': 'uno', 'two': 'dos'}
        self.assertEqual(2, len(babel_fish))

    def test_accessing_dictionaries(self):
        babel_fish = {'one': 'uno', 'two': 'dos'}
        self.assertEqual('uno', babel_fish['one'])
        self.assertEqual('dos', babel_fish['two'])

    def test_changing_dictionaries(self):
        babel_fish = {'one': 'uno', 'two': 'dos'}
        babel_fish['one'] = 'eins'

        expected = {'two': 'dos', 'one': 'eins'}
        self.assertEqual(expected, babel_fish)

    def test_dictionary_is_unordered(self):
        dict1 = {'one': 'uno', 'two': 'dos'}
        dict2 = {'two': 'dos', 'one': 'uno'}

        self.assertEqual(True, dict1 == dict2)

    def test_dictionary_keys_and_values(self):
        babel_fish = {'one': 'uno', 'two': 'dos'}
        self.assertEqual(2, len(babel_fish.keys()))
        self.assertEqual(2, len(babel_fish.values()))
        self.assertEqual(True, 'one' in babel_fish.keys())
        self.assertEqual(False, 'two' in babel_fish.values())
        self.assertEqual(False, 'uno' in babel_fish.keys())
        self.assertEqual(True, 'dos' in babel_fish.values())

    def test_making_a_dictionary_from_a_sequence_of_keys(self):
        cards = {}.fromkeys(
            ('red warrior', 'green elf', 'blue valkyrie', 'yellow dwarf',
             'confused looking zebra'),
            42)

        self.assertEqual(5, len(cards))
        self.assertEqual(42, cards['green elf'])
        self.assertEqual(42, cards['yellow dwarf'])
<|endoftext|>


<commit_before>"""
cclib (http://cclib.sf.net) is (c) 2007, the cclib development team
and licensed under the LGPL (http://www.gnu.org/copyleft/lgpl.html).
"""

import cPickle as pickle
import os
import sys

import numpy

class ccData(object):
    """Class for objects containing data from cclib parsers and methods.

    Description of cclib attributes:
        aonames -- atomic orbital names (list)
        aooverlaps -- atomic orbital overlap matrix (array[2])
        atombasis -- indices of atomic orbitals on each atom (list of lists)
        atomcoords -- atom coordinates (array[3], angstroms)
        atomnos -- atomic numbers (array[1])
        charge -- net charge of the system (integer)
        ccenergies -- molecular energies with Coupled-Cluster corrections (array[2], eV)
        coreelectrons -- number of core electrons in atom pseudopotentials (array[1])
        etenergies -- energies of electronic transitions (array[1], 1/cm)
        etoscs -- oscillator strengths of electronic transitions (array[1])
        etrotats -- rotatory strengths of electronic transitions (array[1], ??)
        etsecs -- singly-excited configurations for electronic transitions (list of lists)
        etsyms -- symmetries of electronic transitions (list)
        fonames -- fragment orbital names (list)
        fooverlaps -- fragment orbital overlap matrix (array[2])
        fragnames -- names of fragments (list)
        frags -- indices of atoms in a fragment (list of lists)
        gbasis -- coefficients and exponents of Gaussian basis functions (PyQuante format)
        geotargets -- targets for convergence of geometry optimization (array[1])
        geovalues -- current values for convergence of geometry optmization (array[1])
        hessian -- elements of the force constant matrix (array[1])
        homos -- molecular orbital indices of HOMO(s) (array[1])
        mocoeffs -- molecular orbital coefficients (list of arrays[2])
        moenergies -- molecular orbital energies (list of arrays[1], eV)
        mosyms -- orbital symmetries (list of lists)
        mpenergies -- molecular electronic energies with Moller-Plesset corrections (array[2], eV)
        mult -- multiplicity of the system (integer)
        natom -- number of atoms (integer)
        nbasis -- number of basis functions (integer)
        nmo -- number of molecular orbitals (integer)
        nocoeffs -- natural orbital coefficients (array[2])
        scfenergies -- molecular electronic energies after SCF (Hartree-Fock, DFT) (array[1], eV)
        scftargets -- targets for convergence of the SCF (array[2])
        scfvalues -- current values for convergence of the SCF (list of arrays[2])
        vibdisps -- cartesian displacement vectors (array[3], delta angstrom)
        vibfreqs -- vibrational frequencies (array[1], 1/cm)
        vibirs -- IR intensities (array[1], km/mol)
        vibramans -- Raman intensities (array[1], A^4/Da)
        vibsyms -- symmetries of vibrations (list)
    (1) The term 'array' refers to a numpy array
    (2) The number of dimensions of an array is given in square brackets
    (3) Python indexes arrays/lists starting at zero, so if homos==[10], then
            the 11th molecular orbital is the HOMO
    """

    def __init__(self, attributes=None):
        """Initialize the cclibData object.

        Normally called in the parse() method of a Logfile subclass.

        Inputs:
            attributes - dictionary of attributes to load
        """

        # Names of all supported attributes.
        self._attrlist = ['aonames', 'aooverlaps', 'atombasis',
                          'atomcoords', 'atomnos',
                          'ccenergies', 'charge', 'coreelectrons',
                          'etenergies', 'etoscs', 'etrotats', 'etsecs', 'etsyms',
                          'fonames', 'fooverlaps', 'fragnames', 'frags',
                          'gbasis', 'geotargets', 'geovalues', 'grads',
                          'hessian', 'homos',
                          'mocoeffs', 'moenergies', 'mosyms', 'mpenergies', 'mult',
                          'natom', 'nbasis', 'nmo', 'nocoeffs',
                          'scfenergies', 'scftargets', 'scfvalues',
                          'vibdisps', 'vibfreqs', 'vibirs', 'vibramans', 'vibsyms']

        # The expected types for all supported attributes.
        self._attrtypes = { "aonames":        list,
                            "aooverlaps":     numpy.ndarray,
                            "atombasis":      list,
                            "atomcoords":     numpy.ndarray,
                            "atomnos":        numpy.ndarray,
                            "charge":         int,
                            "coreelectrons":  numpy.ndarray,
                            "etenergies":     numpy.ndarray,
                            "etoscs":         numpy.ndarray,
                            "etrotats":       numpy.ndarray,
                            "etsecs":         list,
                            "etsyms":         list,
                            'gbasis':         list,
                            "geotargets":     numpy.ndarray,
                            "geovalues":      numpy.ndarray,
                            "grads":          numpy.ndarray,
                            "hessian":        numpy.ndarray,
                            "homos":          numpy.ndarray,
                            "mocoeffs":       list,
                            "moenergies":     list,
                            "mosyms":         list,
                            "mpenergies":     numpy.ndarray,
                            "mult":           int,
                            "natom":          int,
                            "nbasis":         int,
                            "nmo":            int,
                            "nocoeffs":       numpy.ndarray,
                            "scfenergies":    numpy.ndarray,
                            "scftargets":     numpy.ndarray,
                            "scfvalues":      list,
                            "vibdisps":       numpy.ndarray,
                            "vibfreqs":       numpy.ndarray,
                            "vibirs":         numpy.ndarray,
                            "vibramans":      numpy.ndarray,
                            "vibsyms":        list,
                          }

        # Arrays are double precision by default, but these will be integer arrays.
        self._intarrays = ['atomnos', 'coreelectrons', 'homos']

        # Attributes that should be lists of arrays (double precision).
        self._listsofarrays = ['mocoeffs', 'moenergies', 'scfvalues']

        if attributes:
            self.setattributes(attributes)

    def listify(self):
        """Converts all attributes that are arrays or lists of arrays to lists."""

        for k, v in self._attrtypes.iteritems():
            if hasattr(self, k):
                if v == numpy.ndarray:
                    setattr(self, k, getattr(self, k).tolist())
                elif v == list and k in self._listsofarrays:
                    setattr(self, k, [x.tolist() for x in getattr(self, k)])

    def arrayify(self):
        """Converts appropriate attributes to arrays or lists of arrays."""

        for k, v in self._attrtypes.iteritems():
            if hasattr(self, k):
                precision = 'd'
                if k in self._intarrays:
                    precision = 'i'
                if v == numpy.ndarray:
                    setattr(self, k, numpy.array(getattr(self, k), precision))
                elif v == list and k in self._listsofarrays:
                    setattr(self, k, [numpy.array(x, precision)
                                      for x in getattr(self, k)])

    def getattributes(self, tolists=False):
        """Returns a dictionary of existing data attributes.

        Inputs:
            tolists - flag to convert attributes to lists where applicable
        """

        if tolists:
            self.listify()
        attributes = {}
        for attr in self._attrlist:
            if hasattr(self, attr):
                attributes[attr] = getattr(self,attr)
        if tolists:
            self.arrayofy()
        return attributes

    def setattributes(self, attributes):
        """Sets data attributes given in a dictionary.

        Inputs:
            attributes - dictionary of attributes to set
        Outputs:
            invalid - list of attributes names that were not set, which
                      means they are not specified in self._attrlist
        """

        if type(attributes) is not dict:
            raise TypeError, "attributes must be in a dictionary"

        valid = [a for a in attributes if a in self._attrlist]
        invalid = [a for a in attributes if a not in self._attrlist]

        for attr in valid:
            setattr(self, attr, attributes[attr])
        self.arrayify()
        return invalid
<commit_msg>Added atommasses to ccData.<commit_after>"""
cclib (http://cclib.sf.net) is (c) 2007, the cclib development team
and licensed under the LGPL (http://www.gnu.org/copyleft/lgpl.html).
"""

import cPickle as pickle
import os
import sys

import numpy

class ccData(object):
    """Class for objects containing data from cclib parsers and methods.

    Description of cclib attributes:
        aonames -- atomic orbital names (list)
        aooverlaps -- atomic orbital overlap matrix (array[2])
        atombasis -- indices of atomic orbitals on each atom (list of lists)
        atomcoords -- atom coordinates (array[3], angstroms)
        atommasses -- atom masses (array[1], daltons)
        atomnos -- atomic numbers (array[1])
        charge -- net charge of the system (integer)
        ccenergies -- molecular energies with Coupled-Cluster corrections (array[2], eV)
        coreelectrons -- number of core electrons in atom pseudopotentials (array[1])
        etenergies -- energies of electronic transitions (array[1], 1/cm)
        etoscs -- oscillator strengths of electronic transitions (array[1])
        etrotats -- rotatory strengths of electronic transitions (array[1], ??)
        etsecs -- singly-excited configurations for electronic transitions (list of lists)
        etsyms -- symmetries of electronic transitions (list)
        fonames -- fragment orbital names (list)
        fooverlaps -- fragment orbital overlap matrix (array[2])
        fragnames -- names of fragments (list)
        frags -- indices of atoms in a fragment (list of lists)
        gbasis -- coefficients and exponents of Gaussian basis functions (PyQuante format)
        geotargets -- targets for convergence of geometry optimization (array[1])
        geovalues -- current values for convergence of geometry optmization (array[1])
        hessian -- elements of the force constant matrix (array[1])
        homos -- molecular orbital indices of HOMO(s) (array[1])
        mocoeffs -- molecular orbital coefficients (list of arrays[2])
        moenergies -- molecular orbital energies (list of arrays[1], eV)
        mosyms -- orbital symmetries (list of lists)
        mpenergies -- molecular electronic energies with Moller-Plesset corrections (array[2], eV)
        mult -- multiplicity of the system (integer)
        natom -- number of atoms (integer)
        nbasis -- number of basis functions (integer)
        nmo -- number of molecular orbitals (integer)
        nocoeffs -- natural orbital coefficients (array[2])
        scfenergies -- molecular electronic energies after SCF (Hartree-Fock, DFT) (array[1], eV)
        scftargets -- targets for convergence of the SCF (array[2])
        scfvalues -- current values for convergence of the SCF (list of arrays[2])
        vibdisps -- cartesian displacement vectors (array[3], delta angstrom)
        vibfreqs -- vibrational frequencies (array[1], 1/cm)
        vibirs -- IR intensities (array[1], km/mol)
        vibramans -- Raman intensities (array[1], A^4/Da)
        vibsyms -- symmetries of vibrations (list)
    (1) The term 'array' refers to a numpy array
    (2) The number of dimensions of an array is given in square brackets
    (3) Python indexes arrays/lists starting at zero, so if homos==[10], then
            the 11th molecular orbital is the HOMO
    """

    def __init__(self, attributes=None):
        """Initialize the cclibData object.

        Normally called in the parse() method of a Logfile subclass.

        Inputs:
            attributes - dictionary of attributes to load
        """

        # Names of all supported attributes.
        self._attrlist = ['aonames', 'aooverlaps', 'atombasis',
                          'atomcoords', 'atommasses', 'atomnos',
                          'ccenergies', 'charge', 'coreelectrons',
                          'etenergies', 'etoscs', 'etrotats', 'etsecs', 'etsyms',
                          'fonames', 'fooverlaps', 'fragnames', 'frags',
                          'gbasis', 'geotargets', 'geovalues', 'grads',
                          'hessian', 'homos',
                          'mocoeffs', 'moenergies', 'mosyms', 'mpenergies', 'mult',
                          'natom', 'nbasis', 'nmo', 'nocoeffs',
                          'scfenergies', 'scftargets', 'scfvalues',
                          'vibdisps', 'vibfreqs', 'vibirs', 'vibramans', 'vibsyms']

        # The expected types for all supported attributes.
        self._attrtypes = { "aonames":        list,
                            "aooverlaps":     numpy.ndarray,
                            "atombasis":      list,
                            "atomcoords":     numpy.ndarray,
                            "atommasses":     numpy.ndarray,
                            "atomnos":        numpy.ndarray,
                            "charge":         int,
                            "coreelectrons":  numpy.ndarray,
                            "etenergies":     numpy.ndarray,
                            "etoscs":         numpy.ndarray,
                            "etrotats":       numpy.ndarray,
                            "etsecs":         list,
                            "etsyms":         list,
                            'gbasis':         list,
                            "geotargets":     numpy.ndarray,
                            "geovalues":      numpy.ndarray,
                            "grads":          numpy.ndarray,
                            "hessian":        numpy.ndarray,
                            "homos":          numpy.ndarray,
                            "mocoeffs":       list,
                            "moenergies":     list,
                            "mosyms":         list,
                            "mpenergies":     numpy.ndarray,
                            "mult":           int,
                            "natom":          int,
                            "nbasis":         int,
                            "nmo":            int,
                            "nocoeffs":       numpy.ndarray,
                            "scfenergies":    numpy.ndarray,
                            "scftargets":     numpy.ndarray,
                            "scfvalues":      list,
                            "vibdisps":       numpy.ndarray,
                            "vibfreqs":       numpy.ndarray,
                            "vibirs":         numpy.ndarray,
                            "vibramans":      numpy.ndarray,
                            "vibsyms":        list,
                          }

        # Arrays are double precision by default, but these will be integer arrays.
        self._intarrays = ['atomnos', 'coreelectrons', 'homos']

        # Attributes that should be lists of arrays (double precision).
        self._listsofarrays = ['mocoeffs', 'moenergies', 'scfvalues']

        if attributes:
            self.setattributes(attributes)

    def listify(self):
        """Converts all attributes that are arrays or lists of arrays to lists."""

        for k, v in self._attrtypes.iteritems():
            if hasattr(self, k):
                if v == numpy.ndarray:
                    setattr(self, k, getattr(self, k).tolist())
                elif v == list and k in self._listsofarrays:
                    setattr(self, k, [x.tolist() for x in getattr(self, k)])

    def arrayify(self):
        """Converts appropriate attributes to arrays or lists of arrays."""

        for k, v in self._attrtypes.iteritems():
            if hasattr(self, k):
                precision = 'd'
                if k in self._intarrays:
                    precision = 'i'
                if v == numpy.ndarray:
                    setattr(self, k, numpy.array(getattr(self, k), precision))
                elif v == list and k in self._listsofarrays:
                    setattr(self, k, [numpy.array(x, precision)
                                      for x in getattr(self, k)])

    def getattributes(self, tolists=False):
        """Returns a dictionary of existing data attributes.

        Inputs:
            tolists - flag to convert attributes to lists where applicable
        """

        if tolists:
            self.listify()
        attributes = {}
        for attr in self._attrlist:
            if hasattr(self, attr):
                attributes[attr] = getattr(self,attr)
        if tolists:
            self.arrayofy()
        return attributes

    def setattributes(self, attributes):
        """Sets data attributes given in a dictionary.

        Inputs:
            attributes - dictionary of attributes to set
        Outputs:
            invalid - list of attributes names that were not set, which
                      means they are not specified in self._attrlist
        """

        if type(attributes) is not dict:
            raise TypeError, "attributes must be in a dictionary"

        valid = [a for a in attributes if a in self._attrlist]
        invalid = [a for a in attributes if a not in self._attrlist]

        for attr in valid:
            setattr(self, attr, attributes[attr])
        self.arrayify()
        return invalid
<|endoftext|>


<commit_before>import inspect
from itertools import dropwhile, islice
import logging
import sys

import docker
import yaml

client = docker.from_env()

logging.basicConfig(level=logging.INFO)
log = logging.getLogger()

def compose_project():
    import compose.cli.main as compose_main
    from compose.cli.command import project_from_options
    from docopt import docopt

    # Forward all args after "--"
    forwards = list(islice(dropwhile(lambda x: x != '--', sys.argv), 1, None))
    compose_opts = docopt(
        inspect.getdoc(compose_main.TopLevelCommand),
        argv=forwards,
        help=False)
    return project_from_options('.', compose_opts)

def project_containers(project):
    return tuple(c for c in client.containers.list() if
                 c.labels.get('com.docker.compose.project', None) == project)

def project_networks(project):
    return tuple(
        n for n in client.networks.list() if
        n.attrs['Labels'].get('com.docker.compose.project', None) == project)

def interface_in_net(c, n):
    """ Get interface of container c attached to net n. """
    mac = n.attrs['Containers'][c.id]['MacAddress']
    devices = c.exec_run(['ls', '-1', '/sys/class/net']).decode()
    addrs = c.exec_run(['sh', '-c', 'cat /sys/class/net/*/address']).decode()
    addr_to_dev = dict(zip(addrs.splitlines(), devices.splitlines()))
    return addr_to_dev[mac]

def replace_netem(c, n, cmd):
    """ Apply netem on container c for network n using command cmd. """
    result = c.exec_run(
        ['tc', 'qdisc', 'replace', 'dev', interface_in_net(c, n), 'root',
         'netem'] + cmd.split(' '))
    if result:
        raise RuntimeError(result.decode())

if __name__ == '__main__':
    with open('netz.yml') as f:
        config = yaml.safe_load(f.read())

    name = compose_project().name
    log.info('Using project name "{}".'.format(name))

    nets = {n.attrs['Labels']['com.docker.compose.network']: n for n in
            project_networks(name)}
    cons = {c.labels['com.docker.compose.service']: c for c in
            project_containers(name)}

    for net_name, net_cfg in config.items():
        try:
            net = nets[net_name]
        except KeyError as e:
            log.error('Network "{}" not found.'.format(net_name))
            exit(1)
        for con_name, cmd in net_cfg.items():
            try:
                con = cons[con_name]
            except KeyError as e:
                log.error('Container "{}" not found.'.format(net_name))
                exit(1)
            try:
                replace_netem(con, net, cmd)
            except RuntimeError as e:
                log.error(('Error applying "{}" to container "{}" in network '
                           '"{}": {}').format(cmd, con_name, net_name, e))
                exit(1)
<commit_msg>Moar comments.<commit_after>import inspect
from itertools import dropwhile, islice
import logging
import sys

import docker
import yaml

client = docker.from_env()

logging.basicConfig(level=logging.INFO)
log = logging.getLogger()

def compose_project():
    import compose.cli.main as compose_main
    from compose.cli.command import project_from_options
    from docopt import docopt

    # Forward all args after "--".
    forwards = list(islice(dropwhile(lambda x: x != '--', sys.argv), 1, None))
    # Parse into a Docker Compose project.
    compose_opts = docopt(
        inspect.getdoc(compose_main.TopLevelCommand),
        argv=forwards,
        help=False)
    return project_from_options('.', compose_opts)

def project_containers(project):
    """ Get all containers that belong to a Compose project. """
    return tuple(c for c in client.containers.list() if
                 c.labels.get('com.docker.compose.project', None) == project)

def project_networks(project):
    """ Get all networks that belong to a Compose project. """
    return tuple(
        n for n in client.networks.list() if
        n.attrs['Labels'].get('com.docker.compose.project', None) == project)

def interface_in_net(c, n):
    """ Get interface at which container c is attached to network n. """
    # Container MAC address in network.
    mac = n.attrs['Containers'][c.id]['MacAddress']
    # List of all network interfaces on the container.
    devices = c.exec_run(['ls', '-1', '/sys/class/net']).decode()
    # List of their MAC addresses.
    addrs = c.exec_run(['sh', '-c', 'cat /sys/class/net/*/address']).decode()
    # Build MAC->interface dict and get the interface we're looking for.
    addr_to_dev = dict(zip(addrs.splitlines(), devices.splitlines()))
    return addr_to_dev[mac]

def replace_netem(c, n, cmd):
    """ Apply netem on container c for network n using command cmd. """
    result = c.exec_run(
        ['tc', 'qdisc', 'replace', 'dev', interface_in_net(c, n), 'root',
         'netem'] + cmd.split(' '))
    if result:
        raise RuntimeError(result.decode())

if __name__ == '__main__':
    with open('netz.yml') as f:
        config = yaml.safe_load(f.read())

    name = compose_project().name
    log.info('Using project name "{}".'.format(name))

    # name->instance for all networks in the Compose project.
    nets = {n.attrs['Labels']['com.docker.compose.network']: n for n in
            project_networks(name)}
    # Ditto for containers.
    cons = {c.labels['com.docker.compose.service']: c for c in
            project_containers(name)}

    # Iterate top level (network names).
    for net_name, net_cfg in config.items():
        # Get network instance by name.
        try:
            net = nets[net_name]
        except KeyError as e:
            log.error('Network "{}" not found.'.format(net_name))
            exit(1)
        # Iterate second level (container name -> netem options)
        for con_name, cmd in net_cfg.items():
            # Get container instance by name.
            try:
                con = cons[con_name]
            except KeyError as e:
                log.error('Container "{}" not found.'.format(net_name))
                exit(1)
            # Execute the command on the container.
            try:
                replace_netem(con, net, cmd)
            except RuntimeError as e:
                log.error(('Error applying "{}" to container "{}" in network '
                           '"{}": {}').format(cmd, con_name, net_name, e))
                exit(1)
<|endoftext|>


<commit_before>import logging
import os

import jinja2

from pyvivado import vivado_project

logger = logging.getLogger(__name__)

def make_jtagtestbench(top_entity, generics):
    # Make records for inputs and outputs.
    template_fn = os.path.join(os.path.dirname(__file__), 'templates', 'jtag_testbench.vhd')
    with open(template_fn, 'r') as f:
        filetestbench_template = jinja2.Template(f.read())
    jtagtestbench = filetestbench_template.render(
        dut_name=top_entity,
        dut_generics=generics,
        )
    return jtagtestbench

def prepare_files(directory, filenames, top_entity, generics):
    jtb = make_jtagtestbench(top_entity, generics)
    jtb_fn = os.path.join(directory, '{}_jtag.vhd'.format(top_entity))
    with open(jtb_fn, 'w') as f:
        f.write(jtb)
    new_fns = []
    new_fns.append(jtb_fn)
    return new_fns

def get_files_and_ip(directory, filenames, top_entity, generics, board_params,
                     frequency):
    new_fns = prepare_files(directory, filenames, top_entity, generics)
    xdc_file = board_params['xdc_filename']
    ips = (
        ('clk_wiz', {
            'PRIM_IN_FREQ': board_params['clock_frequency'],
            'PRIM_SOURCE': board_params['clock_type'],
            'CLKOUT1_REQUESTED_OUT_FREQ': frequency,
        }, 'clk_wiz_0'),
        ('jtag_axi', {
            'PROTOCOL': 2,
        }, 'jtag_axi_0'),
        )
    files_and_ip = {
        'design_files': new_fns + filenames + [xdc_file],
        'simulation_files': [],
        'ips': ips,
        'top_module': top_entity + '_jtag'
        }
    return files_and_ip

if __name__ == '__main__':
    from slvcodec import config as slvcodec_config
    import axilent
    from axilent.examples import axi_adder
    work_root = '/home/ben/Code/local/deleteme_test_axi_adder'
    corename = 'axi_adder'
    entityname = 'axi_adder'
    slvcodec_config.setup_fusesoc([axilent.coresdir])

    p = vivado_project.VivadoProject.from_fusesoc_core(
        directory=work_root,
        corename=corename,
        entityname=entityname,
        generics={},
        top_params={},
        boardname='xilinx:vc709',
        frequency=100,
        overwrite_ok=True,
    )
    tests = [axi_adder.AxiAdderTest()]
    p.implement_deploy_and_run_tests(tests)
<commit_msg>Remove circulant dependency.<commit_after>import logging
import os

import jinja2

logger = logging.getLogger(__name__)

def make_jtagtestbench(top_entity, generics):
    # Make records for inputs and outputs.
    template_fn = os.path.join(os.path.dirname(__file__), 'templates', 'jtag_testbench.vhd')
    with open(template_fn, 'r') as f:
        filetestbench_template = jinja2.Template(f.read())
    jtagtestbench = filetestbench_template.render(
        dut_name=top_entity,
        dut_generics=generics,
        )
    return jtagtestbench

def prepare_files(directory, filenames, top_entity, generics):
    jtb = make_jtagtestbench(top_entity, generics)
    jtb_fn = os.path.join(directory, '{}_jtag.vhd'.format(top_entity))
    with open(jtb_fn, 'w') as f:
        f.write(jtb)
    new_fns = []
    new_fns.append(jtb_fn)
    return new_fns

def get_files_and_ip(directory, filenames, top_entity, generics, board_params,
                     frequency):
    new_fns = prepare_files(directory, filenames, top_entity, generics)
    xdc_file = board_params['xdc_filename']
    ips = (
        ('clk_wiz', {
            'PRIM_IN_FREQ': board_params['clock_frequency'],
            'PRIM_SOURCE': board_params['clock_type'],
            'CLKOUT1_REQUESTED_OUT_FREQ': frequency,
        }, 'clk_wiz_0'),
        ('jtag_axi', {
            'PROTOCOL': 2,
        }, 'jtag_axi_0'),
        )
    files_and_ip = {
        'design_files': new_fns + filenames + [xdc_file],
        'simulation_files': [],
        'ips': ips,
        'top_module': top_entity + '_jtag'
        }
    return files_and_ip

if __name__ == '__main__':
    from slvcodec import config as slvcodec_config
    import axilent
    from axilent.examples import axi_adder
    work_root = '/home/ben/Code/local/deleteme_test_axi_adder'
    corename = 'axi_adder'
    entityname = 'axi_adder'
    slvcodec_config.setup_fusesoc([axilent.coresdir])

    from pyvivado import vivado_project
    p = vivado_project.VivadoProject.from_fusesoc_core(
        directory=work_root,
        corename=corename,
        entityname=entityname,
        generics={},
        top_params={},
        boardname='xilinx:vc709',
        frequency=100,
        overwrite_ok=True,
    )
    tests = [axi_adder.AxiAdderTest()]
    p.implement_deploy_and_run_tests(tests)
<|endoftext|>


<commit_before>from abc import ABC, ABCMeta, abstractmethod
from collections import OrderedDict
from copy import deepcopy
from itertools import chain
from types import MappingProxyType

class MetaQuery(type):
    """
    Metaclass for all Queries
    """
    def __new__(mcs, name, bases, attrs):
        cls = super(MetaQuery, mcs).__new__(mcs, name, bases, attrs)
        try:
            Model = attrs['Model']
        except KeyError:
            for base in bases:
                if hasattr(base, 'Model'):
                    Model = base.Model
                    break
            else:
                raise TypeError('Query without Model!')

        if Model is not None:
            Model.Query = cls

        cls._settings_defaults = OrderedDict()
        for base in cls.__bases__:
            cls._settings_defaults.update(getattr(base, '_settings_defaults', {}))
        if '_settings_defaults' in attrs:
            cls._settings_defaults.update(attrs['_settings_defaults'])

        return cls

class QuerySettingsProxy:
    """
    Proxy for Query settings that allows read-only access (used bei Query.settings)
    """
    def __init__(self, settings):
        self._settings = settings

    def __setattr__(self, name, value):
        if name != '_settings' or hasattr(self, name):
            raise TypeError('Can not set settings directly, set them using methods!')
        super().__setattr__(name, value)

    def __getattr__(self, name):
        try:
            return self._settings[name]
        except KeyError:
            raise AttributeError

    def __delattr__(self, name):
        raise TypeError

class Query(metaclass=MetaQuery):
    """
    A Query for a specific Model.

    A Query has settings and all attributes of its model.
    Neither of these values ever change. All methods by which the query can be altered return a new query.
    It can be executed using .execute() or by accessing its results like an iterable.
    """
    Model = None
    _settings_defaults = {'limit': None}

    def __init__(self, api):
        if self.__class__ == Query:
            raise TypeError('only subclasses of Query can be initialised')

        if isinstance(self, BoundAPIQuery):
            if not isinstance(api, self.API):
                raise ValueError('api has to be an %s subclass, got %s instead' % repr(self.API), repr(api))
        else:
            if api is not None:
                raise ValueError('api has to be None, got %s instead.' % repr(api))

        self.api = api
        self._obj = self.Model()
        self._settings = self._settings_defaults.copy()
        self._cached_results = []
        self._results_generator = None
        self._results_done = False

    def copy(self):
        """
        Returns a deep copy of the query.
        """
        result = self.__class__(self.api)
        result._obj = deepcopy(self._obj)
        result._settings = self._settings
        return result

    def where(self, **kwargs):
        """
        Returns a new Query with the Model attributes updated.

        Example:
        >>> query.where(name='Berlin Hbf').execute()
        """
        result = self.copy()

        for name, value in kwargs.items():
            if name not in self.Model._fields:
                raise TypeError('invalid field: %s.%s' % (self.Model.__name__, name))

            setattr(result._obj, name, value)

        return result

    def serialize(self):
        return {
            'api': self.api.serialize() if self.api else None,
            'obj': self._obj.serialize(),
            'settings': self._settings,
        }

    @classmethod
    def unserialize(cls, data):
        from ..apis.base import API
        result = cls(API.unserialize(data.get('api')))
        result._obj = cls.Model.unserialize(data.get('obj', {}))
        for name, value in data.get('settings', {}).items():
            result = getattr(result, name)(value)
        return result

    @property
    def settings(self):
        """
        Returns the current query settings as a read-only QuerySettingsProxy object.

        Example:
        >>> query.settings.limit
        """
        return MappingProxyType(self._settings)

    def get(self, obj):
        """
        Retrieve the given object from the API.
        Returns the retrieved object or raises Model.NotFound.

        Example:
        >>> try:
        ...     query.get(Stop(name='Essen Hbf'))
        ... except Stop.NotFound:
        ...     pass
        """
        if not isinstance(obj, self.Model):
            raise TypeError('Expected %s instance, got %s' % (self.Model.__name__, repr(obj)))

        result = self.copy()
        result._obj = deepcopy(obj)
        r = result.limit(1).execute()
        if not r:
            raise self.Model.NotFound
        return next(iter(r))

    def _execute(self):
        """
        This is the only method that an APIs should overwrite.
        It has to return iterable (or generator) over instances of the query's model.
        Some Query types (e.g. GeoLocationQuery) require different result types.
        """
        raise TypeError('Cannot execute query not bound to an API')

    def limit(self, limit):
        """
        Set the maximum number of results. Returns a new query.
        None means unlimited (although APIs can have an internal limit)
        """
        if limit is not None and (not isinstance(limit, int) or limit < 1):
            raise TypeError('limit has to be None or int >= 1')
        self._update_setting('limit', limit)
        return self

    def _update_setting(self, name, value):
        result = self.copy()
        result._settings[name] = value
        return result

    def execute(self):
        """
        Execute the query. Returns the query itself.
        If the Query was already executed, nothing happens.
        """
        if self._results_generator is None:
            self._results_generator = self._execute()
        return self

    def _full_iter(self):
        """
        Returns an iterator over the query results.
        Used by __iter__() which is overwritten by some Queries which supply metadata, like any GeoPointQuery.
        """
        self.execute()

        if self._results_done:
            return iter(self._cached_results)

        return chain(self._cached_results, self._next_result())

    def __iter__(self):
        """
        Iterate over the query results. Each item is _alwayss_ a subclass of the query's Model.
        """
        return self._full_iter

    def _next_result(self):
        """
        Iterate over the results generator. Used by _full_iter().
        """
        for result in self._results_generator:
            self._cached_results.append(result)
            yield result
        self._results_done = True

    def __getattr__(self, name):
        """
        Deliver attribute values of the underlying object.
        """
        if name in self.Model._fields:
            return getattr(self._obj, name)

        if name in self.__class__._settings_defaults:
            raise TypeError('Use .settings to get settings')

        raise AttributeError(name)

    def __setattr__(self, name, value):
        """
        Raise correct exceptions if someone tries to set settings or model attributes.
        """
        if name in self.Model._fields:
            raise TypeError('Can not set fields, use .where()')

        if name in self._settings_defaults:
            raise TypeError('Can not set settings directly, set them using their methods')

        super().__setattr__(name, value)

    def __delattr__(self, name):
        """
        Raise correct exceptions if some tries to delete settings or model attributes.
        """
        if name in self.Model._fields:
            raise TypeError('Can not delete fields, use .where(%s=None)' % name)

        if name in self._settings_defaults:
            raise TypeError('Can not delete settings')

        super().__delattr__(name)

class MetaBoundAPIQuery(ABCMeta, MetaQuery):
    pass

class BoundAPIQuery(ABC, Query, metaclass=MetaBoundAPIQuery):
    API = None

    @abstractmethod
    def _execute(self):
        pass
<commit_msg>serialize Query to ordereddict<commit_after>from abc import ABC, ABCMeta, abstractmethod
from collections import OrderedDict
from copy import deepcopy
from itertools import chain
from types import MappingProxyType

class MetaQuery(type):
    """
    Metaclass for all Queries
    """
    def __new__(mcs, name, bases, attrs):
        cls = super(MetaQuery, mcs).__new__(mcs, name, bases, attrs)
        try:
            Model = attrs['Model']
        except KeyError:
            for base in bases:
                if hasattr(base, 'Model'):
                    Model = base.Model
                    break
            else:
                raise TypeError('Query without Model!')

        if Model is not None:
            Model.Query = cls

        cls._settings_defaults = OrderedDict()
        for base in cls.__bases__:
            cls._settings_defaults.update(getattr(base, '_settings_defaults', {}))
        if '_settings_defaults' in attrs:
            cls._settings_defaults.update(attrs['_settings_defaults'])

        return cls

class QuerySettingsProxy:
    """
    Proxy for Query settings that allows read-only access (used bei Query.settings)
    """
    def __init__(self, settings):
        self._settings = settings

    def __setattr__(self, name, value):
        if name != '_settings' or hasattr(self, name):
            raise TypeError('Can not set settings directly, set them using methods!')
        super().__setattr__(name, value)

    def __getattr__(self, name):
        try:
            return self._settings[name]
        except KeyError:
            raise AttributeError

    def __delattr__(self, name):
        raise TypeError

class Query(metaclass=MetaQuery):
    """
    A Query for a specific Model.

    A Query has settings and all attributes of its model.
    Neither of these values ever change. All methods by which the query can be altered return a new query.
    It can be executed using .execute() or by accessing its results like an iterable.
    """
    Model = None
    _settings_defaults = {'limit': None}

    def __init__(self, api):
        if self.__class__ == Query:
            raise TypeError('only subclasses of Query can be initialised')

        if isinstance(self, BoundAPIQuery):
            if not isinstance(api, self.API):
                raise ValueError('api has to be an %s subclass, got %s instead' % repr(self.API), repr(api))
        else:
            if api is not None:
                raise ValueError('api has to be None, got %s instead.' % repr(api))

        self.api = api
        self._obj = self.Model()
        self._settings = self._settings_defaults.copy()
        self._cached_results = []
        self._results_generator = None
        self._results_done = False

    def copy(self):
        """
        Returns a deep copy of the query.
        """
        result = self.__class__(self.api)
        result._obj = deepcopy(self._obj)
        result._settings = self._settings
        return result

    def where(self, **kwargs):
        """
        Returns a new Query with the Model attributes updated.

        Example:
        >>> query.where(name='Berlin Hbf').execute()
        """
        result = self.copy()

        for name, value in kwargs.items():
            if name not in self.Model._fields:
                raise TypeError('invalid field: %s.%s' % (self.Model.__name__, name))

            setattr(result._obj, name, value)

        return result

    def serialize(self):
        return OrderedDict((
            ('api', self.api.serialize() if self.api else None),
            ('obj', self._obj.serialize()),
            ('settings', self._settings),
        ))

    @classmethod
    def unserialize(cls, data):
        from ..apis.base import API
        result = cls(API.unserialize(data.get('api')))
        result._obj = cls.Model.unserialize(data.get('obj', {}))
        for name, value in data.get('settings', {}).items():
            result = getattr(result, name)(value)
        return result

    @property
    def settings(self):
        """
        Returns the current query settings as a read-only QuerySettingsProxy object.

        Example:
        >>> query.settings.limit
        """
        return MappingProxyType(self._settings)

    def get(self, obj):
        """
        Retrieve the given object from the API.
        Returns the retrieved object or raises Model.NotFound.

        Example:
        >>> try:
        ...     query.get(Stop(name='Essen Hbf'))
        ... except Stop.NotFound:
        ...     pass
        """
        if not isinstance(obj, self.Model):
            raise TypeError('Expected %s instance, got %s' % (self.Model.__name__, repr(obj)))

        result = self.copy()
        result._obj = deepcopy(obj)
        r = result.limit(1).execute()
        if not r:
            raise self.Model.NotFound
        return next(iter(r))

    def _execute(self):
        """
        This is the only method that an APIs should overwrite.
        It has to return iterable (or generator) over instances of the query's model.
        Some Query types (e.g. GeoLocationQuery) require different result types.
        """
        raise TypeError('Cannot execute query not bound to an API')

    def limit(self, limit):
        """
        Set the maximum number of results. Returns a new query.
        None means unlimited (although APIs can have an internal limit)
        """
        if limit is not None and (not isinstance(limit, int) or limit < 1):
            raise TypeError('limit has to be None or int >= 1')
        self._update_setting('limit', limit)
        return self

    def _update_setting(self, name, value):
        result = self.copy()
        result._settings[name] = value
        return result

    def execute(self):
        """
        Execute the query. Returns the query itself.
        If the Query was already executed, nothing happens.
        """
        if self._results_generator is None:
            self._results_generator = self._execute()
        return self

    def _full_iter(self):
        """
        Returns an iterator over the query results.
        Used by __iter__() which is overwritten by some Queries which supply metadata, like any GeoPointQuery.
        """
        self.execute()

        if self._results_done:
            return iter(self._cached_results)

        return chain(self._cached_results, self._next_result())

    def __iter__(self):
        """
        Iterate over the query results. Each item is _alwayss_ a subclass of the query's Model.
        """
        return self._full_iter

    def _next_result(self):
        """
        Iterate over the results generator. Used by _full_iter().
        """
        for result in self._results_generator:
            self._cached_results.append(result)
            yield result
        self._results_done = True

    def __getattr__(self, name):
        """
        Deliver attribute values of the underlying object.
        """
        if name in self.Model._fields:
            return getattr(self._obj, name)

        if name in self.__class__._settings_defaults:
            raise TypeError('Use .settings to get settings')

        raise AttributeError(name)

    def __setattr__(self, name, value):
        """
        Raise correct exceptions if someone tries to set settings or model attributes.
        """
        if name in self.Model._fields:
            raise TypeError('Can not set fields, use .where()')

        if name in self._settings_defaults:
            raise TypeError('Can not set settings directly, set them using their methods')

        super().__setattr__(name, value)

    def __delattr__(self, name):
        """
        Raise correct exceptions if some tries to delete settings or model attributes.
        """
        if name in self.Model._fields:
            raise TypeError('Can not delete fields, use .where(%s=None)' % name)

        if name in self._settings_defaults:
            raise TypeError('Can not delete settings')

        super().__delattr__(name)

class MetaBoundAPIQuery(ABCMeta, MetaQuery):
    pass

class BoundAPIQuery(ABC, Query, metaclass=MetaBoundAPIQuery):
    API = None

    @abstractmethod
    def _execute(self):
        pass
<|endoftext|>


<commit_before>from pyinsect.documentModel.representations.DocumentNGramGraph import DocumentNGramGraph

class DocumentNGramHSubGraph(DocumentNGramGraph):
    def __init__(self, n=3, Dwin=2, Data=[], GPrintVerbose=True):
        self.lookup_table = []

        super().__init__(n=n, Dwin=Dwin, Data=Data, GPrintVerbose=GPrintVerbose)

    def buildGraph(self, verbose=False, d=[]):
        super().buildGraph(verbose=verbose, d=d)

        for node in self._Graph.nodes:
            subset = [node] + list(self._Graph.neighbors(node))
            subgraph = self._Graph.subgraph(subset)
            self.lookup_table.append(subgraph)

        return self._Graph

    def addEdgeInc(self, a, b, w=1):
        # A = repr(a)#str(a)
        # B = repr(b)#str(b)
        # merging can also be done in other ways
        # add an extra class variable
        A, B = tuple(a), tuple(b)
        if (A, B) in self._edges:
            edata = self._Graph.get_edge_data(A, B)
            # DEBUG LINES
            # print "updating edge between (",A,B,")"
            # print "to weight",(edata['weight']+1)

            r = edata["weight"] + w
        else:
            # DEBUG LINES
            # print "adding edge between (",A,B,")"

            r = w
        # update/add edge weight
        self.setEdge(A, B, r)

class DocumentNGramHGraph(DocumentNGramHSubGraph):
    def __init__(self, levels, n=3, Dwin=2, Data=[], GPrintVerbose=True):
        self.levels = levels
        self.original_data = Data

        super().__init__(n=n, Dwin=Dwin, Data=Data, GPrintVerbose=GPrintVerbose)

    def buildGraph(self, verbose=False, d=[]):
        sequence = self._Data
        for level in range(1, self.levels):
            window_size = int(self._Dwin * level)

            document_n_gram_graph = DocumentNGramHSubGraph(
                n=self._n, Dwin=window_size, Data=sequence, GPrintVerbose=True
            )

            sequence = document_n_gram_graph.lookup_table

        self._Dwin = int(self._Dwin * (level + 1))
        super().buildGraph(verbose=verbose, d=sequence)

        return self._Graph
<commit_msg>feat: Added HPG subgraph array<commit_after>from functools import reduce

from pyinsect.documentModel.representations.DocumentNGramGraph import DocumentNGramGraph

class DocumentNGramHSubGraph(DocumentNGramGraph):
    def __init__(self, n=3, Dwin=2, Data=[], GPrintVerbose=True):
        self.lookup_table = []

        super().__init__(n=n, Dwin=Dwin, Data=Data, GPrintVerbose=GPrintVerbose)

    def buildGraph(self, verbose=False, d=[]):
        super().buildGraph(verbose=verbose, d=d)

        for node in self._Graph.nodes:
            subset = [node] + list(self._Graph.neighbors(node))
            subgraph = self._Graph.subgraph(subset)
            self.lookup_table.append(subgraph)

        return self._Graph

    def addEdgeInc(self, a, b, w=1):
        # A = repr(a)#str(a)
        # B = repr(b)#str(b)
        # merging can also be done in other ways
        # add an extra class variable
        A, B = tuple(a), tuple(b)
        if (A, B) in self._edges:
            edata = self._Graph.get_edge_data(A, B)
            # DEBUG LINES
            # print "updating edge between (",A,B,")"
            # print "to weight",(edata['weight']+1)

            r = edata["weight"] + w
        else:
            # DEBUG LINES
            # print "adding edge between (",A,B,")"

            r = w
        # update/add edge weight
        self.setEdge(A, B, r)

class DocumentNGramHGraph(DocumentNGramHSubGraph):
    def __init__(self, levels, n=3, Dwin=2, Data=[], GPrintVerbose=True):
        self.levels = levels
        self.original_data = Data
        self.subgraphs = []

        super().__init__(n=n, Dwin=Dwin, Data=Data, GPrintVerbose=GPrintVerbose)

    def buildGraph(self, verbose=False, d=[]):
        sequence = self._Data
        for level in range(1, self.levels):
            window_size = int(self._Dwin * level)

            document_n_gram_graph = DocumentNGramHSubGraph(
                n=self._n, Dwin=window_size, Data=sequence, GPrintVerbose=True
            )

            self.subgraphs.append(document_n_gram_graph)
            sequence = document_n_gram_graph.lookup_table

        if self.levels > 1:
            self._Dwin = int(self._Dwin * (level + 1))

        self.subgraphs = self.subgraphs[::-1]

        return super().buildGraph(verbose=verbose, d=sequence)
<|endoftext|>


<commit_before>import codecs
import os
import re
from django.core.management.base import LabelCommand
from django.conf import settings

class Command(LabelCommand):
    help = ("Goes through the collected LESS files after collectstatic is run "
            "and makes sure the relative paths in the imports that reference "
            "../../../../../style/static/style/less/ are changed to "
            "../../style/less/ in order to match the new directory structure of"
            " the collected static files so that less compiles properly.")
    args = ""

    def handle(self, *args, **options):
        # all the less files collected during collectstatic
        collectedless = [os.path.join(root, file)
                         for root, dir, files in os.walk(settings.STATIC_ROOT)
                         for file in files if file.endswith('.less')]
        for less_file in collectedless:
            with codecs.open(less_file, 'r', 'utf-8') as fd:
                content = fd.read()
                if content is not None:
                    B3_REGEX = r"@\{b3-import-[a-z]+-[a-z]+\}"
                    p = re.search(B3_REGEX, content)
                    if p is not None:
                        path_def = p.group(0)
                        path_file = path_def.split('-')[-2]
                        new_path = settings.LESS_B3_PATHS.get(path_file)
                        print ("[CORRECTING] %(f_name)s path in %(less_file)s: "
                               "%(new_path)s" % {
                                   'f_name': path_file,
                                   'less_file': less_file,
                                   'new_path': new_path,
                               })
                        if new_path is not None:
                            content = re.sub(B3_REGEX, new_path, content)
                        else:
                            content = None
                    else:
                        content = None
            if content is not None:
                with open(less_file, 'w') as fd:
                    fd.write(content)
<commit_msg>pr response<commit_after>import codecs
import os
import re
from django.core.management.base import LabelCommand
from django.conf import settings

class Command(LabelCommand):
    help = ("check to see which files are using the b3 import variable"
            " and replace the variable with the appropriate static file"
            " path once collectstatic is run on production"
            "the variable is necessary for less.js debug mode.")
    args = ""

    def handle(self, *args, **options):
        # all the less files collected during collectstatic
        collectedless = [os.path.join(root, file)
                         for root, dir, files in os.walk(settings.STATIC_ROOT)
                         for file in files if file.endswith('.less')]
        for less_file in collectedless:
            with codecs.open(less_file, 'r', 'utf-8') as fd:
                content = fd.read()
                if content is not None:
                    B3_REGEX = r"@\{b3-import-[a-z]+-[a-z]+\}"
                    p = re.search(B3_REGEX, content)
                    if p is not None:
                        path_def = p.group(0)
                        path_file = path_def.split('-')[-2]
                        new_path = settings.LESS_B3_PATHS.get(path_file)
                        if new_path is not None:
                            print ("[CORRECTING] %(f_name)s path in %(less_file)s: "
                                   "%(new_path)s" % {
                                       'f_name': path_file,
                                       'less_file': less_file,
                                       'new_path': new_path,
                                   })
                            content = re.sub(B3_REGEX, new_path, content)
                        else:
                            content = None
                    else:
                        content = None
            if content is not None:
                with open(less_file, 'w') as fd:
                    fd.write(content)
<|endoftext|>


<commit_before>import datetime

from typing import Dict, Optional

def handle_result(created_on: datetime.datetime, category: str,
                  test_suite: str, test_name: str, status: str, results: Dict,
                  artifacts: Dict, last_logs: str) -> Optional[str]:
    assert test_suite == "long_running_tests"

    # elapsed_time = results.get("elapsed_time", 0.)
    last_update_diff = results.get("last_update_diff", float("inf"))

    if test_name in [
            "actor_deaths", "many_actor_tasks", "many_drivers", "many_tasks",
            "many_tasks_serialized_ids", "node_failures",
            "object_spilling_shuffle", "serve", "serve_failure"
    ]:
        # Core tests
        target_update_diff = 120

    elif test_name in ["apex", "impala", "many_ppo", "pbt"]:
        # Tune/RLLib style tests
        target_update_diff = 360
    else:
        return None

    if last_update_diff > target_update_diff:
        return f"Last update to results json was too long ago " \
               f"({last_update_diff:.2f} > {target_update_diff})"

    return None
<commit_msg>[ci/release] Bump long running release test timeouts to 6 minutes (#19458)<commit_after>import datetime

from typing import Dict, Optional

def handle_result(created_on: datetime.datetime, category: str,
                  test_suite: str, test_name: str, status: str, results: Dict,
                  artifacts: Dict, last_logs: str) -> Optional[str]:
    assert test_suite == "long_running_tests"

    # elapsed_time = results.get("elapsed_time", 0.)
    last_update_diff = results.get("last_update_diff", float("inf"))

    if test_name in [
            "actor_deaths",
            "many_actor_tasks",
            "many_drivers",
            "many_tasks",
            "many_tasks_serialized_ids",
            "node_failures",
            "object_spilling_shuffle",
    ]:
        # Core tests
        target_update_diff = 120

    elif test_name in ["apex", "impala", "many_ppo", "pbt"]:
        # Tune/RLLib style tests
        target_update_diff = 360
    elif test_name in ["serve", "serve_failure"]:
        # Serve tests have workload logs every five minutes.
        # Leave up to 60 seconds overhead.
        target_update_diff = 360
    else:
        return None

    if last_update_diff > target_update_diff:
        return f"Last update to results json was too long ago " \
               f"({last_update_diff:.2f} > {target_update_diff})"

    return None
<|endoftext|>


<commit_before>'''
Package *.uproject and release it to output location defined in ue4config.py
'''
import os, sys, argparse, json, shutil
import ue4util, gitutil, ziputil, uploadutil

def package(project_file, project_output_folder):
    '''
    Build project
    '''

    UATScriptTemplate = '{UATScript} BuildCookRun -project={ProjectFile} -archivedirectory={OutputFolder} -noP4 -platform={Platform} -clientconfig=Development -serverconfig=Development -cook -allmaps -stage -pak -archive -build'
    # See help information in Engine/Source/Programs/AutomationTool/AutomationUtils/ProjectParams.cs

    # cmd = UATScriptTemplate.format(
    #     UATScript = ue4util.get_UAT_script(),
    #     Platform = ue4util.get_platform_name(),
    #     OutputFolder = ue4util.get_real_abspath(project_output_folder),
    #     ProjectFile = project_file
    #     )
    cmd = [
        ue4util.get_UAT_script(),
        'BuildCookRun',
        '-project=%s' % project_file,
        '-archivedirectory=%s' % ue4util.get_real_abspath(project_output_folder),
        '-noP4',
        '-platform=%s' % ue4util.get_platform_name(),
        '-clientconfig=Development',
        '-serverconfig=Development',
        '-cook',
        '-allmaps',
        '-stage',
        '-pak',
        '-archive',
        '-build'
    ]

    # ue4util.run_ue4cmd(cmd, False)
    subprocess.call(cmd)

def save_version_info(info_filename, project_file, project_output_folder, plugin_version):
    ''' Save the version info of UnrealCV plugin and the game for easier issue tracking'''
    project_name = ue4util.get_project_name(project_file)
    project_folder = os.path.dirname(project_file)

    if gitutil.is_dirty(project_folder):
        return False

    # if gitutil.is_dirty(plugin_folder):
    #     return False
    # plugin_version = gitutil.get_short_version(plugin_folder)
    assert(len(plugin_version) == 7 or len(plugin_version) == 0)
    project_version = gitutil.get_short_version(project_folder)
    assert(len(project_version) == 7 or len(project_version) == 0)

    info = dict(
        project_name = project_name,
        project_version = project_version,
        plugin_version = plugin_version,
        platform = ue4util.get_platform_name(),
    )

    ue4util.mkdirp(os.path.dirname(info_filename))

    with open(info_filename, 'w') as f:
        json.dump(info, f, indent = 4)

    print 'Save project info to file %s' % info_filename
    # print json.dumps(info, indent = 4)

    return True

def get_files_win(output_folder, project_name):
    info_filename = os.path.join(output_folder, '%s-info.txt' % project_name)
    root = os.path.join(output_folder, 'WindowsNoEditor')
    files = [
        os.path.join(root, '%s.exe' % project_name),
        os.path.join(root, '%s/' % project_name),
        os.path.join(root, 'Engine/'),
        info_filename,
    ]
    return files

def get_files_mac(output_folder, project_name):
    info_filename = os.path.join(output_folder, '%s-info.txt' % project_name)
    root = os.path.join(output_folder, 'MacNoEditor')
    files = [
        os.path.join(root, '%s.app' % project_name),
        info_filename
    ]
    return files

def get_files_linux(output_folder, project_name):
    info_filename = os.path.join(output_folder, '%s-info.txt' % project_name)
    root = os.path.join(output_folder, 'LinuxNoEditor')
    files = [
        os.path.join(root, 'Engine/'),
        os.path.join(root, project_name), #binary
        info_filename
    ]
    return files

get_files = ue4util.platform_function(
    Mac = get_files_mac,
    Win64 = get_files_win,
    Linux = get_files_linux,
)

def get_zipfilename_from_infofile(project_infofile, plugin_infofile):
    with open(project_infofile, 'r') as f:
        project_info = json.load(f)
    with open(plugin_infofile, 'r') as f:
        plugin_info = json.load(f)
    info = project_info.copy()
    info.update(plugin_info)
    return '{project_name}-{platform}-{project_version}-{plugin_version}.zip'.format(**info)

def check_files_ok(files):
    isok = True
    for file in files:
        if os.path.isfile(file) or os.path.isdir(file):
            continue
        else:
            print 'File %s not exist, stop packaging' % file
            isok = False
    return isok

def zip_project(zipfilename, project_file, project_output_folder):
    project_name = ue4util.get_project_name(project_file)
    files = get_files(project_output_folder, project_name)

    zip_root_folder = ue4util.get_real_abspath(project_output_folder) + '/'
    if check_files_ok(files): # Make sure all files we want exist
        all_files = ziputil.get_all_files(files)
        print 'Start zipping files to %s, Zip root folder is %s' % (zipfilename, zip_root_folder)
        ziputil.zipfiles(all_files, zip_root_folder, zipfilename, verbose=False)
        return True
    else:
        return False

def main():
    # Files is relative to this python script
    cur_dir = os.path.dirname(os.path.abspath(__file__))

    # Parse command line arguments
    parser = argparse.ArgumentParser()
    parser.add_argument('project_file')
    parser.add_argument('--engine_path') # Add here just as a placeholder
    # args = parser.parse_args()
    args, _ = parser.parse_known_args()

    # Read project information
    project_file = ue4util.get_real_abspath(args.project_file)
    project_name = ue4util.get_project_name(project_file)
    project_output_folder = os.path.join(cur_dir, 'built_project/%s' % project_name)
    info_filename = os.path.join(project_output_folder, '%s-info.txt' % project_name)

    # Get plugin information
    project_folder = os.path.dirname(project_file)
    plugin_infofile = os.path.join(project_folder, 'Plugins', 'unrealcv', 'unrealcv-info.txt')
    if not os.path.isfile(plugin_infofile):
        print 'Plugin not exist'
        return
    else:
        with open(plugin_infofile, 'r') as f:
            plugin_info = json.load(f)
            plugin_version = plugin_info['plugin_version']

    if os.path.isdir(project_output_folder):
        print 'Output directory %s exist, delete it first' % project_output_folder
    else:
        # Package game and save version info
        package(project_file, project_output_folder)

        # Save version info after build finished
        save_version_info(info_filename, project_file, project_output_folder, plugin_version)

    # Zip files
    # zipfilename = os.path.join(project_output_folder, get_zipfilename_from_infofile(info_filename, plugin_infofile))
    # zip_project(zipfilename, project_file, project_output_folder)

    # Upload built games to output targets
    # upload_confs = ue4config.conf['ProjectOutput']
    # upload_handlers = dict(
    #     scp = uploadutil.upload_scp,
    #     s3 = uploadutil.upload_s3,
    # )
    # for upload_conf in upload_confs:
    #     tgt_type = upload_conf['Type']
    #     upload_handlers[tgt_type](upload_conf, [zipfilename], os.path.dirname(zipfilename))

if __name__ == '__main__':
    main()
<commit_msg>Fix missing import.<commit_after>'''
Package *.uproject and release it to output location defined in ue4config.py
'''
import os, sys, argparse, json, shutil, subprocess
import ue4util, gitutil, ziputil, uploadutil

def package(project_file, project_output_folder):
    '''
    Build project
    '''

    UATScriptTemplate = '{UATScript} BuildCookRun -project={ProjectFile} -archivedirectory={OutputFolder} -noP4 -platform={Platform} -clientconfig=Development -serverconfig=Development -cook -allmaps -stage -pak -archive -build'
    # See help information in Engine/Source/Programs/AutomationTool/AutomationUtils/ProjectParams.cs

    # cmd = UATScriptTemplate.format(
    #     UATScript = ue4util.get_UAT_script(),
    #     Platform = ue4util.get_platform_name(),
    #     OutputFolder = ue4util.get_real_abspath(project_output_folder),
    #     ProjectFile = project_file
    #     )
    cmd = [
        ue4util.get_UAT_script(),
        'BuildCookRun',
        '-project=%s' % project_file,
        '-archivedirectory=%s' % ue4util.get_real_abspath(project_output_folder),
        '-noP4',
        '-platform=%s' % ue4util.get_platform_name(),
        '-clientconfig=Development',
        '-serverconfig=Development',
        '-cook',
        '-allmaps',
        '-stage',
        '-pak',
        '-archive',
        '-build'
    ]

    # ue4util.run_ue4cmd(cmd, False)
    subprocess.call(cmd)

def save_version_info(info_filename, project_file, project_output_folder, plugin_version):
    ''' Save the version info of UnrealCV plugin and the game for easier issue tracking'''
    project_name = ue4util.get_project_name(project_file)
    project_folder = os.path.dirname(project_file)

    if gitutil.is_dirty(project_folder):
        return False

    # if gitutil.is_dirty(plugin_folder):
    #     return False
    # plugin_version = gitutil.get_short_version(plugin_folder)
    assert(len(plugin_version) == 7 or len(plugin_version) == 0)
    project_version = gitutil.get_short_version(project_folder)
    assert(len(project_version) == 7 or len(project_version) == 0)

    info = dict(
        project_name = project_name,
        project_version = project_version,
        plugin_version = plugin_version,
        platform = ue4util.get_platform_name(),
    )

    ue4util.mkdirp(os.path.dirname(info_filename))

    with open(info_filename, 'w') as f:
        json.dump(info, f, indent = 4)

    print 'Save project info to file %s' % info_filename
    # print json.dumps(info, indent = 4)

    return True

def get_files_win(output_folder, project_name):
    info_filename = os.path.join(output_folder, '%s-info.txt' % project_name)
    root = os.path.join(output_folder, 'WindowsNoEditor')
    files = [
        os.path.join(root, '%s.exe' % project_name),
        os.path.join(root, '%s/' % project_name),
        os.path.join(root, 'Engine/'),
        info_filename,
    ]
    return files

def get_files_mac(output_folder, project_name):
    info_filename = os.path.join(output_folder, '%s-info.txt' % project_name)
    root = os.path.join(output_folder, 'MacNoEditor')
    files = [
        os.path.join(root, '%s.app' % project_name),
        info_filename
    ]
    return files

def get_files_linux(output_folder, project_name):
    info_filename = os.path.join(output_folder, '%s-info.txt' % project_name)
    root = os.path.join(output_folder, 'LinuxNoEditor')
    files = [
        os.path.join(root, 'Engine/'),
        os.path.join(root, project_name), #binary
        info_filename
    ]
    return files

get_files = ue4util.platform_function(
    Mac = get_files_mac,
    Win64 = get_files_win,
    Linux = get_files_linux,
)

def get_zipfilename_from_infofile(project_infofile, plugin_infofile):
    with open(project_infofile, 'r') as f:
        project_info = json.load(f)
    with open(plugin_infofile, 'r') as f:
        plugin_info = json.load(f)
    info = project_info.copy()
    info.update(plugin_info)
    return '{project_name}-{platform}-{project_version}-{plugin_version}.zip'.format(**info)

def check_files_ok(files):
    isok = True
    for file in files:
        if os.path.isfile(file) or os.path.isdir(file):
            continue
        else:
            print 'File %s not exist, stop packaging' % file
            isok = False
    return isok

def zip_project(zipfilename, project_file, project_output_folder):
    project_name = ue4util.get_project_name(project_file)
    files = get_files(project_output_folder, project_name)

    zip_root_folder = ue4util.get_real_abspath(project_output_folder) + '/'
    if check_files_ok(files): # Make sure all files we want exist
        all_files = ziputil.get_all_files(files)
        print 'Start zipping files to %s, Zip root folder is %s' % (zipfilename, zip_root_folder)
        ziputil.zipfiles(all_files, zip_root_folder, zipfilename, verbose=False)
        return True
    else:
        return False

def main():
    # Files is relative to this python script
    cur_dir = os.path.dirname(os.path.abspath(__file__))

    # Parse command line arguments
    parser = argparse.ArgumentParser()
    parser.add_argument('project_file')
    parser.add_argument('--engine_path') # Add here just as a placeholder
    # args = parser.parse_args()
    args, _ = parser.parse_known_args()

    # Read project information
    project_file = ue4util.get_real_abspath(args.project_file)
    project_name = ue4util.get_project_name(project_file)
    project_output_folder = os.path.join(cur_dir, 'built_project/%s' % project_name)
    info_filename = os.path.join(project_output_folder, '%s-info.txt' % project_name)

    # Get plugin information
    project_folder = os.path.dirname(project_file)
    plugin_infofile = os.path.join(project_folder, 'Plugins', 'unrealcv', 'unrealcv-info.txt')
    if not os.path.isfile(plugin_infofile):
        print 'Plugin not exist'
        return
    else:
        with open(plugin_infofile, 'r') as f:
            plugin_info = json.load(f)
            plugin_version = plugin_info['plugin_version']

    if os.path.isdir(project_output_folder):
        print 'Output directory %s exist, delete it first' % project_output_folder
    else:
        # Package game and save version info
        package(project_file, project_output_folder)

        # Save version info after build finished
        save_version_info(info_filename, project_file, project_output_folder, plugin_version)

    # Zip files
    # zipfilename = os.path.join(project_output_folder, get_zipfilename_from_infofile(info_filename, plugin_infofile))
    # zip_project(zipfilename, project_file, project_output_folder)

    # Upload built games to output targets
    # upload_confs = ue4config.conf['ProjectOutput']
    # upload_handlers = dict(
    #     scp = uploadutil.upload_scp,
    #     s3 = uploadutil.upload_s3,
    # )
    # for upload_conf in upload_confs:
    #     tgt_type = upload_conf['Type']
    #     upload_handlers[tgt_type](upload_conf, [zipfilename], os.path.dirname(zipfilename))

if __name__ == '__main__':
    main()
<|endoftext|>