<commit_before>#MAEC Grouping Relationship Class

#Copyright (c) 2013, The MITRE Corporation
#All rights reserved.

#Compatible with MAEC v4.0
#Last updated 05/15/2013

import maec
import maec.bindings.maec_package as package_binding
from maec.package.malware_subject_reference import MalwareSubjectReference

class GroupingRelationship(maec.Entity):

    def init(self):
        super(GroupingRelationship, self).__init__()
        self.type = None
        self.malware_family_name = None
        self.malware_toolkit_name = None
        self.clustering_metadata = None

    def to_obj(self):
        grouping_relationship_obj = package_binding.GroupingRelationshipType()
        if self.type is not None : grouping_relationship_obj.set_Type(self.type.to_obj())
        if self.malware_family_name is not None : grouping_relationship_obj.set_Malware_Family_Name(self.malware_family_name)
        if self.malware_toolkit_name is not None : grouping_relationship_obj.set_Malware_Toolkit_Name(self.malware_toolkit_name)
        if self.clustering_metadata is not None : grouping_relationship_obj.set_Clustering_Metadata(self.clustering_metadata.to_obj())
        return grouping_relationship_obj

    def to_dict(self):
        grouping_relationship_dict = {}
        if self.type is not None : grouping_relationship_dict['type'] = self.type.to_dict()
        if self.malware_family_name is not None : grouping_relationship_dict['malware_family_name'] = self.malware_family_name
        if self.malware_toolkit_name is not None : grouping_relationship_dict['malware_toolkit_name'] = self.malware_family_name
        if self.clustering_metadata is not None : grouping_relationship_dict['clustering_metadata'] = self.clustering_metadata.to_dict()
        return grouping_relationship_dict

    @staticmethod
    def from_dict(grouping_relationship_dict):
        if not grouping_relationship_dict:
            return None
        grouping_relationship_ = GroupingRelationship()
        grouping_relationship_.type = None #TODO: add support
        grouping_relationship_.malware_family_name = grouping_relationship_dict.get('malware_family_name')
        grouping_relationship_.malware_toolkit_name = grouping_relationship_dict.get('malware_toolkit_name')
        grouping_relationship_.clustering_metadata = ClusteringMetadata.from_dict(grouping_relationship_dict.get('clustering_metadata'))
        return grouping_relationship_

    @staticmethod
    def from_obj(grouping_relationship_obj):
        if not grouping_relationship_obj:
            return None
        grouping_relationship_ = GroupingRelationship()
        grouping_relationship_.type = None #TODO: add support
        grouping_relationship_.malware_family_name = grouping_relationship_obj.get_Malware_Family_Name()
        grouping_relationship_.malware_toolkit_name = grouping_relationship_obj.get_Malware_Toolkit_Name()
        grouping_relationship_.clustering_metadata = ClusteringMetadata.from_obj(grouping_relationship_obj.get_Clustering_Metadata())
        return grouping_relationship_

class GroupingRelationshipList(maec.EntityList):
    _contained_type = ObjectEquivalence
    _binding_class = package_binding.ObjectEquivalenceListType

    def __init__(self):
        super(ObjectEquivalenceList, self).__init__()

    @staticmethod
    def _set_list(binding_obj, list_):
        binding_obj.set_Object_Equivalence(list_)

    @staticmethod
    def _get_list(binding_obj):
        return binding_obj.get_Object_Equivalence()

class ClusteringMetadata(maec.Entity):

    def __init__(self):
        super(ClusteringMetadata, self).__init__()
        self.algorithm_name = None
        self.algorithm_version = None
        self.algorithm_parameters = None
        self.cluster_size = None
        self.cluster_description = None
        self.cluster_composition = None

    def to_obj(self):
        clustering_metadata_obj = package_binding.ClusteringMetadataType()
        if self.algorithm_name is not None : clustering_metadata_obj.set_Algorithm_Name(self.algorithm_name)
        if self.algorithm_version is not None : clustering_metadata_obj.set_Algorithm_Version(self.algorithm_version)
        if self.algorithm_parameters is not None : clustering_metadata_obj.set_Algorithm_Parameters(self.algorithm_parameters.to_obj())
        if self.cluster_size is not None : clustering_metadata_obj.set_Cluster_Size(self.cluster_size)
        if self.cluster_description is not None : clustering_metadata_obj.set_Cluster_Description(self.cluster_description)
        if self.cluster_composition is not None : clustering_metadata_obj.set_Cluster_Composition(self.cluster_composition.to_obj())
        return clustering_metadata_obj

    def to_dict(self):
        clustering_metadata_dict = {}
        if self.algorithm_name is not None : clustering_metadata_dict['algorithm_name'] = self.algorithm_name
        if self.algorithm_version is not None : clustering_metadata_dict['algorithm_version'] = self.algorithm_version
        if self.algorithm_parameters is not None : clustering_metadata_dict['algorithm_parameters'] = self.algorithm_parameters.to_dict()
        if self.cluster_size is not None : clustering_metadata_dict['cluster_size'] = self.cluster_size
        if self.cluster_description is not None : clustering_metadata_dict['cluster_description'] = self.cluster_description
        if self.cluster_composition is not None : clustering_metadata_dict['cluster_composition'] = self.cluster_composition.to_dict()
        return clustering_metadata_dict

    @staticmethod
    def from_dict(clustering_metadata_dict):
        if not clustering_metadata_dict:
            return None
        clustering_metadata_ = ClusteringMetadata()
        clustering_metadata_.algorithm_name = clustering_metadata_dict.get('algorithm_name')
        clustering_metadata_.algorithm_version = clustering_metadata_dict.get('algorithm_version')
        clustering_metadata_.algorithm_parameters = ClusteringAlgorithmParameters.from_dict(clustering_metadata_dict.get('algorithm_parameters'))
        clustering_metadata_.cluster_size = clustering_metadata_dict.get('cluster_size')
        clustering_metadata_.cluster_description = clustering_metadata_dict.get('cluster_description')
        clustering_metadata_.cluster_composition = ClusterComposition.from_dict(clustering_metadata_dict.get('cluster_composition'))
        return clustering_metadata_

    @staticmethod
    def from_obj(clustering_metadata_obj):
        if not clustering_metadata_obj:
            return None
        clustering_metadata_ = ClusteringMetadata()
        clustering_metadata_.algorithm_name = clustering_metadata_obj.get_Algorithm_Name()
        clustering_metadata_.algorithm_version = clustering_metadata_obj.get_Algorithm_Version()
        clustering_metadata_.algorithm_parameters = ClusteringAlgorithmParameters.from_obj(clustering_metadata_obj.get_Algorithm_Parameters())
        clustering_metadata_.cluster_size = clustering_metadata_obj.get_Cluster_Size()
        clustering_metadata_.cluster_description = clustering_metadata_obj.get_Cluster_Description()
        clustering_metadata_.cluster_composition = ClusterComposition.from_obj(clustering_metadata_obj.get_Cluster_Composition())
        return clustering_metadata_

class ClusteringAlgorithmParameters(maec.Entity):

    def __init__(self):
        super(ClusteringAlgorithmParameters, self).__init__()
        self.distance_threshold = None
        self.number_of_iterations = None

    def to_obj(self):
        clustering_algorithm_parameters_obj = package_binding.ClusteringAlgorithmParametersType()
        if self.distance_threshold is not None : clustering_algorithm_parameters_obj.set_Distance_Threshold(self.distance_threshold)
        if self.number_of_iterations is not None : clustering_algorithm_parameters_obj.set_Number_of_Iterations(self.number_of_iterations)
        return clustering_algorithm_parameters_obj

    def to_dict(self):
        clustering_algorithm_parameters_dict = {}
        if self.distance_threshold is not None : clustering_algorithm_parameters_dict['distance_threshold'] = self.distance_threshold
        if self.number_of_iterations is not None : clustering_algorithm_parameters_dict['number_of_iterations'] = self.number_of_iterations
        return clustering_algorithm_parameters_dict

    @staticmethod
    def from_dict(clustering_algorithm_parameters_dict):
        if not clustering_algorithm_parameters_dict:
            return None
        clustering_algorithm_parameters_ = ClusteringAlgorithmParameters()
        clustering_algorithm_parameters_.distance_threshold = clustering_algorithm_parameters_dict.get('distance_threshold')
        clustering_algorithm_parameters_.number_of_iterations = clustering_algorithm_parameters_dict.get('number_of_iterations')
        return clustering_algorithm_parameters_

    @staticmethod
    def from_obj(clustering_algorithm_parameters_obj):
        if not clustering_algorithm_parameters_obj:
            return None
        clustering_algorithm_parameters_ = ClusteringAlgorithmParameters()
        clustering_algorithm_parameters_.distance_threshold = clustering_algorithm_parameters_obj.get_Distance_Threshold()
        clustering_algorithm_parameters_.number_of_iterations = clustering_algorithm_parameters_obj.get_Number_of_Iterations()
        return clustering_algorithm_parameters_

class ClusterComposition(maec.Entity):

    def __init__(self):
        super(ClusterComposition, self).__init__()
        self.score_type = None
        self.edge_node_pairs = []

    def to_obj(self):
        cluster_composition_obj = package_binding.ClusterCompositionType()
        if self.score_type is not None : cluster_composition_obj.set_score_type(self.score_type)
        if len(edge_node_pairs) > 0:
            for edge_node_pair in self.edge_node_pairs: cluster_composition_obj.add_Edge_Node_Pair(edge_node_pair.to_obj())
        return cluster_composition_obj

    def to_dict(self):
        cluster_composition_dict = {}
        if self.score_type is not None : cluster_composition_dict['score_type'] = self.score_type
        if len(edge_node_pairs) > 0:
            for edge_node_pair in self.edge_node_pairs: cluster_composition_obj.add_Edge_Node_Pair(edge_node_pair.to_obj())
        return cluster_composition_dict

    @staticmethod
    def from_dict(cluster_composition_dict):
        if not cluster_composition_dict:
            return None
        cluster_composition_ = ClusterComposition()
        cluster_composition_.score_type = cluster_composition_dict.get('score_type')
        cluster_composition_.edge_node_pairs = [ClusterEdgeNodePair.from_dict(x) for x in cluster_composition_dict.get('edge_node_pairs',[])]
        return cluster_composition_

    @staticmethod
    def from_obj(cluster_composition_obj):
        if not cluster_composition_obj:
            return None
        cluster_composition_ = ClusterComposition()
        cluster_composition_.score_type = cluster_composition_obj.get_score_type()
        cluster_composition_.edge_node_pairs = [ClusterEdgeNodePair.from_obj(x) for x in cluster_composition_obj.get_Edge_Node_Pair()]
        return cluster_composition_

class ClusterEdgeNodePair(maec.Entity):

    def __init__(self):
        super(ClusterEdgeNodePair, self).__init__()
        self.similarity_index = None
        self.similarity_distance = None
        self.malware_subject_node_a = None
        self.malware_subject_node_b = None

    def to_obj(self):
        cluster_edge_node_pair_obj = package_binding.ClusterEdgeNodePairType()
        if self.similarity_index is not None : cluster_edge_node_pair_obj.set_similarity_index(self.similarity_index)
        if self.similarity_distance is not None : cluster_edge_node_pair_obj.set_similarity_distance(self.similarity_distance)
        if self.malware_subject_node_a is not None : cluster_edge_node_pair_obj.set_Malware_Subject_Node_A(self.malware_subject_node_a.to_obj())
        if self.malware_subject_node_b is not None : cluster_edge_node_pair_obj.set_Malware_Subject_Node_B(self.malware_subject_node_b.to_obj())
        return cluster_edge_node_pair_obj

    def to_dict(self):
        cluster_edge_node_pair_dict = {}
        if self.similarity_index is not None : cluster_edge_node_pair_dict['similarity_index'] = self.similarity_index
        if self.similarity_distance is not None : cluster_edge_node_pair_dict['similarity_distance'] = self.similarity_distance
        if self.malware_subject_node_a is not None : cluster_edge_node_pair_dict['malware_subject_node_a'] = self.malware_subject_node_a.to_dict()
        if self.malware_subject_node_b is not None : cluster_edge_node_pair_dict['malware_subject_node_b'] = self.malware_subject_node_b.to_dict()
        return cluster_edge_node_pair_dict

    @staticmethod
    def from_dict(cluster_edge_node_pair_dict):
        if not cluster_edge_node_pair_dict:
            return None
        cluster_edge_node_pair_ = ClusterEdgeNodePair()
        cluster_edge_node_pair_.similarity_index = cluster_edge_node_pair_dict.get('similarity_index')
        cluster_edge_node_pair_.similarity_distance = cluster_edge_node_pair_dict.get('similarity_distance')
        cluster_edge_node_pair_.malware_subject_node_a = MalwareSubjectReference.from_dict(cluster_edge_node_pair_dict.get('malware_subject_node_a'))
        cluster_edge_node_pair_.malware_subject_node_b = MalwareSubjectReference.from_dict(cluster_edge_node_pair_dict.get('malware_subject_node_b'))
        return cluster_edge_node_pair_

    @staticmethod
    def from_obj(cluster_edge_node_pair_obj):
        if not cluster_edge_node_pair_obj:
            return None
        cluster_edge_node_pair_ = ClusterEdgeNodePair()
        cluster_edge_node_pair_.similarity_index = cluster_edge_node_pair_obj.get_similarity_index()
        cluster_edge_node_pair_.similarity_distance = cluster_edge_node_pair_obj.get_similarity_distance()
        cluster_edge_node_pair_.malware_subject_node_a = MalwareSubjectReference.from_obj(cluster_edge_node_pair_obj.get_Malware_Subject_Node_A())
        cluster_edge_node_pair_.malware_subject_node_b = MalwareSubjectReference.from_obj(cluster_edge_node_pair_obj.get_Malware_Subject_Node_B())
        return cluster_edge_node_pair_
<commit_msg>Updated GroupingRelationshipList for accuracy<commit_after>#MAEC Grouping Relationship Class

#Copyright (c) 2013, The MITRE Corporation
#All rights reserved.

#Compatible with MAEC v4.0
#Last updated 05/15/2013

import maec
import maec.bindings.maec_package as package_binding
from maec.package.malware_subject_reference import MalwareSubjectReference

class GroupingRelationship(maec.Entity):

    def init(self):
        super(GroupingRelationship, self).__init__()
        self.type = None
        self.malware_family_name = None
        self.malware_toolkit_name = None
        self.clustering_metadata = None

    def to_obj(self):
        grouping_relationship_obj = package_binding.GroupingRelationshipType()
        if self.type is not None : grouping_relationship_obj.set_Type(self.type.to_obj())
        if self.malware_family_name is not None : grouping_relationship_obj.set_Malware_Family_Name(self.malware_family_name)
        if self.malware_toolkit_name is not None : grouping_relationship_obj.set_Malware_Toolkit_Name(self.malware_toolkit_name)
        if self.clustering_metadata is not None : grouping_relationship_obj.set_Clustering_Metadata(self.clustering_metadata.to_obj())
        return grouping_relationship_obj

    def to_dict(self):
        grouping_relationship_dict = {}
        if self.type is not None : grouping_relationship_dict['type'] = self.type.to_dict()
        if self.malware_family_name is not None : grouping_relationship_dict['malware_family_name'] = self.malware_family_name
        if self.malware_toolkit_name is not None : grouping_relationship_dict['malware_toolkit_name'] = self.malware_family_name
        if self.clustering_metadata is not None : grouping_relationship_dict['clustering_metadata'] = self.clustering_metadata.to_dict()
        return grouping_relationship_dict

    @staticmethod
    def from_dict(grouping_relationship_dict):
        if not grouping_relationship_dict:
            return None
        grouping_relationship_ = GroupingRelationship()
        grouping_relationship_.type = None #TODO: add support
        grouping_relationship_.malware_family_name = grouping_relationship_dict.get('malware_family_name')
        grouping_relationship_.malware_toolkit_name = grouping_relationship_dict.get('malware_toolkit_name')
        grouping_relationship_.clustering_metadata = ClusteringMetadata.from_dict(grouping_relationship_dict.get('clustering_metadata'))
        return grouping_relationship_

    @staticmethod
    def from_obj(grouping_relationship_obj):
        if not grouping_relationship_obj:
            return None
        grouping_relationship_ = GroupingRelationship()
        grouping_relationship_.type = None #TODO: add support
        grouping_relationship_.malware_family_name = grouping_relationship_obj.get_Malware_Family_Name()
        grouping_relationship_.malware_toolkit_name = grouping_relationship_obj.get_Malware_Toolkit_Name()
        grouping_relationship_.clustering_metadata = ClusteringMetadata.from_obj(grouping_relationship_obj.get_Clustering_Metadata())
        return grouping_relationship_

class GroupingRelationshipList(maec.EntityList):
    _contained_type = GroupingRelationship
    _binding_class = package_binding.GroupingRelationshipListType

    def __init__(self):
        super(GroupingRelationshipList, self).__init__()

    @staticmethod
    def _set_list(binding_obj, list_):
        binding_obj.set_Grouping_Relationship(list_)

    @staticmethod
    def _get_list(binding_obj):
        return binding_obj.get_Grouping_Relationship()

class ClusteringMetadata(maec.Entity):

    def __init__(self):
        super(ClusteringMetadata, self).__init__()
        self.algorithm_name = None
        self.algorithm_version = None
        self.algorithm_parameters = None
        self.cluster_size = None
        self.cluster_description = None
        self.cluster_composition = None

    def to_obj(self):
        clustering_metadata_obj = package_binding.ClusteringMetadataType()
        if self.algorithm_name is not None : clustering_metadata_obj.set_Algorithm_Name(self.algorithm_name)
        if self.algorithm_version is not None : clustering_metadata_obj.set_Algorithm_Version(self.algorithm_version)
        if self.algorithm_parameters is not None : clustering_metadata_obj.set_Algorithm_Parameters(self.algorithm_parameters.to_obj())
        if self.cluster_size is not None : clustering_metadata_obj.set_Cluster_Size(self.cluster_size)
        if self.cluster_description is not None : clustering_metadata_obj.set_Cluster_Description(self.cluster_description)
        if self.cluster_composition is not None : clustering_metadata_obj.set_Cluster_Composition(self.cluster_composition.to_obj())
        return clustering_metadata_obj

    def to_dict(self):
        clustering_metadata_dict = {}
        if self.algorithm_name is not None : clustering_metadata_dict['algorithm_name'] = self.algorithm_name
        if self.algorithm_version is not None : clustering_metadata_dict['algorithm_version'] = self.algorithm_version
        if self.algorithm_parameters is not None : clustering_metadata_dict['algorithm_parameters'] = self.algorithm_parameters.to_dict()
        if self.cluster_size is not None : clustering_metadata_dict['cluster_size'] = self.cluster_size
        if self.cluster_description is not None : clustering_metadata_dict['cluster_description'] = self.cluster_description
        if self.cluster_composition is not None : clustering_metadata_dict['cluster_composition'] = self.cluster_composition.to_dict()
        return clustering_metadata_dict

    @staticmethod
    def from_dict(clustering_metadata_dict):
        if not clustering_metadata_dict:
            return None
        clustering_metadata_ = ClusteringMetadata()
        clustering_metadata_.algorithm_name = clustering_metadata_dict.get('algorithm_name')
        clustering_metadata_.algorithm_version = clustering_metadata_dict.get('algorithm_version')
        clustering_metadata_.algorithm_parameters = ClusteringAlgorithmParameters.from_dict(clustering_metadata_dict.get('algorithm_parameters'))
        clustering_metadata_.cluster_size = clustering_metadata_dict.get('cluster_size')
        clustering_metadata_.cluster_description = clustering_metadata_dict.get('cluster_description')
        clustering_metadata_.cluster_composition = ClusterComposition.from_dict(clustering_metadata_dict.get('cluster_composition'))
        return clustering_metadata_

    @staticmethod
    def from_obj(clustering_metadata_obj):
        if not clustering_metadata_obj:
            return None
        clustering_metadata_ = ClusteringMetadata()
        clustering_metadata_.algorithm_name = clustering_metadata_obj.get_Algorithm_Name()
        clustering_metadata_.algorithm_version = clustering_metadata_obj.get_Algorithm_Version()
        clustering_metadata_.algorithm_parameters = ClusteringAlgorithmParameters.from_obj(clustering_metadata_obj.get_Algorithm_Parameters())
        clustering_metadata_.cluster_size = clustering_metadata_obj.get_Cluster_Size()
        clustering_metadata_.cluster_description = clustering_metadata_obj.get_Cluster_Description()
        clustering_metadata_.cluster_composition = ClusterComposition.from_obj(clustering_metadata_obj.get_Cluster_Composition())
        return clustering_metadata_

class ClusteringAlgorithmParameters(maec.Entity):

    def __init__(self):
        super(ClusteringAlgorithmParameters, self).__init__()
        self.distance_threshold = None
        self.number_of_iterations = None

    def to_obj(self):
        clustering_algorithm_parameters_obj = package_binding.ClusteringAlgorithmParametersType()
        if self.distance_threshold is not None : clustering_algorithm_parameters_obj.set_Distance_Threshold(self.distance_threshold)
        if self.number_of_iterations is not None : clustering_algorithm_parameters_obj.set_Number_of_Iterations(self.number_of_iterations)
        return clustering_algorithm_parameters_obj

    def to_dict(self):
        clustering_algorithm_parameters_dict = {}
        if self.distance_threshold is not None : clustering_algorithm_parameters_dict['distance_threshold'] = self.distance_threshold
        if self.number_of_iterations is not None : clustering_algorithm_parameters_dict['number_of_iterations'] = self.number_of_iterations
        return clustering_algorithm_parameters_dict

    @staticmethod
    def from_dict(clustering_algorithm_parameters_dict):
        if not clustering_algorithm_parameters_dict:
            return None
        clustering_algorithm_parameters_ = ClusteringAlgorithmParameters()
        clustering_algorithm_parameters_.distance_threshold = clustering_algorithm_parameters_dict.get('distance_threshold')
        clustering_algorithm_parameters_.number_of_iterations = clustering_algorithm_parameters_dict.get('number_of_iterations')
        return clustering_algorithm_parameters_

    @staticmethod
    def from_obj(clustering_algorithm_parameters_obj):
        if not clustering_algorithm_parameters_obj:
            return None
        clustering_algorithm_parameters_ = ClusteringAlgorithmParameters()
        clustering_algorithm_parameters_.distance_threshold = clustering_algorithm_parameters_obj.get_Distance_Threshold()
        clustering_algorithm_parameters_.number_of_iterations = clustering_algorithm_parameters_obj.get_Number_of_Iterations()
        return clustering_algorithm_parameters_

class ClusterComposition(maec.Entity):

    def __init__(self):
        super(ClusterComposition, self).__init__()
        self.score_type = None
        self.edge_node_pairs = []

    def to_obj(self):
        cluster_composition_obj = package_binding.ClusterCompositionType()
        if self.score_type is not None : cluster_composition_obj.set_score_type(self.score_type)
        if len(edge_node_pairs) > 0:
            for edge_node_pair in self.edge_node_pairs: cluster_composition_obj.add_Edge_Node_Pair(edge_node_pair.to_obj())
        return cluster_composition_obj

    def to_dict(self):
        cluster_composition_dict = {}
        if self.score_type is not None : cluster_composition_dict['score_type'] = self.score_type
        if len(edge_node_pairs) > 0:
            for edge_node_pair in self.edge_node_pairs: cluster_composition_obj.add_Edge_Node_Pair(edge_node_pair.to_obj())
        return cluster_composition_dict

    @staticmethod
    def from_dict(cluster_composition_dict):
        if not cluster_composition_dict:
            return None
        cluster_composition_ = ClusterComposition()
        cluster_composition_.score_type = cluster_composition_dict.get('score_type')
        cluster_composition_.edge_node_pairs = [ClusterEdgeNodePair.from_dict(x) for x in cluster_composition_dict.get('edge_node_pairs',[])]
        return cluster_composition_

    @staticmethod
    def from_obj(cluster_composition_obj):
        if not cluster_composition_obj:
            return None
        cluster_composition_ = ClusterComposition()
        cluster_composition_.score_type = cluster_composition_obj.get_score_type()
        cluster_composition_.edge_node_pairs = [ClusterEdgeNodePair.from_obj(x) for x in cluster_composition_obj.get_Edge_Node_Pair()]
        return cluster_composition_

class ClusterEdgeNodePair(maec.Entity):

    def __init__(self):
        super(ClusterEdgeNodePair, self).__init__()
        self.similarity_index = None
        self.similarity_distance = None
        self.malware_subject_node_a = None
        self.malware_subject_node_b = None

    def to_obj(self):
        cluster_edge_node_pair_obj = package_binding.ClusterEdgeNodePairType()
        if self.similarity_index is not None : cluster_edge_node_pair_obj.set_similarity_index(self.similarity_index)
        if self.similarity_distance is not None : cluster_edge_node_pair_obj.set_similarity_distance(self.similarity_distance)
        if self.malware_subject_node_a is not None : cluster_edge_node_pair_obj.set_Malware_Subject_Node_A(self.malware_subject_node_a.to_obj())
        if self.malware_subject_node_b is not None : cluster_edge_node_pair_obj.set_Malware_Subject_Node_B(self.malware_subject_node_b.to_obj())
        return cluster_edge_node_pair_obj

    def to_dict(self):
        cluster_edge_node_pair_dict = {}
        if self.similarity_index is not None : cluster_edge_node_pair_dict['similarity_index'] = self.similarity_index
        if self.similarity_distance is not None : cluster_edge_node_pair_dict['similarity_distance'] = self.similarity_distance
        if self.malware_subject_node_a is not None : cluster_edge_node_pair_dict['malware_subject_node_a'] = self.malware_subject_node_a.to_dict()
        if self.malware_subject_node_b is not None : cluster_edge_node_pair_dict['malware_subject_node_b'] = self.malware_subject_node_b.to_dict()
        return cluster_edge_node_pair_dict

    @staticmethod
    def from_dict(cluster_edge_node_pair_dict):
        if not cluster_edge_node_pair_dict:
            return None
        cluster_edge_node_pair_ = ClusterEdgeNodePair()
        cluster_edge_node_pair_.similarity_index = cluster_edge_node_pair_dict.get('similarity_index')
        cluster_edge_node_pair_.similarity_distance = cluster_edge_node_pair_dict.get('similarity_distance')
        cluster_edge_node_pair_.malware_subject_node_a = MalwareSubjectReference.from_dict(cluster_edge_node_pair_dict.get('malware_subject_node_a'))
        cluster_edge_node_pair_.malware_subject_node_b = MalwareSubjectReference.from_dict(cluster_edge_node_pair_dict.get('malware_subject_node_b'))
        return cluster_edge_node_pair_

    @staticmethod
    def from_obj(cluster_edge_node_pair_obj):
        if not cluster_edge_node_pair_obj:
            return None
        cluster_edge_node_pair_ = ClusterEdgeNodePair()
        cluster_edge_node_pair_.similarity_index = cluster_edge_node_pair_obj.get_similarity_index()
        cluster_edge_node_pair_.similarity_distance = cluster_edge_node_pair_obj.get_similarity_distance()
        cluster_edge_node_pair_.malware_subject_node_a = MalwareSubjectReference.from_obj(cluster_edge_node_pair_obj.get_Malware_Subject_Node_A())
        cluster_edge_node_pair_.malware_subject_node_b = MalwareSubjectReference.from_obj(cluster_edge_node_pair_obj.get_Malware_Subject_Node_B())
        return cluster_edge_node_pair_
<|endoftext|>


<commit_before>#!/usr/bin/env python3

#
# This file is part of LiteX-Boards.
#
# Copyright (c) 2020 Florent Kermarrec <florent@enjoy-digital.fr>
# SPDX-License-Identifier: BSD-2-Clause

# Build/Use ----------------------------------------------------------------------------------------
# Build/Load bitstream:
# ./sqrl_acorn.py --uart-name=crossover --with-pcie --build --driver --load (or --flash)
#
#.Build the kernel and load it:
# cd build/<platform>/driver/kernel
# make
# sudo ./init.sh
#
# Test userspace utilities:
# cd build/<platform>/driver/user
# make
# ./litepcie_util info
# ./litepcie_util scratch_test
# ./litepcie_util dma_test
# ./litepcie_util uart_test

import os

from migen import *

from litex_boards.platforms import sqrl_acorn

from litex.soc.interconnect.csr import *
from litex.soc.integration.soc_core import *
from litex.soc.integration.builder import *

from litex.soc.cores.clock import *
from litex.soc.cores.led import LedChaser
from litex.soc.cores.xadc import XADC
from litex.soc.cores.dna  import DNA

from litedram.modules import MT41K512M16
from litedram.phy import s7ddrphy

from litepcie.phy.s7pciephy import S7PCIEPHY
from litepcie.software import generate_litepcie_software

# CRG ----------------------------------------------------------------------------------------------

class CRG(Module):
    def __init__(self, platform, sys_clk_freq):
        self.rst = Signal()
        self.clock_domains.cd_sys       = ClockDomain()
        self.clock_domains.cd_sys4x     = ClockDomain()
        self.clock_domains.cd_sys4x_dqs = ClockDomain()
        self.clock_domains.cd_idelay    = ClockDomain()

        # Clk/Rst
        clk200 = platform.request("clk200")

        # PLL
        self.submodules.pll = pll = S7PLL()
        self.comb += pll.reset.eq(self.rst)
        pll.register_clkin(clk200, 200e6)
        pll.create_clkout(self.cd_sys,       sys_clk_freq)
        pll.create_clkout(self.cd_sys4x,     4*sys_clk_freq)
        pll.create_clkout(self.cd_sys4x_dqs, 4*sys_clk_freq, phase=90)
        pll.create_clkout(self.cd_idelay,    200e6)
        platform.add_false_path_constraints(self.cd_sys.clk, pll.clkin) # Ignore sys_clk to pll.clkin path created by SoC's rst.

        self.submodules.idelayctrl = S7IDELAYCTRL(self.cd_idelay)

# BaseSoC -----------------------------------------------------------------------------------------

class BaseSoC(SoCCore):
    def __init__(self, variant="cle-215+", sys_clk_freq=int(100e6), with_led_chaser=True,
                 with_pcie=False, with_sata=False, **kwargs):
        platform = sqrl_acorn.Platform(variant=variant)

        # CRG --------------------------------------------------------------------------------------
        self.submodules.crg = CRG(platform, sys_clk_freq)

        # SoCCore ----------------------------------------------------------------------------------
        SoCCore.__init__(self, platform, sys_clk_freq, ident="LiteX SoC on Acorn CLE-101/215(+)", **kwargs)

        # XADC -------------------------------------------------------------------------------------
        self.submodules.xadc = XADC()

        # DNA --------------------------------------------------------------------------------------
        self.submodules.dna = DNA()
        self.dna.add_timing_constraints(platform, sys_clk_freq, self.crg.cd_sys.clk)

        # DDR3 SDRAM -------------------------------------------------------------------------------
        if not self.integrated_main_ram_size:
            self.submodules.ddrphy = s7ddrphy.A7DDRPHY(platform.request("ddram"),
                memtype          = "DDR3",
                nphases          = 4,
                sys_clk_freq     = sys_clk_freq,
                iodelay_clk_freq = 200e6)
            self.add_sdram("sdram",
                phy           = self.ddrphy,
                module        = MT41K512M16(sys_clk_freq, "1:4"),
                l2_cache_size = kwargs.get("l2_size", 8192)
            )

        # PCIe -------------------------------------------------------------------------------------
        if with_pcie:
            self.submodules.pcie_phy = S7PCIEPHY(platform, platform.request("pcie_x1"),
                data_width = 128,
                bar0_size  = 0x20000)
            self.add_pcie(phy=self.pcie_phy, ndmas=1)
            # FIXME: Apply it to all targets (integrate it in LitePCIe?).
            platform.add_period_constraint(self.crg.cd_sys.clk, 1e9/sys_clk_freq)
            platform.toolchain.pre_placement_commands.add("set_clock_groups -group [get_clocks {sys_clk}] -group [get_clocks userclk2] -asynchronous", sys_clk=self.crg.cd_sys.clk)
            platform.toolchain.pre_placement_commands.add("set_clock_groups -group [get_clocks {sys_clk}] -group [get_clocks clk_125mhz] -asynchronous", sys_clk=self.crg.cd_sys.clk)
            platform.toolchain.pre_placement_commands.add("set_clock_groups -group [get_clocks {sys_clk}] -group [get_clocks clk_250mhz] -asynchronous", sys_clk=self.crg.cd_sys.clk)
            platform.toolchain.pre_placement_commands.add("set_clock_groups -group [get_clocks clk_125mhz] -group [get_clocks clk_250mhz] -asynchronous")

            # ICAP (For FPGA reload over PCIe).
            from litex.soc.cores.icap import ICAP
            self.submodules.icap = ICAP()
            self.icap.add_reload()
            self.icap.add_timing_constraints(platform, sys_clk_freq, self.crg.cd_sys.clk)

            # Flash (For SPIFlash update over PCIe).
            from litex.soc.cores.gpio import GPIOOut
            from litex.soc.cores.spi_flash import S7SPIFlash
            self.submodules.flash_cs_n = GPIOOut(platform.request("flash_cs_n"))
            self.submodules.flash      = S7SPIFlash(platform.request("flash"), sys_clk_freq, 25e6)

        # SATA -------------------------------------------------------------------------------------
        if with_sata:
            from litex.build.generic_platform import Subsignal, Pins
            from litesata.phy import LiteSATAPHY

            # IOs
            _sata_io = [
                 # PCIe 2 SATA Custom Adapter (With PCIe Riser / SATA cable mod).
                ("pcie2sata", 0,
                    Subsignal("tx_p",  Pins("B6")),
                    Subsignal("tx_n",  Pins("A6")),
                    Subsignal("rx_p",  Pins("B10")),
                    Subsignal("rx_n",  Pins("A10")),
                ),
            ]
            platform.add_extension(_sata_io)

            # RefClk, Generate 150MHz from PLL.
            self.clock_domains.cd_sata_refclk = ClockDomain()
            self.crg.pll.create_clkout(self.cd_sata_refclk, 150e6)
            sata_refclk = ClockSignal("sata_refclk")
            platform.add_platform_command("set_property SEVERITY {{Warning}} [get_drc_checks REQP-49]")

            # PHY
            self.submodules.sata_phy = LiteSATAPHY(platform.device,
                refclk     = sata_refclk,
                pads       = platform.request("pcie2sata"),
                gen        = "gen1",
                clk_freq   = sys_clk_freq,
                data_width = 16)

            # Core
            self.add_sata(phy=self.sata_phy, mode="read+write")

        # Leds -------------------------------------------------------------------------------------
        if with_led_chaser:
            self.submodules.leds = LedChaser(
                pads         = platform.request_all("user_led"),
                sys_clk_freq = sys_clk_freq)

# Build --------------------------------------------------------------------------------------------

def main():
    from litex.soc.integration.soc import LiteXSoCArgumentParser
    parser = LiteXSoCArgumentParser(description="LiteX SoC on Acorn CLE-101/215(+)")
    target_group = parser.add_argument_group(title="Target options")
    target_group.add_argument("--build",           action="store_true", help="Build design.")
    target_group.add_argument("--load",            action="store_true", help="Load bitstream.")
    target_group.add_argument("--flash",           action="store_true", help="Flash bitstream.")
    target_group.add_argument("--variant",         default="cle-215+",  help="Board variant (cle-215+, cle-215 or cle-101).")
    target_group.add_argument("--sys-clk-freq",    default=100e6,       help="System clock frequency.")
    pcieopts = target_group.add_mutually_exclusive_group()
    pcieopts.add_argument("--with-pcie",     action="store_true", help="Enable PCIe support.")
    target_group.add_argument("--driver",          action="store_true", help="Generate PCIe driver.")
    target_group.add_argument("--with-spi-sdcard", action="store_true", help="Enable SPI-mode SDCard support (requires SDCard adapter on P2).")
    pcieopts.add_argument("--with-sata",     action="store_true", help="Enable SATA support (over PCIe2SATA).")
    builder_args(parser)
    soc_core_args(parser)
    args = parser.parse_args()

    soc = BaseSoC(
        variant      = args.variant,
        sys_clk_freq = int(float(args.sys_clk_freq)),
        with_pcie    = args.with_pcie,
        with_sata    = args.with_sata,
        **soc_core_argdict(args)
    )
    if args.with_spi_sdcard:
        soc.add_spi_sdcard()

    builder  = Builder(soc, **builder_argdict(args))
    if args.build:
        builder.build()

    if args.driver:
        generate_litepcie_software(soc, os.path.join(builder.output_dir, "driver"))

    if args.load:
        prog = soc.platform.create_programmer()
        prog.load_bitstream(builder.get_bitstream_filename(mode="sram"))

    if args.flash:
        prog = soc.platform.create_programmer()
        prog.flash(0, builder.get_bitstream_filename(mode="flash"))

if __name__ == "__main__":
    main()
<commit_msg>sqrl_acorn: Switch back to PCIe Gen2 X4 and enable 64-bit addressing (Not really useful except for 64-bit addressing tests).<commit_after>#!/usr/bin/env python3

#
# This file is part of LiteX-Boards.
#
# Copyright (c) 2020 Florent Kermarrec <florent@enjoy-digital.fr>
# SPDX-License-Identifier: BSD-2-Clause

# Build/Use ----------------------------------------------------------------------------------------
# Build/Load bitstream:
# ./sqrl_acorn.py --uart-name=crossover --with-pcie --build --driver --load (or --flash)
#
#.Build the kernel and load it:
# cd build/<platform>/driver/kernel
# make
# sudo ./init.sh
#
# Test userspace utilities:
# cd build/<platform>/driver/user
# make
# ./litepcie_util info
# ./litepcie_util scratch_test
# ./litepcie_util dma_test
# ./litepcie_util uart_test

import os

from migen import *

from litex_boards.platforms import sqrl_acorn

from litex.soc.interconnect.csr import *
from litex.soc.integration.soc_core import *
from litex.soc.integration.builder import *

from litex.soc.cores.clock import *
from litex.soc.cores.led import LedChaser
from litex.soc.cores.xadc import XADC
from litex.soc.cores.dna  import DNA

from litedram.modules import MT41K512M16
from litedram.phy import s7ddrphy

from litepcie.phy.s7pciephy import S7PCIEPHY
from litepcie.software import generate_litepcie_software

# CRG ----------------------------------------------------------------------------------------------

class CRG(Module):
    def __init__(self, platform, sys_clk_freq):
        self.rst = Signal()
        self.clock_domains.cd_sys       = ClockDomain()
        self.clock_domains.cd_sys4x     = ClockDomain()
        self.clock_domains.cd_sys4x_dqs = ClockDomain()
        self.clock_domains.cd_idelay    = ClockDomain()

        # Clk/Rst
        clk200 = platform.request("clk200")

        # PLL
        self.submodules.pll = pll = S7PLL()
        self.comb += pll.reset.eq(self.rst)
        pll.register_clkin(clk200, 200e6)
        pll.create_clkout(self.cd_sys,       sys_clk_freq)
        pll.create_clkout(self.cd_sys4x,     4*sys_clk_freq)
        pll.create_clkout(self.cd_sys4x_dqs, 4*sys_clk_freq, phase=90)
        pll.create_clkout(self.cd_idelay,    200e6)
        platform.add_false_path_constraints(self.cd_sys.clk, pll.clkin) # Ignore sys_clk to pll.clkin path created by SoC's rst.

        self.submodules.idelayctrl = S7IDELAYCTRL(self.cd_idelay)

# BaseSoC -----------------------------------------------------------------------------------------

class BaseSoC(SoCCore):
    def __init__(self, variant="cle-215+", sys_clk_freq=int(100e6), with_led_chaser=True,
                 with_pcie=False, with_sata=False, **kwargs):
        platform = sqrl_acorn.Platform(variant=variant)

        # CRG --------------------------------------------------------------------------------------
        self.submodules.crg = CRG(platform, sys_clk_freq)

        # SoCCore ----------------------------------------------------------------------------------
        SoCCore.__init__(self, platform, sys_clk_freq, ident="LiteX SoC on Acorn CLE-101/215(+)", **kwargs)

        # XADC -------------------------------------------------------------------------------------
        self.submodules.xadc = XADC()

        # DNA --------------------------------------------------------------------------------------
        self.submodules.dna = DNA()
        self.dna.add_timing_constraints(platform, sys_clk_freq, self.crg.cd_sys.clk)

        # DDR3 SDRAM -------------------------------------------------------------------------------
        if not self.integrated_main_ram_size:
            self.submodules.ddrphy = s7ddrphy.A7DDRPHY(platform.request("ddram"),
                memtype          = "DDR3",
                nphases          = 4,
                sys_clk_freq     = sys_clk_freq,
                iodelay_clk_freq = 200e6)
            self.add_sdram("sdram",
                phy           = self.ddrphy,
                module        = MT41K512M16(sys_clk_freq, "1:4"),
                l2_cache_size = kwargs.get("l2_size", 8192)
            )

        # PCIe -------------------------------------------------------------------------------------
        if with_pcie:
            self.submodules.pcie_phy = S7PCIEPHY(platform, platform.request("pcie_x4"),
                data_width = 128,
                bar0_size  = 0x20000)
            self.add_pcie(phy=self.pcie_phy, ndmas=1, address_width=64)
            # FIXME: Apply it to all targets (integrate it in LitePCIe?).
            platform.add_period_constraint(self.crg.cd_sys.clk, 1e9/sys_clk_freq)
            platform.toolchain.pre_placement_commands.add("set_clock_groups -group [get_clocks {sys_clk}] -group [get_clocks userclk2] -asynchronous", sys_clk=self.crg.cd_sys.clk)
            platform.toolchain.pre_placement_commands.add("set_clock_groups -group [get_clocks {sys_clk}] -group [get_clocks clk_125mhz] -asynchronous", sys_clk=self.crg.cd_sys.clk)
            platform.toolchain.pre_placement_commands.add("set_clock_groups -group [get_clocks {sys_clk}] -group [get_clocks clk_250mhz] -asynchronous", sys_clk=self.crg.cd_sys.clk)
            platform.toolchain.pre_placement_commands.add("set_clock_groups -group [get_clocks clk_125mhz] -group [get_clocks clk_250mhz] -asynchronous")

            # ICAP (For FPGA reload over PCIe).
            from litex.soc.cores.icap import ICAP
            self.submodules.icap = ICAP()
            self.icap.add_reload()
            self.icap.add_timing_constraints(platform, sys_clk_freq, self.crg.cd_sys.clk)

            # Flash (For SPIFlash update over PCIe).
            from litex.soc.cores.gpio import GPIOOut
            from litex.soc.cores.spi_flash import S7SPIFlash
            self.submodules.flash_cs_n = GPIOOut(platform.request("flash_cs_n"))
            self.submodules.flash      = S7SPIFlash(platform.request("flash"), sys_clk_freq, 25e6)

        # SATA -------------------------------------------------------------------------------------
        if with_sata:
            from litex.build.generic_platform import Subsignal, Pins
            from litesata.phy import LiteSATAPHY

            # IOs
            _sata_io = [
                 # PCIe 2 SATA Custom Adapter (With PCIe Riser / SATA cable mod).
                ("pcie2sata", 0,
                    Subsignal("tx_p",  Pins("B6")),
                    Subsignal("tx_n",  Pins("A6")),
                    Subsignal("rx_p",  Pins("B10")),
                    Subsignal("rx_n",  Pins("A10")),
                ),
            ]
            platform.add_extension(_sata_io)

            # RefClk, Generate 150MHz from PLL.
            self.clock_domains.cd_sata_refclk = ClockDomain()
            self.crg.pll.create_clkout(self.cd_sata_refclk, 150e6)
            sata_refclk = ClockSignal("sata_refclk")
            platform.add_platform_command("set_property SEVERITY {{Warning}} [get_drc_checks REQP-49]")

            # PHY
            self.submodules.sata_phy = LiteSATAPHY(platform.device,
                refclk     = sata_refclk,
                pads       = platform.request("pcie2sata"),
                gen        = "gen1",
                clk_freq   = sys_clk_freq,
                data_width = 16)

            # Core
            self.add_sata(phy=self.sata_phy, mode="read+write")

        # Leds -------------------------------------------------------------------------------------
        if with_led_chaser:
            self.submodules.leds = LedChaser(
                pads         = platform.request_all("user_led"),
                sys_clk_freq = sys_clk_freq)

# Build --------------------------------------------------------------------------------------------

def main():
    from litex.soc.integration.soc import LiteXSoCArgumentParser
    parser = LiteXSoCArgumentParser(description="LiteX SoC on Acorn CLE-101/215(+)")
    target_group = parser.add_argument_group(title="Target options")
    target_group.add_argument("--build",           action="store_true", help="Build design.")
    target_group.add_argument("--load",            action="store_true", help="Load bitstream.")
    target_group.add_argument("--flash",           action="store_true", help="Flash bitstream.")
    target_group.add_argument("--variant",         default="cle-215+",  help="Board variant (cle-215+, cle-215 or cle-101).")
    target_group.add_argument("--sys-clk-freq",    default=100e6,       help="System clock frequency.")
    pcieopts = target_group.add_mutually_exclusive_group()
    pcieopts.add_argument("--with-pcie",     action="store_true", help="Enable PCIe support.")
    target_group.add_argument("--driver",          action="store_true", help="Generate PCIe driver.")
    target_group.add_argument("--with-spi-sdcard", action="store_true", help="Enable SPI-mode SDCard support (requires SDCard adapter on P2).")
    pcieopts.add_argument("--with-sata",     action="store_true", help="Enable SATA support (over PCIe2SATA).")
    builder_args(parser)
    soc_core_args(parser)
    args = parser.parse_args()

    soc = BaseSoC(
        variant      = args.variant,
        sys_clk_freq = int(float(args.sys_clk_freq)),
        with_pcie    = args.with_pcie,
        with_sata    = args.with_sata,
        **soc_core_argdict(args)
    )
    if args.with_spi_sdcard:
        soc.add_spi_sdcard()

    builder  = Builder(soc, **builder_argdict(args))
    if args.build:
        builder.build()

    if args.driver:
        generate_litepcie_software(soc, os.path.join(builder.output_dir, "driver"))

    if args.load:
        prog = soc.platform.create_programmer()
        prog.load_bitstream(builder.get_bitstream_filename(mode="sram"))

    if args.flash:
        prog = soc.platform.create_programmer()
        prog.flash(0, builder.get_bitstream_filename(mode="flash"))

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before># This file is part of the MapProxy project.
# Copyright (C) 2011 Omniscale <http://omniscale.de>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from __future__ import with_statement

import os
import shutil
import tempfile
import time

from cStringIO import StringIO

from PIL import Image

from mapproxy.cache.tile import Tile
from mapproxy.cache.file import FileCache
from mapproxy.cache.mbtiles import MBTilesCache
from mapproxy.image import ImageSource
from mapproxy.image.opts import ImageOptions
from mapproxy.test.image import create_tmp_image_buf, is_png

tile_image = create_tmp_image_buf((256, 256), color='blue')
tile_image2 = create_tmp_image_buf((256, 256), color='red')

def timestamp_is_now(timestamp, delta=5):
    return abs(timestamp - time.time()) <= delta

class TileCacheTestBase(object):
    always_loads_metadata = False

    def setup(self):
        self.cache_dir = tempfile.mkdtemp()

    def teardown(self):
        if hasattr(self, 'cache_dir') and os.path.exists(self.cache_dir):
            shutil.rmtree(self.cache_dir)

    def create_tile(self, coord=(0, 0, 4)):
        return Tile(coord,
            ImageSource(tile_image,
                image_opts=ImageOptions(format='image/png')))

    def create_another_tile(self, coord=(0, 0, 4)):
        return Tile(coord,
            ImageSource(tile_image2,
                image_opts=ImageOptions(format='image/png')))

    def test_is_cached_miss(self):
        assert not self.cache.is_cached(Tile((0, 0, 4)))

    def test_is_cached_hit(self):
        tile = self.create_tile()
        self.create_cached_tile(tile)
        assert self.cache.is_cached(Tile((0, 0, 4)))

    def test_is_cached_none(self):
        assert self.cache.is_cached(Tile(None))

    def test_load_tile_none(self):
        assert self.cache.load_tile(Tile(None))

    def test_load_tile_not_cached(self):
        tile = Tile((0, 0, 4))
        assert not self.cache.load_tile(tile)
        assert tile.source is None
        assert tile.is_missing()

    def test_load_tile_cached(self):
        tile = self.create_tile()
        self.create_cached_tile(tile)
        tile = Tile((0, 0, 4))
        assert self.cache.load_tile(tile) == True
        assert not tile.is_missing()

    def test_load_tiles(self):
        tile = self.create_tile((1, 0, 4))
        self.create_cached_tile(tile)
        tiles = [Tile(None), Tile((0, 0, 4)), Tile((1, 0, 4))]
        assert self.cache.load_tiles(tiles) == False
        assert not tiles[0].is_missing()
        assert tiles[1].is_missing()
        assert not tiles[2].is_missing()

    def test_store_tiles(self):
        tiles = [self.create_tile((x, 0, 4)) for x in range(4)]
        tiles[0].stored = True
        self.cache.store_tiles(tiles)

        tiles = [Tile((x, 0, 4)) for x in range(4)]
        assert tiles[0].is_missing()
        assert self.cache.load_tile(tiles[0]) == False
        assert tiles[0].is_missing()

        for tile in tiles[1:]:
            assert tile.is_missing()
            assert self.cache.load_tile(tile) == True
            assert not tile.is_missing()

    def test_load_stored_tile(self):
        tile = self.create_tile((5, 12, 4))
        self.cache.store_tile(tile)
        size = tile.size

        # check stored tile
        tile = Tile((5, 12, 4))
        assert tile.source is None

        assert self.cache.load_tile(tile)
        if not self.always_loads_metadata:
            assert tile.source is not None
            assert tile.timestamp is None
            assert tile.size is None
        stored_size = len(tile.source.as_buffer().read())
        assert stored_size == size

        # check loading of metadata (timestamp, size)
        tile = Tile((5, 12, 4))
        assert tile.source is None
        assert self.cache.load_tile(tile, with_metadata=True)
        assert tile.source is not None
        if tile.timestamp:
            assert timestamp_is_now(tile.timestamp, delta=5)
        if tile.size:
            assert tile.size == size

    def test_overwrite_tile(self):
        tile = self.create_tile((5, 12, 4))
        self.cache.store_tile(tile)

        tile = Tile((5, 12, 4))
        self.cache.load_tile(tile)
        tile1_content = tile.source.as_buffer().read()
        assert tile1_content == tile_image.getvalue()

        tile = self.create_another_tile((5, 12, 4))
        self.cache.store_tile(tile)

        tile = Tile((5, 12, 4))
        self.cache.load_tile(tile)
        tile2_content = tile.source.as_buffer().read()
        assert tile2_content == tile_image2.getvalue()

        assert tile1_content != tile2_content

    def test_store_tile_already_stored(self):
        # tile object is marked as stored,
        # check that is is not stored 'again'
        # (used for disable_storage)
        tile = Tile((0, 0, 4), ImageSource(StringIO('foo')))
        tile.stored = True
        self.cache.store_tile(tile)

        assert self.cache.is_cached(tile)

        tile = Tile((0, 0, 4))
        assert not self.cache.is_cached(tile)

    def test_remove(self):
        tile = self.create_tile((1, 0, 4))
        self.create_cached_tile(tile)
        assert self.cache.is_cached(Tile((1, 0, 4)))

        self.cache.remove_tile(Tile((1, 0, 4)))
        assert not self.cache.is_cached(Tile((1, 0, 4)))

    def create_cached_tile(self, tile):
        self.cache.store_tile(tile)

class TestFileTileCache(TileCacheTestBase):
    def setup(self):
        TileCacheTestBase.setup(self)
        self.cache = FileCache(self.cache_dir, 'png')

    def test_store_tile(self):
        tile = self.create_tile((5, 12, 4))
        self.cache.store_tile(tile)
        tile_location = os.path.join(self.cache_dir,
            '04', '000', '000', '005', '000', '000', '012.png' )
        assert os.path.exists(tile_location), tile_location

    def test_single_color_tile_store(self):
        img = Image.new('RGB', (256, 256), color='#ff0105')
        tile = Tile((0, 0, 4), ImageSource(img, image_opts=ImageOptions(format='image/png')))
        self.cache.link_single_color_images = True
        self.cache.store_tile(tile)
        assert self.cache.is_cached(tile)
        loc = self.cache.tile_location(tile)
        assert os.path.islink(loc)
        assert os.path.realpath(loc).endswith('ff0105.png')
        assert is_png(open(loc, 'rb'))

        tile2 = Tile((0, 0, 1), ImageSource(img))
        self.cache.store_tile(tile2)
        assert self.cache.is_cached(tile2)
        loc2 = self.cache.tile_location(tile2)
        assert os.path.islink(loc2)
        assert os.path.realpath(loc2).endswith('ff0105.png')
        assert is_png(open(loc2, 'rb'))

        assert loc != loc2
        assert os.path.samefile(loc, loc2)

    def test_single_color_tile_store_w_alpha(self):
        img = Image.new('RGBA', (256, 256), color='#ff0105')
        tile = Tile((0, 0, 4), ImageSource(img, image_opts=ImageOptions(format='image/png')))
        self.cache.link_single_color_images = True
        self.cache.store_tile(tile)
        assert self.cache.is_cached(tile)
        loc = self.cache.tile_location(tile)
        assert os.path.islink(loc)
        assert os.path.realpath(loc).endswith('ff0105ff.png')
        assert is_png(open(loc, 'rb'))

    def create_cached_tile(self, tile):
        loc = self.cache.tile_location(tile, create_dir=True)
        with open(loc, 'w') as f:
            f.write('foo')

class TestMBTileCache(TileCacheTestBase):
    def setup(self):
        TileCacheTestBase.setup(self)
        self.cache = MBTilesCache(os.path.join(self.cache_dir, 'tmp.mbtiles'))
      <commit_msg>add tile cache tests<commit_after># This file is part of the MapProxy project.
# Copyright (C) 2011 Omniscale <http://omniscale.de>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from __future__ import with_statement

import os
import shutil
import tempfile
import time

from cStringIO import StringIO

from PIL import Image

from mapproxy.cache.tile import Tile
from mapproxy.cache.file import FileCache
from mapproxy.cache.mbtiles import MBTilesCache
from mapproxy.image import ImageSource
from mapproxy.image.opts import ImageOptions
from mapproxy.test.image import create_tmp_image_buf, is_png

tile_image = create_tmp_image_buf((256, 256), color='blue')
tile_image2 = create_tmp_image_buf((256, 256), color='red')

def timestamp_is_now(timestamp, delta=5):
    return abs(timestamp - time.time()) <= delta

class TileCacheTestBase(object):
    always_loads_metadata = False

    def setup(self):
        self.cache_dir = tempfile.mkdtemp()

    def teardown(self):
        if hasattr(self, 'cache_dir') and os.path.exists(self.cache_dir):
            shutil.rmtree(self.cache_dir)

    def create_tile(self, coord=(0, 0, 4)):
        return Tile(coord,
            ImageSource(tile_image,
                image_opts=ImageOptions(format='image/png')))

    def create_another_tile(self, coord=(0, 0, 4)):
        return Tile(coord,
            ImageSource(tile_image2,
                image_opts=ImageOptions(format='image/png')))

    def test_is_cached_miss(self):
        assert not self.cache.is_cached(Tile((0, 0, 4)))

    def test_is_cached_hit(self):
        tile = self.create_tile()
        self.create_cached_tile(tile)
        assert self.cache.is_cached(Tile((0, 0, 4)))

    def test_is_cached_none(self):
        assert self.cache.is_cached(Tile(None))

    def test_load_tile_none(self):
        assert self.cache.load_tile(Tile(None))

    def test_load_tile_not_cached(self):
        tile = Tile((0, 0, 4))
        assert not self.cache.load_tile(tile)
        assert tile.source is None
        assert tile.is_missing()

    def test_load_tile_cached(self):
        tile = self.create_tile()
        self.create_cached_tile(tile)
        tile = Tile((0, 0, 4))
        assert self.cache.load_tile(tile) == True
        assert not tile.is_missing()

    def test_store_tiles(self):
        tiles = [self.create_tile((x, 0, 4)) for x in range(4)]
        tiles[0].stored = True
        self.cache.store_tiles(tiles)

        tiles = [Tile((x, 0, 4)) for x in range(4)]
        assert tiles[0].is_missing()
        assert self.cache.load_tile(tiles[0]) == False
        assert tiles[0].is_missing()

        for tile in tiles[1:]:
            assert tile.is_missing()
            assert self.cache.load_tile(tile) == True
            assert not tile.is_missing()

    def test_load_tiles_cached(self):
        self.cache.store_tile(self.create_tile((0, 0, 1)))
        self.cache.store_tile(self.create_tile((0, 1, 1)))
        tiles = [Tile((0, 0, 1)), Tile((0, 1, 1))]
        assert self.cache.load_tiles(tiles)

        assert not tiles[0].is_missing()
        assert not tiles[1].is_missing()

    def test_load_tiles_mixed(self):
        tile = self.create_tile((1, 0, 4))
        self.create_cached_tile(tile)
        tiles = [Tile(None), Tile((0, 0, 4)), Tile((1, 0, 4))]
        assert self.cache.load_tiles(tiles) == False
        assert not tiles[0].is_missing()
        assert tiles[1].is_missing()
        assert not tiles[2].is_missing()

    def test_load_stored_tile(self):
        tile = self.create_tile((5, 12, 4))
        self.cache.store_tile(tile)
        size = tile.size

        # check stored tile
        tile = Tile((5, 12, 4))
        assert tile.source is None

        assert self.cache.load_tile(tile)
        if not self.always_loads_metadata:
            assert tile.source is not None
            assert tile.timestamp is None
            assert tile.size is None
        stored_size = len(tile.source.as_buffer().read())
        assert stored_size == size

        # check loading of metadata (timestamp, size)
        tile = Tile((5, 12, 4))
        assert tile.source is None
        assert self.cache.load_tile(tile, with_metadata=True)
        assert tile.source is not None
        if tile.timestamp:
            assert timestamp_is_now(tile.timestamp, delta=10)
        if tile.size:
            assert tile.size == size

    def test_overwrite_tile(self):
        tile = self.create_tile((5, 12, 4))
        self.cache.store_tile(tile)

        tile = Tile((5, 12, 4))
        self.cache.load_tile(tile)
        tile1_content = tile.source.as_buffer().read()
        assert tile1_content == tile_image.getvalue()

        tile = self.create_another_tile((5, 12, 4))
        self.cache.store_tile(tile)

        tile = Tile((5, 12, 4))
        self.cache.load_tile(tile)
        tile2_content = tile.source.as_buffer().read()
        assert tile2_content == tile_image2.getvalue()

        assert tile1_content != tile2_content

    def test_store_tile_already_stored(self):
        # tile object is marked as stored,
        # check that is is not stored 'again'
        # (used for disable_storage)
        tile = Tile((0, 0, 4), ImageSource(StringIO('foo')))
        tile.stored = True
        self.cache.store_tile(tile)

        assert self.cache.is_cached(tile)

        tile = Tile((0, 0, 4))
        assert not self.cache.is_cached(tile)

    def test_remove(self):
        tile = self.create_tile((1, 0, 4))
        self.create_cached_tile(tile)
        assert self.cache.is_cached(Tile((1, 0, 4)))

        self.cache.remove_tile(Tile((1, 0, 4)))
        assert not self.cache.is_cached(Tile((1, 0, 4)))

    def create_cached_tile(self, tile):
        self.cache.store_tile(tile)

class TestFileTileCache(TileCacheTestBase):
    def setup(self):
        TileCacheTestBase.setup(self)
        self.cache = FileCache(self.cache_dir, 'png')

    def test_store_tile(self):
        tile = self.create_tile((5, 12, 4))
        self.cache.store_tile(tile)
        tile_location = os.path.join(self.cache_dir,
            '04', '000', '000', '005', '000', '000', '012.png' )
        assert os.path.exists(tile_location), tile_location

    def test_single_color_tile_store(self):
        img = Image.new('RGB', (256, 256), color='#ff0105')
        tile = Tile((0, 0, 4), ImageSource(img, image_opts=ImageOptions(format='image/png')))
        self.cache.link_single_color_images = True
        self.cache.store_tile(tile)
        assert self.cache.is_cached(tile)
        loc = self.cache.tile_location(tile)
        assert os.path.islink(loc)
        assert os.path.realpath(loc).endswith('ff0105.png')
        assert is_png(open(loc, 'rb'))

        tile2 = Tile((0, 0, 1), ImageSource(img))
        self.cache.store_tile(tile2)
        assert self.cache.is_cached(tile2)
        loc2 = self.cache.tile_location(tile2)
        assert os.path.islink(loc2)
        assert os.path.realpath(loc2).endswith('ff0105.png')
        assert is_png(open(loc2, 'rb'))

        assert loc != loc2
        assert os.path.samefile(loc, loc2)

    def test_single_color_tile_store_w_alpha(self):
        img = Image.new('RGBA', (256, 256), color='#ff0105')
        tile = Tile((0, 0, 4), ImageSource(img, image_opts=ImageOptions(format='image/png')))
        self.cache.link_single_color_images = True
        self.cache.store_tile(tile)
        assert self.cache.is_cached(tile)
        loc = self.cache.tile_location(tile)
        assert os.path.islink(loc)
        assert os.path.realpath(loc).endswith('ff0105ff.png')
        assert is_png(open(loc, 'rb'))

    def create_cached_tile(self, tile):
        loc = self.cache.tile_location(tile, create_dir=True)
        with open(loc, 'w') as f:
            f.write('foo')

class TestMBTileCache(TileCacheTestBase):
    def setup(self):
        TileCacheTestBase.setup(self)
        self.cache = MBTilesCache(os.path.join(self.cache_dir, 'tmp.mbtiles'))
      <|endoftext|>


<commit_before># Copyright (c) 2012 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""Utility class to build the Skia master BuildFactory's.

Based on gclient_factory.py and adds Skia-specific steps."""

import ntpath
import posixpath

from buildbot.process.properties import WithProperties
from master.factory import gclient_factory

from skia_master_scripts import commands as skia_commands
import config

SKIA_SVN_BASEURL = 'https://skia.googlecode.com/svn'
AUTOGEN_SVN_BASEURL = 'https://skia-autogen.googlecode.com/svn'

# TODO(epoger): My intent is to make the build steps identical on all platforms
# and thus remove the need for the whole target_platform parameter.
# For now, these must match the target_platform values used in
# third_party/chromium_buildbot/scripts/master/factory/gclient_factory.py ,
# because we pass these values into GClientFactory.__init__() .
TARGET_PLATFORM_LINUX = 'linux'
TARGET_PLATFORM_MAC = 'mac'
TARGET_PLATFORM_WIN32 = 'win32'

class SkiaFactory(gclient_factory.GClientFactory):
  """Encapsulates data and methods common to the Skia master.cfg files."""

  def __init__(self, do_upload_results=False,
               build_subdir='trunk', other_subdirs=None,
               target_platform=None, configuration='Debug',
               default_timeout=600,
               environment_variables=None, gm_image_subdir=None,
               perf_output_basedir=None, builder_name=None, make_flags=None,
               test_args=None, gm_args=None, bench_args=None):
    """Instantiates a SkiaFactory as appropriate for this target_platform.

    do_upload_results: whether we should upload bench/gm results
    build_subdir: subdirectory to check out and then build within
    other_subdirs: list of other subdirectories to also check out (or None)
    target_platform: a string such as TARGET_PLATFORM_LINUX
    configuration: 'Debug' or 'Release'
    default_timeout: default timeout for each command, in seconds
    environment_variables: dictionary of environment variables that should
        be passed to all commands
    gm_image_subdir: directory containing images for comparison against results
        of gm tool
    perf_output_basedir: path to directory under which to store performance
        data, or None if we don't want to store performance data
    builder_name: name of the builder associated with this factory
    make_flags: list of extra flags to pass to the compile step
    test_args: list of extra flags to pass to the 'tests' executable
    gm_args: list of extra flags to pass to the 'gm' executable
    bench_args: list of extra flags to pass to the 'bench' executable
    """

    if not make_flags:
      make_flags = []
    self._make_flags = make_flags
    # Platform-specific stuff.
    if target_platform == TARGET_PLATFORM_WIN32:
      self.TargetPathJoin = ntpath.join
    else:
      self.TargetPathJoin = posixpath.join
      self._make_flags += ['--jobs', '--max-load=4.0']

    # Create gclient solutions corresponding to the main build_subdir
    # and other directories we also wish to check out.
    solutions = [gclient_factory.GClientSolution(
        svn_url=config.Master.skia_url + build_subdir, name=build_subdir)]
    if not other_subdirs:
      other_subdirs = []
    if gm_image_subdir:
      other_subdirs += ['gm-expected/%s' % gm_image_subdir, 'skp']
    for other_subdir in other_subdirs:
      solutions.append(gclient_factory.GClientSolution(
          svn_url=config.Master.skia_url + other_subdir,
          name=self.TargetPathJoin(*other_subdir.split('/'))))
    gclient_factory.GClientFactory.__init__(
        self, build_dir='', solutions=solutions,
        target_platform=target_platform)

    self._factory = self.BaseFactory(factory_properties=None)

    # Set _default_clobber based on config.Master
    self._default_clobber = getattr(config.Master, 'default_clobber', False)

    self._do_upload_results = do_upload_results
    self._make_bench_graphs = perf_output_basedir != None

    # Get an implementation of SkiaCommands as appropriate for
    # this target_platform.
    workdir = self.TargetPathJoin('build', build_subdir)
    self._skia_cmd_obj = skia_commands.SkiaCommands(
        target_platform=target_platform, factory=self._factory,
        configuration=configuration, workdir=workdir,
        target_arch=None, default_timeout=default_timeout,
        environment_variables=environment_variables)

    if not perf_output_basedir:
      perf_output_basedir = 'None'

    if not gm_image_subdir:
      gm_image_subdir = 'None'

    self._skia_svn_username_file = '.skia_svn_username'
    self._skia_svn_password_file = '.skia_svn_password'
    self._autogen_svn_username_file = '.autogen_svn_username'
    self._autogen_svn_password_file = '.autogen_svn_password'
    self._builder_name = builder_name

    if not test_args:
      test_args = []
    if not gm_args:
      gm_args = []
    if not bench_args:
      bench_args = []
    self._common_args = ['--autogen_svn_baseurl', AUTOGEN_SVN_BASEURL,
                         '--configuration', configuration,
                         '--gm_image_subdir', gm_image_subdir,
                         '--builder_name', builder_name,
                         '--target_platform', target_platform,
                         '--revision', WithProperties('%(got_revision)s'),
                         '--perf_output_basedir', perf_output_basedir,
                         '--make_flags', '"%s"' % ' '.join(self._make_flags),
                         '--test_args', '"%s"' % ' '.join(test_args),
                         '--gm_args', '"%s"' % ' '.join(gm_args),
                         '--bench_args', '"%s"' % ' '.join(bench_args),
                         ]

  def AddSlaveScript(self, script, description, args=[], timeout=None):
    self._skia_cmd_obj.AddSlaveScript(script=script,
                                      args=self._common_args + args,
                                      description=description,
                                      timeout=timeout)

  def Make(self, target, description):
    """Build a single target."""
    args = ['--target', target]
    self.AddSlaveScript(script='compile.py', args=args,
                        description=description, timeout=1200)

  def Compile(self):
    """Compile step.  Build everything. """
    self.Make('core',  'BuildCore')
    self.Make('tests', 'BuildTests')
    self.Make('gm',    'BuildGM')
    self.Make('tools', 'BuildTools')
    self.Make('bench', 'BuildBench')
    self.Make('all',   'BuildAllOtherTargets')

  def RunTests(self):
    """ Run the unit tests. """
    self.AddSlaveScript(script='run_tests.py', description='RunTests')

  def RunGM(self):
    """ Run the "GM" tool, saving the images to disk. """
    self.AddSlaveScript(script='run_gm.py', description='GenerateGMs')

  def RenderPictures(self):
    """ Run the "render_pictures" tool to generate images from .skp's. """
    self.AddSlaveScript(script='render_pictures.py',
                        description='RenderPictures')

  def CompareGMs(self):
    """ Run the "skdiff" tool to compare the "actual" GM images we just
    generated to the baselines in _gm_image_subdir. """
    self.AddSlaveScript(script='compare_gms.py', description='CompareGMs')

  def RunBench(self):
    """ Run "bench", piping the output somewhere so we can graph
    results over time. """
    self.AddSlaveScript(script='run_bench.py', description='RunBench',
                        timeout=1200)

  def BenchPictures(self):
    """ Run "bench_pictures" """
    self.AddSlaveScript(script='bench_pictures.py',
                        description='BenchPictures')

  def BenchGraphs(self):
    """ Generate and upload bench performance graphs (but only if we have been
    recording bench output for this build type). """
    self.AddSlaveScript(script='generate_bench_graphs.py',
                        description='GenerateBenchGraphs')
    if self._do_upload_results:
      self.AddSlaveScript(script='upload_bench_graphs.py',
                          description='UploadBenchGraphs')

  def UploadGMResults(self):
    """ Upload the images generated by GM """
    args = ['--autogen_svn_username_file', self._autogen_svn_username_file,
            '--autogen_svn_password_file', self._autogen_svn_password_file]
    self.AddSlaveScript(script='upload_gm_results.py', args=args,
                        description='UploadGMResults', timeout=1200)

  def Build(self, clobber=None):
    """Build and return the complete BuildFactory.

    clobber: boolean indicating whether we should clean before building
    """
    # Do all the build steps first, so we will find out about build breakages
    # as soon as possible.
    if clobber is None:
      clobber = self._default_clobber
    if clobber:
      self.AddSlaveScript(script='clean.py', description='Clean')
    self.Compile()
    self.RunTests()
    self.RunGM()
    self.RenderPictures()
    if self._do_upload_results:
      self.UploadGMResults()
    self.CompareGMs()
    self.RunBench()
    self.BenchPictures()
    if self._make_bench_graphs:
      self.BenchGraphs()

    return self._factory
<commit_msg>Fix gclient issue on Windows<commit_after># Copyright (c) 2012 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""Utility class to build the Skia master BuildFactory's.

Based on gclient_factory.py and adds Skia-specific steps."""

import ntpath
import posixpath

from buildbot.process.properties import WithProperties
from master.factory import gclient_factory

from skia_master_scripts import commands as skia_commands
import config

SKIA_SVN_BASEURL = 'https://skia.googlecode.com/svn'
AUTOGEN_SVN_BASEURL = 'https://skia-autogen.googlecode.com/svn'

# TODO(epoger): My intent is to make the build steps identical on all platforms
# and thus remove the need for the whole target_platform parameter.
# For now, these must match the target_platform values used in
# third_party/chromium_buildbot/scripts/master/factory/gclient_factory.py ,
# because we pass these values into GClientFactory.__init__() .
TARGET_PLATFORM_LINUX = 'linux'
TARGET_PLATFORM_MAC = 'mac'
TARGET_PLATFORM_WIN32 = 'win32'

class SkiaFactory(gclient_factory.GClientFactory):
  """Encapsulates data and methods common to the Skia master.cfg files."""

  def __init__(self, do_upload_results=False,
               build_subdir='trunk', other_subdirs=None,
               target_platform=None, configuration='Debug',
               default_timeout=600,
               environment_variables=None, gm_image_subdir=None,
               perf_output_basedir=None, builder_name=None, make_flags=None,
               test_args=None, gm_args=None, bench_args=None):
    """Instantiates a SkiaFactory as appropriate for this target_platform.

    do_upload_results: whether we should upload bench/gm results
    build_subdir: subdirectory to check out and then build within
    other_subdirs: list of other subdirectories to also check out (or None)
    target_platform: a string such as TARGET_PLATFORM_LINUX
    configuration: 'Debug' or 'Release'
    default_timeout: default timeout for each command, in seconds
    environment_variables: dictionary of environment variables that should
        be passed to all commands
    gm_image_subdir: directory containing images for comparison against results
        of gm tool
    perf_output_basedir: path to directory under which to store performance
        data, or None if we don't want to store performance data
    builder_name: name of the builder associated with this factory
    make_flags: list of extra flags to pass to the compile step
    test_args: list of extra flags to pass to the 'tests' executable
    gm_args: list of extra flags to pass to the 'gm' executable
    bench_args: list of extra flags to pass to the 'bench' executable
    """

    if not make_flags:
      make_flags = []
    self._make_flags = make_flags
    # Platform-specific stuff.
    if target_platform == TARGET_PLATFORM_WIN32:
      self.TargetPathJoin = ntpath.join
    else:
      self.TargetPathJoin = posixpath.join
      self._make_flags += ['--jobs', '--max-load=4.0']

    # Create gclient solutions corresponding to the main build_subdir
    # and other directories we also wish to check out.
    solutions = [gclient_factory.GClientSolution(
        svn_url=config.Master.skia_url + build_subdir, name=build_subdir)]
    if not other_subdirs:
      other_subdirs = []
    if gm_image_subdir:
      other_subdirs += ['gm-expected/%s' % gm_image_subdir, 'skp']
    for other_subdir in other_subdirs:
      solutions.append(gclient_factory.GClientSolution(
          svn_url=config.Master.skia_url + other_subdir,
          name=other_subdir))
    gclient_factory.GClientFactory.__init__(
        self, build_dir='', solutions=solutions,
        target_platform=target_platform)

    self._factory = self.BaseFactory(factory_properties=None)

    # Set _default_clobber based on config.Master
    self._default_clobber = getattr(config.Master, 'default_clobber', False)

    self._do_upload_results = do_upload_results
    self._make_bench_graphs = perf_output_basedir != None

    # Get an implementation of SkiaCommands as appropriate for
    # this target_platform.
    workdir = self.TargetPathJoin('build', build_subdir)
    self._skia_cmd_obj = skia_commands.SkiaCommands(
        target_platform=target_platform, factory=self._factory,
        configuration=configuration, workdir=workdir,
        target_arch=None, default_timeout=default_timeout,
        environment_variables=environment_variables)

    if not perf_output_basedir:
      perf_output_basedir = 'None'

    if not gm_image_subdir:
      gm_image_subdir = 'None'

    self._skia_svn_username_file = '.skia_svn_username'
    self._skia_svn_password_file = '.skia_svn_password'
    self._autogen_svn_username_file = '.autogen_svn_username'
    self._autogen_svn_password_file = '.autogen_svn_password'
    self._builder_name = builder_name

    if not test_args:
      test_args = []
    if not gm_args:
      gm_args = []
    if not bench_args:
      bench_args = []
    self._common_args = ['--autogen_svn_baseurl', AUTOGEN_SVN_BASEURL,
                         '--configuration', configuration,
                         '--gm_image_subdir', gm_image_subdir,
                         '--builder_name', builder_name,
                         '--target_platform', target_platform,
                         '--revision', WithProperties('%(got_revision)s'),
                         '--perf_output_basedir', perf_output_basedir,
                         '--make_flags', '"%s"' % ' '.join(self._make_flags),
                         '--test_args', '"%s"' % ' '.join(test_args),
                         '--gm_args', '"%s"' % ' '.join(gm_args),
                         '--bench_args', '"%s"' % ' '.join(bench_args),
                         ]

  def AddSlaveScript(self, script, description, args=[], timeout=None):
    self._skia_cmd_obj.AddSlaveScript(script=script,
                                      args=self._common_args + args,
                                      description=description,
                                      timeout=timeout)

  def Make(self, target, description):
    """Build a single target."""
    args = ['--target', target]
    self.AddSlaveScript(script='compile.py', args=args,
                        description=description, timeout=1200)

  def Compile(self):
    """Compile step.  Build everything. """
    self.Make('core',  'BuildCore')
    self.Make('tests', 'BuildTests')
    self.Make('gm',    'BuildGM')
    self.Make('tools', 'BuildTools')
    self.Make('bench', 'BuildBench')
    self.Make('all',   'BuildAllOtherTargets')

  def RunTests(self):
    """ Run the unit tests. """
    self.AddSlaveScript(script='run_tests.py', description='RunTests')

  def RunGM(self):
    """ Run the "GM" tool, saving the images to disk. """
    self.AddSlaveScript(script='run_gm.py', description='GenerateGMs')

  def RenderPictures(self):
    """ Run the "render_pictures" tool to generate images from .skp's. """
    self.AddSlaveScript(script='render_pictures.py',
                        description='RenderPictures')

  def CompareGMs(self):
    """ Run the "skdiff" tool to compare the "actual" GM images we just
    generated to the baselines in _gm_image_subdir. """
    self.AddSlaveScript(script='compare_gms.py', description='CompareGMs')

  def RunBench(self):
    """ Run "bench", piping the output somewhere so we can graph
    results over time. """
    self.AddSlaveScript(script='run_bench.py', description='RunBench',
                        timeout=1200)

  def BenchPictures(self):
    """ Run "bench_pictures" """
    self.AddSlaveScript(script='bench_pictures.py',
                        description='BenchPictures')

  def BenchGraphs(self):
    """ Generate and upload bench performance graphs (but only if we have been
    recording bench output for this build type). """
    self.AddSlaveScript(script='generate_bench_graphs.py',
                        description='GenerateBenchGraphs')
    if self._do_upload_results:
      self.AddSlaveScript(script='upload_bench_graphs.py',
                          description='UploadBenchGraphs')

  def UploadGMResults(self):
    """ Upload the images generated by GM """
    args = ['--autogen_svn_username_file', self._autogen_svn_username_file,
            '--autogen_svn_password_file', self._autogen_svn_password_file]
    self.AddSlaveScript(script='upload_gm_results.py', args=args,
                        description='UploadGMResults', timeout=1200)

  def Build(self, clobber=None):
    """Build and return the complete BuildFactory.

    clobber: boolean indicating whether we should clean before building
    """
    # Do all the build steps first, so we will find out about build breakages
    # as soon as possible.
    if clobber is None:
      clobber = self._default_clobber
    if clobber:
      self.AddSlaveScript(script='clean.py', description='Clean')
    self.Compile()
    self.RunTests()
    self.RunGM()
    self.RenderPictures()
    if self._do_upload_results:
      self.UploadGMResults()
    self.CompareGMs()
    self.RunBench()
    self.BenchPictures()
    if self._make_bench_graphs:
      self.BenchGraphs()

    return self._factory
<|endoftext|>