<commit_before>#!/usr/bin/env python
# -*- coding: UTF-8 -*-
# (c) 2016 Mike Lewis
import logging; log = logging.getLogger(__name__)

from . import BaseAuthenticatedEndpointTestCase, BaseUserlessEndpointTestCase

class VenuesEndpointTestCase(BaseAuthenticatedEndpointTestCase):
    """
    General
    """
    def test_venue(self):
        response = self.api.venues(self.default_venueid)
        assert 'venue' in response

    def test_categories(self):
        response = self.api.venues.categories()
        assert 'categories' in response

    def test_explore(self):
        response = self.api.venues.explore(params={'ll': self.default_geo})
        assert 'groups' in response

    def test_explore_radius(self):
        response = self.api.venues.explore(params={'ll': self.default_geo, 'radius': 30})
        assert 'groups' in response

    def test_explore_section(self):
        response = self.api.venues.explore(params={'ll': self.default_geo, 'section': u'café'})
        assert 'groups' in response

    def test_explore_query(self):
        response = self.api.venues.explore(params={'ll': self.default_geo, 'query': 'donuts'})
        assert 'groups' in response

    def test_explore_limit(self):
        response = self.api.venues.explore(params={'ll': self.default_geo, 'limit': 10})
        assert 'groups' in response

    def test_explore_intent(self):
        response = self.api.venues.explore(params={'ll': self.default_geo, 'intent': 'specials'})
        assert 'groups' in response

    def test_managed(self):
        response = self.api.venues.managed()
        assert 'venues' in response

    def test_search(self):
        response = self.api.venues.search(params={'ll': self.default_geo})
        assert 'venues' in response

    def test_search_query(self):
        response = self.api.venues.search(params={'ll': self.default_geo, 'query': 'donuts'})
        assert 'venues' in response

    def test_search_limit(self):
        response = self.api.venues.search(params={'ll': self.default_geo, 'limit': 10})
        assert 'venues' in response

    def test_search_browse(self):
        response = self.api.venues.search(params={'ll': self.default_geo, 'radius': self.default_geo_radius, 'intent': 'browse'})
        assert 'venues' in response

    def test_claim_venues(self):
        response = self.api.venues.claim(self.default_venueid, params={"description": "Venue is Claimed"})
        assert 'venues' in response

    def test_suggestcompletion(self):
        response = self.api.venues.suggestcompletion(params={'ll': self.default_geo, 'query': 'cof'})
        assert 'minivenues' in response

    def test_trending(self):
        response = self.api.venues.trending(params={'ll': self.default_geo})
        assert 'venues' in response

    def test_trending_limit(self):
        response = self.api.venues.trending(params={'ll': self.default_geo, 'limit': 10})
        assert 'venues' in response

    def test_trending_radius(self):
        response = self.api.venues.trending(params={'ll': self.default_geo, 'radius': 100})
        assert 'venues' in response

    """
    Aspects
    """
    def test_event(self):
        response = self.api.venues.events(self.default_venueid)
        assert 'events' in response

    def test_herenow(self):
        response = self.api.venues.herenow(self.default_venueid)
        assert 'hereNow' in response

    def test_herenow_limit(self):
        response = self.api.venues.herenow(self.default_venueid, params={'limit': 10})
        assert 'hereNow' in response

    def test_herenow_offset(self):
        response = self.api.venues.herenow(self.default_venueid, params={'offset': 3})
        assert 'hereNow' in response

    def test_listed(self):
        response = self.api.venues.listed(self.default_venueid)
        assert 'lists' in response

    def test_listed_group(self):
        response = self.api.venues.listed(self.default_venueid, params={'group': 'friends'})
        assert 'lists' in response

    def test_listed_limit(self):
        response = self.api.venues.listed(self.default_venueid, params={'limit': 10})
        assert 'lists' in response

    def test_listed_offset(self):
        response = self.api.venues.listed(self.default_venueid, params={'offset': 3})
        assert 'lists' in response

    def test_photos(self):
        response = self.api.venues.photos(self.default_venueid, params={'group': 'venue'})
        assert 'photos' in response

    def test_photos_limit(self):
        response = self.api.venues.photos(self.default_venueid, params={'limit': 10})
        assert 'photos' in response

    def test_photos_offset(self):
        response = self.api.venues.photos(self.default_venueid, params={'offset': 3})
        assert 'photos' in response

    def test_similar(self):
        response = self.api.venues.similar(self.default_venueid)
        assert 'similarVenues' in response

    def test_tips(self):
        response = self.api.venues.tips(self.default_venueid)
        assert 'tips' in response

    def test_tips_group(self):
        response = self.api.venues.tips(self.default_venueid, params={'sort': 'popular'})
        assert 'tips' in response

    def test_tips_limit(self):
        response = self.api.venues.tips(self.default_venueid, params={'limit': 10})
        assert 'tips' in response

    def test_tips_offset(self):
        response = self.api.venues.tips(self.default_venueid, params={'offset': 3})
        assert 'tips' in response

class VenuesUserlessEndpointTestCase(BaseUserlessEndpointTestCase):
    """
    General
    """
    def test_venue(self):
        response = self.api.venues(self.default_venueid)
        assert 'venue' in response

    def test_categories(self):
        response = self.api.venues.categories()
        assert 'categories' in response

    def test_explore(self):
        response = self.api.venues.explore(params={'ll': self.default_geo})
        assert 'groups' in response

    def test_explore_radius(self):
        response = self.api.venues.explore(params={'ll': self.default_geo, 'radius': 30})
        assert 'groups' in response

    def test_explore_section(self):
        response = self.api.venues.explore(params={'ll': self.default_geo, 'section': 'coffee'})
        assert 'groups' in response

    def test_explore_query(self):
        response = self.api.venues.explore(params={'ll': self.default_geo, 'query': 'donuts'})
        assert 'groups' in response

    def test_explore_limit(self):
        response = self.api.venues.explore(params={'ll': self.default_geo, 'limit': 10})
        assert 'groups' in response

    def test_explore_intent(self):
        response = self.api.venues.explore(params={'ll': self.default_geo, 'intent': 'specials'})
        assert 'groups' in response

    def test_search(self):
        response = self.api.venues.search(params={'ll': self.default_geo})
        assert 'venues' in response

    def test_search_query(self):
        response = self.api.venues.search(params={'ll': self.default_geo, 'query': 'donuts'})
        assert 'venues' in response

    def test_search_limit(self):
        response = self.api.venues.search(params={'ll': self.default_geo, 'limit': 10})
        assert 'venues' in response

    def test_search_browse(self):
        response = self.api.venues.search(params={'ll': self.default_geo, 'radius': self.default_geo_radius, 'intent': 'browse'})
        assert 'venues' in response

    def test_search_ampersand(self):
        response = self.api.venues.search(params={'query': u'Mirch Masala Restaurant & Bar', 'll': u'22.52,88.36'})
        assert 'venues' in response
        assert len(response['venues']) # Make sure there's at least one result

    def test_trending(self):
        response = self.api.venues.trending(params={'ll': self.default_geo})
        assert 'venues' in response

    def test_trending_limit(self):
        response = self.api.venues.trending(params={'ll': self.default_geo, 'limit': 10})
        assert 'venues' in response

    def test_trending_radius(self):
        response = self.api.venues.trending(params={'ll': self.default_geo, 'radius': 100})
        assert 'venues' in response

    """
    Aspects
    """
    def test_listed(self):
        response = self.api.venues.listed(self.default_venueid)
        assert 'lists' in response

    def test_listed_group(self):
        response = self.api.venues.listed(self.default_venueid, params={'group': 'other'})
        assert 'lists' in response

    def test_listed_limit(self):
        response = self.api.venues.listed(self.default_venueid, params={'limit': 10})
        assert 'lists' in response

    def test_listed_offset(self):
        response = self.api.venues.listed(self.default_venueid, params={'offset': 3})
        assert 'lists' in response

    def test_photos(self):
        response = self.api.venues.photos(self.default_venueid, params={'group': 'venue'})
        assert 'photos' in response

    def test_photos_limit(self):
        response = self.api.venues.photos(self.default_venueid, params={'limit': 10})
        assert 'photos' in response

    def test_photos_offset(self):
        response = self.api.venues.photos(self.default_venueid, params={'offset': 3})
        assert 'photos' in response

    def test_tips(self):
        response = self.api.venues.tips(self.default_venueid)
        assert 'tips' in response

    def test_tips_group(self):
        response = self.api.venues.tips(self.default_venueid, params={'sort': 'popular'})
        assert 'tips' in response

    def test_tips_limit(self):
        response = self.api.venues.tips(self.default_venueid, params={'limit': 10})
        assert 'tips' in response

    def test_tips_offset(self):
        response = self.api.venues.tips(self.default_venueid, params={'offset': 3})
        assert 'tips' in response
<commit_msg>drop the venue claim test<commit_after>#!/usr/bin/env python
# -*- coding: UTF-8 -*-
# (c) 2016 Mike Lewis
import logging; log = logging.getLogger(__name__)

from . import BaseAuthenticatedEndpointTestCase, BaseUserlessEndpointTestCase

class VenuesEndpointTestCase(BaseAuthenticatedEndpointTestCase):
    """
    General
    """
    def test_venue(self):
        response = self.api.venues(self.default_venueid)
        assert 'venue' in response

    def test_categories(self):
        response = self.api.venues.categories()
        assert 'categories' in response

    def test_explore(self):
        response = self.api.venues.explore(params={'ll': self.default_geo})
        assert 'groups' in response

    def test_explore_radius(self):
        response = self.api.venues.explore(params={'ll': self.default_geo, 'radius': 30})
        assert 'groups' in response

    def test_explore_section(self):
        response = self.api.venues.explore(params={'ll': self.default_geo, 'section': u'café'})
        assert 'groups' in response

    def test_explore_query(self):
        response = self.api.venues.explore(params={'ll': self.default_geo, 'query': 'donuts'})
        assert 'groups' in response

    def test_explore_limit(self):
        response = self.api.venues.explore(params={'ll': self.default_geo, 'limit': 10})
        assert 'groups' in response

    def test_explore_intent(self):
        response = self.api.venues.explore(params={'ll': self.default_geo, 'intent': 'specials'})
        assert 'groups' in response

    def test_managed(self):
        response = self.api.venues.managed()
        assert 'venues' in response

    def test_search(self):
        response = self.api.venues.search(params={'ll': self.default_geo})
        assert 'venues' in response

    def test_search_query(self):
        response = self.api.venues.search(params={'ll': self.default_geo, 'query': 'donuts'})
        assert 'venues' in response

    def test_search_limit(self):
        response = self.api.venues.search(params={'ll': self.default_geo, 'limit': 10})
        assert 'venues' in response

    def test_search_browse(self):
        response = self.api.venues.search(params={'ll': self.default_geo, 'radius': self.default_geo_radius, 'intent': 'browse'})
        assert 'venues' in response

    def test_suggestcompletion(self):
        response = self.api.venues.suggestcompletion(params={'ll': self.default_geo, 'query': 'cof'})
        assert 'minivenues' in response

    def test_trending(self):
        response = self.api.venues.trending(params={'ll': self.default_geo})
        assert 'venues' in response

    def test_trending_limit(self):
        response = self.api.venues.trending(params={'ll': self.default_geo, 'limit': 10})
        assert 'venues' in response

    def test_trending_radius(self):
        response = self.api.venues.trending(params={'ll': self.default_geo, 'radius': 100})
        assert 'venues' in response

    """
    Aspects
    """
    def test_event(self):
        response = self.api.venues.events(self.default_venueid)
        assert 'events' in response

    def test_herenow(self):
        response = self.api.venues.herenow(self.default_venueid)
        assert 'hereNow' in response

    def test_herenow_limit(self):
        response = self.api.venues.herenow(self.default_venueid, params={'limit': 10})
        assert 'hereNow' in response

    def test_herenow_offset(self):
        response = self.api.venues.herenow(self.default_venueid, params={'offset': 3})
        assert 'hereNow' in response

    def test_listed(self):
        response = self.api.venues.listed(self.default_venueid)
        assert 'lists' in response

    def test_listed_group(self):
        response = self.api.venues.listed(self.default_venueid, params={'group': 'friends'})
        assert 'lists' in response

    def test_listed_limit(self):
        response = self.api.venues.listed(self.default_venueid, params={'limit': 10})
        assert 'lists' in response

    def test_listed_offset(self):
        response = self.api.venues.listed(self.default_venueid, params={'offset': 3})
        assert 'lists' in response

    def test_photos(self):
        response = self.api.venues.photos(self.default_venueid, params={'group': 'venue'})
        assert 'photos' in response

    def test_photos_limit(self):
        response = self.api.venues.photos(self.default_venueid, params={'limit': 10})
        assert 'photos' in response

    def test_photos_offset(self):
        response = self.api.venues.photos(self.default_venueid, params={'offset': 3})
        assert 'photos' in response

    def test_similar(self):
        response = self.api.venues.similar(self.default_venueid)
        assert 'similarVenues' in response

    def test_tips(self):
        response = self.api.venues.tips(self.default_venueid)
        assert 'tips' in response

    def test_tips_group(self):
        response = self.api.venues.tips(self.default_venueid, params={'sort': 'popular'})
        assert 'tips' in response

    def test_tips_limit(self):
        response = self.api.venues.tips(self.default_venueid, params={'limit': 10})
        assert 'tips' in response

    def test_tips_offset(self):
        response = self.api.venues.tips(self.default_venueid, params={'offset': 3})
        assert 'tips' in response

class VenuesUserlessEndpointTestCase(BaseUserlessEndpointTestCase):
    """
    General
    """
    def test_venue(self):
        response = self.api.venues(self.default_venueid)
        assert 'venue' in response

    def test_categories(self):
        response = self.api.venues.categories()
        assert 'categories' in response

    def test_explore(self):
        response = self.api.venues.explore(params={'ll': self.default_geo})
        assert 'groups' in response

    def test_explore_radius(self):
        response = self.api.venues.explore(params={'ll': self.default_geo, 'radius': 30})
        assert 'groups' in response

    def test_explore_section(self):
        response = self.api.venues.explore(params={'ll': self.default_geo, 'section': 'coffee'})
        assert 'groups' in response

    def test_explore_query(self):
        response = self.api.venues.explore(params={'ll': self.default_geo, 'query': 'donuts'})
        assert 'groups' in response

    def test_explore_limit(self):
        response = self.api.venues.explore(params={'ll': self.default_geo, 'limit': 10})
        assert 'groups' in response

    def test_explore_intent(self):
        response = self.api.venues.explore(params={'ll': self.default_geo, 'intent': 'specials'})
        assert 'groups' in response

    def test_search(self):
        response = self.api.venues.search(params={'ll': self.default_geo})
        assert 'venues' in response

    def test_search_query(self):
        response = self.api.venues.search(params={'ll': self.default_geo, 'query': 'donuts'})
        assert 'venues' in response

    def test_search_limit(self):
        response = self.api.venues.search(params={'ll': self.default_geo, 'limit': 10})
        assert 'venues' in response

    def test_search_browse(self):
        response = self.api.venues.search(params={'ll': self.default_geo, 'radius': self.default_geo_radius, 'intent': 'browse'})
        assert 'venues' in response

    def test_search_ampersand(self):
        response = self.api.venues.search(params={'query': u'Mirch Masala Restaurant & Bar', 'll': u'22.52,88.36'})
        assert 'venues' in response
        assert len(response['venues']) # Make sure there's at least one result

    def test_trending(self):
        response = self.api.venues.trending(params={'ll': self.default_geo})
        assert 'venues' in response

    def test_trending_limit(self):
        response = self.api.venues.trending(params={'ll': self.default_geo, 'limit': 10})
        assert 'venues' in response

    def test_trending_radius(self):
        response = self.api.venues.trending(params={'ll': self.default_geo, 'radius': 100})
        assert 'venues' in response

    """
    Aspects
    """
    def test_listed(self):
        response = self.api.venues.listed(self.default_venueid)
        assert 'lists' in response

    def test_listed_group(self):
        response = self.api.venues.listed(self.default_venueid, params={'group': 'other'})
        assert 'lists' in response

    def test_listed_limit(self):
        response = self.api.venues.listed(self.default_venueid, params={'limit': 10})
        assert 'lists' in response

    def test_listed_offset(self):
        response = self.api.venues.listed(self.default_venueid, params={'offset': 3})
        assert 'lists' in response

    def test_photos(self):
        response = self.api.venues.photos(self.default_venueid, params={'group': 'venue'})
        assert 'photos' in response

    def test_photos_limit(self):
        response = self.api.venues.photos(self.default_venueid, params={'limit': 10})
        assert 'photos' in response

    def test_photos_offset(self):
        response = self.api.venues.photos(self.default_venueid, params={'offset': 3})
        assert 'photos' in response

    def test_tips(self):
        response = self.api.venues.tips(self.default_venueid)
        assert 'tips' in response

    def test_tips_group(self):
        response = self.api.venues.tips(self.default_venueid, params={'sort': 'popular'})
        assert 'tips' in response

    def test_tips_limit(self):
        response = self.api.venues.tips(self.default_venueid, params={'limit': 10})
        assert 'tips' in response

    def test_tips_offset(self):
        response = self.api.venues.tips(self.default_venueid, params={'offset': 3})
        assert 'tips' in response
<|endoftext|>


<commit_before># Copyright (c) 2006-2007 The Regents of The University of Michigan
# Copyright (c) 2009 Advanced Micro Devices, Inc.
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met: redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer;
# redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in the
# documentation and/or other materials provided with the distribution;
# neither the name of the copyright holders nor the names of its
# contributors may be used to endorse or promote products derived from
# this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#
# Authors: Brad Beckmann

import m5
from m5.objects import *
from m5.defines import buildEnv
from m5.util import addToPath

#
# Note: the L1 Cache latency is only used by the sequencer on fast path hits
#
class L1Cache(RubyCache):
    latency = 3

#
# Note: the L2 Cache latency is not currently used
#
class L2Cache(RubyCache):
    latency = 15

def create_system(options, phys_mem, piobus, dma_devices):

    if buildEnv['PROTOCOL'] != 'MOESI_hammer':
        panic("This script requires the MOESI_hammer protocol to be built.")

    cpu_sequencers = []

    #
    # The ruby network creation expects the list of nodes in the system to be
    # consistent with the NetDest list.  Therefore the l1 controller nodes must be
    # listed before the directory nodes and directory nodes before dma nodes, etc.
    #
    l1_cntrl_nodes = []
    dir_cntrl_nodes = []
    dma_cntrl_nodes = []

    #
    # Must create the individual controllers before the network to ensure the
    # controller constructors are called before the network constructor
    #

    for i in xrange(options.num_cpus):
        #
        # First create the Ruby objects associated with this cpu
        #
        l1i_cache = L1Cache(size = options.l1i_size,
                            assoc = options.l1i_assoc)
        l1d_cache = L1Cache(size = options.l1d_size,
                            assoc = options.l1d_assoc)
        l2_cache = L2Cache(size = options.l2_size,
                           assoc = options.l2_assoc)

        cpu_seq = RubySequencer(version = i,
                                icache = l1i_cache,
                                dcache = l1d_cache,
                                physMemPort = phys_mem.port,
                                physmem = phys_mem)

        if piobus != None:
            cpu_seq.pio_port = piobus.port

        l1_cntrl = L1Cache_Controller(version = i,
                                      sequencer = cpu_seq,
                                      L1IcacheMemory = l1i_cache,
                                      L1DcacheMemory = l1d_cache,
                                      L2cacheMemory = l2_cache)
        #
        # Add controllers and sequencers to the appropriate lists
        #
        cpu_sequencers.append(cpu_seq)
        l1_cntrl_nodes.append(l1_cntrl)

    phys_mem_size = long(phys_mem.range.second) - long(phys_mem.range.first) + 1
    mem_module_size = phys_mem_size / options.num_dirs

    for i in xrange(options.num_dirs):
        #
        # Create the Ruby objects associated with the directory controller
        #

        mem_cntrl = RubyMemoryControl(version = i)

        dir_size = MemorySize('0B')
        dir_size.value = mem_module_size

        dir_cntrl = Directory_Controller(version = i,
                                         directory = \
                                         RubyDirectoryMemory(version = i,
                                                             size = dir_size),
                                         memBuffer = mem_cntrl)

        dir_cntrl_nodes.append(dir_cntrl)

    for i, dma_device in enumerate(dma_devices):
        #
        # Create the Ruby objects associated with the dma controller
        #
        dma_seq = DMASequencer(version = i,
                               physMemPort = phys_mem.port,
                               physmem = phys_mem)

        dma_cntrl = DMA_Controller(version = i,
                                   dma_sequencer = dma_seq)

        dma_cntrl.dma_sequencer.port = dma_device.dma
        dma_cntrl_nodes.append(dma_cntrl)

    all_cntrls = l1_cntrl_nodes + dir_cntrl_nodes + dma_cntrl_nodes

    return (cpu_sequencers, dir_cntrl_nodes, all_cntrls)
<commit_msg>added sparse memory support to hammer<commit_after># Copyright (c) 2006-2007 The Regents of The University of Michigan
# Copyright (c) 2009 Advanced Micro Devices, Inc.
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met: redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer;
# redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in the
# documentation and/or other materials provided with the distribution;
# neither the name of the copyright holders nor the names of its
# contributors may be used to endorse or promote products derived from
# this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#
# Authors: Brad Beckmann

import m5
from m5.objects import *
from m5.defines import buildEnv
from m5.util import addToPath

#
# Note: the L1 Cache latency is only used by the sequencer on fast path hits
#
class L1Cache(RubyCache):
    latency = 3

#
# Note: the L2 Cache latency is not currently used
#
class L2Cache(RubyCache):
    latency = 15

def create_system(options, phys_mem, piobus, dma_devices):

    if buildEnv['PROTOCOL'] != 'MOESI_hammer':
        panic("This script requires the MOESI_hammer protocol to be built.")

    cpu_sequencers = []

    #
    # The ruby network creation expects the list of nodes in the system to be
    # consistent with the NetDest list.  Therefore the l1 controller nodes must be
    # listed before the directory nodes and directory nodes before dma nodes, etc.
    #
    l1_cntrl_nodes = []
    dir_cntrl_nodes = []
    dma_cntrl_nodes = []

    #
    # Must create the individual controllers before the network to ensure the
    # controller constructors are called before the network constructor
    #

    for i in xrange(options.num_cpus):
        #
        # First create the Ruby objects associated with this cpu
        #
        l1i_cache = L1Cache(size = options.l1i_size,
                            assoc = options.l1i_assoc)
        l1d_cache = L1Cache(size = options.l1d_size,
                            assoc = options.l1d_assoc)
        l2_cache = L2Cache(size = options.l2_size,
                           assoc = options.l2_assoc)

        cpu_seq = RubySequencer(version = i,
                                icache = l1i_cache,
                                dcache = l1d_cache,
                                physMemPort = phys_mem.port,
                                physmem = phys_mem)

        if piobus != None:
            cpu_seq.pio_port = piobus.port

        l1_cntrl = L1Cache_Controller(version = i,
                                      sequencer = cpu_seq,
                                      L1IcacheMemory = l1i_cache,
                                      L1DcacheMemory = l1d_cache,
                                      L2cacheMemory = l2_cache)
        #
        # Add controllers and sequencers to the appropriate lists
        #
        cpu_sequencers.append(cpu_seq)
        l1_cntrl_nodes.append(l1_cntrl)

    phys_mem_size = long(phys_mem.range.second) - long(phys_mem.range.first) + 1
    mem_module_size = phys_mem_size / options.num_dirs

    for i in xrange(options.num_dirs):
        #
        # Create the Ruby objects associated with the directory controller
        #

        mem_cntrl = RubyMemoryControl(version = i)

        dir_size = MemorySize('0B')
        dir_size.value = mem_module_size

        dir_cntrl = Directory_Controller(version = i,
                                         directory = \
                                         RubyDirectoryMemory(version = i,
                                               size = dir_size,
                                               use_map = options.use_map,
                                               map_levels = options.map_levels),
                                         memBuffer = mem_cntrl)

        dir_cntrl_nodes.append(dir_cntrl)

    for i, dma_device in enumerate(dma_devices):
        #
        # Create the Ruby objects associated with the dma controller
        #
        dma_seq = DMASequencer(version = i,
                               physMemPort = phys_mem.port,
                               physmem = phys_mem)

        dma_cntrl = DMA_Controller(version = i,
                                   dma_sequencer = dma_seq)

        dma_cntrl.dma_sequencer.port = dma_device.dma
        dma_cntrl_nodes.append(dma_cntrl)

    all_cntrls = l1_cntrl_nodes + dir_cntrl_nodes + dma_cntrl_nodes

    return (cpu_sequencers, dir_cntrl_nodes, all_cntrls)
<|endoftext|>


<commit_before>import argparse
import os
import subprocess

parser = argparse.ArgumentParser()
parser.add_argument('-o', dest='out')
parser.add_argument('-s', dest='stamp')
args = parser.parse_args()

def gen_list(out, name, obj_dirs):
    out.write(name + " = [\n")
    for base_dir in obj_dirs:
        for dir, subdirs, files in os.walk(os.path.join('obj', base_dir)):
            for f in files:
                if f.endswith('.obj') or f.endswith('.o'):
                    out.write('"' + os.path.abspath(os.path.join(dir, f)) + '",\n')
    out.write("]\n")

with open(args.out, 'w') as out:
    gen_list(
        out,
        "obj_libchromiumcontent",
        [
            "build",
            "chrome/browser/ui/libgtkui",
            "content",
            "crypto",
            "dbus",
            "device",
            "gin",
            "google_apis",
            "gpu",
            "ipc",
            "jingle",
            "mojo",
            "pdf",
            "printing",
            "sandbox",
            "sdch",
            "sql/sql",
            "storage",
            "third_party/adobe",
            "third_party/boringssl",
            "third_party/brotli/common",
            "third_party/brotli/dec",
            "third_party/ced/ced",
            "third_party/decklink",
            "third_party/expat",
            "third_party/flac",
            "third_party/harfbuzz-ng",
            "third_party/iaccessible2",
            "third_party/iccjpeg",
            "third_party/isimpledom",
            "third_party/leveldatabase",
            "third_party/libXNVCtrl",
            "third_party/libjingle",
            "third_party/libjpeg_turbo",
            "third_party/libpng",
            "third_party/libsrtp",
            "third_party/libusb",
            "third_party/libvpx",
            "third_party/libwebm",
            "third_party/libwebp",
            "third_party/libxml",
            "third_party/libxslt",
            "third_party/libyuv",
            "third_party/mesa",
            "third_party/modp_b64",
            "third_party/mozilla",
            "third_party/openh264",
            "third_party/openmax_dl",
            "third_party/opus",
            "third_party/ots",
            "third_party/protobuf/protobuf_lite",
            "third_party/qcms",
            "third_party/re2",
            "third_party/sfntly",
            "third_party/smhasher",
            "third_party/snappy",
            "third_party/sqlite",
            "third_party/sudden_motion_sensor",
            "third_party/usrsctp",
            "third_party/woff2",
            "third_party/zlib",
            "tools",
            "ui",
            "url",
        ])

    gen_list(
        out,
        "obj_base",
        [
            "base/allocator",
            "base/base",
            "base/base_paths",
            "base/base_static",
            "base/build_utf8_validator_tables",
            "base/i18n",
            "base/third_party",
        ])

    gen_list(
        out,
        "obj_cc",
        [
            "cc/animation",
            "cc/base",
            "cc/blink",
            "cc/cc",
            "cc/ipc",
            "cc/paint",
            "cc/proto",
            "cc/surfaces",
        ])

    gen_list(
        out,
        "obj_components",
        [
            "components/bitmap_uploader",
            "components/cdm",
            "components/cookie_config",
            "components/crash/core/common",
            "components/discardable_memory",
            "components/display_compositor",
            "components/filesystem",
            "components/leveldb",
            "components/link_header_util",
            "components/memory_coordinator",
            "components/mime_util",
            "components/mus/clipboard",
            "components/mus/common",
            "components/mus/gles2",
            "components/mus/gpu",
            "components/mus/input_devices",
            "components/mus/public",
            "components/os_crypt",
            "components/payments",
            "components/prefs",
            "components/rappor",
            "components/scheduler/common",
            "components/scheduler/scheduler",
            "components/security_state",
            "components/tracing/proto",
            "components/tracing/startup_tracing",
            "components/tracing/tracing",
            "components/url_formatter",
            "components/variations",
            "components/webcrypto",
            "components/webmessaging",
        ])

    gen_list(
        out,
        "obj_ppapi",
        [
            "ppapi/cpp/objects",
            "ppapi/cpp/private",
            "ppapi/host",
            "ppapi/proxy",
            "ppapi/shared_impl",
            "ppapi/thunk",
        ])

    gen_list(
        out,
        "obj_media",
        [
            "media",
        ])

    gen_list(
        out,
        "obj_net",
        [
            "net",
        ])

    gen_list(
        out,
        "obj_services",
        [
            "services/catalog",
            "services/device",
            "services/file",
            "services/service_manager/public/cpp/sources",
            "services/service_manager/public/interfaces",
            "services/service_manager/runner",
            "services/service_manager/service_manager",
            "services/shape_detection",
            "services/shell/public",
            "services/shell/runner",
            "services/shell/shell",
            "services/tracing/public",
            "services/ui/public",
            "services/ui/gpu",
            "services/user",
        ])

    gen_list(
        out,
        "obj_skia",
        [
            "skia",
        ])

    gen_list(
        out,
        "obj_angle",
        [
            "third_party/angle/angle_common",
            "third_party/angle/angle_image_util",
            "third_party/angle/libANGLE",
            "third_party/angle/libEGL",
            "third_party/angle/libGLESv2",
            "third_party/angle/preprocessor",
            "third_party/angle/translator",
            "third_party/angle/translator_lib",
        ])

    gen_list(
        out,
        "obj_pdfium",
        [
            "third_party/pdfium",
        ])

    gen_list(
        out,
        "obj_webkit",
        [
            "third_party/WebKit/public",
            "third_party/WebKit/Source/core",
            "third_party/WebKit/Source/platform/heap",
            "third_party/WebKit/Source/platform/blink_common",
            "third_party/WebKit/Source/platform/loader",
            "third_party/WebKit/Source/platform/platform",
            "third_party/WebKit/Source/web",
            "third_party/WebKit/Source/wtf",
        ])

    gen_list(
        out,
        "obj_webkitbindings",
        [
            "third_party/WebKit/Source/bindings",
        ])

    gen_list(
        out,
        "obj_webkitmodules",
        [
            "third_party/WebKit/Source/modules",
        ])

    gen_list(
        out,
        "obj_webrtc",
        [
            "third_party/webrtc",
            "third_party/webrtc_overrides",
        ])

    gen_list(
        out,
        "obj_v8",
        [
            "v8/src/inspector",
            "v8/v8_base",
            "v8/v8_external_snapshot",
            "v8/v8_libbase",
            "v8/v8_libplatform",
            "v8/v8_libsampler",
            "third_party/icu",
        ])

os.environ['CHROMIUMCONTENT_2ND_PASS'] = '1'
subprocess.check_call(['ninja', 'chromiumcontent:libs'])

open(args.stamp, 'w')
<commit_msg>standalone_service is still needed on mac<commit_after>import argparse
import os
import subprocess
import sys

parser = argparse.ArgumentParser()
parser.add_argument('-o', dest='out')
parser.add_argument('-s', dest='stamp')
args = parser.parse_args()

def gen_list(out, name, obj_dirs):
    out.write(name + " = [\n")
    for base_dir in obj_dirs:
        for dir, subdirs, files in os.walk(os.path.join('obj', base_dir)):
            for f in files:
                if f.endswith('.obj') or f.endswith('.o'):
                    out.write('"' + os.path.abspath(os.path.join(dir, f)) + '",\n')
    out.write("]\n")

with open(args.out, 'w') as out:
    gen_list(
        out,
        "obj_libchromiumcontent",
        [
            "build",
            "chrome/browser/ui/libgtkui",
            "content",
            "crypto",
            "dbus",
            "device",
            "gin",
            "google_apis",
            "gpu",
            "ipc",
            "jingle",
            "mojo",
            "pdf",
            "printing",
            "sandbox",
            "sdch",
            "sql/sql",
            "storage",
            "third_party/adobe",
            "third_party/boringssl",
            "third_party/brotli/common",
            "third_party/brotli/dec",
            "third_party/ced/ced",
            "third_party/decklink",
            "third_party/expat",
            "third_party/flac",
            "third_party/harfbuzz-ng",
            "third_party/iaccessible2",
            "third_party/iccjpeg",
            "third_party/isimpledom",
            "third_party/leveldatabase",
            "third_party/libXNVCtrl",
            "third_party/libjingle",
            "third_party/libjpeg_turbo",
            "third_party/libpng",
            "third_party/libsrtp",
            "third_party/libusb",
            "third_party/libvpx",
            "third_party/libwebm",
            "third_party/libwebp",
            "third_party/libxml",
            "third_party/libxslt",
            "third_party/libyuv",
            "third_party/mesa",
            "third_party/modp_b64",
            "third_party/mozilla",
            "third_party/openh264",
            "third_party/openmax_dl",
            "third_party/opus",
            "third_party/ots",
            "third_party/protobuf/protobuf_lite",
            "third_party/qcms",
            "third_party/re2",
            "third_party/sfntly",
            "third_party/smhasher",
            "third_party/snappy",
            "third_party/sqlite",
            "third_party/sudden_motion_sensor",
            "third_party/usrsctp",
            "third_party/woff2",
            "third_party/zlib",
            "tools",
            "ui",
            "url",
        ])

    gen_list(
        out,
        "obj_base",
        [
            "base/allocator",
            "base/base",
            "base/base_paths",
            "base/base_static",
            "base/build_utf8_validator_tables",
            "base/i18n",
            "base/third_party",
        ])

    gen_list(
        out,
        "obj_cc",
        [
            "cc/animation",
            "cc/base",
            "cc/blink",
            "cc/cc",
            "cc/ipc",
            "cc/paint",
            "cc/proto",
            "cc/surfaces",
        ])

    gen_list(
        out,
        "obj_components",
        [
            "components/bitmap_uploader",
            "components/cdm",
            "components/cookie_config",
            "components/crash/core/common",
            "components/discardable_memory",
            "components/display_compositor",
            "components/filesystem",
            "components/leveldb",
            "components/link_header_util",
            "components/memory_coordinator",
            "components/mime_util",
            "components/mus/clipboard",
            "components/mus/common",
            "components/mus/gles2",
            "components/mus/gpu",
            "components/mus/input_devices",
            "components/mus/public",
            "components/os_crypt",
            "components/payments",
            "components/prefs",
            "components/rappor",
            "components/scheduler/common",
            "components/scheduler/scheduler",
            "components/security_state",
            "components/tracing/proto",
            "components/tracing/startup_tracing",
            "components/tracing/tracing",
            "components/url_formatter",
            "components/variations",
            "components/webcrypto",
            "components/webmessaging",
        ])

    gen_list(
        out,
        "obj_ppapi",
        [
            "ppapi/cpp/objects",
            "ppapi/cpp/private",
            "ppapi/host",
            "ppapi/proxy",
            "ppapi/shared_impl",
            "ppapi/thunk",
        ])

    gen_list(
        out,
        "obj_media",
        [
            "media",
        ])

    gen_list(
        out,
        "obj_net",
        [
            "net",
        ])

    additional_services = []
    if sys.platform == 'darwin':
        additional_services = [
            "services/service_manager/public/cpp/standalone_service",
        ]
    gen_list(
        out,
        "obj_services",
        [
            "services/catalog",
            "services/device",
            "services/file",
            "services/service_manager/public/cpp/sources",
            "services/service_manager/public/interfaces",
            "services/service_manager/runner",
            "services/service_manager/service_manager",
            "services/shape_detection",
            "services/shell/public",
            "services/shell/runner",
            "services/shell/shell",
            "services/tracing/public",
            "services/ui/public",
            "services/ui/gpu",
            "services/user",
        ] + additional_services)

    gen_list(
        out,
        "obj_skia",
        [
            "skia",
        ])

    gen_list(
        out,
        "obj_angle",
        [
            "third_party/angle/angle_common",
            "third_party/angle/angle_image_util",
            "third_party/angle/libANGLE",
            "third_party/angle/libEGL",
            "third_party/angle/libGLESv2",
            "third_party/angle/preprocessor",
            "third_party/angle/translator",
            "third_party/angle/translator_lib",
        ])

    gen_list(
        out,
        "obj_pdfium",
        [
            "third_party/pdfium",
        ])

    gen_list(
        out,
        "obj_webkit",
        [
            "third_party/WebKit/public",
            "third_party/WebKit/Source/core",
            "third_party/WebKit/Source/platform/heap",
            "third_party/WebKit/Source/platform/blink_common",
            "third_party/WebKit/Source/platform/loader",
            "third_party/WebKit/Source/platform/platform",
            "third_party/WebKit/Source/web",
            "third_party/WebKit/Source/wtf",
        ])

    gen_list(
        out,
        "obj_webkitbindings",
        [
            "third_party/WebKit/Source/bindings",
        ])

    gen_list(
        out,
        "obj_webkitmodules",
        [
            "third_party/WebKit/Source/modules",
        ])

    gen_list(
        out,
        "obj_webrtc",
        [
            "third_party/webrtc",
            "third_party/webrtc_overrides",
        ])

    gen_list(
        out,
        "obj_v8",
        [
            "v8/src/inspector",
            "v8/v8_base",
            "v8/v8_external_snapshot",
            "v8/v8_libbase",
            "v8/v8_libplatform",
            "v8/v8_libsampler",
            "third_party/icu",
        ])

os.environ['CHROMIUMCONTENT_2ND_PASS'] = '1'
subprocess.check_call(['ninja', 'chromiumcontent:libs'])

open(args.stamp, 'w')
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
import magic
import os
import mimetypes
import shlex
import subprocess
import time

from kittengroomer import FileBase, KittenGroomerBase, main

UNOCONV = '/usr/bin/unoconv'
LIBREOFFICE = '/usr/bin/libreoffice'
GS = '/usr/bin/gs'
PDF2HTMLEX = '/usr/bin/pdf2htmlEX'
SEVENZ = '/usr/bin/7z'

# Prepare application/<subtype>
mimes_office = ['msword', 'vnd.openxmlformats-officedocument.', 'vnd.ms-',
                'vnd.oasis.opendocument']
mimes_pdf = ['pdf']
mimes_xml = ['xml']
mimes_ms = ['x-dosexec']
mimes_compressed = ['zip', 'x-rar', 'x-bzip2', 'x-lzip', 'x-lzma', 'x-lzop',
                    'x-xz', 'x-compress', 'x-gzip', 'x-tar', 'compressed']
mimes_data = ['octet-stream']

# Aliases
aliases = {
    # Win executables
    'application/x-msdos-program': 'application/x-dosexec',
    'application/x-dosexec': 'application/x-msdos-program'
}

# Sometimes, mimetypes.guess_type is giving unexpected results, such as for the .tar.gz files:
# In [12]: mimetypes.guess_type('toot.tar.gz', strict=False)
# Out[12]: ('application/x-tar', 'gzip')
# It works as expected if you do mimetypes.guess_type('application/gzip', strict=False)
propertype = {'.gz': 'application/gzip'}

class File(FileBase):

    def __init__(self, src_path, dst_path):
        ''' Init file object, set the mimetype '''
        super(File, self).__init__(src_path, dst_path)

        mimetype = magic.from_file(src_path, mime=True).decode("utf-8")
        self.main_type, self.sub_type = mimetype.split('/')
        a, self.extension = os.path.splitext(src_path)

        self.log_details.update({'maintype': self.main_type, 'subtype': self.sub_type, 'extension': self.extension})

        # Check correlation known extension => actual mime type
        if propertype.get(self.extension) is not None:
            expected_mimetype = propertype.get(self.extension)
        else:
            expected_mimetype, encoding = mimetypes.guess_type(self.src_path, strict=False)
            if aliases.get(expected_mimetype) is not None:
                expected_mimetype = aliases.get(expected_mimetype)

        is_known_extension = self.extension in mimetypes.types_map.keys()
        if is_known_extension and expected_mimetype != mimetype:
            self.log_details.update({'expected_mimetype': expected_mimetype})
            self.make_dangerous()

        # check correlation actual mime type => known extensions
        if aliases.get(mimetype) is not None:
            mimetype = aliases.get(mimetype)
        expected_extensions = mimetypes.guess_all_extensions(mimetype, strict=False)
        if expected_extensions is not None:
            if len(self.extension) > 0 and self.extension not in expected_extensions:
                self.log_details.update({'expected_extensions': expected_extensions})
                self.make_dangerous()
        else:
            # there are no known extensions associated to this mimetype.
            pass

        self.is_recursive = False

class KittenGroomer(KittenGroomerBase):

    def __init__(self, root_src=None, root_dst=None, max_recursive=5):
        '''
            Initialize the basics of the conversion process
        '''
        if root_src is None:
            root_src = os.path.join(os.sep, 'media', 'src')
        if root_dst is None:
            root_dst = os.path.join(os.sep, 'media', 'dst')
        super(KittenGroomer, self).__init__(root_src, root_dst)

        self.recursive = 0
        self.max_recursive = max_recursive

        subtypes_apps = [
            (mimes_office, self._office_related),
            (mimes_pdf, self._pdf),
            (mimes_xml, self._office_related),
            (mimes_ms, self._executables),
            (mimes_compressed, self._archive),
            (mimes_data, self._binary_app),
        ]
        self.subtypes_application = self._init_subtypes_application(subtypes_apps)

        self.mime_processing_options = {
            'text': self.text,
            'audio': self.audio,
            'image': self.image,
            'video': self.video,
            'application': self.application,
            'example': self.example,
            'message': self.message,
            'model': self.model,
            'multipart': self.multipart,
            'inode': self.inode,
        }

        unoconv_listener = UNOCONV + ' --listener'
        self._run_process(unoconv_listener, background=True)

    # ##### Helpers #####
    def _init_subtypes_application(self, subtypes_application):
        '''
            Create the Dict to pick the right function based on the sub mime type
        '''
        to_return = {}
        for list_subtypes, fct in subtypes_application:
            for st in list_subtypes:
                to_return[st] = fct
        return to_return

    def _print_log(self):
        '''
            Print the logs related to the current file being processed
        '''
        tmp_log = self.log_name.fields(**self.cur_file.log_details)
        if self.cur_file.log_details.get('dangerous'):
            tmp_log.warning(self.cur_file.log_string)
        elif self.cur_file.log_details.get('unknown') or self.cur_file.log_details.get('binary'):
            tmp_log.info(self.cur_file.log_string)
        else:
            tmp_log.debug(self.cur_file.log_string)

    def _run_process(self, command_line, timeout=0, background=False):
        '''Run subprocess, wait until it finishes'''
        if timeout != 0:
            deadline = time.time() + timeout
        else:
            deadline = None
        args = shlex.split(command_line)
        p = subprocess.Popen(args)
        if background:
            # FIXME: This timer is here to make sure the unoconv listener is properly started.
            time.sleep(10)
            return True
        while True:
            code = p.poll()
            if code is not None:
                break
            if deadline is not None and time.time() > deadline:
                p.kill()
                break
            time.sleep(1)
        return True

    #######################

    # ##### Discarded mime types, reason in the comments ######
    def inode(self):
        ''' Usually empty file. No reason (?) to copy it on the dest key'''
        self.cur_file.log_string += 'Inode file'

    def unknown(self):
        ''' This main type is unknown, that should not happen '''
        self.cur_file.log_string += 'Unknown file'

    # ##### Threated as malicious, no reason to have it on a USB key ######
    def example(self):
        '''Way to process example file'''
        self.cur_file.log_string += 'Example file'
        self.cur_file.make_dangerous()
        self._safe_copy()

    def message(self):
        '''Way to process message file'''
        self.cur_file.log_string += 'Message file'
        self.cur_file.make_dangerous()
        self._safe_copy()

    def model(self):
        '''Way to process model file'''
        self.cur_file.log_string += 'Model file'
        self.cur_file.make_dangerous()
        self._safe_copy()

    def multipart(self):
        '''Way to process multipart file'''
        self.cur_file.log_string += 'Multipart file'
        self.cur_file.make_dangerous()
        self._safe_copy()

    #######################

    # ##### Converted ######
    def text(self):
        ''' LibreOffice should be able to open all the files '''
        self.cur_file.log_string += 'Text file'
        self._office_related()

    def application(self):
        ''' Everything can be there, using the subtype to decide '''
        for subtype, fct in list(self.subtypes_application.items()):
            if subtype in self.cur_file.sub_type:
                fct()
                self.cur_file.log_string += 'Application file'
                return
        self.cur_file.log_string += 'Unknown Application file'
        self._unknown_app()

    def _executables(self):
        '''Way to process executable file'''
        self.cur_file.add_log_details('processing_type', 'executable')
        self.cur_file.make_dangerous()
        self._safe_copy()

    def _office_related(self):
        '''Way to process all the files LibreOffice can handle'''
        self.cur_file.add_log_details('processing_type', 'office')
        dst_dir, filename = os.path.split(self.cur_file.dst_path)
        tmpdir = os.path.join(dst_dir, 'temp')
        name, ext = os.path.splitext(filename)
        tmppath = os.path.join(tmpdir, name + '.pdf')
        self._safe_mkdir(tmpdir)
        lo_command = '{} --format pdf -eSelectPdfVersion=1 --output {} {}'.format(
            UNOCONV, tmppath, self.cur_file.src_path)
        self._run_process(lo_command)
        self._pdfa(tmppath)
        self._safe_rmtree(tmpdir)

    def _pdfa(self, tmpsrcpath):
        '''Way to process PDF/A file'''
        pdf_command = '{} --dest-dir / {} {}'.format(PDF2HTMLEX, tmpsrcpath,
                                                     self.cur_file.dst_path + '.html')
        self._run_process(pdf_command)

    def _pdf(self):
        '''Way to process PDF file'''
        self.cur_file.add_log_details('processing_type', 'pdf')
        dst_dir, filename = os.path.split(self.cur_file.dst_path)
        tmpdir = os.path.join(dst_dir, 'temp')
        tmppath = os.path.join(tmpdir, filename)
        self._safe_mkdir(tmpdir)
        gs_command = '{} -dPDFA -dBATCH -dNOPAUSE -sDEVICE=pdfwrite -sOutputFile={} {}'.format(
            GS, tmppath, self.cur_file.src_path)
        self._run_process(gs_command)
        self._pdfa(tmppath)
        self._safe_rmtree(tmpdir)

    def _archive(self):
        '''Way to process Archive'''
        self.cur_file.add_log_details('processing_type', 'archive')
        self.cur_file.is_recursive = True
        self.cur_file.log_string += 'Archive extracted, processing content.'
        tmpdir = self.cur_file.dst_path + '_temp'
        self._safe_mkdir(tmpdir)
        extract_command = '{} -p1 x {} -o{} -bd -aoa'.format(SEVENZ, self.cur_file.src_path, tmpdir)
        self._run_process(extract_command)
        self.recursive += 1
        self.processdir(tmpdir, self.cur_file.dst_path)
        self.recursive -= 1
        self._safe_rmtree(tmpdir)

    def _unknown_app(self):
        '''Way to process an unknown file'''
        self.cur_file.make_unknown()
        self._safe_copy()

    def _binary_app(self):
        '''Way to process an unknown binary file'''
        self.cur_file.make_binary()
        self._safe_copy()

    #######################

    # ##### Not converted, checking the mime type ######
    def audio(self):
        '''Way to process an audio file'''
        self.cur_file.log_string += 'Audio file'
        self._media_processing()

    def image(self):
        '''Way to process an image'''
        self.cur_file.log_string += 'Image file'
        self._media_processing()

    def video(self):
        '''Way to process a video'''
        self.cur_file.log_string += 'Video file'
        self._media_processing()

    def _media_processing(self):
        '''Generic way to process all the media files'''
        self.cur_log.fields(processing_type='media')
        self._safe_copy()

    #######################

    def processdir(self, src_dir=None, dst_dir=None):
        '''
            Main function doing the processing
        '''
        if src_dir is None:
            src_dir = self.src_root_dir
        if dst_dir is None:
            dst_dir = self.dst_root_dir

        if self.recursive > 0:
            self._print_log()

        if self.recursive >= self.max_recursive:
            self.cur_log.warning('ARCHIVE BOMB.')
            self.cur_log.warning('The content of the archive contains recursively other archives.')
            self.cur_log.warning('This is a bad sign so the archive is not extracted to the destination key.')
            self._safe_rmtree(src_dir)
            if src_dir.endswith('_temp'):
                archbomb_path = src_dir[:-len('_temp')]
                self._safe_remove(archbomb_path)

        for srcpath in self._list_all_files(src_dir):
            self.cur_file = File(srcpath, srcpath.replace(src_dir, dst_dir))

            self.log_name.info('Processing {} ({}/{})', srcpath.replace(src_dir + '/', ''),
                               self.cur_file.main_type, self.cur_file.sub_type)
            if self.cur_file.log_details.get('dangerous') is None:
                self.mime_processing_options.get(self.cur_file.main_type, self.unknown)()
            else:
                self._safe_copy()
            if not self.cur_file.is_recursive:
                self._print_log()

if __name__ == '__main__':
    main(KittenGroomer, 'Generic version of the KittenGroomer. Convert and rename files.')
<commit_msg>Fix bug with media processing<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
import magic
import os
import mimetypes
import shlex
import subprocess
import time

from kittengroomer import FileBase, KittenGroomerBase, main

UNOCONV = '/usr/bin/unoconv'
LIBREOFFICE = '/usr/bin/libreoffice'
GS = '/usr/bin/gs'
PDF2HTMLEX = '/usr/bin/pdf2htmlEX'
SEVENZ = '/usr/bin/7z'

# Prepare application/<subtype>
mimes_office = ['msword', 'vnd.openxmlformats-officedocument.', 'vnd.ms-',
                'vnd.oasis.opendocument']
mimes_pdf = ['pdf']
mimes_xml = ['xml']
mimes_ms = ['x-dosexec']
mimes_compressed = ['zip', 'x-rar', 'x-bzip2', 'x-lzip', 'x-lzma', 'x-lzop',
                    'x-xz', 'x-compress', 'x-gzip', 'x-tar', 'compressed']
mimes_data = ['octet-stream']

# Aliases
aliases = {
    # Win executables
    'application/x-msdos-program': 'application/x-dosexec',
    'application/x-dosexec': 'application/x-msdos-program'
}

# Sometimes, mimetypes.guess_type is giving unexpected results, such as for the .tar.gz files:
# In [12]: mimetypes.guess_type('toot.tar.gz', strict=False)
# Out[12]: ('application/x-tar', 'gzip')
# It works as expected if you do mimetypes.guess_type('application/gzip', strict=False)
propertype = {'.gz': 'application/gzip'}

class File(FileBase):

    def __init__(self, src_path, dst_path):
        ''' Init file object, set the mimetype '''
        super(File, self).__init__(src_path, dst_path)

        mimetype = magic.from_file(src_path, mime=True).decode("utf-8")
        self.main_type, self.sub_type = mimetype.split('/')
        a, self.extension = os.path.splitext(src_path)

        self.log_details.update({'maintype': self.main_type, 'subtype': self.sub_type, 'extension': self.extension})

        # Check correlation known extension => actual mime type
        if propertype.get(self.extension) is not None:
            expected_mimetype = propertype.get(self.extension)
        else:
            expected_mimetype, encoding = mimetypes.guess_type(self.src_path, strict=False)
            if aliases.get(expected_mimetype) is not None:
                expected_mimetype = aliases.get(expected_mimetype)

        is_known_extension = self.extension in mimetypes.types_map.keys()
        if is_known_extension and expected_mimetype != mimetype:
            self.log_details.update({'expected_mimetype': expected_mimetype})
            self.make_dangerous()

        # check correlation actual mime type => known extensions
        if aliases.get(mimetype) is not None:
            mimetype = aliases.get(mimetype)
        expected_extensions = mimetypes.guess_all_extensions(mimetype, strict=False)
        if expected_extensions is not None:
            if len(self.extension) > 0 and self.extension not in expected_extensions:
                self.log_details.update({'expected_extensions': expected_extensions})
                self.make_dangerous()
        else:
            # there are no known extensions associated to this mimetype.
            pass

        self.is_recursive = False

class KittenGroomer(KittenGroomerBase):

    def __init__(self, root_src=None, root_dst=None, max_recursive=5):
        '''
            Initialize the basics of the conversion process
        '''
        if root_src is None:
            root_src = os.path.join(os.sep, 'media', 'src')
        if root_dst is None:
            root_dst = os.path.join(os.sep, 'media', 'dst')
        super(KittenGroomer, self).__init__(root_src, root_dst)

        self.recursive = 0
        self.max_recursive = max_recursive

        subtypes_apps = [
            (mimes_office, self._office_related),
            (mimes_pdf, self._pdf),
            (mimes_xml, self._office_related),
            (mimes_ms, self._executables),
            (mimes_compressed, self._archive),
            (mimes_data, self._binary_app),
        ]
        self.subtypes_application = self._init_subtypes_application(subtypes_apps)

        self.mime_processing_options = {
            'text': self.text,
            'audio': self.audio,
            'image': self.image,
            'video': self.video,
            'application': self.application,
            'example': self.example,
            'message': self.message,
            'model': self.model,
            'multipart': self.multipart,
            'inode': self.inode,
        }

        unoconv_listener = UNOCONV + ' --listener'
        self._run_process(unoconv_listener, background=True)

    # ##### Helpers #####
    def _init_subtypes_application(self, subtypes_application):
        '''
            Create the Dict to pick the right function based on the sub mime type
        '''
        to_return = {}
        for list_subtypes, fct in subtypes_application:
            for st in list_subtypes:
                to_return[st] = fct
        return to_return

    def _print_log(self):
        '''
            Print the logs related to the current file being processed
        '''
        tmp_log = self.log_name.fields(**self.cur_file.log_details)
        if self.cur_file.log_details.get('dangerous'):
            tmp_log.warning(self.cur_file.log_string)
        elif self.cur_file.log_details.get('unknown') or self.cur_file.log_details.get('binary'):
            tmp_log.info(self.cur_file.log_string)
        else:
            tmp_log.debug(self.cur_file.log_string)

    def _run_process(self, command_line, timeout=0, background=False):
        '''Run subprocess, wait until it finishes'''
        if timeout != 0:
            deadline = time.time() + timeout
        else:
            deadline = None
        args = shlex.split(command_line)
        p = subprocess.Popen(args)
        if background:
            # FIXME: This timer is here to make sure the unoconv listener is properly started.
            time.sleep(10)
            return True
        while True:
            code = p.poll()
            if code is not None:
                break
            if deadline is not None and time.time() > deadline:
                p.kill()
                break
            time.sleep(1)
        return True

    #######################

    # ##### Discarded mime types, reason in the comments ######
    def inode(self):
        ''' Usually empty file. No reason (?) to copy it on the dest key'''
        self.cur_file.log_string += 'Inode file'

    def unknown(self):
        ''' This main type is unknown, that should not happen '''
        self.cur_file.log_string += 'Unknown file'

    # ##### Threated as malicious, no reason to have it on a USB key ######
    def example(self):
        '''Way to process example file'''
        self.cur_file.log_string += 'Example file'
        self.cur_file.make_dangerous()
        self._safe_copy()

    def message(self):
        '''Way to process message file'''
        self.cur_file.log_string += 'Message file'
        self.cur_file.make_dangerous()
        self._safe_copy()

    def model(self):
        '''Way to process model file'''
        self.cur_file.log_string += 'Model file'
        self.cur_file.make_dangerous()
        self._safe_copy()

    def multipart(self):
        '''Way to process multipart file'''
        self.cur_file.log_string += 'Multipart file'
        self.cur_file.make_dangerous()
        self._safe_copy()

    #######################

    # ##### Converted ######
    def text(self):
        ''' LibreOffice should be able to open all the files '''
        self.cur_file.log_string += 'Text file'
        self._office_related()

    def application(self):
        ''' Everything can be there, using the subtype to decide '''
        for subtype, fct in list(self.subtypes_application.items()):
            if subtype in self.cur_file.sub_type:
                fct()
                self.cur_file.log_string += 'Application file'
                return
        self.cur_file.log_string += 'Unknown Application file'
        self._unknown_app()

    def _executables(self):
        '''Way to process executable file'''
        self.cur_file.add_log_details('processing_type', 'executable')
        self.cur_file.make_dangerous()
        self._safe_copy()

    def _office_related(self):
        '''Way to process all the files LibreOffice can handle'''
        self.cur_file.add_log_details('processing_type', 'office')
        dst_dir, filename = os.path.split(self.cur_file.dst_path)
        tmpdir = os.path.join(dst_dir, 'temp')
        name, ext = os.path.splitext(filename)
        tmppath = os.path.join(tmpdir, name + '.pdf')
        self._safe_mkdir(tmpdir)
        lo_command = '{} --format pdf -eSelectPdfVersion=1 --output {} {}'.format(
            UNOCONV, tmppath, self.cur_file.src_path)
        self._run_process(lo_command)
        self._pdfa(tmppath)
        self._safe_rmtree(tmpdir)

    def _pdfa(self, tmpsrcpath):
        '''Way to process PDF/A file'''
        pdf_command = '{} --dest-dir / {} {}'.format(PDF2HTMLEX, tmpsrcpath,
                                                     self.cur_file.dst_path + '.html')
        self._run_process(pdf_command)

    def _pdf(self):
        '''Way to process PDF file'''
        self.cur_file.add_log_details('processing_type', 'pdf')
        dst_dir, filename = os.path.split(self.cur_file.dst_path)
        tmpdir = os.path.join(dst_dir, 'temp')
        tmppath = os.path.join(tmpdir, filename)
        self._safe_mkdir(tmpdir)
        gs_command = '{} -dPDFA -dBATCH -dNOPAUSE -sDEVICE=pdfwrite -sOutputFile={} {}'.format(
            GS, tmppath, self.cur_file.src_path)
        self._run_process(gs_command)
        self._pdfa(tmppath)
        self._safe_rmtree(tmpdir)

    def _archive(self):
        '''Way to process Archive'''
        self.cur_file.add_log_details('processing_type', 'archive')
        self.cur_file.is_recursive = True
        self.cur_file.log_string += 'Archive extracted, processing content.'
        tmpdir = self.cur_file.dst_path + '_temp'
        self._safe_mkdir(tmpdir)
        extract_command = '{} -p1 x {} -o{} -bd -aoa'.format(SEVENZ, self.cur_file.src_path, tmpdir)
        self._run_process(extract_command)
        self.recursive += 1
        self.processdir(tmpdir, self.cur_file.dst_path)
        self.recursive -= 1
        self._safe_rmtree(tmpdir)

    def _unknown_app(self):
        '''Way to process an unknown file'''
        self.cur_file.make_unknown()
        self._safe_copy()

    def _binary_app(self):
        '''Way to process an unknown binary file'''
        self.cur_file.make_binary()
        self._safe_copy()

    #######################

    # ##### Not converted, checking the mime type ######
    def audio(self):
        '''Way to process an audio file'''
        self.cur_file.log_string += 'Audio file'
        self._media_processing()

    def image(self):
        '''Way to process an image'''
        self.cur_file.log_string += 'Image file'
        self._media_processing()

    def video(self):
        '''Way to process a video'''
        self.cur_file.log_string += 'Video file'
        self._media_processing()

    def _media_processing(self):
        '''Generic way to process all the media files'''
        self.cur_file.add_log_details('processing_type', 'media')
        self._safe_copy()

    #######################

    def processdir(self, src_dir=None, dst_dir=None):
        '''
            Main function doing the processing
        '''
        if src_dir is None:
            src_dir = self.src_root_dir
        if dst_dir is None:
            dst_dir = self.dst_root_dir

        if self.recursive > 0:
            self._print_log()

        if self.recursive >= self.max_recursive:
            self.cur_log.warning('ARCHIVE BOMB.')
            self.cur_log.warning('The content of the archive contains recursively other archives.')
            self.cur_log.warning('This is a bad sign so the archive is not extracted to the destination key.')
            self._safe_rmtree(src_dir)
            if src_dir.endswith('_temp'):
                archbomb_path = src_dir[:-len('_temp')]
                self._safe_remove(archbomb_path)

        for srcpath in self._list_all_files(src_dir):
            self.cur_file = File(srcpath, srcpath.replace(src_dir, dst_dir))

            self.log_name.info('Processing {} ({}/{})', srcpath.replace(src_dir + '/', ''),
                               self.cur_file.main_type, self.cur_file.sub_type)
            if self.cur_file.log_details.get('dangerous') is None:
                self.mime_processing_options.get(self.cur_file.main_type, self.unknown)()
            else:
                self._safe_copy()
            if not self.cur_file.is_recursive:
                self._print_log()

if __name__ == '__main__':
    main(KittenGroomer, 'Generic version of the KittenGroomer. Convert and rename files.')
<|endoftext|>


<commit_before>from collections import namedtuple
import logging
from itertools import groupby

from django.db import transaction
from django.utils.translation import ugettext as _
from dimagi.utils.decorators.log_exception import log_exception

from casexml.apps.case.exceptions import IllegalCaseId
from casexml.apps.stock.models import StockTransaction
from corehq.form_processor.casedb_base import AbstractCaseDbCache
from corehq.form_processor.interfaces.processor import FormProcessorInterface
from casexml.apps.stock import const as stockconst
from corehq.form_processor.parsers.ledgers import get_stock_actions

logger = logging.getLogger('commtrack.incoming')

COMMTRACK_LEGACY_REPORT_XMLNS = 'http://commtrack.org/legacy/stock_report'

class StockProcessingResult(object):
    """
    An object used to collect the changes that are made during stock
    processing so that they can be made more atomic
    """

    def __init__(self, xform, relevant_cases=None, stock_report_helpers=None):
        self.domain = xform.domain
        self.xform = xform
        self.relevant_cases = relevant_cases or []
        self.stock_report_helpers = stock_report_helpers or []
        self.models_to_save = None
        self.models_to_delete = None
        self.populated = False

    def populate_models(self):
        self.populated = True

        interface = FormProcessorInterface(domain=self.domain)
        processor = interface.ledger_processor
        ledger_db = interface.ledger_db

        for helper in self.stock_report_helpers:
            assert helper.domain == self.domain

        normal_helpers = [srh for srh in self.stock_report_helpers if not srh.deprecated]
        deprecated_helpers = [srh for srh in self.stock_report_helpers if srh.deprecated]

        models_result = processor.get_models_to_update(
            self.xform.form_id, normal_helpers, deprecated_helpers, ledger_db
        )
        self.models_to_save, self.models_to_delete = models_result

    def commit(self):
        for to_delete in self.models_to_delete:
            to_delete.delete()
        for to_save in self.models_to_save:
            to_save.save()

    def finalize(self):
        """
        Finalize anything else that needs to happen - this runs after models are saved.
        """
        # if cases were changed we should purge the sync token cache
        # this ensures that ledger updates will sync back down
        if self.relevant_cases and self.xform.get_sync_token():
            self.xform.get_sync_token().invalidate_cached_payloads()

LedgerValues = namedtuple('LedgerValues', ['balance', 'delta'])

def compute_ledger_values(lazy_original_balance, report_type, quantity):
    """
    lazy_original_balance:
        a zero-argument function returning original balance
        the original_balance is only used in the case of a transfer
        and it may be computationally expensive for the caller to provide
        putting it behind a function lets compute_ledger_values decide
        whether it's necessary to do that work
    report_type:
        a string in VALID_REPORT_TYPES
        says whether it's a transfer or balance
    quantity:
        the associated quantity, interpreted as a delta for transfers
        and a total balance for balances

    """
    if report_type == stockconst.REPORT_TYPE_BALANCE:
        new_balance = quantity
        # this is currently always 0 because of inferred transactions
        new_delta = 0
    elif report_type == stockconst.REPORT_TYPE_TRANSFER:
        original_balance = lazy_original_balance()
        new_delta = quantity
        new_balance = new_delta + (original_balance if original_balance else 0)
    else:
        raise ValueError()
    return LedgerValues(new_balance, new_delta)

@log_exception()
def process_stock(xforms, case_db=None):
    """
    process the commtrack xml constructs in an incoming submission
    """
    if not case_db:
        case_db = FormProcessorInterface(xforms[0].domain).casedb_cache()
    else:
        assert isinstance(case_db, AbstractCaseDbCache)

    stock_report_helpers = []
    case_action_intents = []
    sorted_forms = sorted(xforms, key=lambda f: 0 if f.is_deprecated else 1)
    for xform in sorted_forms:
        actions_for_form = get_stock_actions(xform)
        stock_report_helpers += actions_for_form.stock_report_helpers
        case_action_intents += actions_for_form.case_action_intents

    # validate the parsed transactions
    for stock_report_helper in stock_report_helpers:
        stock_report_helper.validate()

    relevant_cases = mark_cases_changed(case_action_intents, case_db)
    return StockProcessingResult(
        xform=sorted_forms[-1],
        relevant_cases=relevant_cases,
        stock_report_helpers=stock_report_helpers,
    )

def mark_cases_changed(case_action_intents, case_db):
    relevant_cases = []
    # touch every case for proper ota restore logic syncing to be preserved
    for case_id, intents in groupby(case_action_intents, lambda intent: intent.case_id):
        case = case_db.get(case_id)
        relevant_cases.append(case)
        if case is None:
            raise IllegalCaseId(
                _('Ledger transaction references invalid Case ID "{}"')
                .format(case_id))

        deprecation_intent = None
        intents = list(intents)
        if len(intents) > 1:
            primary_intent, deprecation_intent = sorted(case_action_intents, key=lambda i: i.is_deprecation)
        else:
            [primary_intent] = intents
        case_db.apply_action_intents(case, primary_intent, deprecation_intent)
        case_db.mark_changed(case)

    return relevant_cases

def _compute_ledger_values(original_balance, stock_transaction):
    if stock_transaction.report.type == stockconst.REPORT_TYPE_BALANCE:
        quantity = stock_transaction.stock_on_hand
    elif stock_transaction.report.type == stockconst.REPORT_TYPE_TRANSFER:
        quantity = stock_transaction.quantity
    else:
        raise ValueError()

    ledger_values = compute_ledger_values(
        lambda: original_balance, stock_transaction.report.type, quantity)

    # check that the reported value (either transfer quantity or balance)
    # is not being changed; don't know why it would be, but that would be
    # a red flag
    if stock_transaction.report.type == stockconst.REPORT_TYPE_BALANCE:
        assert stock_transaction.stock_on_hand == ledger_values.balance
    elif stock_transaction.report.type == stockconst.REPORT_TYPE_TRANSFER:
        assert stock_transaction.quantity == ledger_values.delta, \
            (stock_transaction.quantity, ledger_values.delta)
    else:
        raise ValueError()

    return ledger_values

_DeleteStockTransaction = namedtuple(
    '_DeleteStockTransaction', ['stock_transaction'])
_SaveStockTransaction = namedtuple(
    '_SaveStockTransaction',
    ['stock_transaction', 'previous_ledger_values', 'ledger_values'])

def plan_rebuild_stock_state(case_id, section_id, product_id):
    """
    planner for rebuild_stock_state

    yields actions for rebuild_stock_state to take,
    facilitating doing a dry run

    Warning: since some important things are still done through signals
    rather than here explicitly, there may be some effects that aren't
    represented in the plan. For example, inferred transaction creation
    will not be represented, nor will updates to the StockState object.

    """

    # these come out latest first, so reverse them below
    stock_transactions = (
        StockTransaction
        .get_ordered_transactions_for_stock(
            case_id=case_id, section_id=section_id, product_id=product_id)
        .reverse()  # we want earliest transactions first
        .select_related('report__type')
    )
    balance = None
    for stock_transaction in stock_transactions:
        if stock_transaction.subtype == stockconst.TRANSACTION_SUBTYPE_INFERRED:
            yield _DeleteStockTransaction(stock_transaction)
        else:
            before = LedgerValues(balance=stock_transaction.stock_on_hand,
                                  delta=stock_transaction.quantity)
            after = _compute_ledger_values(balance, stock_transaction)
            # update balance for the next iteration
            balance = after.balance
            yield _SaveStockTransaction(stock_transaction, before, after)

@transaction.atomic
def rebuild_stock_state(case_id, section_id, product_id):
    """
    rebuilds the StockState object
    and the quantity and stock_on_hand fields of StockTransaction
    when they are calculated from previous state
    (as opposed to part of the explict transaction)
    """

    for action in plan_rebuild_stock_state(case_id, section_id, product_id):
        if isinstance(action, _DeleteStockTransaction):
            action.stock_transaction.delete()
        elif isinstance(action, _SaveStockTransaction):
            action.stock_transaction.stock_on_hand = action.ledger_values.balance
            action.stock_transaction.quantity = action.ledger_values.delta
            action.stock_transaction.save()
        else:
            raise ValueError()
<commit_msg>add assert<commit_after>from collections import namedtuple
import logging
from itertools import groupby

from django.db import transaction
from django.utils.translation import ugettext as _
from dimagi.utils.decorators.log_exception import log_exception

from casexml.apps.case.exceptions import IllegalCaseId
from casexml.apps.stock.models import StockTransaction
from corehq.form_processor.casedb_base import AbstractCaseDbCache
from corehq.form_processor.interfaces.processor import FormProcessorInterface
from casexml.apps.stock import const as stockconst
from corehq.form_processor.parsers.ledgers import get_stock_actions

logger = logging.getLogger('commtrack.incoming')

COMMTRACK_LEGACY_REPORT_XMLNS = 'http://commtrack.org/legacy/stock_report'

class StockProcessingResult(object):
    """
    An object used to collect the changes that are made during stock
    processing so that they can be made more atomic
    """

    def __init__(self, xform, relevant_cases=None, stock_report_helpers=None):
        self.domain = xform.domain
        self.xform = xform
        self.relevant_cases = relevant_cases or []
        self.stock_report_helpers = stock_report_helpers or []
        self.models_to_save = None
        self.models_to_delete = None
        self.populated = False

    def populate_models(self):
        self.populated = True

        interface = FormProcessorInterface(domain=self.domain)
        processor = interface.ledger_processor
        ledger_db = interface.ledger_db

        for helper in self.stock_report_helpers:
            assert helper.domain == self.domain

        normal_helpers = [srh for srh in self.stock_report_helpers if not srh.deprecated]
        deprecated_helpers = [srh for srh in self.stock_report_helpers if srh.deprecated]

        models_result = processor.get_models_to_update(
            self.xform.form_id, normal_helpers, deprecated_helpers, ledger_db
        )
        self.models_to_save, self.models_to_delete = models_result

    def commit(self):
        assert self.populated
        for to_delete in self.models_to_delete:
            to_delete.delete()
        for to_save in self.models_to_save:
            to_save.save()

    def finalize(self):
        """
        Finalize anything else that needs to happen - this runs after models are saved.
        """
        # if cases were changed we should purge the sync token cache
        # this ensures that ledger updates will sync back down
        if self.relevant_cases and self.xform.get_sync_token():
            self.xform.get_sync_token().invalidate_cached_payloads()

LedgerValues = namedtuple('LedgerValues', ['balance', 'delta'])

def compute_ledger_values(lazy_original_balance, report_type, quantity):
    """
    lazy_original_balance:
        a zero-argument function returning original balance
        the original_balance is only used in the case of a transfer
        and it may be computationally expensive for the caller to provide
        putting it behind a function lets compute_ledger_values decide
        whether it's necessary to do that work
    report_type:
        a string in VALID_REPORT_TYPES
        says whether it's a transfer or balance
    quantity:
        the associated quantity, interpreted as a delta for transfers
        and a total balance for balances

    """
    if report_type == stockconst.REPORT_TYPE_BALANCE:
        new_balance = quantity
        # this is currently always 0 because of inferred transactions
        new_delta = 0
    elif report_type == stockconst.REPORT_TYPE_TRANSFER:
        original_balance = lazy_original_balance()
        new_delta = quantity
        new_balance = new_delta + (original_balance if original_balance else 0)
    else:
        raise ValueError()
    return LedgerValues(new_balance, new_delta)

@log_exception()
def process_stock(xforms, case_db=None):
    """
    process the commtrack xml constructs in an incoming submission
    """
    if not case_db:
        case_db = FormProcessorInterface(xforms[0].domain).casedb_cache()
    else:
        assert isinstance(case_db, AbstractCaseDbCache)

    stock_report_helpers = []
    case_action_intents = []
    sorted_forms = sorted(xforms, key=lambda f: 0 if f.is_deprecated else 1)
    for xform in sorted_forms:
        actions_for_form = get_stock_actions(xform)
        stock_report_helpers += actions_for_form.stock_report_helpers
        case_action_intents += actions_for_form.case_action_intents

    # validate the parsed transactions
    for stock_report_helper in stock_report_helpers:
        stock_report_helper.validate()

    relevant_cases = mark_cases_changed(case_action_intents, case_db)
    return StockProcessingResult(
        xform=sorted_forms[-1],
        relevant_cases=relevant_cases,
        stock_report_helpers=stock_report_helpers,
    )

def mark_cases_changed(case_action_intents, case_db):
    relevant_cases = []
    # touch every case for proper ota restore logic syncing to be preserved
    for case_id, intents in groupby(case_action_intents, lambda intent: intent.case_id):
        case = case_db.get(case_id)
        relevant_cases.append(case)
        if case is None:
            raise IllegalCaseId(
                _('Ledger transaction references invalid Case ID "{}"')
                .format(case_id))

        deprecation_intent = None
        intents = list(intents)
        if len(intents) > 1:
            primary_intent, deprecation_intent = sorted(case_action_intents, key=lambda i: i.is_deprecation)
        else:
            [primary_intent] = intents
        case_db.apply_action_intents(case, primary_intent, deprecation_intent)
        case_db.mark_changed(case)

    return relevant_cases

def _compute_ledger_values(original_balance, stock_transaction):
    if stock_transaction.report.type == stockconst.REPORT_TYPE_BALANCE:
        quantity = stock_transaction.stock_on_hand
    elif stock_transaction.report.type == stockconst.REPORT_TYPE_TRANSFER:
        quantity = stock_transaction.quantity
    else:
        raise ValueError()

    ledger_values = compute_ledger_values(
        lambda: original_balance, stock_transaction.report.type, quantity)

    # check that the reported value (either transfer quantity or balance)
    # is not being changed; don't know why it would be, but that would be
    # a red flag
    if stock_transaction.report.type == stockconst.REPORT_TYPE_BALANCE:
        assert stock_transaction.stock_on_hand == ledger_values.balance
    elif stock_transaction.report.type == stockconst.REPORT_TYPE_TRANSFER:
        assert stock_transaction.quantity == ledger_values.delta, \
            (stock_transaction.quantity, ledger_values.delta)
    else:
        raise ValueError()

    return ledger_values

_DeleteStockTransaction = namedtuple(
    '_DeleteStockTransaction', ['stock_transaction'])
_SaveStockTransaction = namedtuple(
    '_SaveStockTransaction',
    ['stock_transaction', 'previous_ledger_values', 'ledger_values'])

def plan_rebuild_stock_state(case_id, section_id, product_id):
    """
    planner for rebuild_stock_state

    yields actions for rebuild_stock_state to take,
    facilitating doing a dry run

    Warning: since some important things are still done through signals
    rather than here explicitly, there may be some effects that aren't
    represented in the plan. For example, inferred transaction creation
    will not be represented, nor will updates to the StockState object.

    """

    # these come out latest first, so reverse them below
    stock_transactions = (
        StockTransaction
        .get_ordered_transactions_for_stock(
            case_id=case_id, section_id=section_id, product_id=product_id)
        .reverse()  # we want earliest transactions first
        .select_related('report__type')
    )
    balance = None
    for stock_transaction in stock_transactions:
        if stock_transaction.subtype == stockconst.TRANSACTION_SUBTYPE_INFERRED:
            yield _DeleteStockTransaction(stock_transaction)
        else:
            before = LedgerValues(balance=stock_transaction.stock_on_hand,
                                  delta=stock_transaction.quantity)
            after = _compute_ledger_values(balance, stock_transaction)
            # update balance for the next iteration
            balance = after.balance
            yield _SaveStockTransaction(stock_transaction, before, after)

@transaction.atomic
def rebuild_stock_state(case_id, section_id, product_id):
    """
    rebuilds the StockState object
    and the quantity and stock_on_hand fields of StockTransaction
    when they are calculated from previous state
    (as opposed to part of the explict transaction)
    """

    for action in plan_rebuild_stock_state(case_id, section_id, product_id):
        if isinstance(action, _DeleteStockTransaction):
            action.stock_transaction.delete()
        elif isinstance(action, _SaveStockTransaction):
            action.stock_transaction.stock_on_hand = action.ledger_values.balance
            action.stock_transaction.quantity = action.ledger_values.delta
            action.stock_transaction.save()
        else:
            raise ValueError()
<|endoftext|>