<commit_before>'''
Created on Mar 13, 2012

@package: internationalization
@copyright: 2012 Sourcefabric o.p.s.
@license: http://www.gnu.org/licenses/gpl-3.0.txt
@author: Mugur Rus

Implementation for the PO file management.
'''

from babel.messages.catalog import Catalog
from internationalization.api.message import IMessageService
from ally.container import wire
from ally.container.ioc import injected
from internationalization.api.POFileManager import IPOFileManager
from genericpath import isdir, isfile
import os
from internationalization.api.source import ISourceService, QSource
from introspection.api.component import Component
from introspection.api.plugin import Plugin
from ally.api.type import Iter
from internationalization.api.message import Message
from babel.messages.pofile import read_po, write_po
from os.path import join

# --------------------------------------------------------------------

@injected
class POFileManagerAlchemy(IPOFileManager):
    '''
    Implementation for @see: IPOFileManager
    '''

    messageService = IMessageService; wire.entity('messageService')

    sourceService = ISourceService; wire.entity('sourceService')

    localeDirPath = str

    _cacheDir = '_cache'

    _internalDir = '_internal'

    def __init__(self):
        assert isinstance(self.messageService, IMessageService), 'Invalid message service %s' % self.messageService
        assert isinstance(self.sourceService, ISourceService), 'Invalid source file service %s' % self.sourceService
        assert isinstance(self.localeDirPath, str), 'Invalid locale directory %s' % self.localeDirPath
        if not isdir(self.localeDirPath) or not os.access(self.repositoryPath, os.W_OK):
            raise Exception('Unable to access the repository directory %s' % self.localeDirPath)

    def getGlobalPOFile(self, locale):
        '''
        @see: IPOFileManager.getGlobalPOFile
        '''
        poFilePath = self._cachedPOFilePath(locale)
        if self._isSyncPOFile(locale):
            return poFilePath
        keys = self.messageService.getMessages()
        with poFilePath as fileObj:
            return self._buildPOFile(fileObj, locale, keys)
        return poFilePath

    def getComponentPOFile(self, component, locale):
        '''
        @see: IPOFileManager.getComponentPOFile
        '''
        poFilePath = self._cachedPOFilePath(locale, component)
        if self._isSyncPOFile(locale, component):
            return poFilePath
        keys = self.messageService.getComponentMessages(component)
        exceptionsCatalog = self._readPOFile(self._internalPOFilePath(locale, component), locale)
        with poFilePath as fileObj:
            self._buildPOFile(fileObj, locale, keys, exceptionsCatalog)
        return poFilePath

    def getPluginPOFile(self, plugin, locale):
        '''
        @see: IPOFileManager.getPluginPOFile
        '''
        poFilePath = self._cachedPOFilePath(locale, plugin=plugin)
        if self._isSyncPOFile(locale, plugin=plugin):
            return poFilePath
        keys = self.messageService.getPluginMessages(plugin)
        exceptionsCatalog = self._readPOFile(self._internalPOFilePath(locale, plugin=plugin), locale)
        with poFilePath as fileObj:
            self._buildPOFile(fileObj, locale, keys, exceptionsCatalog)
        return poFilePath

    def updateGlobalPOFile(self, locale):
        '''
        @see: IPOFileManager.updateGlobalPOFile
        '''

    def updateComponentPOFile(self, component, locale):
        '''
        @see: IPOFileManager.updateComponentPOFile
        '''

    def updatePluginPOFile(self, plugin, locale):
        '''
        @see: IPOFileManager.updatePluginPOFile
        '''

    def _buildPOFile(self, fileObj, locale:str=None, keys:Iter(Message)=Iter(Message),
                     exceptionsCat:Catalog=Catalog()):
        globalCatalog = self._readPOFile(self._internalPOFilePath(locale), locale)
        for msg in exceptionsCat:
            globalCatalog.add(msg.id, msg.string, msg.locations, msg.flags, msg.auto_comments,
                              msg.user_comments, msg.previous_id, msg.lineno, msg.context)
        templateCatalog = self._buildCatalog(keys, locale)
        globalCatalog.update(templateCatalog)
        write_po(fileObj)

    def _readPOFile(self, path, locale:str=None) -> Catalog:
        if not isfile(path):
            return Catalog()
        with path as fObj:
            return read_po(fObj, locale)

    def _poFileName(self, locale:str=None, component:Component.Id=None, plugin:Plugin.Id=None):
        fileLocale = '_' + locale if locale else ''
        if component:
            return 'component_' + component + fileLocale + '.po'
        elif plugin:
            return 'plugins' + plugin + fileLocale + '.po'
        else:
            return 'global' + fileLocale + '.po'

    def _internalPOFilePath(self, locale:str=None, component:Component.Id=None, plugin:Plugin.Id=None):
        return join(self.localeDirPath, self._internalDir, self._poFileName(locale, component, plugin))

    def _cachedPOFilePath(self, locale:str=None, component:Component.Id=None, plugin:Plugin.Id=None):
        return join(self.localeDirPath, self._cacheDir, self._poFileName(locale, component, plugin))

    def _isSyncPOFile(self, locale:str=None, component:Component.Id=None, plugin:Plugin.Id=None):
        path = self._cachedPOFilePath(locale, component, plugin)
        if not isfile(path):
            return False
        fileMTime = os.stat(path).st_mtime
        return fileMTime >= self._messagesLastModified(component, plugin)

    def _messagesLastModified(self, component:Component.Id=None, plugin:Plugin.Id=None):
        q = QSource()
        q.lastModified.orderDesc()
        if component:
            q.component = component
        if plugin:
            q.plugin = plugin
        sources = self.sourceService.getAll(0, 1, q)
        if sources:
            return sources[0].LastModified
        return None

    def _buildCatalog(self, messages, locale:str=None):
        catalog = Catalog(locale)
        if isinstance(messages, Iter(Message)):
            messages = (messages,)
        for grp in messages:
            for msg in grp:
                msgId = msg.Singular if not msg.Plural else (msg.Singular,) + msg.Plural
                catalog.add(id=msgId, locations=((msg.Source.Path, msg.LineNumber),), flags=(),
                            auto_comments=(msg.Comments), user_comments=(), context=msg.Context)
        return catalog
<commit_msg>NR-21 : Translation management plugin implementation<commit_after>'''
Created on Mar 13, 2012

@package: internationalization
@copyright: 2012 Sourcefabric o.p.s.
@license: http://www.gnu.org/licenses/gpl-3.0.txt
@author: Mugur Rus

Implementation for the PO file management.
'''

from babel.messages.catalog import Catalog
from internationalization.api.message import IMessageService
from ally.container import wire
from ally.container.ioc import injected
from internationalization.api.POFileManager import IPOFileManager
from genericpath import isdir, isfile
import os
from internationalization.api.source import ISourceService, QSource
from introspection.api.component import Component
from introspection.api.plugin import Plugin
from ally.api.type import Iter
from internationalization.api.message import Message
from babel.messages.pofile import read_po, write_po
from os.path import join
from time import localtime, strptime

# --------------------------------------------------------------------

@injected
class POFileManagerAlchemy(IPOFileManager):
    '''
    Implementation for @see: IPOFileManager
    '''

    messageService = IMessageService; wire.entity('messageService')

    sourceService = ISourceService; wire.entity('sourceService')

    localeDirPath = str

    _cacheDir = '_cache'

    _internalDir = '_internal'

    def __init__(self):
        assert isinstance(self.messageService, IMessageService), 'Invalid message service %s' % self.messageService
        assert isinstance(self.sourceService, ISourceService), 'Invalid source file service %s' % self.sourceService
        assert isinstance(self.localeDirPath, str), 'Invalid locale directory %s' % self.localeDirPath
        if not isdir(self.localeDirPath) or not os.access(self.repositoryPath, os.W_OK):
            raise Exception('Unable to access the repository directory %s' % self.localeDirPath)

    def getGlobalPOFile(self, locale):
        '''
        @see: IPOFileManager.getGlobalPOFile
        '''
        poFilePath = self._cachedPOFilePath(locale)
        if self._isSyncPOFile(locale):
            return poFilePath
        keys = self.messageService.getMessages()
        with poFilePath as fileObj:
            return self._buildPOFile(fileObj, locale, keys)
        return poFilePath

    def getComponentPOFile(self, component, locale):
        '''
        @see: IPOFileManager.getComponentPOFile
        '''
        poFilePath = self._cachedPOFilePath(locale, component)
        if self._isSyncPOFile(locale, component):
            return poFilePath
        keys = self.messageService.getComponentMessages(component)
        exceptionsCatalog = self._readPOFile(self._internalPOFilePath(locale, component), locale)
        with poFilePath as fileObj:
            self._buildPOFile(fileObj, locale, keys, exceptionsCatalog)
        return poFilePath

    def getPluginPOFile(self, plugin, locale):
        '''
        @see: IPOFileManager.getPluginPOFile
        '''
        poFilePath = self._cachedPOFilePath(locale, plugin=plugin)
        if self._isSyncPOFile(locale, plugin=plugin):
            return poFilePath
        keys = self.messageService.getPluginMessages(plugin)
        exceptionsCatalog = self._readPOFile(self._internalPOFilePath(locale, plugin=plugin), locale)
        with poFilePath as fileObj:
            self._buildPOFile(fileObj, locale, keys, exceptionsCatalog)
        return poFilePath

    def updateGlobalPOFile(self, poFile, locale):
        '''
        @see: IPOFileManager.updateGlobalPOFile
        '''
        keys = self.messageService.getMessages()
        templateCatalog = self._buildCatalog(keys, locale)
        self._updatePOFile(self._internalPOFilePath(locale), poFile, templateCatalog, locale)

    def updateComponentPOFile(self, poFile, component, locale):
        '''
        @see: IPOFileManager.updateComponentPOFile
        '''
        keys = self.messageService.getComponentMessages(component)
        templateCatalog = self._buildCatalog(keys, locale)
        self._updatePOFile(self._internalPOFilePath(locale, component), poFile, templateCatalog, locale)

    def updatePluginPOFile(self, poFile, plugin, locale):
        '''
        @see: IPOFileManager.updatePluginPOFile
        '''
        keys = self.messageService.getPluginMessages(plugin)
        templateCatalog = self._buildCatalog(keys, locale)
        self._updatePOFile(self._internalPOFilePath(locale, plugin=plugin), poFile, templateCatalog, locale)

    def _updatePOFile(self, path:str, newPOFile, templateCatalog:Catalog, locale:str):
        '''
        Update a PO file from the given file like object.

        @param path: str
            The path of the PO file to be updated
        @param newPOFile: file like object
            The PO file containing the updates
        @param templateCatalog: Catalog
            Catalog containing allowed keys. Keys not existent in this catalog
            will be discarded from the PO file.
        '''
        catalog = self._readPOFile(path, locale)
        newCatalog = read_po(newPOFile, locale)
        for msg in newCatalog:
            if msg not in templateCatalog:
                newCatalog.delete(msg.id, msg.context)
        catalog.update(newCatalog)
        with open(path) as globalPo:
            write_po(globalPo, catalog)

    def _buildPOFile(self, fileObj, locale:str=None, keys:Iter(Message)=Iter(Message),
                     exceptionsCat:Catalog=Catalog()):
        '''
        Builds a PO file from the given file (as file object) to read from, for the
        given locale, using the given template (keys) and exceptions catalog.
        Messages not in the keys list will be discarded while messages from the
        exceptions catalog will overwrite the messages with the same keys that
        existed in the given PO file.

        @param fileObj: file like object
            The PO file to read from
        @param locale: str
            The locale code
        @param keys: Iter(Message)
            The keys used to filter the messages. Only keys from this list will
            be kept in the catalog.
        @param exceptionsCat: Catalog
            Messages that override the generic translations.
        '''
        globalCatalog = self._readPOFile(self._internalPOFilePath(locale), locale)
        for msg in exceptionsCat:
            globalCatalog.add(msg.id, msg.string, msg.locations, msg.flags, msg.auto_comments,
                              msg.user_comments, msg.previous_id, msg.lineno, msg.context)
        templateCatalog = self._buildCatalog(keys, locale)
        globalCatalog.update(templateCatalog)
        write_po(fileObj)

    def _readPOFile(self, path:str, locale:str=None) -> Catalog:
        '''
        Read the file pointed to by the given path into a catalog.

        @param path: str
        @param locale: str
            The locale code
        '''
        if not isfile(path):
            return Catalog()
        with path as fObj:
            return read_po(fObj, locale)

    def _poFileName(self, locale:str=None, component:Component.Id=None, plugin:Plugin.Id=None):
        '''
        Returns the name of the PO file corresponding to the given locale and/or
        component / plugin. If no component of plugin was specified it returns the
        name of the global PO file.

        @param locale: str
            The locale code
        @param component: Component.Id
        @param plugin: Plugin.Id
        '''
        fileLocale = '_' + locale if locale else ''
        if component:
            return 'component_' + component + fileLocale + '.po'
        elif plugin:
            return 'plugins' + plugin + fileLocale + '.po'
        else:
            return 'global' + fileLocale + '.po'

    def _internalPOFilePath(self, locale:str=None, component:Component.Id=None, plugin:Plugin.Id=None):
        '''
        Returns the path to the internal PO file corresponding to the given locale and / or
        component / plugin. If no component of plugin was specified it returns the
        name of the global PO file.

        @param locale: str
            The locale code
        @param component: Component.Id
        @param plugin: Plugin.Id
        '''
        return join(self.localeDirPath, self._internalDir, self._poFileName(locale, component, plugin))

    def _cachedPOFilePath(self, locale:str=None, component:Component.Id=None, plugin:Plugin.Id=None):
        '''
        Returns the path to the cached PO file corresponding to the given locale and / or
        component / plugin. If no component of plugin was specified it returns the
        name of the global PO file.

        @param locale: str
            The locale code
        @param component: Component.Id
        @param plugin: Plugin.Id
        '''
        return join(self.localeDirPath, self._cacheDir, self._poFileName(locale, component, plugin))

    def _isSyncPOFile(self, locale:str=None, component:Component.Id=None, plugin:Plugin.Id=None):
        '''
        Returns true if the PO file corresponding to the given locale and / or
        component / plugin is up to date. If no component of plugin was specified
        it checks the global PO file.

        @param locale: str
            The locale code
        @param component: Component.Id
        @param plugin: Plugin.Id
        '''
        path = self._cachedPOFilePath(locale, component, plugin)
        if not isfile(path):
            return False
        cachedFileMTime = localtime(os.stat(path).st_mtime)
        path = self._internalPOFilePath(locale, component, plugin)
        if not isfile(path):
            return False
        internalFileMTime = localtime(os.stat(path).st_mtime)
        lastMsgTimestamp = strptime(self._messagesLastModified(component, plugin))
        return cachedFileMTime >= lastMsgTimestamp and cachedFileMTime >= internalFileMTime

    def _messagesLastModified(self, component:Component.Id=None, plugin:Plugin.Id=None):
        '''
        Returns the timestamp of the last modified message from the messages table.

        @param component: Component.Id
            Filters by component
        @param plugin: Plugin.Id
            Filters by plugin
        @return: timestamp
        '''
        q = QSource()
        q.lastModified.orderDesc()
        if component:
            q.component = component
        elif plugin:
            q.plugin = plugin
        sources = self.sourceService.getAll(0, 1, q)
        if sources:
            return sources[0].LastModified
        return None

    def _buildCatalog(self, messages, locale:str=None):
        '''
        Builds a catalog from the given messages list.

        @param messages: list of Message entities
        @param locale: str
            The locale code
        @return: Catalog
        '''
        assert isinstance(messages, Iter(Message)) or isinstance(messages, tuple), \
                'Invalid messages list %s' % messages
        catalog = Catalog(locale)
        if isinstance(messages, Iter(Message)):
            messages = (messages,)
        for grp in messages:
            for msg in grp:
                assert isinstance(msg, Message), 'Invalid message %s in list' % msg
                msgId = msg.Singular if not msg.Plural else (msg.Singular,) + msg.Plural
                catalog.add(id=msgId, locations=((msg.Source.Path, msg.LineNumber),), flags=(),
                            auto_comments=(msg.Comments), user_comments=(), context=msg.Context)
        return catalog
<|endoftext|>


<commit_before>from django.core.management.base import BaseCommand
from django.contrib.gis.geos import Point
from django.contrib.gis.geos import GEOSGeometry

from addressbase.models import Address, Blacklist
from pollingstations.models import PollingStation, PollingDistrict, ResidentialAddress
from councils.models import Council

def update_station_point(council_id, station_id, point):
    stations = PollingStation.objects.filter(
        council_id=council_id,
        internal_council_id=station_id
    )
    if len(stations) == 1:
        station = stations[0]
        station.location = point
        station.save()
        print("..updated")
    else:
        print("..NOT updated")

class Command(BaseCommand):

    def handle(self, *args, **kwargs):

        print("updating Westminster Address...")
        westminster = Council.objects.get(pk='E09000033')
        westminster.address = "Electoral Services\nWestminster City Council\n2nd Floor\nCity Hall\n5 Strand\nLondon"
        westminster.postcode = "WC2N 5HR"
        westminster.save()
        print("..updated")

        print("updating Southend email address...")
        southend = Council.objects.get(pk='E06000033')
        southend.email = 'elections@southend.gov.uk'
        southend.save()
        print("..updated")

        print("updating Hull phone number...")
        hull = Council.objects.get(pk='E06000010')
        hull.phone = '01482 300 302'
        hull.save()
        print("..updated")

        print("updating point for: Alf Marshall Community Centre, Hull...")
        update_station_point(
            'E06000010',
            '8515',
            Point(-0.321628, 53.787659, srid=4326))

        print("updating point for: All Saints Church Hall, Hastings...")
        update_station_point(
            'E07000062',
            '23-all-saints-church-hall',
            Point(0.595763, 50.859451, srid=4326))

        print("updating point for: Salvation Army Hall, South Cambs...")
        update_station_point(
            'E07000012',
            '6555',
            Point(0.191974, 52.264660, srid=4326))

        # bug report #5
        print("updating point for: St. Luke's Church Hall, Sefton...")
        update_station_point(
            'E08000014',
            '4529',
            Point(-2.985813, 53.646988, srid=4326))

        # 2x reports from Halton council
        print("updating point for: St Marys Halton Ce Primary School, Halton...")
        update_station_point(
            'E06000006',
            '308',
            Point(-2.690746, 53.331598, srid=4326))

        print("updating: St Edwards Parish Centre, Halton...")
        stations = PollingStation.objects.filter(
            council_id='E06000006',
            internal_council_id='283'
        )
        if len(stations) == 1:
            station = stations[0]
            station.location = Point(-2.728014, 53.335534, srid=4326)
            station.postcode = 'WA7 5PB'
            station.save()
            print("..updated")
        else:
            print("..NOT updated")

        # bug report #6
        print("updating point for: Holy Trinity Church Centre, Sutton...")
        update_station_point(
            'E09000029',
            'OB_1',
            Point(-0.151435, 51.364474, srid=4326))

        # bug report #7
        print("updating: Berkley Street Methodist Church, Huntingdonshire...")
        stations = PollingStation.objects.filter(
            council_id='E07000011',
            internal_council_id='5860'
        )
        if len(stations) == 1:
            station = stations[0]
            station.location = Point(-0.266772, 52.223385, srid=4326)
            station.postcode = 'PE19 2NB'
            station.save()
            print("..updated")
        else:
            print("..NOT updated")

        # request from council
        print("updating: St George`s Church, Reading...")
        stations = PollingStation.objects.filter(
            council_id='E06000038',
            internal_council_id='2701'
        )
        if len(stations) == 1:
            station = stations[0]
            station.address = "St George`s Church\nSt George`s Road\nTilehurst\nReading\nRG30 2RG"
            station.save()
            print("..updated")
        else:
            print("..NOT updated")

        # bug report #8
        print("updating point for: Macaulay CE Primary School, Lambeth...")
        update_station_point(
            'E09000022',
            'VHC',
            Point(-0.148103, 51.463782, srid=4326))

        # bug report #9
        print("updating point for: METHODIST CHURCH HALL, Great Yarmouth...")
        update_station_point(
            'E07000145',
            '24-methodist-church-hall',
            Point(1.714309, 52.572677, srid=4326))

        # 2x reports from Islington council
        print("updating point for: St Joan of Arc Community Centre, Islington...")
        update_station_point(
            'E09000019',
            '1401',
            Point(-0.0966823, 51.5559102, srid=4326))

        print("updating: St. Thomas` Church Hall, Islington...")
        stations = PollingStation.objects.filter(
            council_id='E09000019',
            internal_council_id='1404'
        )
        if len(stations) == 1:
            station = stations[0]
            station.postcode = "N4 2QP"
            station.save()
            print("..updated")
        else:
            print("..NOT updated")

        # bug report #12
        print("updating point for: St Mary and Ambrose Church Hall, Birmingham...")
        update_station_point(
            'E08000025',
            '122-st-mary-and-ambrose-church-hall',
            Point(-1.904365, 52.458623, srid=4326))

        print("updating point for: St Cuthbert`s Centre, Kensington & Chelsea...")
        update_station_point(
            'E09000020',
            '672',
            Point(-0.200082, 51.491363, srid=4326))

        # bug report #13
        print("updating point for: Summertown United Reformed Church, Oxford...")
        update_station_point(
            'E07000178',
            '4793',
            Point(-1.265459, 51.778845, srid=4326))

        # bug report #14
        print("updating point for: Cutteslowe Community Centre, Oxford...")
        update_station_point(
            'E07000178',
            '4796',
            Point(-1.260158, 51.786878, srid=4326))

        # bug report 15
        # correction from council
        print("updating: GUIDE & SCOUT H.Q, North Tyeside...")
        stations = PollingStation.objects.filter(
            council_id='E08000022',
            internal_council_id='OE'
        )
        if len(stations) == 1:
            station = stations[0]
            station.postcode = 'NE30 4RH'
            station.location = Point(-1.429405, 55.019131, srid=4326)
            station.save()
            print("..updated")
        else:
            print("..NOT updated")

        # bug report 16
        print("updating point for: Ellenborough Table Tennis Club, Enfield...")
        update_station_point(
            'E09000010',
            '5624',
            Point(-0.069540, 51.652642, srid=4326))

        # feedback
        print("updating point for: All Saints Community Centre, Lewisham...")
        update_station_point(
            'E09000023',
            '14514',
            Point(-0.048807, 51.476468, srid=4326))

        # bug report 18
        print("updating: Whiston & Goldsmith Community Hall, Hackney...")
        stations = PollingStation.objects.filter(
            council_id='E09000012',
            internal_council_id='3055'
        )
        if len(stations) == 1:
            station = stations[0]
            station.postcode = 'E2 8QY'
            station.location = Point(-0.064828, 51.533471, srid=4326)
            station.save()
            print("..updated")
        else:
            print("..NOT updated")

        # bug report #19
        print("updating: Castle Street Centre, South Lakeland...")
        update_station_point(
            'E07000031',
            '73-castle-street-centre',
            Point(-2.739782, 54.330235, srid=4326))

        # feedback
        print("updating: Ainsworth Parish Church Hall, Bury...")
        update_station_point('E08000002', '2548', None)

        # bug report #23
        print("updating: Carpenters and Docklands Centre, Newham...")
        update_station_point(
            'E09000025',
            '4000',
            Point(-0.003917, 51.540183, srid=4326))

        deleteme = [
            # nothing yet
        ]
        for council_id in deleteme:
            print('Deleting data for council %s...' % (council_id))
            # check this council exists
            c = Council.objects.get(pk=council_id)
            print(c.name)

            PollingStation.objects.filter(council=council_id).delete()
            PollingDistrict.objects.filter(council=council_id).delete()
            ResidentialAddress.objects.filter(council=council_id).delete()
            print('..deleted')

        print("..done")
<commit_msg>remove dodgy point<commit_after>from django.core.management.base import BaseCommand
from django.contrib.gis.geos import Point
from django.contrib.gis.geos import GEOSGeometry

from addressbase.models import Address, Blacklist
from pollingstations.models import PollingStation, PollingDistrict, ResidentialAddress
from councils.models import Council

def update_station_point(council_id, station_id, point):
    stations = PollingStation.objects.filter(
        council_id=council_id,
        internal_council_id=station_id
    )
    if len(stations) == 1:
        station = stations[0]
        station.location = point
        station.save()
        print("..updated")
    else:
        print("..NOT updated")

class Command(BaseCommand):

    def handle(self, *args, **kwargs):

        print("updating Westminster Address...")
        westminster = Council.objects.get(pk='E09000033')
        westminster.address = "Electoral Services\nWestminster City Council\n2nd Floor\nCity Hall\n5 Strand\nLondon"
        westminster.postcode = "WC2N 5HR"
        westminster.save()
        print("..updated")

        print("updating Southend email address...")
        southend = Council.objects.get(pk='E06000033')
        southend.email = 'elections@southend.gov.uk'
        southend.save()
        print("..updated")

        print("updating Hull phone number...")
        hull = Council.objects.get(pk='E06000010')
        hull.phone = '01482 300 302'
        hull.save()
        print("..updated")

        print("updating point for: Alf Marshall Community Centre, Hull...")
        update_station_point(
            'E06000010',
            '8515',
            Point(-0.321628, 53.787659, srid=4326))

        print("updating point for: All Saints Church Hall, Hastings...")
        update_station_point(
            'E07000062',
            '23-all-saints-church-hall',
            Point(0.595763, 50.859451, srid=4326))

        print("updating point for: Salvation Army Hall, South Cambs...")
        update_station_point(
            'E07000012',
            '6555',
            Point(0.191974, 52.264660, srid=4326))

        # bug report #5
        print("updating point for: St. Luke's Church Hall, Sefton...")
        update_station_point(
            'E08000014',
            '4529',
            Point(-2.985813, 53.646988, srid=4326))

        # 2x reports from Halton council
        print("updating point for: St Marys Halton Ce Primary School, Halton...")
        update_station_point(
            'E06000006',
            '308',
            Point(-2.690746, 53.331598, srid=4326))

        print("updating: St Edwards Parish Centre, Halton...")
        stations = PollingStation.objects.filter(
            council_id='E06000006',
            internal_council_id='283'
        )
        if len(stations) == 1:
            station = stations[0]
            station.location = Point(-2.728014, 53.335534, srid=4326)
            station.postcode = 'WA7 5PB'
            station.save()
            print("..updated")
        else:
            print("..NOT updated")

        # bug report #6
        print("updating point for: Holy Trinity Church Centre, Sutton...")
        update_station_point(
            'E09000029',
            'OB_1',
            Point(-0.151435, 51.364474, srid=4326))

        # bug report #7
        print("updating: Berkley Street Methodist Church, Huntingdonshire...")
        stations = PollingStation.objects.filter(
            council_id='E07000011',
            internal_council_id='5860'
        )
        if len(stations) == 1:
            station = stations[0]
            station.location = Point(-0.266772, 52.223385, srid=4326)
            station.postcode = 'PE19 2NB'
            station.save()
            print("..updated")
        else:
            print("..NOT updated")

        # request from council
        print("updating: St George`s Church, Reading...")
        stations = PollingStation.objects.filter(
            council_id='E06000038',
            internal_council_id='2701'
        )
        if len(stations) == 1:
            station = stations[0]
            station.address = "St George`s Church\nSt George`s Road\nTilehurst\nReading\nRG30 2RG"
            station.save()
            print("..updated")
        else:
            print("..NOT updated")

        # bug report #8
        print("updating point for: Macaulay CE Primary School, Lambeth...")
        update_station_point(
            'E09000022',
            'VHC',
            Point(-0.148103, 51.463782, srid=4326))

        # bug report #9
        print("updating point for: METHODIST CHURCH HALL, Great Yarmouth...")
        update_station_point(
            'E07000145',
            '24-methodist-church-hall',
            Point(1.714309, 52.572677, srid=4326))

        # 2x reports from Islington council
        print("updating point for: St Joan of Arc Community Centre, Islington...")
        update_station_point(
            'E09000019',
            '1401',
            Point(-0.0966823, 51.5559102, srid=4326))

        print("updating: St. Thomas` Church Hall, Islington...")
        stations = PollingStation.objects.filter(
            council_id='E09000019',
            internal_council_id='1404'
        )
        if len(stations) == 1:
            station = stations[0]
            station.postcode = "N4 2QP"
            station.save()
            print("..updated")
        else:
            print("..NOT updated")

        # bug report #12
        print("updating point for: St Mary and Ambrose Church Hall, Birmingham...")
        update_station_point(
            'E08000025',
            '122-st-mary-and-ambrose-church-hall',
            Point(-1.904365, 52.458623, srid=4326))

        print("updating point for: St Cuthbert`s Centre, Kensington & Chelsea...")
        update_station_point(
            'E09000020',
            '672',
            Point(-0.200082, 51.491363, srid=4326))

        # bug report #13
        print("updating point for: Summertown United Reformed Church, Oxford...")
        update_station_point(
            'E07000178',
            '4793',
            Point(-1.265459, 51.778845, srid=4326))

        # bug report #14
        print("updating point for: Cutteslowe Community Centre, Oxford...")
        update_station_point(
            'E07000178',
            '4796',
            Point(-1.260158, 51.786878, srid=4326))

        # bug report 15
        # correction from council
        print("updating: GUIDE & SCOUT H.Q, North Tyeside...")
        stations = PollingStation.objects.filter(
            council_id='E08000022',
            internal_council_id='OE'
        )
        if len(stations) == 1:
            station = stations[0]
            station.postcode = 'NE30 4RH'
            station.location = Point(-1.429405, 55.019131, srid=4326)
            station.save()
            print("..updated")
        else:
            print("..NOT updated")

        # bug report 16
        print("updating point for: Ellenborough Table Tennis Club, Enfield...")
        update_station_point(
            'E09000010',
            '5624',
            Point(-0.069540, 51.652642, srid=4326))

        # feedback
        print("updating point for: All Saints Community Centre, Lewisham...")
        update_station_point(
            'E09000023',
            '14514',
            Point(-0.048807, 51.476468, srid=4326))

        # bug report 18
        print("updating: Whiston & Goldsmith Community Hall, Hackney...")
        stations = PollingStation.objects.filter(
            council_id='E09000012',
            internal_council_id='3055'
        )
        if len(stations) == 1:
            station = stations[0]
            station.postcode = 'E2 8QY'
            station.location = Point(-0.064828, 51.533471, srid=4326)
            station.save()
            print("..updated")
        else:
            print("..NOT updated")

        # bug report #19
        print("updating: Castle Street Centre, South Lakeland...")
        update_station_point(
            'E07000031',
            '73-castle-street-centre',
            Point(-2.739782, 54.330235, srid=4326))

        # feedback
        print("updating: Ainsworth Parish Church Hall, Bury...")
        update_station_point('E08000002', '2548', None)

        # bug report #23
        print("updating: Carpenters and Docklands Centre, Newham...")
        update_station_point(
            'E09000025',
            '4000',
            Point(-0.003917, 51.540183, srid=4326))

        # feedback
        print("updating: Osprey Tenants Hall, Southwark...")
        update_station_point('E09000028', '10692', None)

        deleteme = [
            # nothing yet
        ]
        for council_id in deleteme:
            print('Deleting data for council %s...' % (council_id))
            # check this council exists
            c = Council.objects.get(pk=council_id)
            print(c.name)

            PollingStation.objects.filter(council=council_id).delete()
            PollingDistrict.objects.filter(council=council_id).delete()
            ResidentialAddress.objects.filter(council=council_id).delete()
            print('..deleted')

        print("..done")
<|endoftext|>


<commit_before># Copyright (C) 2016 Anne Mulhern
#
# This copyrighted material is made available to anyone wishing to use,
# modify, copy, or redistribute it subject to the terms and conditions of
# the GNU General Public License v.2, or (at your option) any later version.
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY expressed or implied, including the implied warranties of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General
# Public License for more details.  You should have received a copy of the
# GNU General Public License along with this program; if not, write to the
# Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
# Anne Mulhern <mulhern@cs.wisc.edu>

""" Test utilities. """

from __future__ import absolute_import

import itertools

from justbases import BaseConfig
from justbases import DisplayConfig
from justbases import Radix
from justbases import StripConfig

from hypothesis import strategies

def build_nat(base, max_len):
    """
    Build a well-formed nat strategy from ``base``.
    """
    ints = strategies.integers(min_value=0, max_value=(base - 1))
    nats = strategies.lists(ints, min_size=1, max_size=max_len)
    return \
       nats.map(lambda l: list(itertools.dropwhile(lambda x: x == 0, l)))

def build_base(max_base):
    """
    Builds a base.

    :param int max_base: the maximum base
    """
    return strategies.integers(min_value=2, max_value=max_base)

def build_nat_with_base(max_base, max_len):
    """
    Builds a nat and its base.
    :param int max_base: the maximum base
    :param int max_len: the maximum number of digits in the nat

    :returns: a strategy from which can be drawn a pair, a nat and its base
    """
    return build_base(max_base).flatmap(
       lambda n: strategies.tuples(
          build_nat(n, max_len),
          strategies.just(n)
       )
    )

def build_nat_with_base_and_carry(max_base, max_len):
    """
    Build a nat with a base and also a valid carry in value.
    :param int max_base: the maximum base
    :param int max_len: the maximum number of digits in the nat

    :returns: a strategy from which is drawn a triple, nat, base, carry-in
    """
    def the_func(x):
        """
        From a nat and base generally a valid carry-in value.

        :param x: a nat and a corresponding base
        :type x: tuple of (list of int) * int
        :returns: strategies that yields triple of nat, base, carry-in
        """
        (nat, base) = x
        return strategies.tuples(
           strategies.just(nat),
           strategies.just(base),
           strategies.integers(min_value=0, max_value=base - 1)
        )

    return build_nat_with_base(max_base, max_len).flatmap(the_func)

def build_sign():
    """
    Build a sign value.
    """
    return strategies.integers(min_value=-1, max_value=1)

build_relation = build_sign

def build_radix(max_base, max_len):
    """
    Build a well-formed Radix strategy.

    :param int max_base: maximum value for a numeric base
    :param int max_len: the maximum length for the component lists
    """

    def make_radix(base):
        """
        Build a radix from base.

        :param int base: the base of the radix
        """
        list1 = build_nat(base, max_len)
        list2 = build_nat(base, max_len)
        list3 = build_nat(base, max_len)
        if list1 == [] and list2 == [] and list3 == []:
            return strategies.builds(
               Radix,
               strategies.just(0),
               list1,
               list2,
               list3,
               strategies.just(base)
            )
        else:
            return strategies.builds(
               Radix,
               strategies.sampled_from((-1, 1)),
               list1,
               list2,
               list3,
               strategies.just(base)
            )

    return build_base(max_base).flatmap(make_radix)

def build_display_config(base_config, digits_config, strip_config):
    """
    Builds a well-formed display configuration.

    :param BaseConfig base_config: the base config
    :param DigitsConfig digits_config: the digits config
    :param StripConfig strip_config: the strip config
    """
    return strategies.builds(
       DisplayConfig,
       show_approx_str=strategies.booleans(),
       base_config=base_config,
       digits_config=digits_config,
       strip_config=strip_config
    )

def build_strip_config():
    """
    Build strip config.
    """
    return strategies.builds(
       StripConfig,
       strategies.booleans(),
       strategies.booleans(),
       strategies.booleans()
    )

def build_base_config():
    """
    Build base config.
    """
    return strategies.builds(
       BaseConfig,
       strategies.booleans(),
       strategies.booleans()
    )
<commit_msg>Add a strategy for precision.<commit_after># Copyright (C) 2016 Anne Mulhern
#
# This copyrighted material is made available to anyone wishing to use,
# modify, copy, or redistribute it subject to the terms and conditions of
# the GNU General Public License v.2, or (at your option) any later version.
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY expressed or implied, including the implied warranties of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General
# Public License for more details.  You should have received a copy of the
# GNU General Public License along with this program; if not, write to the
# Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
# Anne Mulhern <mulhern@cs.wisc.edu>

""" Test utilities. """

from __future__ import absolute_import

import itertools

from justbases import BaseConfig
from justbases import DisplayConfig
from justbases import Radix
from justbases import StripConfig

from hypothesis import strategies

def build_nat(base, max_len):
    """
    Build a well-formed nat strategy from ``base``.
    """
    ints = strategies.integers(min_value=0, max_value=(base - 1))
    nats = strategies.lists(ints, min_size=1, max_size=max_len)
    return \
       nats.map(lambda l: list(itertools.dropwhile(lambda x: x == 0, l)))

def build_base(max_base):
    """
    Builds a base.

    :param int max_base: the maximum base
    """
    return strategies.integers(min_value=2, max_value=max_base)

def build_nat_with_base(max_base, max_len):
    """
    Builds a nat and its base.
    :param int max_base: the maximum base
    :param int max_len: the maximum number of digits in the nat

    :returns: a strategy from which can be drawn a pair, a nat and its base
    """
    return build_base(max_base).flatmap(
       lambda n: strategies.tuples(
          build_nat(n, max_len),
          strategies.just(n)
       )
    )

def build_nat_with_base_and_carry(max_base, max_len):
    """
    Build a nat with a base and also a valid carry in value.
    :param int max_base: the maximum base
    :param int max_len: the maximum number of digits in the nat

    :returns: a strategy from which is drawn a triple, nat, base, carry-in
    """
    def the_func(x):
        """
        From a nat and base generally a valid carry-in value.

        :param x: a nat and a corresponding base
        :type x: tuple of (list of int) * int
        :returns: strategies that yields triple of nat, base, carry-in
        """
        (nat, base) = x
        return strategies.tuples(
           strategies.just(nat),
           strategies.just(base),
           strategies.integers(min_value=0, max_value=base - 1)
        )

    return build_nat_with_base(max_base, max_len).flatmap(the_func)

def build_precision(min_value, max_value):
    """
    Build a precision value.

    :param int min_value: the minimum allowable precision, may be negative
    :param int max_value: the maximum allowable precision, may be negative

    May return None, which means no precision specified.
    """
    return strategies.one_of(
       strategies.just(None),
       strategies.integers(min_value, max_value)
    )

def build_sign():
    """
    Build a sign value.
    """
    return strategies.integers(min_value=-1, max_value=1)

build_relation = build_sign

def build_radix(max_base, max_len):
    """
    Build a well-formed Radix strategy.

    :param int max_base: maximum value for a numeric base
    :param int max_len: the maximum length for the component lists
    """

    def make_radix(base):
        """
        Build a radix from base.

        :param int base: the base of the radix
        """
        list1 = build_nat(base, max_len)
        list2 = build_nat(base, max_len)
        list3 = build_nat(base, max_len)
        if list1 == [] and list2 == [] and list3 == []:
            return strategies.builds(
               Radix,
               strategies.just(0),
               list1,
               list2,
               list3,
               strategies.just(base)
            )
        else:
            return strategies.builds(
               Radix,
               strategies.sampled_from((-1, 1)),
               list1,
               list2,
               list3,
               strategies.just(base)
            )

    return build_base(max_base).flatmap(make_radix)

def build_display_config(base_config, digits_config, strip_config):
    """
    Builds a well-formed display configuration.

    :param BaseConfig base_config: the base config
    :param DigitsConfig digits_config: the digits config
    :param StripConfig strip_config: the strip config
    """
    return strategies.builds(
       DisplayConfig,
       show_approx_str=strategies.booleans(),
       base_config=base_config,
       digits_config=digits_config,
       strip_config=strip_config
    )

def build_strip_config():
    """
    Build strip config.
    """
    return strategies.builds(
       StripConfig,
       strategies.booleans(),
       strategies.booleans(),
       strategies.booleans()
    )

def build_base_config():
    """
    Build base config.
    """
    return strategies.builds(
       BaseConfig,
       strategies.booleans(),
       strategies.booleans()
    )
<|endoftext|>


<commit_before>import csv
from pathlib import Path

from django.contrib.gis.geos import Point
from django.db import connection

from addressbase.models import UprnToCouncil, Address
from polling_stations.settings.constants.councils import NIR_IDS
from councils.models import Council
from data_importers.base_importers import BaseStationsImporter, CsvMixin
from data_importers.data_types import StationSet
from pollingstations.models import PollingStation, CustomFinder

ADDRESSES_FIELDS = ["uprn", "address", "postcode", "location", "addressbase_postal"]
UPRN_FIELDS = ["uprn", "lad", "polling_station_id", "advance_voting_station_id"]
STATION_FIELDS = [
    "internal_council_id",
    "address",
    "postcode",
    "location",
    "council_id",
    "sample_uprn",
]

UPRN_TO_COUNCIL_CACHE = {}

class Command(BaseStationsImporter, CsvMixin):
    council_id = "ANN"  # This is a hack to make the command initialise
    stations_name = "eoni_stations.csv"
    stations_filetype = "csv"
    csv_encoding = "utf-8"
    eoni_csv_encoding = "latin-1"
    additional_report_councils = NIR_IDS

    def add_arguments(self, parser):
        parser.add_argument("eoni_csv", help="The path to the EONI export csv")
        parser.add_argument(
            "--stations-only",
            help="Don't process address, only update stations",
            action="store_true",
        )
        parser.add_argument(
            "--cleanup", help="Delete intermediate CSVs", action="store_true"
        )

    def handle(self, *args, **options):
        self.eoni_export_path = Path(options["eoni_csv"])
        self.eoni_data_root = self.eoni_export_path.absolute().parent

        self.paths = {
            "addresses": self.eoni_data_root / "eoni_address.csv",
            "uprn_to_council": self.eoni_data_root / "eoni_uprn_to_council.csv",
            "stations": self.eoni_data_root / "eoni_stations.csv",
        }
        self.stations_only = options.get("stations_only")
        self.pre_process_data()
        self.clear_old_data()
        self.copy_data()
        self.assign_uprn_to_councils()
        super().handle(*args, **options)
        if options.get("cleanup"):
            [path.unlink() for path in self.paths.values() if path.exists()]

    def teardown(self, council):
        # Only remove old polling stations, as the UPRN to Council table
        # is populated by this command previously, so deleting data form it
        # isn't useful.
        CustomFinder.objects.filter(area_code="N07000001").delete()
        PollingStation.objects.filter(council_id__in=NIR_IDS).delete()

    def get_base_folder_path(self):
        return str(self.eoni_data_root)

    def pre_process_data(self):
        if not self.stations_only:
            addresses = csv.DictWriter(
                open(self.paths["addresses"], "w", encoding=self.csv_encoding),
                fieldnames=ADDRESSES_FIELDS,
            )
            addresses.writeheader()
            uprns = csv.DictWriter(
                open(self.paths["uprn_to_council"], "w", encoding=self.csv_encoding),
                fieldnames=UPRN_FIELDS,
            )
            uprns.writeheader()

        stations = csv.DictWriter(
            open(self.paths["stations"], "w", encoding=self.csv_encoding),
            fieldnames=STATION_FIELDS,
        )
        stations.writeheader()
        station_data = {}

        with self.eoni_export_path.open(
            "r", encoding=self.eoni_csv_encoding
        ) as eoni_csv:
            # Do a single loop over the whole data file
            reader = csv.DictReader(eoni_csv)
            for row in reader:
                if not self.stations_only:
                    address_location = Point(
                        int(row["PRO_X_COR"]), int(row["PRO_Y_COR"]), srid=29902
                    )
                    address_location.transform(4326)
                    addresses.writerow(
                        {
                            "uprn": row["PRO_UPRN"].strip(),
                            "address": row["PRO_FULLADDRESS"].strip(),
                            "postcode": row["PRO_POSTCODE"].strip(),
                            "location": address_location.ewkt,
                            "addressbase_postal": "D",
                        }
                    )

                    uprns.writerow(
                        {
                            "uprn": row["PRO_UPRN"],
                            "lad": "EONI",
                            "polling_station_id": row["PREM_ID"],
                            "advance_voting_station_id": "NULL",
                        }
                    )

                station_data[row["PREM_ID"]] = {
                    "internal_council_id": row["PREM_ID"],
                    "postcode": row["PREM_POSTCODE"].strip(),
                    "address": f'{row["PREM_NAME"].strip()}, {row["PREM_FULLADDRESS"].strip()}',
                    "location_y": row["PREM_Y_COR"],
                    "location_x": row["PREM_X_COR"],
                    "council_id": "EONI",
                    "sample_uprn": row["PRO_UPRN"].strip(),
                }
            for internal_council_id, station in station_data.items():
                location = Point(
                    int(station.pop("location_x")),
                    int(station.pop("location_y")),
                    srid=29902,
                )
                location.transform(4326)
                station["location"] = location.ewkt
                stations.writerow(station)

    def clear_old_data(self):
        # Polling stations use reg codes
        PollingStation.objects.filter(council_id__in=NIR_IDS).delete()

        if not self.stations_only:
            # UprnToCouncil uses GSS codes
            NIR_AND_EONI = [
                c.geography.gss
                for c in Council.objects.filter(council_id__in=NIR_IDS).select_related(
                    "geography"
                )
            ]
            NIR_AND_EONI.append("EONI")  # Include fake 'EONI' gss just in case
            Address.objects.filter(uprntocouncil__lad__in=NIR_AND_EONI).delete()
            UprnToCouncil.objects.filter(lad__in=NIR_AND_EONI).delete()

    def copy_data(self):
        if not self.stations_only:
            cursor = connection.cursor()
            cursor.copy_expert(
                "COPY addressbase_address FROM STDIN CSV HEADER",
                open(self.paths["addresses"]),
            )
            cursor.copy_expert(
                "COPY addressbase_uprntocouncil FROM STDIN WITH NULL AS 'NULL' CSV HEADER",
                open(self.paths["uprn_to_council"]),
            )

    def assign_uprn_to_councils(self):
        ni_councils = Council.objects.filter(council_id__in=NIR_IDS).select_related(
            "geography"
        )
        for council in ni_councils:
            uprns_in_council = UprnToCouncil.objects.filter(lad="EONI").filter(
                uprn__location__within=council.geography.geography
            )
            uprns_in_council.update(lad=council.geography.gss)

    def station_record_to_dict(self, record):
        if not record.sample_uprn in UPRN_TO_COUNCIL_CACHE:
            UPRN_TO_COUNCIL_CACHE[record.sample_uprn] = Council.objects.get(
                identifiers__contains=[
                    UprnToCouncil.objects.get(uprn=record.sample_uprn).lad
                ]
            )
        station_council = UPRN_TO_COUNCIL_CACHE[record.sample_uprn]
        if station_council == self.council:
            return {
                "internal_council_id": getattr(record, "internal_council_id").strip(),
                "postcode": record.postcode,
                "address": record.address,
                "location": Point().from_ewkt(record.location),
            }

    def import_data(self):
        # We only need to import stations as addresses and UPRN to Council
        # is added using COPY
        ni_councils = Council.objects.filter(council_id__in=NIR_IDS)
        for council in ni_councils:
            self.council = council
            self.council_id = council.council_id
            self.stations = StationSet()
            self.import_polling_stations()
            self.stations.save()

    def check_in_council_bounds(self, station_record):
        if not station_record["postcode"].startswith("BT"):
            return False
        return True
<commit_msg>Add an flag to import_eoni indictating 4326 coords available<commit_after>import csv
import subprocess
from pathlib import Path

from django.contrib.gis.geos import Point
from django.db import connection

from addressbase.models import UprnToCouncil, Address
from polling_stations.settings.constants.councils import NIR_IDS
from councils.models import Council
from data_importers.base_importers import BaseStationsImporter, CsvMixin
from data_importers.data_types import StationSet
from pollingstations.models import PollingStation, CustomFinder

ADDRESSES_FIELDS = ["uprn", "address", "postcode", "location", "addressbase_postal"]
UPRN_FIELDS = ["uprn", "lad", "polling_station_id", "advance_voting_station_id"]
STATION_FIELDS = [
    "internal_council_id",
    "address",
    "postcode",
    "location",
    "council_id",
    "sample_uprn",
]

UPRN_TO_COUNCIL_CACHE = {}

class Command(BaseStationsImporter, CsvMixin):
    council_id = "ANN"  # This is a hack to make the command initialise
    stations_name = "eoni_stations.csv"
    stations_filetype = "csv"
    csv_encoding = "utf-8"
    eoni_csv_encoding = "latin-1"
    additional_report_councils = NIR_IDS

    def add_arguments(self, parser):
        parser.add_argument("eoni_csv", help="The path to the EONI export csv")
        parser.add_argument(
            "--stations-only",
            help="Don't process address, only update stations",
            action="store_true",
        )
        parser.add_argument(
            "--cleanup", help="Delete intermediate CSVs", action="store_true"
        )
        parser.add_argument(
            "--reprojected",
            help="Coordinates available in 4326. Fieldnames: PRO_{X,Y}_4326 & PREM_{X,Y}_4326",
            action="store_true",
        )

    def handle(self, *args, **options):
        self.eoni_export_path = Path(options["eoni_csv"])
        self.eoni_data_root = self.eoni_export_path.absolute().parent

        self.paths = {
            "addresses": self.eoni_data_root / "eoni_address.csv",
            "uprn_to_council": self.eoni_data_root / "eoni_uprn_to_council.csv",
            "stations": self.eoni_data_root / "eoni_stations.csv",
        }
        self.stations_only = options.get("stations_only")
        self.pre_process_data(reprojected=options["reprojected"])
        self.clear_old_data()
        self.copy_data()
        self.assign_uprn_to_councils()
        super().handle(*args, **options)
        if options.get("cleanup"):
            [path.unlink() for path in self.paths.values() if path.exists()]

    def teardown(self, council):
        # Only remove old polling stations, as the UPRN to Council table
        # is populated by this command previously, so deleting data form it
        # isn't useful.
        CustomFinder.objects.filter(area_code="N07000001").delete()
        PollingStation.objects.filter(council_id__in=NIR_IDS).delete()

    def get_base_folder_path(self):
        return str(self.eoni_data_root)

    def pre_process_data(self, reprojected=False):
        if not self.stations_only:
            addresses = csv.DictWriter(
                open(self.paths["addresses"], "w", encoding=self.csv_encoding),
                fieldnames=ADDRESSES_FIELDS,
            )
            addresses.writeheader()
            uprns = csv.DictWriter(
                open(self.paths["uprn_to_council"], "w", encoding=self.csv_encoding),
                fieldnames=UPRN_FIELDS,
            )
            uprns.writeheader()

        stations = csv.DictWriter(
            open(self.paths["stations"], "w", encoding=self.csv_encoding),
            fieldnames=STATION_FIELDS,
        )
        stations.writeheader()
        station_data = {}
        with self.eoni_export_path.open(
            "r", encoding=self.eoni_csv_encoding
        ) as eoni_csv:
            # Do a single loop over the whole data file
            reader = csv.DictReader(eoni_csv)
            for row in reader:
                if not self.stations_only:
                    if reprojected:
                        address_location_ewkt = (
                            f"SRID=4326;POINT({row['PRO_X_4326']} {row['PRO_Y_4326']})"
                        )
                    else:
                        address_location = Point(
                            int(row["PRO_X_COR"]), int(row["PRO_Y_COR"]), srid=29902
                        )
                        address_location.transform(4326)
                        address_location_ewkt = address_location.ewkt
                    addresses.writerow(
                        {
                            "uprn": row["PRO_UPRN"].strip(),
                            "address": row["PRO_FULLADDRESS"].strip(),
                            "postcode": row["PRO_POSTCODE"].strip(),
                            "location": address_location_ewkt,
                            "addressbase_postal": "D",
                        }
                    )

                    uprns.writerow(
                        {
                            "uprn": row["PRO_UPRN"],
                            "lad": "EONI",
                            "polling_station_id": row["PREM_ID"],
                            "advance_voting_station_id": "NULL",
                        }
                    )

                if row["PREM_ID"] not in station_data:
                    if reprojected:
                        station_location_ewkt = f"SRID=4326;POINT({row['PREM_X_4326']} {row['PREM_Y_4326']})"
                    else:
                        location = Point(
                            int(row["PREM_X_COR"]),
                            int(row["PREM_Y_COR"]),
                            srid=29902,
                        )
                        location.transform(4326)
                        station_location_ewkt = location.ewkt
                    station_data[row["PREM_ID"]] = {
                        "internal_council_id": row["PREM_ID"],
                        "postcode": row["PREM_POSTCODE"].strip(),
                        "address": f'{row["PREM_NAME"].strip()}, {row["PREM_FULLADDRESS"].strip()}',
                        "location": station_location_ewkt,
                        "council_id": "EONI",
                        "sample_uprn": row["PRO_UPRN"].strip(),
                    }
            for internal_council_id, station in station_data.items():
                stations.writerow(station)

    def clear_old_data(self):
        # Polling stations use reg codes
        PollingStation.objects.filter(council_id__in=NIR_IDS).delete()

        if not self.stations_only:
            # UprnToCouncil uses GSS codes
            NIR_AND_EONI = [
                c.geography.gss
                for c in Council.objects.filter(council_id__in=NIR_IDS).select_related(
                    "geography"
                )
            ]
            NIR_AND_EONI.append("EONI")  # Include fake 'EONI' gss just in case
            Address.objects.filter(uprntocouncil__lad__in=NIR_AND_EONI).delete()
            UprnToCouncil.objects.filter(lad__in=NIR_AND_EONI).delete()

    def copy_data(self):
        if not self.stations_only:
            cursor = connection.cursor()
            cursor.copy_expert(
                "COPY addressbase_address FROM STDIN CSV HEADER",
                open(self.paths["addresses"]),
            )
            cursor.copy_expert(
                "COPY addressbase_uprntocouncil FROM STDIN WITH NULL AS 'NULL' CSV HEADER",
                open(self.paths["uprn_to_council"]),
            )

    def assign_uprn_to_councils(self):
        ni_councils = Council.objects.filter(council_id__in=NIR_IDS).select_related(
            "geography"
        )
        for council in ni_councils:
            uprns_in_council = UprnToCouncil.objects.filter(lad="EONI").filter(
                uprn__location__within=council.geography.geography
            )
            uprns_in_council.update(lad=council.geography.gss)

    def station_record_to_dict(self, record):
        if not record.sample_uprn in UPRN_TO_COUNCIL_CACHE:
            UPRN_TO_COUNCIL_CACHE[record.sample_uprn] = Council.objects.get(
                identifiers__contains=[
                    UprnToCouncil.objects.get(uprn=record.sample_uprn).lad
                ]
            )
        station_council = UPRN_TO_COUNCIL_CACHE[record.sample_uprn]
        if station_council == self.council:
            return {
                "internal_council_id": getattr(record, "internal_council_id").strip(),
                "postcode": record.postcode,
                "address": record.address,
                "location": Point().from_ewkt(record.location),
            }

    def import_data(self):
        # We only need to import stations as addresses and UPRN to Council
        # is added using COPY
        ni_councils = Council.objects.filter(council_id__in=NIR_IDS)
        for council in ni_councils:
            self.council = council
            self.council_id = council.council_id
            self.stations = StationSet()
            self.import_polling_stations()
            self.stations.save()

    def check_in_council_bounds(self, station_record):
        if not station_record["postcode"].startswith("BT"):
            return False
        return True
<|endoftext|>


<commit_before>import io
import subprocess

import piexif
import PIL.Image
import pytest

import madam.core
from madam.future import subprocess_run

def image_rgb(width, height, transpositions=None):
    if not transpositions:
        transpositions = []
    image = PIL.Image.new('RGB', (width, height))
    # Fill the image with a shape which is (probably) not invariant towards
    # rotations or flips as long as the image has a size of (2, 2) or greater
    for y in range(0, height):
        for x in range(0, width):
            color = (255, 255, 255) if y == 0 or x == 0 else (0, 0, 0)
            image.putpixel((x, y), color)
    for transposition in transpositions:
        image = image.transpose(transposition)
    return image

def jpeg_rgb(width=4, height=3, transpositions=None):
    if not transpositions:
        transpositions = []
    image = image_rgb(width=width, height=height, transpositions=transpositions)
    image_data = io.BytesIO()
    image.save(image_data, 'JPEG', quality=100)
    image_data.seek(0)
    return image_data

def png_rgb(width, height):
    image = image_rgb(width, height)
    image_data = io.BytesIO()
    image.save(image_data, 'PNG')
    image_data.seek(0)
    return image_data

def gif_rgb(width, height):
    image = image_rgb(width, height)
    image_data = io.BytesIO()
    image.save(image_data, 'GIF')
    image_data.seek(0)
    return image_data

def add_exif_to_jpeg(exif, image_data):
    exif_bytes = piexif.dump(exif)
    image_with_exif_metadata = io.BytesIO()
    piexif.insert(exif_bytes, image_data.read(), image_with_exif_metadata)
    return image_with_exif_metadata

@pytest.fixture
def jpeg_asset(width=4, height=3, transpositions=None, **additional_metadata):
    if not transpositions:
        transpositions = []
    essence = jpeg_rgb(width=width, height=height, transpositions=transpositions)
    metadata = dict(
        exif={'0th': {piexif.ImageIFD.Artist: b'Test artist'}},
        width=width,
        height=height,
        mime_type='image/jpeg'
    )
    metadata.update(additional_metadata)
    asset = madam.core.Asset(essence, **metadata)
    return asset

@pytest.fixture
def png_asset():
    width = 4
    height = 3
    essence = png_rgb(width, height)
    metadata = dict(
        width=width,
        height=height,
        mime_type='image/png'
    )
    asset = madam.core.Asset(essence, **metadata)
    return asset

@pytest.fixture
def gif_asset():
    width = 4
    height = 3
    essence = gif_rgb(width, height)
    metadata = dict(
        width=width,
        height=height,
        mime_type='image/gif'
    )
    asset = madam.core.Asset(essence, **metadata)
    return asset

@pytest.fixture(params=['jpeg_asset', 'png_asset', 'gif_asset'])
def image_asset(request, jpeg_asset, png_asset, gif_asset):
    if request.param == 'jpeg_asset':
        return jpeg_asset
    elif request.param == 'png_asset':
        return png_asset
    else:
        return gif_asset

@pytest.fixture(scope='session')
def wav_asset():
    with open('tests/resources/16-bit-mono.wav', 'rb') as file:
        essence = file.read()
    return madam.core.Asset(essence=io.BytesIO(essence),
                            mime_type='audio/wav',
                            duration=0.1)

@pytest.fixture(scope='session')
def mp3_asset():
    with open('tests/resources/64kbits.mp3', 'rb') as file:
        essence = file.read()
    return madam.core.Asset(essence=io.BytesIO(essence),
                            mime_type='audio/mpeg',
                            duration=0.144)

@pytest.fixture(params=['mp3_asset', 'wav_asset'])
def audio_asset(request, mp3_asset, wav_asset):
    if request.param == 'mp3_asset':
        return mp3_asset
    else:
        return wav_asset

@pytest.fixture
def mp4_asset(tmpdir):
    duration = 0.2
    command = ('ffmpeg -loglevel error -f lavfi -i color=color=red:duration=%.1f:rate=15 '
               '-c:v libx264 -preset ultrafast -qp 0 -f mp4' % duration).split()
    tmpfile = tmpdir.join('lossless.mp4')
    command.append(str(tmpfile))
    subprocess_run(command, check=True, stderr=subprocess.PIPE)
    with tmpfile.open('rb') as file:
        essence = file.read()
    return madam.core.Asset(essence=io.BytesIO(essence),
                            mime_type='video/mp4',
                            duration=duration)

@pytest.fixture(scope='session')
def y4m_asset():
    duration = 0.2
    command = ('ffmpeg -loglevel error -f lavfi -i color=color=red:duration=%.1f:rate=15 '
               '-f yuv4mpegpipe pipe:' % duration).split()
    ffmpeg = subprocess_run(command, check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
    return madam.core.Asset(essence=io.BytesIO(ffmpeg.stdout),
                            mime_type='video/x-yuv4mpegpipe',
                            duration=duration)

@pytest.fixture(params=['mp4_asset', 'y4m_asset'])
def video_asset(request, mp4_asset, y4m_asset):
    if request.param == 'mp4_asset':
        return mp4_asset
    else:
        return y4m_asset

@pytest.fixture(params=['jpeg_asset', 'png_asset', 'mp3_asset', 'wav_asset', 'mp4_asset', 'y4m_asset'])
def asset(request, jpeg_asset, png_asset, gif_asset, mp3_asset, wav_asset, mp4_asset, y4m_asset):
    if request.param == 'jpeg_asset':
        return jpeg_asset
    elif request.param == 'png_asset':
        return png_asset
    elif request.param == 'gif_asset':
        return gif_asset
    elif request.param == 'mp3_asset':
        return mp3_asset
    elif request.param == 'wav_asset':
        return wav_asset
    elif request.param == 'mp4_asset':
        return mp4_asset
    else:
        return y4m_asset

@pytest.fixture
def unknown_asset():
    random_data = b'\x07]>e\x10\n+Y\x07\xd8\xf4\x90%\r\xbbK\xb8+\xf3v%\x0f\x11'
    return madam.core.Asset(essence=io.BytesIO(random_data),
                            mime_type='application/octet-stream')
<commit_msg>test assets: mp4_asset, y4m_asset, and video_asset are now class-scoped fixtures.<commit_after>import io
import subprocess

import piexif
import PIL.Image
import pytest

import madam.core
from madam.future import subprocess_run

def image_rgb(width, height, transpositions=None):
    if not transpositions:
        transpositions = []
    image = PIL.Image.new('RGB', (width, height))
    # Fill the image with a shape which is (probably) not invariant towards
    # rotations or flips as long as the image has a size of (2, 2) or greater
    for y in range(0, height):
        for x in range(0, width):
            color = (255, 255, 255) if y == 0 or x == 0 else (0, 0, 0)
            image.putpixel((x, y), color)
    for transposition in transpositions:
        image = image.transpose(transposition)
    return image

def jpeg_rgb(width=4, height=3, transpositions=None):
    if not transpositions:
        transpositions = []
    image = image_rgb(width=width, height=height, transpositions=transpositions)
    image_data = io.BytesIO()
    image.save(image_data, 'JPEG', quality=100)
    image_data.seek(0)
    return image_data

def png_rgb(width, height):
    image = image_rgb(width, height)
    image_data = io.BytesIO()
    image.save(image_data, 'PNG')
    image_data.seek(0)
    return image_data

def gif_rgb(width, height):
    image = image_rgb(width, height)
    image_data = io.BytesIO()
    image.save(image_data, 'GIF')
    image_data.seek(0)
    return image_data

def add_exif_to_jpeg(exif, image_data):
    exif_bytes = piexif.dump(exif)
    image_with_exif_metadata = io.BytesIO()
    piexif.insert(exif_bytes, image_data.read(), image_with_exif_metadata)
    return image_with_exif_metadata

@pytest.fixture
def jpeg_asset(width=4, height=3, transpositions=None, **additional_metadata):
    if not transpositions:
        transpositions = []
    essence = jpeg_rgb(width=width, height=height, transpositions=transpositions)
    metadata = dict(
        exif={'0th': {piexif.ImageIFD.Artist: b'Test artist'}},
        width=width,
        height=height,
        mime_type='image/jpeg'
    )
    metadata.update(additional_metadata)
    asset = madam.core.Asset(essence, **metadata)
    return asset

@pytest.fixture
def png_asset():
    width = 4
    height = 3
    essence = png_rgb(width, height)
    metadata = dict(
        width=width,
        height=height,
        mime_type='image/png'
    )
    asset = madam.core.Asset(essence, **metadata)
    return asset

@pytest.fixture
def gif_asset():
    width = 4
    height = 3
    essence = gif_rgb(width, height)
    metadata = dict(
        width=width,
        height=height,
        mime_type='image/gif'
    )
    asset = madam.core.Asset(essence, **metadata)
    return asset

@pytest.fixture(params=['jpeg_asset', 'png_asset', 'gif_asset'])
def image_asset(request, jpeg_asset, png_asset, gif_asset):
    if request.param == 'jpeg_asset':
        return jpeg_asset
    elif request.param == 'png_asset':
        return png_asset
    else:
        return gif_asset

@pytest.fixture(scope='session')
def wav_asset():
    with open('tests/resources/16-bit-mono.wav', 'rb') as file:
        essence = file.read()
    return madam.core.Asset(essence=io.BytesIO(essence),
                            mime_type='audio/wav',
                            duration=0.1)

@pytest.fixture(scope='session')
def mp3_asset():
    with open('tests/resources/64kbits.mp3', 'rb') as file:
        essence = file.read()
    return madam.core.Asset(essence=io.BytesIO(essence),
                            mime_type='audio/mpeg',
                            duration=0.144)

@pytest.fixture(params=['mp3_asset', 'wav_asset'])
def audio_asset(request, mp3_asset, wav_asset):
    if request.param == 'mp3_asset':
        return mp3_asset
    else:
        return wav_asset

@pytest.fixture(scope='class')
def mp4_asset(tmpdir_factory):
    duration = 0.2
    command = ('ffmpeg -loglevel error -f lavfi -i color=color=red:duration=%.1f:rate=15 '
               '-c:v libx264 -preset ultrafast -qp 0 -f mp4' % duration).split()
    tmpfile = tmpdir_factory.mktemp('mp4_asset').join('lossless.mp4')
    command.append(str(tmpfile))
    subprocess_run(command, check=True, stderr=subprocess.PIPE)
    with tmpfile.open('rb') as file:
        essence = file.read()
    return madam.core.Asset(essence=io.BytesIO(essence),
                            mime_type='video/mp4',
                            duration=duration)

@pytest.fixture(scope='class')
def y4m_asset():
    duration = 0.2
    command = ('ffmpeg -loglevel error -f lavfi -i color=color=red:duration=%.1f:rate=15 '
               '-f yuv4mpegpipe pipe:' % duration).split()
    ffmpeg = subprocess_run(command, check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
    return madam.core.Asset(essence=io.BytesIO(ffmpeg.stdout),
                            mime_type='video/x-yuv4mpegpipe',
                            duration=duration)

@pytest.fixture(scope='class', params=['mp4_asset', 'y4m_asset'])
def video_asset(request, mp4_asset, y4m_asset):
    if request.param == 'mp4_asset':
        return mp4_asset
    else:
        return y4m_asset

@pytest.fixture(params=['jpeg_asset', 'png_asset', 'mp3_asset', 'wav_asset', 'mp4_asset', 'y4m_asset'])
def asset(request, jpeg_asset, png_asset, gif_asset, mp3_asset, wav_asset, mp4_asset, y4m_asset):
    if request.param == 'jpeg_asset':
        return jpeg_asset
    elif request.param == 'png_asset':
        return png_asset
    elif request.param == 'gif_asset':
        return gif_asset
    elif request.param == 'mp3_asset':
        return mp3_asset
    elif request.param == 'wav_asset':
        return wav_asset
    elif request.param == 'mp4_asset':
        return mp4_asset
    else:
        return y4m_asset

@pytest.fixture
def unknown_asset():
    random_data = b'\x07]>e\x10\n+Y\x07\xd8\xf4\x90%\r\xbbK\xb8+\xf3v%\x0f\x11'
    return madam.core.Asset(essence=io.BytesIO(random_data),
                            mime_type='application/octet-stream')
<|endoftext|>


<commit_before>import PIL.Image
import io

import piexif
import PIL.Image
import pytest

import madam.core

def image_rgb(width, height, transpositions=None):
    if not transpositions:
        transpositions = []
    image = PIL.Image.new('RGB', (width, height))
    # Fill the image with a shape which is (probably) not invariant towards
    # rotations or flips as long as the image has a size of (2, 2) or greater
    for y in range(0, height):
        for x in range(0, width):
            color = (255, 255, 255) if y == 0 or x == 0 else (0, 0, 0)
            image.putpixel((x, y), color)
    for transposition in transpositions:
        image = image.transpose(transposition)
    return image

def jpeg_rgb(width=4, height=3, transpositions=None):
    if not transpositions:
        transpositions = []
    image = image_rgb(width=width, height=height, transpositions=transpositions)
    image_data = io.BytesIO()
    image.save(image_data, 'JPEG', quality=100)
    image_data.seek(0)
    return image_data

def png_rgb():
    image = image_rgb(width=4, height=3)
    image_data = io.BytesIO()
    image.save(image_data, 'PNG')
    image_data.seek(0)
    return image_data

def add_exif_to_jpeg(exif, image_data):
    exif_bytes = piexif.dump(exif)
    image_with_exif_metadata = io.BytesIO()
    piexif.insert(exif_bytes, image_data.read(), image_with_exif_metadata)
    return image_with_exif_metadata

@pytest.fixture
def jpeg_asset(width=4, height=3, transpositions=None):
    if not transpositions:
        transpositions = []
    essence = jpeg_rgb(width=width, height=height, transpositions=transpositions).read()
    metadata = {
        'exif': {'0th': {piexif.ImageIFD.Artist: b'Test artist'}},
        'width': width,
        'height': height,
        'mime_type': 'image/jpeg'
    }
    asset = madam.core.Asset(essence, metadata)
    return asset

@pytest.fixture
def png_asset():
    essence = png_rgb().read()
    metadata = {
        'width': 0,
        'height': 0,
        'mime_type': 'image/png'
    }
    asset = madam.core.Asset(essence, metadata)
    return asset

@pytest.fixture(params=['jpeg_asset', 'png_asset'])
def image_asset(request, jpeg_asset, png_asset):
    if request.param == 'jpeg_asset':
        return jpeg_asset
    else:
        return png_asset
<commit_msg>assets.png_rgb requires width and height to be passed.<commit_after>import PIL.Image
import io

import piexif
import PIL.Image
import pytest

import madam.core

def image_rgb(width, height, transpositions=None):
    if not transpositions:
        transpositions = []
    image = PIL.Image.new('RGB', (width, height))
    # Fill the image with a shape which is (probably) not invariant towards
    # rotations or flips as long as the image has a size of (2, 2) or greater
    for y in range(0, height):
        for x in range(0, width):
            color = (255, 255, 255) if y == 0 or x == 0 else (0, 0, 0)
            image.putpixel((x, y), color)
    for transposition in transpositions:
        image = image.transpose(transposition)
    return image

def jpeg_rgb(width=4, height=3, transpositions=None):
    if not transpositions:
        transpositions = []
    image = image_rgb(width=width, height=height, transpositions=transpositions)
    image_data = io.BytesIO()
    image.save(image_data, 'JPEG', quality=100)
    image_data.seek(0)
    return image_data

def png_rgb(width, height):
    image = image_rgb(width, height)
    image_data = io.BytesIO()
    image.save(image_data, 'PNG')
    image_data.seek(0)
    return image_data

def add_exif_to_jpeg(exif, image_data):
    exif_bytes = piexif.dump(exif)
    image_with_exif_metadata = io.BytesIO()
    piexif.insert(exif_bytes, image_data.read(), image_with_exif_metadata)
    return image_with_exif_metadata

@pytest.fixture
def jpeg_asset(width=4, height=3, transpositions=None):
    if not transpositions:
        transpositions = []
    essence = jpeg_rgb(width=width, height=height, transpositions=transpositions).read()
    metadata = {
        'exif': {'0th': {piexif.ImageIFD.Artist: b'Test artist'}},
        'width': width,
        'height': height,
        'mime_type': 'image/jpeg'
    }
    asset = madam.core.Asset(essence, metadata)
    return asset

@pytest.fixture
def png_asset():
    width = 4
    height = 3
    essence = png_rgb(width, height).read()
    metadata = {
        'width': width,
        'height': height,
        'mime_type': 'image/png'
    }
    asset = madam.core.Asset(essence, metadata)
    return asset

@pytest.fixture(params=['jpeg_asset', 'png_asset'])
def image_asset(request, jpeg_asset, png_asset):
    if request.param == 'jpeg_asset':
        return jpeg_asset
    else:
        return png_asset
<|endoftext|>


<commit_before>import deploy_utils

ALL_JOBS = ["mapreduce"]

def get_mapreduce_service_config(args):
  args.mapreduce_config = deploy_utils.get_service_config(args)

def install(args):
  get_mapreduce_service_config(args)
  deploy_utils.install_service(args, "mapreduce", args.mapreduce_config, "hadoop")

def bootstrap_job(args, host, job_name, host_id, instance_id, cleanup_token):
  deploy_utils.bootstrap_job(args, "hadoop", "mapreduce",
      args.mapreduce_config, host, job_name, instance_id, cleanup_token, '0')
  start_job(args, host, job_name, host_id, instance_id)

def bootstrap(args):
  get_mapreduce_service_config(args)
  cleanup_token = deploy_utils.confirm_bootstrap("mapreduce", args.mapreduce_config)

  for job_name in args.job or ALL_JOBS:
    hosts = args.mapreduce_config.jobs[job_name].hosts
    args.task_map = deploy_utils.parse_args_host_and_task(args, hosts)
    for host_id in args.task_map.keys() or hosts.keys():
      for instance_id in args.task_map.get(host_id) or range(hosts[host_id].instance_num):
        instance_id = -1 if not deploy_utils.is_multiple_instances(host_id, hosts) else instance_id
        bootstrap_job(args, hosts[host_id].ip, job_name, host_id, instance_id, cleanup_token)

def start_job(args, host, job_name, host_id, instance_id):
  config_files = {}
  start_script = ""
  http_url = deploy_utils.get_http_service_uri(host,
    args.mapreduce_config.jobs[job_name].base_port, instance_id)
  deploy_utils.start_job(args, "hadoop", "mapreduce", args.mapreduce_config,
      host, job_name, instance_id, start_script, http_url, **config_files)

def start(args):
  if not args.skip_confirm:
    deploy_utils.confirm_start(args)
  get_mapreduce_service_config(args)

  for job_name in args.job or ALL_JOBS:
    hosts = args.mapreduce_config.jobs[job_name].hosts
    args.task_map = deploy_utils.parse_args_host_and_task(args, hosts)
    for host_id in args.task_map.keys() or hosts.keys():
      for instance_id in args.task_map.get(host_id) or range(hosts[host_id].instance_num):
        instance_id = -1 if not deploy_utils.is_multiple_instances(host_id, hosts) else instance_id
        start_job(args, hosts[host_id].ip, job_name, host_id, instance_id)

if __name__ == '__main__':
  test()
<commit_msg>remove start operation<commit_after>import deploy_utils

ALL_JOBS = ["mapreduce"]

def get_mapreduce_service_config(args):
  args.mapreduce_config = deploy_utils.get_service_config(args)

def install(args):
  get_mapreduce_service_config(args)
  deploy_utils.install_service(args, "mapreduce", args.mapreduce_config, "hadoop")

def bootstrap_job(args, host, job_name, host_id, instance_id, cleanup_token):
  deploy_utils.bootstrap_job(args, "hadoop", "mapreduce",
      args.mapreduce_config, host, job_name, instance_id, cleanup_token, '0')

def bootstrap(args):
  get_mapreduce_service_config(args)
  cleanup_token = deploy_utils.confirm_bootstrap("mapreduce", args.mapreduce_config)

  for job_name in args.job or ALL_JOBS:
    hosts = args.mapreduce_config.jobs[job_name].hosts
    args.task_map = deploy_utils.parse_args_host_and_task(args, hosts)
    for host_id in args.task_map.keys() or hosts.keys():
      for instance_id in args.task_map.get(host_id) or range(hosts[host_id].instance_num):
        instance_id = -1 if not deploy_utils.is_multiple_instances(host_id, hosts) else instance_id
        bootstrap_job(args, hosts[host_id].ip, job_name, host_id, instance_id, cleanup_token)

def start(args):
  Log.print_critical("'start' command is not supported!")

def cleanup(args):
  Log.print_critical("'cleanup' command is not supported!")

def show(args):
  Log.print_critical("'show' command is not supported!")

if __name__ == '__main__':
  test()
<|endoftext|>