<commit_before>from __future__ import absolute_import
from __future__ import division

from abc import abstractmethod
from collections import OrderedDict
from datetime import datetime
import logging
import os

import dateutil.parser
import dateutil.tz

from osgeo import gdal

from .core import SourceFileIngester
from .abstract_bandstack import AbstractBandstack
from .abstract_dataset import AbstractDataset

_LOG = logging.getLogger(__name__)

def _get_extent_gdal(dataset):
    """ Get the corner coordinates for a gdal Dataset
    :type dataset: gdal.Dataset
    :rtype: list of (list of float)
    :return: List of four corner coords: ul, ll, lr, ur
    """
    return _get_extent(dataset.GetGeoTransform(), dataset.RasterXSize, dataset.RasterYSize)

def _get_extent(gt, cols, rows):
    """ Return the corner coordinates from a geotransform

    :param gt: geotransform
    :type gt: (float, float, float, float, float, float)
    :param cols: number of columns in the dataset
    :type cols: int
    :param rows: number of rows in the dataset
    :type rows: int
    :rtype: list of (list of float)
    :return: List of four corner coords: ul, ll, lr, ur

    >>> gt = (144.0, 0.00025, 0.0, -36.0, 0.0, -0.00025)
    >>> cols = 4000
    >>> rows = 4000
    >>> _get_extent(gt, cols, rows)
    [[144.0, -36.0], [144.0, -37.0], [145.0, -37.0], [145.0, -36.0]]
    """
    ext = []
    xarr = [0, cols]
    yarr = [0, rows]

    for px in xarr:
        for py in yarr:
            x = gt[0] + (px * gt[1]) + (py * gt[2])
            y = gt[3] + (px * gt[4]) + (py * gt[5])
            ext.append([x, y])
        yarr.reverse()
    return ext

def _get_file_size(path):
    """ File size in KBs.
    :type path: str
    :rtype: int
    """
    if os.path.isdir(path):
        raise NotImplementedError('Directory size not yet supported: {!r}'.format(path))

    return os.path.getsize(path) // 1024

class PreTiledBandstack(AbstractBandstack):
    """Simple bandstack for pretiled ingestion.

    PreTiled ingestion doesn't need VRTs etc. This class can probably be eliminated
    in the future after a refactor of the framework.
    """

    def __init__(self, dataset, band_dict):
        """The bandstack allows for the construction of a list, or stack, of
            bands from the given dataset."""
        super(PreTiledBandstack, self).__init__(dataset.metadata_dict)
        self.dataset = dataset
        self.band_dict = OrderedDict(sorted(band_dict.items(), key=lambda t: t[0]))
        self.source_file_list = None
        self.nodata_list = None
        self.vrt_name = None
        self.vrt_band_stack = None

    def list_source_files(self):
        """Given the dictionary of band source information, form a list
        of scene file names from which a vrt can be constructed. Also return a
        list of nodata values for use by add_metadata"""

        file_list = []
        nodata_list = []
        for file_number in self.band_dict:
            pattern = self.band_dict[file_number]['file_pattern']
            this_file = self.dataset.find_band_file(pattern)
            file_list.append(this_file)
            nodata_list.append(self.band_dict[file_number]['nodata_value'])

        return file_list, nodata_list

    def get_vrt_name(self, vrt_dir):
        """Use the dataset's metadata to form the vrt file name"""
        raise RuntimeError('No VRT for pre-tiled datasets.')

    def add_metadata(self, vrt_filename):
        """Add metadata to the VRT."""
        raise RuntimeError('No VRT for pre-tiled datasets.')

    def buildvrt(self, temporary_directory):
        raise RuntimeError('Pre-tiled ingestion does not need reprojection: No VRT option available.')

class PreTiledDataset(AbstractDataset):
    """
    A dataset that has already been tiled: It should provide type
    and footprint (x,y) information.
    """
    @abstractmethod
    def get_tile_type_id(self):
        """The tile type id.
        :rtype int"""
        raise NotImplementedError

    @abstractmethod
    def get_tile_footprint(self):
        """(x,y) footprint of tile.
        :rtype: (int, int)
        """
        raise NotImplementedError

class GdalMdDataset(PreTiledDataset):
    """
    A Pre-tiled Dataset whose metadata is read from Gdal metadata.
    """

    def __init__(self, dataset_path):
        #: :type: gdal.Dataset
        self._path = dataset_path
        self._ds = gdal.Open(dataset_path)

        self._md = self._ds.GetMetadata_Dict()

        self._ul, self._ll, self._lr, self._ur = _get_extent_gdal(self._ds)

        super(GdalMdDataset, self).__init__()

    def get_processing_level(self):
        return self._md.get('level_name')

    def get_x_pixels(self):
        return self._ds.RasterXSize

    def get_y_pixels(self):
        return self._ds.RasterYSize

    def get_datetime_processed(self):
        t = os.path.getctime(self._path)
        return datetime.utcfromtimestamp(t)

    def get_geo_transform(self):
        return self._ds.GetGeoTransform()

    def get_x_ref(self):
        # Path for landsat, otherwise none. Should we support other Satellite schemes?
        return self._get_int_param('path')

    def get_y_ref(self):
        # TODO: WoFS has a row range, not a specific row, which doesn't fit this model.
        return self._get_int_param('row', 'start_row')

    def get_satellite_tag(self):
        return self._md.get('satellite_tag')

    def get_gcp_count(self):
        return self._get_int_param('gcp_count')

    def get_dataset_size(self):
        return _get_file_size(self._path)

    def get_xml_text(self):
        return None  # N/A?

    def get_dataset_path(self):
        return self._path

    def get_sensor_name(self):
        sensor_name = self._md.get('sensor_name')

        # FIXME: Hard-coded correction.
        # This difference is common across our systems. Maybe add an 'alias' column to sensor table?
        if sensor_name == 'ETM':
            return 'ETM+'

        return sensor_name

    def get_mtl_text(self):
        return None  # N/A?

    def _get_date_param(self, param_name):
        val = self._md.get(param_name)
        if not val or val == 'None':
            return None
        return dateutil.parser.parse(val, tzinfos=dateutil.tz.tzutc)

    def get_start_datetime(self):
        return self._get_date_param('start_datetime')

    def get_end_datetime(self):
        return self._get_date_param('end_datetime')

    def _get_float_param(self, param_name):
        val = self._md.get(param_name)
        if not val or val == 'None':
            return None

        return float(val)

    def _get_int_param(self, *param_names):
        for param_name in param_names:
            val = self._md.get(param_name)
            if not val or val == 'None':
                continue

            return int(val)

        return None

    def get_cloud_cover(self):
        return self._get_float_param('cloud_cover')

    def get_projection(self):
        return self._ds.GetProjection()

    def get_ll_lat(self):
        return self._ll[1]

    def get_ll_lon(self):
        return self._ll[0]

    def get_ul_lat(self):
        return self._ul[0]

    def get_ul_lon(self):
        return self._ul[1]

    def get_lr_lat(self):
        return self._lr[0]

    def get_lr_lon(self):
        return self._lr[1]

    def get_ur_lat(self):
        return self._ur[0]

    def get_ur_lon(self):
        return self._ur[1]

    # ?
    get_ll_x = get_ll_lon
    get_ll_y = get_ll_lat
    get_lr_x = get_lr_lon
    get_lr_y = get_lr_lat
    get_ur_x = get_ur_lon
    get_ur_y = get_ur_lat
    get_ul_x = get_ul_lon
    get_ul_y = get_ul_lat

    def find_band_file(self, file_pattern):
        return self._path

    def stack_bands(self, band_dict):
        return PreTiledBandstack(self, band_dict)

    # Pre-tiled information:

    def get_tile_type_id(self):
        return int(self._md['tile_type_id'])

    def get_tile_footprint(self):
        return int(self._md['x_index']), int(self._md['y_index'])

class PreTiledIngester(SourceFileIngester):
    """Ingest data that is already tiled"""

    def __init__(self, is_valid_file, datacube=None, collection=None):
        super(PreTiledIngester, self).__init__(is_valid_file, datacube, collection)
        self.collection.is_read_only = True

    def tile(self, dataset_record, dataset):
        """Create tiles for a newly created or updated dataset.

        This is similar to the parent method, but performs
        no reprojections, and adds the tile in place (no file moves).

        TODO: This could share more code with parent.

        :type dataset_record: DatasetRecord
        :type dataset: PreTiledDataset
        """
        tile_type_id = dataset.get_tile_type_id()

        tile_bands = dataset_record.get_tile_bands(tile_type_id)
        band_stack = dataset.stack_bands(tile_bands)

        tile_contents = dataset_record.collection.create_tile_contents(
            tile_type_id,
            dataset.get_tile_footprint(),
            band_stack,
            # Use existing path: we're not creating new tiles.
            tile_output_path=dataset.get_dataset_path()
        )

        with self.collection.lock_datasets([dataset_record.dataset_id]):
            with self.collection.transaction():
                dataset_record.store_tiles([tile_contents])
<commit_msg>WOfS timezone handling should match other ingesters<commit_after>from __future__ import absolute_import
from __future__ import division

from abc import abstractmethod
from collections import OrderedDict
from datetime import datetime
import logging
import os

import dateutil.parser
import dateutil.tz
from osgeo import gdal

from .core import SourceFileIngester
from .abstract_bandstack import AbstractBandstack
from .abstract_dataset import AbstractDataset

_LOG = logging.getLogger(__name__)

def _get_extent_gdal(dataset):
    """ Get the corner coordinates for a gdal Dataset
    :type dataset: gdal.Dataset
    :rtype: list of (list of float)
    :return: List of four corner coords: ul, ll, lr, ur
    """
    return _get_extent(dataset.GetGeoTransform(), dataset.RasterXSize, dataset.RasterYSize)

def _get_extent(gt, cols, rows):
    """ Return the corner coordinates from a geotransform

    :param gt: geotransform
    :type gt: (float, float, float, float, float, float)
    :param cols: number of columns in the dataset
    :type cols: int
    :param rows: number of rows in the dataset
    :type rows: int
    :rtype: list of (list of float)
    :return: List of four corner coords: ul, ll, lr, ur

    >>> gt = (144.0, 0.00025, 0.0, -36.0, 0.0, -0.00025)
    >>> cols = 4000
    >>> rows = 4000
    >>> _get_extent(gt, cols, rows)
    [[144.0, -36.0], [144.0, -37.0], [145.0, -37.0], [145.0, -36.0]]
    """
    ext = []
    xarr = [0, cols]
    yarr = [0, rows]

    for px in xarr:
        for py in yarr:
            x = gt[0] + (px * gt[1]) + (py * gt[2])
            y = gt[3] + (px * gt[4]) + (py * gt[5])
            ext.append([x, y])
        yarr.reverse()
    return ext

def _get_file_size(path):
    """ File size in KBs.
    :type path: str
    :rtype: int
    """
    if os.path.isdir(path):
        raise NotImplementedError('Directory size not yet supported: {!r}'.format(path))

    return os.path.getsize(path) // 1024

class PreTiledBandstack(AbstractBandstack):
    """Simple bandstack for pretiled ingestion.

    PreTiled ingestion doesn't need VRTs etc. This class can probably be eliminated
    in the future after a refactor of the framework.
    """

    def __init__(self, dataset, band_dict):
        """The bandstack allows for the construction of a list, or stack, of
            bands from the given dataset."""
        super(PreTiledBandstack, self).__init__(dataset.metadata_dict)
        self.dataset = dataset
        self.band_dict = OrderedDict(sorted(band_dict.items(), key=lambda t: t[0]))
        self.source_file_list = None
        self.nodata_list = None
        self.vrt_name = None
        self.vrt_band_stack = None

    def list_source_files(self):
        """Given the dictionary of band source information, form a list
        of scene file names from which a vrt can be constructed. Also return a
        list of nodata values for use by add_metadata"""

        file_list = []
        nodata_list = []
        for file_number in self.band_dict:
            pattern = self.band_dict[file_number]['file_pattern']
            this_file = self.dataset.find_band_file(pattern)
            file_list.append(this_file)
            nodata_list.append(self.band_dict[file_number]['nodata_value'])

        return file_list, nodata_list

    def get_vrt_name(self, vrt_dir):
        """Use the dataset's metadata to form the vrt file name"""
        raise RuntimeError('No VRT for pre-tiled datasets.')

    def add_metadata(self, vrt_filename):
        """Add metadata to the VRT."""
        raise RuntimeError('No VRT for pre-tiled datasets.')

    def buildvrt(self, temporary_directory):
        raise RuntimeError('Pre-tiled ingestion does not need reprojection: No VRT option available.')

class PreTiledDataset(AbstractDataset):
    """
    A dataset that has already been tiled: It should provide type
    and footprint (x,y) information.
    """
    @abstractmethod
    def get_tile_type_id(self):
        """The tile type id.
        :rtype int"""
        raise NotImplementedError

    @abstractmethod
    def get_tile_footprint(self):
        """(x,y) footprint of tile.
        :rtype: (int, int)
        """
        raise NotImplementedError

class GdalMdDataset(PreTiledDataset):
    """
    A Pre-tiled Dataset whose metadata is read from Gdal metadata.
    """

    def __init__(self, dataset_path):
        #: :type: gdal.Dataset
        self._path = dataset_path
        self._ds = gdal.Open(dataset_path)

        self._md = self._ds.GetMetadata_Dict()

        self._ul, self._ll, self._lr, self._ur = _get_extent_gdal(self._ds)

        super(GdalMdDataset, self).__init__()

    def get_processing_level(self):
        return self._md.get('level_name')

    def get_x_pixels(self):
        return self._ds.RasterXSize

    def get_y_pixels(self):
        return self._ds.RasterYSize

    def get_datetime_processed(self):
        t = os.path.getctime(self._path)
        return datetime.utcfromtimestamp(t)

    def get_geo_transform(self):
        return self._ds.GetGeoTransform()

    def get_x_ref(self):
        # Path for landsat, otherwise none. Should we support other Satellite schemes?
        return self._get_int_param('path')

    def get_y_ref(self):
        # TODO: WoFS has a row range, not a specific row, which doesn't fit this model.
        return self._get_int_param('row', 'start_row')

    def get_satellite_tag(self):
        return self._md.get('satellite_tag')

    def get_gcp_count(self):
        return self._get_int_param('gcp_count')

    def get_dataset_size(self):
        return _get_file_size(self._path)

    def get_xml_text(self):
        return None  # N/A?

    def get_dataset_path(self):
        return self._path

    def get_sensor_name(self):
        sensor_name = self._md.get('sensor_name')

        # FIXME: Hard-coded correction.
        # This difference is common across our systems. Maybe add an 'alias' column to sensor table?
        if sensor_name == 'ETM':
            return 'ETM+'

        return sensor_name

    def get_mtl_text(self):
        return None  # N/A?

    def _get_date_param(self, param_name):
        val = self._md.get(param_name)
        if not val or val == 'None':
            return None
        return dateutil.parser.parse(val)

    def get_start_datetime(self):
        return self._get_date_param('start_datetime')

    def get_end_datetime(self):
        return self._get_date_param('end_datetime')

    def _get_float_param(self, param_name):
        val = self._md.get(param_name)
        if not val or val == 'None':
            return None

        return float(val)

    def _get_int_param(self, *param_names):
        for param_name in param_names:
            val = self._md.get(param_name)
            if not val or val == 'None':
                continue

            return int(val)

        return None

    def get_cloud_cover(self):
        return self._get_float_param('cloud_cover')

    def get_projection(self):
        return self._ds.GetProjection()

    def get_ll_lat(self):
        return self._ll[1]

    def get_ll_lon(self):
        return self._ll[0]

    def get_ul_lat(self):
        return self._ul[0]

    def get_ul_lon(self):
        return self._ul[1]

    def get_lr_lat(self):
        return self._lr[0]

    def get_lr_lon(self):
        return self._lr[1]

    def get_ur_lat(self):
        return self._ur[0]

    def get_ur_lon(self):
        return self._ur[1]

    # ?
    get_ll_x = get_ll_lon
    get_ll_y = get_ll_lat
    get_lr_x = get_lr_lon
    get_lr_y = get_lr_lat
    get_ur_x = get_ur_lon
    get_ur_y = get_ur_lat
    get_ul_x = get_ul_lon
    get_ul_y = get_ul_lat

    def find_band_file(self, file_pattern):
        return self._path

    def stack_bands(self, band_dict):
        return PreTiledBandstack(self, band_dict)

    # Pre-tiled information:

    def get_tile_type_id(self):
        return int(self._md['tile_type_id'])

    def get_tile_footprint(self):
        return int(self._md['x_index']), int(self._md['y_index'])

class PreTiledIngester(SourceFileIngester):
    """Ingest data that is already tiled"""

    def __init__(self, is_valid_file, datacube=None, collection=None):
        super(PreTiledIngester, self).__init__(is_valid_file, datacube, collection)
        self.collection.is_read_only = True

    def tile(self, dataset_record, dataset):
        """Create tiles for a newly created or updated dataset.

        This is similar to the parent method, but performs
        no reprojections, and adds the tile in place (no file moves).

        TODO: This could share more code with parent.

        :type dataset_record: DatasetRecord
        :type dataset: PreTiledDataset
        """
        tile_type_id = dataset.get_tile_type_id()

        tile_bands = dataset_record.get_tile_bands(tile_type_id)
        band_stack = dataset.stack_bands(tile_bands)

        tile_contents = dataset_record.collection.create_tile_contents(
            tile_type_id,
            dataset.get_tile_footprint(),
            band_stack,
            # Use existing path: we're not creating new tiles.
            tile_output_path=dataset.get_dataset_path()
        )

        with self.collection.lock_datasets([dataset_record.dataset_id]):
            with self.collection.transaction():
                dataset_record.store_tiles([tile_contents])
<|endoftext|>


<commit_before>"""
The Application object. One application should be available.

All important services are present in the application object:
 - plugin manager
 - undo manager
 - main window
 - UML element factory
 - action sets
"""

from __future__ import annotations

import inspect
import logging
from typing import Dict, Iterator, Optional, Set, Tuple, Type, TypeVar

import importlib_metadata

from gaphor import transaction
from gaphor.abc import ActionProvider, Service
from gaphor.action import action
from gaphor.event import (
    ServiceInitializedEvent,
    ServiceShutdownEvent,
    SessionShutdownRequested,
)

T = TypeVar("T")

logger = logging.getLogger(__name__)

def distribution():
    """
    The PkgResources distribution for Gaphor
    """
    return importlib_metadata.distribution("gaphor")

class NotInitializedError(Exception):
    pass

class ComponentLookupError(LookupError):
    pass

class Application(Service, ActionProvider):
    """
    The Gaphor application is started from the gaphor.ui module.

    This application instance is used to maintain application wide references
    to services and sessions (opened models). It behaves like a singleton in many ways.

    The Application is responsible for loading services and plugins. Services
    are registered in the "component_registry" service.
    """

    def __init__(self, appservices=None):
        self.active_session: Optional[Session] = None
        self.sessions: Set[Session] = set()

        uninitialized_services = load_services("gaphor.appservices", appservices)
        self._services_by_name = init_services(uninitialized_services, application=self)

        transaction.subscribers.add(self._transaction_proxy)

    def new_session(self, services=None):
        """
        Initialize an application session.
        """
        session = Session()
        self.sessions.add(session)
        self.active_session = session

        return session

    def has_sessions(self):
        return bool(self.active_session)

    def shutdown_session(self, session):
        assert session
        session.shutdown()
        self.sessions.discard(session)
        if session is self.active_session:
            self.active_session = None

    def shutdown(self):
        """
        Forcibly shut down all sessions. No questions asked.

        This is mainly for testing purposes.
        """
        transaction.subscribers.discard(self._transaction_proxy)

        while self.sessions:
            self.shutdown_session(self.sessions.pop())

        for c in self._services_by_name.values():
            if c is not self:
                c.shutdown()
        self._services_by_name.clear()

    @action(name="app.quit", shortcut="<Primary>q")
    def quit(self):
        """
        The user's application Quit command.
        """
        for session in list(self.sessions):
            self.active_session = session
            event_manager = session.get_service("event_manager")
            event_manager.handle(SessionShutdownRequested(self))
            if self.active_session == session:
                logger.info("Window not closed, abort quit operation")
                return
        self.shutdown()

    def all(self, base: Type[T]) -> Iterator[Tuple[str, T]]:
        return (
            (n, c) for n, c in self._services_by_name.items() if isinstance(c, base)
        )

    def _transaction_proxy(self, event):
        if self.active_session:
            self.active_session.event_manager.handle(event)

class Session:
    """
    A user context is a set of services (including UI services)
    that define a window with loaded model.
    """

    def __init__(self, services=None):
        """
        Initialize the application.
        """
        uninitialized_services = load_services("gaphor.services", services)
        services_by_name = init_services(uninitialized_services)

        self.event_manager = services_by_name["event_manager"]
        self.component_registry = services_by_name["component_registry"]

        for name, srv in services_by_name.items():
            logger.info(f"Initializing service {name}")
            self.component_registry.register(name, srv)
            self.event_manager.handle(ServiceInitializedEvent(name, srv))

    def get_service(self, name):
        if not self.component_registry:
            raise NotInitializedError("Session is no longer alive")

        return self.component_registry.get_service(name)

    def shutdown(self):
        if self.component_registry:
            for name, _srv in self.component_registry.all(Service):
                self.shutdown_service(name)

        self.component_registry = None
        self.event_manager = None

    def shutdown_service(self, name):
        logger.info(f"Shutting down service {name}")
        assert self.component_registry
        assert self.event_manager

        srv = self.component_registry.get_service(name)
        self.event_manager.handle(ServiceShutdownEvent(name, srv))
        self.component_registry.unregister(srv)
        srv.shutdown()

def load_services(scope, services=None) -> Dict[str, Type[Service]]:
    """
    Load services from resources.

    Service should provide an interface `gaphor.abc.Service`.
    """
    uninitialized_services = {}
    for ep in importlib_metadata.entry_points()[scope]:
        cls = ep.load()
        if isinstance(cls, Service):
            raise NameError(f"Entry point {ep.name} doesnt provide Service")
        if not services or ep.name in services:
            logger.debug(f'found service entry point "{ep.name}"')
            uninitialized_services[ep.name] = cls
    return uninitialized_services

def init_services(uninitialized_services, **known_services):
    """
    Instantiate service definitions, taking into account dependencies
    defined in the constructor.

    Given a dictionary `{name: service-class}`,
    return a map `{name: service-instance}`.
    """
    ready: Dict[str, Service] = dict(known_services)

    def pop(name):
        try:
            return uninitialized_services.pop(name)
        except KeyError:
            return None

    def init(name, cls):
        kwargs = {}
        for dep in inspect.signature(cls).parameters:
            if dep not in ready:
                depcls = pop(dep)
                if depcls:
                    kwargs[dep] = init(dep, depcls)
                else:
                    logger.info(
                        f"Service {name} parameter {dep} does not reference a service"
                    )
            else:
                kwargs[dep] = ready[dep]
        srv = cls(**kwargs)
        ready[name] = srv
        return srv

    while uninitialized_services:
        name = next(iter(uninitialized_services.keys()))
        cls = pop(name)
        init(name, cls)

    return ready
<commit_msg>Update application module docstring<commit_after>"""
The Application object. One application should be available.

An application can host multiple sessions. From a user point of
view a session is represented as a window in which a diagram
can be edited.
"""

from __future__ import annotations

import inspect
import logging
from typing import Dict, Iterator, Optional, Set, Tuple, Type, TypeVar

import importlib_metadata

from gaphor import transaction
from gaphor.abc import ActionProvider, Service
from gaphor.action import action
from gaphor.event import (
    ServiceInitializedEvent,
    ServiceShutdownEvent,
    SessionShutdownRequested,
)

T = TypeVar("T")

logger = logging.getLogger(__name__)

def distribution():
    """
    The PkgResources distribution for Gaphor
    """
    return importlib_metadata.distribution("gaphor")

class NotInitializedError(Exception):
    pass

class ComponentLookupError(LookupError):
    pass

class Application(Service, ActionProvider):
    """
    The Gaphor application is started from the gaphor.ui module.

    This application instance is used to maintain application wide references
    to services and sessions (opened models). It behaves like a singleton in many ways.

    The Application is responsible for loading services and plugins. Services
    are registered in the "component_registry" service.
    """

    def __init__(self, appservices=None):
        self.active_session: Optional[Session] = None
        self.sessions: Set[Session] = set()

        uninitialized_services = load_services("gaphor.appservices", appservices)
        self._services_by_name = init_services(uninitialized_services, application=self)

        transaction.subscribers.add(self._transaction_proxy)

    def new_session(self, services=None):
        """
        Initialize an application session.
        """
        session = Session()
        self.sessions.add(session)
        self.active_session = session

        return session

    def has_sessions(self):
        return bool(self.active_session)

    def shutdown_session(self, session):
        assert session
        session.shutdown()
        self.sessions.discard(session)
        if session is self.active_session:
            self.active_session = None

    def shutdown(self):
        """
        Forcibly shut down all sessions. No questions asked.

        This is mainly for testing purposes.
        """
        transaction.subscribers.discard(self._transaction_proxy)

        while self.sessions:
            self.shutdown_session(self.sessions.pop())

        for c in self._services_by_name.values():
            if c is not self:
                c.shutdown()
        self._services_by_name.clear()

    @action(name="app.quit", shortcut="<Primary>q")
    def quit(self):
        """
        The user's application Quit command.
        """
        for session in list(self.sessions):
            self.active_session = session
            event_manager = session.get_service("event_manager")
            event_manager.handle(SessionShutdownRequested(self))
            if self.active_session == session:
                logger.info("Window not closed, abort quit operation")
                return
        self.shutdown()

    def all(self, base: Type[T]) -> Iterator[Tuple[str, T]]:
        return (
            (n, c) for n, c in self._services_by_name.items() if isinstance(c, base)
        )

    def _transaction_proxy(self, event):
        if self.active_session:
            self.active_session.event_manager.handle(event)

class Session:
    """
    A user context is a set of services (including UI services)
    that define a window with loaded model.
    """

    def __init__(self, services=None):
        """
        Initialize the application.
        """
        uninitialized_services = load_services("gaphor.services", services)
        services_by_name = init_services(uninitialized_services)

        self.event_manager = services_by_name["event_manager"]
        self.component_registry = services_by_name["component_registry"]

        for name, srv in services_by_name.items():
            logger.info(f"Initializing service {name}")
            self.component_registry.register(name, srv)
            self.event_manager.handle(ServiceInitializedEvent(name, srv))

    def get_service(self, name):
        if not self.component_registry:
            raise NotInitializedError("Session is no longer alive")

        return self.component_registry.get_service(name)

    def shutdown(self):
        if self.component_registry:
            for name, _srv in self.component_registry.all(Service):
                self.shutdown_service(name)

        self.component_registry = None
        self.event_manager = None

    def shutdown_service(self, name):
        logger.info(f"Shutting down service {name}")
        assert self.component_registry
        assert self.event_manager

        srv = self.component_registry.get_service(name)
        self.event_manager.handle(ServiceShutdownEvent(name, srv))
        self.component_registry.unregister(srv)
        srv.shutdown()

def load_services(scope, services=None) -> Dict[str, Type[Service]]:
    """
    Load services from resources.

    Service should provide an interface `gaphor.abc.Service`.
    """
    uninitialized_services = {}
    for ep in importlib_metadata.entry_points()[scope]:
        cls = ep.load()
        if isinstance(cls, Service):
            raise NameError(f"Entry point {ep.name} doesnt provide Service")
        if not services or ep.name in services:
            logger.debug(f'found service entry point "{ep.name}"')
            uninitialized_services[ep.name] = cls
    return uninitialized_services

def init_services(uninitialized_services, **known_services):
    """
    Instantiate service definitions, taking into account dependencies
    defined in the constructor.

    Given a dictionary `{name: service-class}`,
    return a map `{name: service-instance}`.
    """
    ready: Dict[str, Service] = dict(known_services)

    def pop(name):
        try:
            return uninitialized_services.pop(name)
        except KeyError:
            return None

    def init(name, cls):
        kwargs = {}
        for dep in inspect.signature(cls).parameters:
            if dep not in ready:
                depcls = pop(dep)
                if depcls:
                    kwargs[dep] = init(dep, depcls)
                else:
                    logger.info(
                        f"Service {name} parameter {dep} does not reference a service"
                    )
            else:
                kwargs[dep] = ready[dep]
        srv = cls(**kwargs)
        ready[name] = srv
        return srv

    while uninitialized_services:
        name = next(iter(uninitialized_services.keys()))
        cls = pop(name)
        init(name, cls)

    return ready
<|endoftext|>


<commit_before>from pyramid.compat import escape

import logging
LOGGER = logging.getLogger("PHOENIX")

def escape_output(output):
    if output:
        if isinstance(output, list):
            return map(escape, output)
        else:
            return escape(output)
    else:
        return output

def output_details(request, output):
    wms_dataset_path = None
    # get proxy_reference and wms_dataset_path
    LOGGER.debug("output reference: %s", output.reference)
    if output.reference and 'wpsoutputs' in output.reference:
        wps_output_url = request.registry.settings.get('wps.output.url')
        if request.map_activated and output.mimeType and 'netcdf' in output.mimeType:
            wms_dataset_path = "outputs" + output.reference.split('wpsoutputs', 1)[1]
        if wps_output_url and output.reference.startswith(wps_output_url):
            proxy_reference = request.route_url(
                'download_wpsoutputs',
                subpath=output.reference.split(wps_output_url, 1)[1])
            LOGGER.debug("proxy reference: %s", proxy_reference)
    else:
        proxy_reference = output.reference
    # get category and data
    if output.mimeType:
        category = 'ComplexType'
        data = output.data
    elif output.dataType == 'BoundingBoxData':
        category = 'BoundingBoxType'
        data = ["{0.minx},{0.miny},{0.maxx},{0.maxy}".format(bbox) for bbox in output.data]
    else:
        category = 'LiteralType'
        data = output.data
    return dict(title=output.title,
                abstract=output.abstract,
                identifier=output.identifier,
                mime_type=output.mimeType,
                data=escape_output(data),
                reference=escape_output(output.reference),
                proxy_reference=escape_output(proxy_reference),
                wms_dataset_path=escape_output(wms_dataset_path),
                category=category)
<commit_msg>fixed proxy_reference<commit_after>from pyramid.compat import escape

import logging
LOGGER = logging.getLogger("PHOENIX")

def escape_output(output):
    if output:
        if isinstance(output, list):
            return map(escape, output)
        else:
            return escape(output)
    else:
        return output

def output_details(request, output):
    wms_dataset_path = None
    # get proxy_reference and wms_dataset_path
    LOGGER.debug("output reference: %s", output.reference)
    proxy_reference = output.reference
    if output.reference and 'wpsoutputs' in output.reference:
        wps_output_url = request.registry.settings.get('wps.output.url')
        if request.map_activated and output.mimeType and 'netcdf' in output.mimeType:
            wms_dataset_path = "outputs" + output.reference.split('wpsoutputs', 1)[1]
        if wps_output_url and output.reference.startswith(wps_output_url):
            proxy_reference = request.route_url(
                'download_wpsoutputs',
                subpath=output.reference.split(wps_output_url, 1)[1])
            LOGGER.debug("proxy reference: %s", proxy_reference)
    # get category and data
    if output.mimeType:
        category = 'ComplexType'
        data = output.data
    elif output.dataType == 'BoundingBoxData':
        category = 'BoundingBoxType'
        data = ["{0.minx},{0.miny},{0.maxx},{0.maxy}".format(bbox) for bbox in output.data]
    else:
        category = 'LiteralType'
        data = output.data
    return dict(title=output.title,
                abstract=output.abstract,
                identifier=output.identifier,
                mime_type=output.mimeType,
                data=escape_output(data),
                reference=escape_output(output.reference),
                proxy_reference=escape_output(proxy_reference),
                wms_dataset_path=escape_output(wms_dataset_path),
                category=category)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""
Updates the WikiProject Directory
Copyright (C) 2015 James Hare, Betacommand, Merlijn Van Deen
Licensed under MIT License: http://mitlicense.org
"""

import re
import time
import pywikibot
import json
import operator
import mwparserfromhell as mwph
from collections import Counter
from project_index import WikiProjectTools
from category_tree import WikiProjectCategories

def listpull(wptools, projects, directoryrow, key):
    query = wptools.query('wiki', 'select distinct page.page_title from categorylinks join page on categorylinks.cl_from=page.page_id where page_namespace = 4 and cl_to = "{0}" order by page.page_title'.format(key), None)
    output = ''
    outputlist = []
    for row in query:
        proj = row[0].decode('utf-8')
        if proj in projects:  # This check is to filter against query results like "WikiProject_Stupid/talkheader" from being considered as projects.
            outputlist.append(proj)
    if len(outputlist) > 0:  # i.e., if we actually *have* WikiProjects to append to the list or just garbage
        for proj in outputlist:
            output += directoryrow[proj]
        output = "{{WikiProject directory top}}\n" + output + "|}\n\n"
    return output

def treeiterator(wptools, tree, projects, directoryrow, key, counter=2):
    output = ''
    if len(tree) > 0:
        print("Populating directory page: " + key + " (level " + str(counter - 1) + ")")
        header = "=" * counter  # Python always finds new ways to amaze me.
        for step in tree.keys():
            output += header + step.replace('_', ' ') + header + "\n" + listpull(wptools, projects, directoryrow, step) + "\n"
            if len(tree[step]) > 0:
                output += treeiterator(wptools, tree[step], projects, directoryrow, step, counter=counter+1)
    return output

def main(rootpage):
    # Initializing...
    bot = pywikibot.Site('en', 'wikipedia')
    wptools = WikiProjectTools()
    config = json.loads(wptools.query('index', 'select json from config;', None)[0][0])
    print("Let's get this show on the road!")

    # Get list of people who opted out
    optout = pywikibot.Page(bot, 'User:Reports bot/Opt-out')
    blacklist = []
    regexes =[re.findall('\[\[User:(.*?)\|',optout.text,re.I), re.findall('\{\{user\|(.*?)\}\}',optout.text,re.I), re.findall('\[\[:User:(.*?)\]',optout.text,re.I), re.findall('\[\[:User talk:(.*?)\]',optout.text,re.I)]
    for results in regexes:
        for user in results:
            blacklist.append(user)
    print(str(len(blacklist)) + ' users opting out')
    # Bots are to be excluded
    for result in wptools.query('wiki', "select user_name from user_groups left join user on user_id = ug_user where ug_group = 'bot';", None):
        blacklist.append(result[0].decode('utf-8'))
    print('With bots, there are ' + str(len(blacklist)) + ' usernames on the blacklist.')

    # Loading the Project Index
    print("Loading the Project Index...")
    projectindex = wptools.query('index', 'select pi_page, pi_project from projectindex;', None)

    # List of projects we are working on
    # Methodology: List from Project Index + List from Formal Definition, minus duplicates
    # This will cover all of our bases.
    projects = []
    articles = {}
    for pair in projectindex:
        pair = list(pair)  # For manipulation
        pair[0] = re.sub(r'(Draft_)?[Tt]alk:', '', pair[0])  # Normalizing by getting rid of namespace
        pair[1] = pair[1][10:]  # Normalizing by getting rid of "Wikipedia:"
        if pair[1] not in projects:
            projects.append(pair[1])
            articles[pair[1]] = [pair[0]]
        else:
            articles[pair[1]].append(pair[0])

    print("Preparing the Formal Definition index...")
    formaldefinition = wptools.query('wiki', 'select distinct page.page_title from page join categorylinks on page.page_id = categorylinks.cl_from left join redirect on page.page_id = redirect.rd_from where page_namespace = 4 and page_title not like "%/%" and rd_title is null and (cl_to in (select page.page_title from page where page_namespace = 14 and page_title like "%\_WikiProjects" and page_title not like "%\_for\_WikiProjects" and page_title not like "%\_of\_WikiProjects") or page_title like "WikiProject\_%");', None)  # http://quarry.wmflabs.org/query/3509
    for row in formaldefinition:
        row = row[0].decode('utf-8')
        if row not in projects:
            projects.append(row)
    projects.sort()
    projects = projects[:150] # DEBUG MODE
    print('There are ' + str(len(projects)) + ' total WikiProjects and task forces.')

    directories = {'All': ''}  # All projects, plus subdirectories to be defined below.
    directoryrow = {}

    # Alright! Let's run some reports!
    for project in projects:
        print("Working on: " + project)

        # Seeding directory row and profile page
        if project not in articles:
            articles[project] = []
        project_normalized = project.replace('_', ' ')

        # List of active project participants (less blacklist)
        wp_editors = []
        start_date = time.strftime('%Y%m%d000000',time.gmtime(time.time()-(60*60*24*90)))  # 90 days
        end_date = time.strftime('%Y%m%d000000',time.gmtime(time.time()))  # Today
        query = "select rev_user_text from page left join revision on page_id = rev_page where (page_namespace = 4 OR page_namespace = 5) and (page_title like \"{0}/%%\" OR page_title = \"{0}\") and rev_timestamp > {1} and rev_timestamp < {2} group by rev_user_text HAVING count(*) > 1;".format(project, start_date, end_date)
        for result in wptools.query('wiki', query, None):
            if result[0] is not None:
                user = result[0].decode('utf-8')
                if user not in blacklist:
                    wp_editors.append(user)
        wp_editors.sort()

        # List of active subject area editors (less blacklist)
        start_date = time.strftime('%Y%m%d000000',time.gmtime(time.time()-(60*60*24*30)))  # 30 days
        end_date = time.strftime('%Y%m%d000000',time.gmtime(time.time()))  # Today

        if len(articles[project]) > 0:
            subject_editors = []
            packages = []
            for i in range(0, len(articles[project]), 10000):
                packages.append(articles[project][i:i+10000])

            counter = 0
            for package in packages:
                counter += 1
                print('Executing batch query no. ' + str(counter))
                query_builder = 'select rev_user_text from page left join revision on page_id = rev_page where page_namespace in (0, 1, 118, 119) and page_title in {0} and rev_timestamp > {1} and rev_timestamp < {2} order by rev_user_text;'.format(tuple(package), start_date, end_date)
                for result in wptools.query('wiki', query_builder, None):
                    if result[0] is not None:
                        subject_editors.append(result[0].decode('utf-8'))

            subject_editors = dict(Counter(subject_editors))  # Convert the list to a dictionary with username as key and edit count as value
            subject_editors_filtered = []
            for user in subject_editors.keys():
                if user not in blacklist:
                    if subject_editors[user] > 4:
                        subject_editors_filtered.append(user)
            subject_editors = subject_editors_filtered   # And now assigned back.
            subject_editors.sort()

        else:
            subject_editors = []

        # Generate and Save Profile Page
        wp_editors_formatted = ""
        subject_editors_formatted = ""
        if len(wp_editors) > 0:
            for editor in wp_editors:
                wp_editors_formatted += "\n* {{{{user|1={0}}}}}".format(editor)
        else:
                wp_editors_formatted = ""
        if len(subject_editors) > 0:
            for editor in subject_editors:
                subject_editors_formatted += "\n* {{{{user|1={0}}}}}".format(editor)
        else:
                subject_editors_formatted = ""

        profilepage = "{{{{WikiProject description page | project = {0} | list_of_active_wikiproject_participants = {1} | list_of_active_subject_area_editors = {2}}}}}".format(project_normalized, wp_editors_formatted, subject_editors_formatted)
        page = pywikibot.Page(bot, rootpage + '/Description/' + project_normalized)
        if profilepage != page.text:  # Checking to see if a change was made to cut down on API queries
            page.text = profilepage
            page.save('Updating', minor=False, async=True)

        # Construct directory entry
        directoryrow[project] = "{{{{WikiProject directory entry | project = {0} | number_of_articles = {1} | wp_editors = {2} | scope_editors = {3}}}}}\n".format(project_normalized, len(articles[project]), len(wp_editors), len(subject_editors))

    # Assign directory entry to relevant directory pages ("All entries" and relevant subdirectory pages)
    print("Populating total directory...")
    for entry in sorted(directoryrow.items(), key=operator.itemgetter(1)):  # Sorting into alphabetical order
        directories['All'] += entry[1]
    directories['All'] = "{{WikiProject directory top}}\n" + directories['All'] + "|}"

    wpcats = WikiProjectCategories()
    tree = wpcats.generate()
    directoryindex = "'''[[{0}/All|All WikiProjects]]'''\n\n".format(rootpage)
    for firstlevel in tree.keys():
        directories[firstlevel] = listpull(wptools, projects, directoryrow, firstlevel)  # For immmedate subcats of WikiProjects_by_area
        directories[firstlevel] += treeiterator(wptools, tree[firstlevel], projects, directoryrow, firstlevel)  # For descendants of those immediate subcats.
        # Updating the directory index
        firstlevel_normalized = firstlevel.replace('_', ' ')
        directoryindex += "'''[[{0}/{1}|{1}]]'''".format(rootpage, firstlevel_normalized)
        if len(tree[firstlevel]) > 0:
            directoryindex += ": "
            for secondlevel in tree[firstlevel].keys():
                directoryindex += "[[{0}/{1}#{2}|{2}]] – ".format(rootpage, firstlevel_normalized, secondlevel.replace('_', ' ')
            directoryindex = directoryindex[:-3]  # Truncates trailing dash and is also a cute smiley face
        directoryindex += "\n\n"
    saveindex = pywikibot.Page(bot, 'Template:WikiProject directory index')
    saveindex.text = directoryindex
    saveindex.save('Updating', minor=False, async=True)

    # Generate directories and save!
    for directory in directories.keys():
        contents = directories[directory]
        page = pywikibot.Page(bot, rootpage + "/" + directory)
        if contents != page.text:  # Checking to see if a change was made to cut down on API save queries
            oldcontents = page.text
            page.text = contents
            page.save('Updating', minor=False, async=True)
            # Cleanup of obsolete description pages
            if directory == 'All':
                oldcontents = mwph.parse(oldcontents)
                oldcontents = oldcontents.filter_templates()
                oldprojectlist = []
                for t in oldcontents:
                    if t.name.strip() == "WikiProject directory entry":
                        oldprojectlist.append(str(t.get('project').value))
                for oldproject in oldprojectlist:
                    oldproject = oldproject.strip().replace(' ', '_')  # Normalizing
                    if oldproject not in projects:
                        deletethis = pywikibot.Page(bot, rootpage + '/Description/' + oldproject)
                        deletethis.text = "{{db-g6|rationale=A bot has automatically tagged this page as obsolete. This means that the WikiProject described on this page has been deleted or made into a redirect}}\n"
                        deletethis.save('Nominating page for deletion', minor=False, async=True)

if __name__ == "__main__":
    main('User:Reports bot/Directory')<commit_msg>Closing parenthesis<commit_after># -*- coding: utf-8 -*-
"""
Updates the WikiProject Directory
Copyright (C) 2015 James Hare, Betacommand, Merlijn Van Deen
Licensed under MIT License: http://mitlicense.org
"""

import re
import time
import pywikibot
import json
import operator
import mwparserfromhell as mwph
from collections import Counter
from project_index import WikiProjectTools
from category_tree import WikiProjectCategories

def listpull(wptools, projects, directoryrow, key):
    query = wptools.query('wiki', 'select distinct page.page_title from categorylinks join page on categorylinks.cl_from=page.page_id where page_namespace = 4 and cl_to = "{0}" order by page.page_title'.format(key), None)
    output = ''
    outputlist = []
    for row in query:
        proj = row[0].decode('utf-8')
        if proj in projects:  # This check is to filter against query results like "WikiProject_Stupid/talkheader" from being considered as projects.
            outputlist.append(proj)
    if len(outputlist) > 0:  # i.e., if we actually *have* WikiProjects to append to the list or just garbage
        for proj in outputlist:
            output += directoryrow[proj]
        output = "{{WikiProject directory top}}\n" + output + "|}\n\n"
    return output

def treeiterator(wptools, tree, projects, directoryrow, key, counter=2):
    output = ''
    if len(tree) > 0:
        print("Populating directory page: " + key + " (level " + str(counter - 1) + ")")
        header = "=" * counter  # Python always finds new ways to amaze me.
        for step in tree.keys():
            output += header + step.replace('_', ' ') + header + "\n" + listpull(wptools, projects, directoryrow, step) + "\n"
            if len(tree[step]) > 0:
                output += treeiterator(wptools, tree[step], projects, directoryrow, step, counter=counter+1)
    return output

def main(rootpage):
    # Initializing...
    bot = pywikibot.Site('en', 'wikipedia')
    wptools = WikiProjectTools()
    config = json.loads(wptools.query('index', 'select json from config;', None)[0][0])
    print("Let's get this show on the road!")

    # Get list of people who opted out
    optout = pywikibot.Page(bot, 'User:Reports bot/Opt-out')
    blacklist = []
    regexes =[re.findall('\[\[User:(.*?)\|',optout.text,re.I), re.findall('\{\{user\|(.*?)\}\}',optout.text,re.I), re.findall('\[\[:User:(.*?)\]',optout.text,re.I), re.findall('\[\[:User talk:(.*?)\]',optout.text,re.I)]
    for results in regexes:
        for user in results:
            blacklist.append(user)
    print(str(len(blacklist)) + ' users opting out')
    # Bots are to be excluded
    for result in wptools.query('wiki', "select user_name from user_groups left join user on user_id = ug_user where ug_group = 'bot';", None):
        blacklist.append(result[0].decode('utf-8'))
    print('With bots, there are ' + str(len(blacklist)) + ' usernames on the blacklist.')

    # Loading the Project Index
    print("Loading the Project Index...")
    projectindex = wptools.query('index', 'select pi_page, pi_project from projectindex;', None)

    # List of projects we are working on
    # Methodology: List from Project Index + List from Formal Definition, minus duplicates
    # This will cover all of our bases.
    projects = []
    articles = {}
    for pair in projectindex:
        pair = list(pair)  # For manipulation
        pair[0] = re.sub(r'(Draft_)?[Tt]alk:', '', pair[0])  # Normalizing by getting rid of namespace
        pair[1] = pair[1][10:]  # Normalizing by getting rid of "Wikipedia:"
        if pair[1] not in projects:
            projects.append(pair[1])
            articles[pair[1]] = [pair[0]]
        else:
            articles[pair[1]].append(pair[0])

    print("Preparing the Formal Definition index...")
    formaldefinition = wptools.query('wiki', 'select distinct page.page_title from page join categorylinks on page.page_id = categorylinks.cl_from left join redirect on page.page_id = redirect.rd_from where page_namespace = 4 and page_title not like "%/%" and rd_title is null and (cl_to in (select page.page_title from page where page_namespace = 14 and page_title like "%\_WikiProjects" and page_title not like "%\_for\_WikiProjects" and page_title not like "%\_of\_WikiProjects") or page_title like "WikiProject\_%");', None)  # http://quarry.wmflabs.org/query/3509
    for row in formaldefinition:
        row = row[0].decode('utf-8')
        if row not in projects:
            projects.append(row)
    projects.sort()
    projects = projects[:150] # DEBUG MODE
    print('There are ' + str(len(projects)) + ' total WikiProjects and task forces.')

    directories = {'All': ''}  # All projects, plus subdirectories to be defined below.
    directoryrow = {}

    # Alright! Let's run some reports!
    for project in projects:
        print("Working on: " + project)

        # Seeding directory row and profile page
        if project not in articles:
            articles[project] = []
        project_normalized = project.replace('_', ' ')

        # List of active project participants (less blacklist)
        wp_editors = []
        start_date = time.strftime('%Y%m%d000000',time.gmtime(time.time()-(60*60*24*90)))  # 90 days
        end_date = time.strftime('%Y%m%d000000',time.gmtime(time.time()))  # Today
        query = "select rev_user_text from page left join revision on page_id = rev_page where (page_namespace = 4 OR page_namespace = 5) and (page_title like \"{0}/%%\" OR page_title = \"{0}\") and rev_timestamp > {1} and rev_timestamp < {2} group by rev_user_text HAVING count(*) > 1;".format(project, start_date, end_date)
        for result in wptools.query('wiki', query, None):
            if result[0] is not None:
                user = result[0].decode('utf-8')
                if user not in blacklist:
                    wp_editors.append(user)
        wp_editors.sort()

        # List of active subject area editors (less blacklist)
        start_date = time.strftime('%Y%m%d000000',time.gmtime(time.time()-(60*60*24*30)))  # 30 days
        end_date = time.strftime('%Y%m%d000000',time.gmtime(time.time()))  # Today

        if len(articles[project]) > 0:
            subject_editors = []
            packages = []
            for i in range(0, len(articles[project]), 10000):
                packages.append(articles[project][i:i+10000])

            counter = 0
            for package in packages:
                counter += 1
                print('Executing batch query no. ' + str(counter))
                query_builder = 'select rev_user_text from page left join revision on page_id = rev_page where page_namespace in (0, 1, 118, 119) and page_title in {0} and rev_timestamp > {1} and rev_timestamp < {2} order by rev_user_text;'.format(tuple(package), start_date, end_date)
                for result in wptools.query('wiki', query_builder, None):
                    if result[0] is not None:
                        subject_editors.append(result[0].decode('utf-8'))

            subject_editors = dict(Counter(subject_editors))  # Convert the list to a dictionary with username as key and edit count as value
            subject_editors_filtered = []
            for user in subject_editors.keys():
                if user not in blacklist:
                    if subject_editors[user] > 4:
                        subject_editors_filtered.append(user)
            subject_editors = subject_editors_filtered   # And now assigned back.
            subject_editors.sort()

        else:
            subject_editors = []

        # Generate and Save Profile Page
        wp_editors_formatted = ""
        subject_editors_formatted = ""
        if len(wp_editors) > 0:
            for editor in wp_editors:
                wp_editors_formatted += "\n* {{{{user|1={0}}}}}".format(editor)
        else:
                wp_editors_formatted = ""
        if len(subject_editors) > 0:
            for editor in subject_editors:
                subject_editors_formatted += "\n* {{{{user|1={0}}}}}".format(editor)
        else:
                subject_editors_formatted = ""

        profilepage = "{{{{WikiProject description page | project = {0} | list_of_active_wikiproject_participants = {1} | list_of_active_subject_area_editors = {2}}}}}".format(project_normalized, wp_editors_formatted, subject_editors_formatted)
        page = pywikibot.Page(bot, rootpage + '/Description/' + project_normalized)
        if profilepage != page.text:  # Checking to see if a change was made to cut down on API queries
            page.text = profilepage
            page.save('Updating', minor=False, async=True)

        # Construct directory entry
        directoryrow[project] = "{{{{WikiProject directory entry | project = {0} | number_of_articles = {1} | wp_editors = {2} | scope_editors = {3}}}}}\n".format(project_normalized, len(articles[project]), len(wp_editors), len(subject_editors))

    # Assign directory entry to relevant directory pages ("All entries" and relevant subdirectory pages)
    print("Populating total directory...")
    for entry in sorted(directoryrow.items(), key=operator.itemgetter(1)):  # Sorting into alphabetical order
        directories['All'] += entry[1]
    directories['All'] = "{{WikiProject directory top}}\n" + directories['All'] + "|}"

    wpcats = WikiProjectCategories()
    tree = wpcats.generate()
    directoryindex = "'''[[{0}/All|All WikiProjects]]'''\n\n".format(rootpage)
    for firstlevel in tree.keys():
        directories[firstlevel] = listpull(wptools, projects, directoryrow, firstlevel)  # For immmedate subcats of WikiProjects_by_area
        directories[firstlevel] += treeiterator(wptools, tree[firstlevel], projects, directoryrow, firstlevel)  # For descendants of those immediate subcats.
        # Updating the directory index
        firstlevel_normalized = firstlevel.replace('_', ' ')
        directoryindex += "'''[[{0}/{1}|{1}]]'''".format(rootpage, firstlevel_normalized)
        if len(tree[firstlevel]) > 0:
            directoryindex += ": "
            for secondlevel in tree[firstlevel].keys():
                directoryindex += "[[{0}/{1}#{2}|{2}]] – ".format(rootpage, firstlevel_normalized, secondlevel.replace('_', ' '))
            directoryindex = directoryindex[:-3]  # Truncates trailing dash and is also a cute smiley face
        directoryindex += "\n\n"
    saveindex = pywikibot.Page(bot, 'Template:WikiProject directory index')
    saveindex.text = directoryindex
    saveindex.save('Updating', minor=False, async=True)

    # Generate directories and save!
    for directory in directories.keys():
        contents = directories[directory]
        page = pywikibot.Page(bot, rootpage + "/" + directory)
        if contents != page.text:  # Checking to see if a change was made to cut down on API save queries
            oldcontents = page.text
            page.text = contents
            page.save('Updating', minor=False, async=True)
            # Cleanup of obsolete description pages
            if directory == 'All':
                oldcontents = mwph.parse(oldcontents)
                oldcontents = oldcontents.filter_templates()
                oldprojectlist = []
                for t in oldcontents:
                    if t.name.strip() == "WikiProject directory entry":
                        oldprojectlist.append(str(t.get('project').value))
                for oldproject in oldprojectlist:
                    oldproject = oldproject.strip().replace(' ', '_')  # Normalizing
                    if oldproject not in projects:
                        deletethis = pywikibot.Page(bot, rootpage + '/Description/' + oldproject)
                        deletethis.text = "{{db-g6|rationale=A bot has automatically tagged this page as obsolete. This means that the WikiProject described on this page has been deleted or made into a redirect}}\n"
                        deletethis.save('Nominating page for deletion', minor=False, async=True)

if __name__ == "__main__":
    main('User:Reports bot/Directory')<|endoftext|>


<commit_before>"""Manipulate access control lists that Cloud Storage provides.

:class:`gcloud.storage.bucket.Bucket` has a getting method that creates
an ACL object under the hood, and you can interact with that using
:func:`gcloud.storage.bucket.Bucket.get_acl`::

  >>> from gcloud import storage
  >>> connection = storage.get_connection(project, email, key_path)
  >>> bucket = connection.get_bucket(bucket_name)
  >>> acl = bucket.get_acl()

Adding and removing permissions can be done with the following methods
(in increasing order of granularity):

- :func:`ACL.all`
  corresponds to access for all users.
- :func:`ACL.all_authenticated` corresponds
  to access for all users that are signed into a Google account.
- :func:`ACL.domain` corresponds to access on a
  per Google Apps domain (ie, ``example.com``).
- :func:`ACL.group` corresponds to access on a
  per group basis (either by ID or e-mail address).
- :func:`ACL.user` corresponds to access on a
  per user basis (either by ID or e-mail address).

And you are able to ``grant`` and ``revoke`` the following roles:

- **Reading**:
  :func:`_ACLEntity.grant_read` and :func:`_ACLEntity.revoke_read`
- **Writing**:
  :func:`_ACLEntity.grant_write` and :func:`_ACLEntity.revoke_write`
- **Owning**:
  :func:`_ACLEntity.grant_owner` and :func:`_ACLEntity.revoke_owner`

You can use any of these like any other factory method (these happen to
be :class:`_ACLEntity` factories)::

  >>> acl.user('me@example.org').grant_read()
  >>> acl.all_authenticated().grant_write()

You can also chain these ``grant_*`` and ``revoke_*`` methods together
for brevity::

  >>> acl.all().grant_read().revoke_write()

After that, you can save any changes you make with the
:func:`gcloud.storage.acl.ACL.save` method::

  >>> acl.save()

You can alternatively save any existing :class:`gcloud.storage.acl.ACL`
object (whether it was created by a factory method or not) with the
:func:`gcloud.storage.bucket.Bucket.save_acl` method::

  >>> bucket.save_acl(acl)

To get the list of ``entity`` and ``role`` for each unique pair, the
:class:`ACL` class is iterable::

  >>> print list(ACL)
  [{'role': 'OWNER', 'entity': 'allUsers'}, ...]

This list of tuples can be used as the ``entity`` and ``role`` fields
when sending metadata for ACLs to the API.
"""

class _ACLEntity(object):
    """Class representing a set of roles for an entity.

    This is a helper class that you likely won't ever construct
    outside of using the factor methods on the :class:`ACL` object.
    """

    READER_ROLE = 'READER'
    WRITER_ROLE = 'WRITER'
    OWNER_ROLE = 'OWNER'

    def __init__(self, entity_type, identifier=None):
        """Entity constructor.

        :type entity_type: string
        :param entity_type: The type of entity (ie, 'group' or 'user').

        :type identifier: string
        :param identifier: The ID or e-mail of the entity. For the special
                           entity types (like 'allUsers') this is optional.
        """
        self.identifier = identifier
        self.roles = set([])
        self.type = entity_type

    def __str__(self):
        if not self.identifier:
            return str(self.type)
        else:
            return '{self.type}-{self.identifier}'.format(self=self)

    def __repr__(self):
        return '<ACL Entity: {self} ({roles})>'.format(
            self=self, roles=', '.join(self.roles))

    def get_roles(self):
        """Get the list of roles permitted by this entity.

        :rtype: list of strings
        :returns: The list of roles associated with this entity.
        """
        return self.roles

    def grant(self, role):
        """Add a role to the entity.

        :type role: string
        :param role: The role to add to the entity.

        :rtype: :class:`_ACLEntity`
        :returns: The entity class.
        """
        self.roles.add(role)
        return self

    def revoke(self, role):
        """Remove a role from the entity.

        :type role: string
        :param role: The role to remove from the entity.

        :rtype: :class:`_ACLEntity`
        :returns: The entity class.
        """
        if role in self.roles:
            self.roles.remove(role)
        return self

    def grant_read(self):
        """Grant read access to the current entity."""

        return self.grant(_ACLEntity.READER_ROLE)

    def grant_write(self):
        """Grant write access to the current entity."""

        return self.grant(_ACLEntity.WRITER_ROLE)

    def grant_owner(self):
        """Grant owner access to the current entity."""

        return self.grant(_ACLEntity.OWNER_ROLE)

    def revoke_read(self):
        """Revoke read access from the current entity."""

        return self.revoke(_ACLEntity.READER_ROLE)

    def revoke_write(self):
        """Revoke write access from the current entity."""

        return self.revoke(_ACLEntity.WRITER_ROLE)

    def revoke_owner(self):
        """Revoke owner access from the current entity."""

        return self.revoke(_ACLEntity.OWNER_ROLE)

class ACL(object):
    """Container class representing a list of access controls."""

    loaded = False

    def __init__(self):
        self.entities = {}

    def reset(self):
        """Remove all entities from the ACL, and clear the ``loaded`` flag."""
        self.entities.clear()
        self.loaded = False

    def __iter__(self):
        for entity in self.entities.itervalues():
            for role in entity.get_roles():
                if role:
                    yield {'entity': str(entity), 'role': role}

    def entity_from_dict(self, entity_dict):
        """Build an _ACLEntity object from a dictionary of data.

        An entity is a mutable object that represents a list of roles
        belonging to either a user or group or the special types for all
        users and all authenticated users.

        :type entity_dict: dict
        :param entity_dict: Dictionary full of data from an ACL lookup.

        :rtype: :class:`_ACLEntity`
        :returns: An Entity constructed from the dictionary.
        """
        entity = entity_dict['entity']
        role = entity_dict['role']

        if entity == 'allUsers':
            entity = self.all()

        elif entity == 'allAuthenticatedUsers':
            entity = self.all_authenticated()

        elif '-' in entity:
            entity_type, identifier = entity.split('-', 1)
            entity = self.entity(entity_type=entity_type,
                                 identifier=identifier)

        if not isinstance(entity, _ACLEntity):
            raise ValueError('Invalid dictionary: %s' % entity_dict)

        return entity.grant(role)

    def has_entity(self, entity):
        """Returns whether or not this ACL has any entries for an entity.

        :type entity: :class:`_ACLEntity`
        :param entity: The entity to check for existence in this ACL.

        :rtype: bool
        :returns: True of the entity exists in the ACL.
        """
        return str(entity) in self.entities

    def get_entity(self, entity, default=None):
        """Gets an entity object from the ACL.

        :type entity: :class:`_ACLEntity` or string
        :param entity: The entity to get lookup in the ACL.

        :type default: anything
        :param default: This value will be returned if the entity
                        doesn't exist.

        :rtype: :class:`_ACLEntity`
        :returns: The corresponding entity or the value provided
                  to ``default``.
        """
        return self.entities.get(str(entity), default)

    def add_entity(self, entity):
        """Add an entity to the ACL.

        :type entity: :class:`_ACLEntity`
        :param entity: The entity to add to this ACL.
        """
        self.entities[str(entity)] = entity
        self.loaded = True

    def entity(self, entity_type, identifier=None):
        """Factory method for creating an Entity.

        If an entity with the same type and identifier already exists,
        this will return a reference to that entity.  If not, it will
        create a new one and add it to the list of known entities for
        this ACL.

        :type entity_type: string
        :param entity_type: The type of entity to create
                            (ie, ``user``, ``group``, etc)

        :type identifier: string
        :param identifier: The ID of the entity (if applicable).
                           This can be either an ID or an e-mail address.

        :rtype: :class:`_ACLEntity`
        :returns: A new Entity or a reference to an existing identical entity.
        """
        entity = _ACLEntity(entity_type=entity_type, identifier=identifier)
        if self.has_entity(entity):
            entity = self.get_entity(entity)
        else:
            self.add_entity(entity)
        return entity

    def user(self, identifier):
        """Factory method for a user Entity.

        :type identifier: string
        :param identifier: An id or e-mail for this particular user.

        :rtype: :class:`_ACLEntity`
        :returns: An Entity corresponding to this user.
        """
        return self.entity('user', identifier=identifier)

    def group(self, identifier):
        """Factory method for a group Entity.

        :type identifier: string
        :param identifier: An id or e-mail for this particular group.

        :rtype: :class:`_ACLEntity`
        :returns: An Entity corresponding to this group.
        """
        return self.entity('group', identifier=identifier)

    def domain(self, domain):
        """Factory method for a domain Entity.

        :type domain: string
        :param domain: The domain for this entity.

        :rtype: :class:`_ACLEntity`
        :returns: An entity corresponding to this domain.
        """
        return self.entity('domain', identifier=domain)

    def all(self):
        """Factory method for an Entity representing all users.

        :rtype: :class:`_ACLEntity`
        :returns: An entity representing all users.
        """
        return self.entity('allUsers')

    def all_authenticated(self):
        """Factory method for an Entity representing all authenticated users.

        :rtype: :class:`_ACLEntity`
        :returns: An entity representing all authenticated users.
        """
        return self.entity('allAuthenticatedUsers')

    def get_entities(self):
        """Get a list of all Entity objects.

        :rtype: list of :class:`_ACLEntity` objects
        :returns: A list of all Entity objects.
        """
        return self.entities.values()

    def reload(self):
        """Reload the ACL data from Cloud Storage.

        :rtype: :class:`ACL`
        :returns: The current ACL.
        """
        raise NotImplementedError

    def save(self, acl=None):
        """A method to be overridden by subclasses.

        :type acl: :class:`gcloud.storage.acl.ACL`, or a compatible list.
        :param acl: The ACL object to save.  If left blank, this will save
                    current entries.

        :raises: NotImplementedError
        """
        raise NotImplementedError

    def clear(self):
        """Remove all entities from the ACL."""
        raise NotImplementedError

class BucketACL(ACL):
    """An ACL specifically for a bucket."""

    _SUBKEY = 'acl'

    def __init__(self, bucket):
        """
        :type bucket: :class:`gcloud.storage.bucket.Bucket`
        :param bucket: The bucket to which this ACL relates.
        """
        super(BucketACL, self).__init__()
        self.bucket = bucket

    def reload(self):
        """Reload the ACL data from Cloud Storage.

        :rtype: :class:`gcloud.storage.acl.BucketACL`
        :returns: The current ACL.
        """
        self.entities.clear()

        url_path = '%s/%s' % (self.bucket.path, self._SUBKEY)
        found = self.bucket.connection.api_request(method='GET', path=url_path)
        for entry in found['items']:
            self.add_entity(self.entity_from_dict(entry))

        # Even if we fetch no entries, the ACL is still loaded.
        self.loaded = True

        return self

    def save(self, acl=None):
        """Save this ACL for the current bucket.

        If called without arguments, this will save the entries
        currently stored on this ACL::

           >>> acl.save()

        You can also provide a specific ACL to save instead of the one
        currently set on the Bucket object::

           >>> acl.save(acl=my_other_acl)

        You can use this to set access controls to be consistent from
        one bucket to another::

          >>> bucket1 = connection.get_bucket(bucket1_name)
          >>> bucket2 = connection.get_bucket(bucket2_name)
          >>> bucket2.acl.save(bucket1.get_acl())

        :type acl: :class:`gcloud.storage.acl.ACL`, or a compatible list.
        :param acl: The ACL object to save.  If left blank, this will save
                    current entries.

        :rtype: :class:`gcloud.storage.acl.BucketACL`
        :returns: The current ACL.
        """
        if acl is None:
            acl = self
            save_to_backend = acl.loaded
        else:
            save_to_backend = True

        if save_to_backend:
            result = self.bucket.connection.api_request(
                method='PATCH', path=self.bucket.path,
                data={self._SUBKEY: list(acl)},
                query_params={'projection': 'full'})
            self.entities.clear()
            for entry in result[self._SUBKEY]:
                self.add_entity(self.entity_from_dict(entry))
            self.loaded = True

        return self

    def clear(self):
        """Remove all ACL entries.

        Note that this won't actually remove *ALL* the rules, but it
        will remove all the non-default rules.  In short, you'll still
        have access to a bucket that you created even after you clear
        ACL rules with this method.

        For example, imagine that you granted access to this bucket to a
        bunch of coworkers::

          >>> acl.user('coworker1@example.org').grant_read()
          >>> acl.user('coworker2@example.org').grant_read()
          >>> acl.save()

        Now they work in another part of the company and you want to
        'start fresh' on who has access::

          >>> acl.clear()

        At this point all the custom rules you created have been removed.

        :rtype: :class:`gcloud.storage.acl.BucketACL`
        :returns: The current ACL.
        """
        return self.save([])

class DefaultObjectACL(BucketACL):
    """A class representing the default object ACL for a bucket."""

    _SUBKEY = 'defaultObjectAcl'

class ObjectACL(ACL):
    """An ACL specifically for a key."""

    def __init__(self, key):
        """
        :type key: :class:`gcloud.storage.key.Key`
        :param key: The key that this ACL corresponds to.
        """
        super(ObjectACL, self).__init__()
        self.key = key

    def reload(self):
        """Reload the ACL data from Cloud Storage.

        :rtype: :class:`ObjectACL`
        :returns: The current ACL.
        """
        self.entities.clear()

        url_path = '%s/acl' % self.key.path
        found = self.key.connection.api_request(method='GET', path=url_path)
        for entry in found['items']:
            self.add_entity(self.entity_from_dict(entry))

        # Even if we fetch no entries, the ACL is still loaded.
        self.loaded = True

        return self

    def save(self, acl=None):
        """Save the ACL data for this key.

        :type acl: :class:`gcloud.storage.acl.ACL`
        :param acl: The ACL object to save.  If left blank, this will
                    save the entries set locally on the ACL.
        """
        if acl is None:
            acl = self
            save_to_backend = acl.loaded
        else:
            save_to_backend = True

        if save_to_backend:
            result = self.key.connection.api_request(
                method='PATCH', path=self.key.path, data={'acl': list(acl)},
                query_params={'projection': 'full'})
            self.entities.clear()
            for entry in result['acl']:
                self.add_entity(self.entity_from_dict(entry))
            self.loaded = True

        return self

    def clear(self):
        """Remove all ACL rules from the key.

        Note that this won't actually remove *ALL* the rules, but it
        will remove all the non-default rules.  In short, you'll still
        have access to a key that you created even after you clear ACL
        rules with this method.
        """
        return self.save([])
<commit_msg>Improve class attr name: '_SUBKEY'->'_URL_PATH_ELEM'.<commit_after>"""Manipulate access control lists that Cloud Storage provides.

:class:`gcloud.storage.bucket.Bucket` has a getting method that creates
an ACL object under the hood, and you can interact with that using
:func:`gcloud.storage.bucket.Bucket.get_acl`::

  >>> from gcloud import storage
  >>> connection = storage.get_connection(project, email, key_path)
  >>> bucket = connection.get_bucket(bucket_name)
  >>> acl = bucket.get_acl()

Adding and removing permissions can be done with the following methods
(in increasing order of granularity):

- :func:`ACL.all`
  corresponds to access for all users.
- :func:`ACL.all_authenticated` corresponds
  to access for all users that are signed into a Google account.
- :func:`ACL.domain` corresponds to access on a
  per Google Apps domain (ie, ``example.com``).
- :func:`ACL.group` corresponds to access on a
  per group basis (either by ID or e-mail address).
- :func:`ACL.user` corresponds to access on a
  per user basis (either by ID or e-mail address).

And you are able to ``grant`` and ``revoke`` the following roles:

- **Reading**:
  :func:`_ACLEntity.grant_read` and :func:`_ACLEntity.revoke_read`
- **Writing**:
  :func:`_ACLEntity.grant_write` and :func:`_ACLEntity.revoke_write`
- **Owning**:
  :func:`_ACLEntity.grant_owner` and :func:`_ACLEntity.revoke_owner`

You can use any of these like any other factory method (these happen to
be :class:`_ACLEntity` factories)::

  >>> acl.user('me@example.org').grant_read()
  >>> acl.all_authenticated().grant_write()

You can also chain these ``grant_*`` and ``revoke_*`` methods together
for brevity::

  >>> acl.all().grant_read().revoke_write()

After that, you can save any changes you make with the
:func:`gcloud.storage.acl.ACL.save` method::

  >>> acl.save()

You can alternatively save any existing :class:`gcloud.storage.acl.ACL`
object (whether it was created by a factory method or not) with the
:func:`gcloud.storage.bucket.Bucket.save_acl` method::

  >>> bucket.save_acl(acl)

To get the list of ``entity`` and ``role`` for each unique pair, the
:class:`ACL` class is iterable::

  >>> print list(ACL)
  [{'role': 'OWNER', 'entity': 'allUsers'}, ...]

This list of tuples can be used as the ``entity`` and ``role`` fields
when sending metadata for ACLs to the API.
"""

class _ACLEntity(object):
    """Class representing a set of roles for an entity.

    This is a helper class that you likely won't ever construct
    outside of using the factor methods on the :class:`ACL` object.
    """

    READER_ROLE = 'READER'
    WRITER_ROLE = 'WRITER'
    OWNER_ROLE = 'OWNER'

    def __init__(self, entity_type, identifier=None):
        """Entity constructor.

        :type entity_type: string
        :param entity_type: The type of entity (ie, 'group' or 'user').

        :type identifier: string
        :param identifier: The ID or e-mail of the entity. For the special
                           entity types (like 'allUsers') this is optional.
        """
        self.identifier = identifier
        self.roles = set([])
        self.type = entity_type

    def __str__(self):
        if not self.identifier:
            return str(self.type)
        else:
            return '{self.type}-{self.identifier}'.format(self=self)

    def __repr__(self):
        return '<ACL Entity: {self} ({roles})>'.format(
            self=self, roles=', '.join(self.roles))

    def get_roles(self):
        """Get the list of roles permitted by this entity.

        :rtype: list of strings
        :returns: The list of roles associated with this entity.
        """
        return self.roles

    def grant(self, role):
        """Add a role to the entity.

        :type role: string
        :param role: The role to add to the entity.

        :rtype: :class:`_ACLEntity`
        :returns: The entity class.
        """
        self.roles.add(role)
        return self

    def revoke(self, role):
        """Remove a role from the entity.

        :type role: string
        :param role: The role to remove from the entity.

        :rtype: :class:`_ACLEntity`
        :returns: The entity class.
        """
        if role in self.roles:
            self.roles.remove(role)
        return self

    def grant_read(self):
        """Grant read access to the current entity."""

        return self.grant(_ACLEntity.READER_ROLE)

    def grant_write(self):
        """Grant write access to the current entity."""

        return self.grant(_ACLEntity.WRITER_ROLE)

    def grant_owner(self):
        """Grant owner access to the current entity."""

        return self.grant(_ACLEntity.OWNER_ROLE)

    def revoke_read(self):
        """Revoke read access from the current entity."""

        return self.revoke(_ACLEntity.READER_ROLE)

    def revoke_write(self):
        """Revoke write access from the current entity."""

        return self.revoke(_ACLEntity.WRITER_ROLE)

    def revoke_owner(self):
        """Revoke owner access from the current entity."""

        return self.revoke(_ACLEntity.OWNER_ROLE)

class ACL(object):
    """Container class representing a list of access controls."""

    loaded = False

    def __init__(self):
        self.entities = {}

    def reset(self):
        """Remove all entities from the ACL, and clear the ``loaded`` flag."""
        self.entities.clear()
        self.loaded = False

    def __iter__(self):
        for entity in self.entities.itervalues():
            for role in entity.get_roles():
                if role:
                    yield {'entity': str(entity), 'role': role}

    def entity_from_dict(self, entity_dict):
        """Build an _ACLEntity object from a dictionary of data.

        An entity is a mutable object that represents a list of roles
        belonging to either a user or group or the special types for all
        users and all authenticated users.

        :type entity_dict: dict
        :param entity_dict: Dictionary full of data from an ACL lookup.

        :rtype: :class:`_ACLEntity`
        :returns: An Entity constructed from the dictionary.
        """
        entity = entity_dict['entity']
        role = entity_dict['role']

        if entity == 'allUsers':
            entity = self.all()

        elif entity == 'allAuthenticatedUsers':
            entity = self.all_authenticated()

        elif '-' in entity:
            entity_type, identifier = entity.split('-', 1)
            entity = self.entity(entity_type=entity_type,
                                 identifier=identifier)

        if not isinstance(entity, _ACLEntity):
            raise ValueError('Invalid dictionary: %s' % entity_dict)

        return entity.grant(role)

    def has_entity(self, entity):
        """Returns whether or not this ACL has any entries for an entity.

        :type entity: :class:`_ACLEntity`
        :param entity: The entity to check for existence in this ACL.

        :rtype: bool
        :returns: True of the entity exists in the ACL.
        """
        return str(entity) in self.entities

    def get_entity(self, entity, default=None):
        """Gets an entity object from the ACL.

        :type entity: :class:`_ACLEntity` or string
        :param entity: The entity to get lookup in the ACL.

        :type default: anything
        :param default: This value will be returned if the entity
                        doesn't exist.

        :rtype: :class:`_ACLEntity`
        :returns: The corresponding entity or the value provided
                  to ``default``.
        """
        return self.entities.get(str(entity), default)

    def add_entity(self, entity):
        """Add an entity to the ACL.

        :type entity: :class:`_ACLEntity`
        :param entity: The entity to add to this ACL.
        """
        self.entities[str(entity)] = entity
        self.loaded = True

    def entity(self, entity_type, identifier=None):
        """Factory method for creating an Entity.

        If an entity with the same type and identifier already exists,
        this will return a reference to that entity.  If not, it will
        create a new one and add it to the list of known entities for
        this ACL.

        :type entity_type: string
        :param entity_type: The type of entity to create
                            (ie, ``user``, ``group``, etc)

        :type identifier: string
        :param identifier: The ID of the entity (if applicable).
                           This can be either an ID or an e-mail address.

        :rtype: :class:`_ACLEntity`
        :returns: A new Entity or a reference to an existing identical entity.
        """
        entity = _ACLEntity(entity_type=entity_type, identifier=identifier)
        if self.has_entity(entity):
            entity = self.get_entity(entity)
        else:
            self.add_entity(entity)
        return entity

    def user(self, identifier):
        """Factory method for a user Entity.

        :type identifier: string
        :param identifier: An id or e-mail for this particular user.

        :rtype: :class:`_ACLEntity`
        :returns: An Entity corresponding to this user.
        """
        return self.entity('user', identifier=identifier)

    def group(self, identifier):
        """Factory method for a group Entity.

        :type identifier: string
        :param identifier: An id or e-mail for this particular group.

        :rtype: :class:`_ACLEntity`
        :returns: An Entity corresponding to this group.
        """
        return self.entity('group', identifier=identifier)

    def domain(self, domain):
        """Factory method for a domain Entity.

        :type domain: string
        :param domain: The domain for this entity.

        :rtype: :class:`_ACLEntity`
        :returns: An entity corresponding to this domain.
        """
        return self.entity('domain', identifier=domain)

    def all(self):
        """Factory method for an Entity representing all users.

        :rtype: :class:`_ACLEntity`
        :returns: An entity representing all users.
        """
        return self.entity('allUsers')

    def all_authenticated(self):
        """Factory method for an Entity representing all authenticated users.

        :rtype: :class:`_ACLEntity`
        :returns: An entity representing all authenticated users.
        """
        return self.entity('allAuthenticatedUsers')

    def get_entities(self):
        """Get a list of all Entity objects.

        :rtype: list of :class:`_ACLEntity` objects
        :returns: A list of all Entity objects.
        """
        return self.entities.values()

    def reload(self):
        """Reload the ACL data from Cloud Storage.

        :rtype: :class:`ACL`
        :returns: The current ACL.
        """
        raise NotImplementedError

    def save(self, acl=None):
        """A method to be overridden by subclasses.

        :type acl: :class:`gcloud.storage.acl.ACL`, or a compatible list.
        :param acl: The ACL object to save.  If left blank, this will save
                    current entries.

        :raises: NotImplementedError
        """
        raise NotImplementedError

    def clear(self):
        """Remove all entities from the ACL."""
        raise NotImplementedError

class BucketACL(ACL):
    """An ACL specifically for a bucket."""

    _URL_PATH_ELEM = 'acl'

    def __init__(self, bucket):
        """
        :type bucket: :class:`gcloud.storage.bucket.Bucket`
        :param bucket: The bucket to which this ACL relates.
        """
        super(BucketACL, self).__init__()
        self.bucket = bucket

    def reload(self):
        """Reload the ACL data from Cloud Storage.

        :rtype: :class:`gcloud.storage.acl.BucketACL`
        :returns: The current ACL.
        """
        self.entities.clear()

        url_path = '%s/%s' % (self.bucket.path, self._URL_PATH_ELEM)
        found = self.bucket.connection.api_request(method='GET', path=url_path)
        for entry in found['items']:
            self.add_entity(self.entity_from_dict(entry))

        # Even if we fetch no entries, the ACL is still loaded.
        self.loaded = True

        return self

    def save(self, acl=None):
        """Save this ACL for the current bucket.

        If called without arguments, this will save the entries
        currently stored on this ACL::

           >>> acl.save()

        You can also provide a specific ACL to save instead of the one
        currently set on the Bucket object::

           >>> acl.save(acl=my_other_acl)

        You can use this to set access controls to be consistent from
        one bucket to another::

          >>> bucket1 = connection.get_bucket(bucket1_name)
          >>> bucket2 = connection.get_bucket(bucket2_name)
          >>> bucket2.acl.save(bucket1.get_acl())

        :type acl: :class:`gcloud.storage.acl.ACL`, or a compatible list.
        :param acl: The ACL object to save.  If left blank, this will save
                    current entries.

        :rtype: :class:`gcloud.storage.acl.BucketACL`
        :returns: The current ACL.
        """
        if acl is None:
            acl = self
            save_to_backend = acl.loaded
        else:
            save_to_backend = True

        if save_to_backend:
            result = self.bucket.connection.api_request(
                method='PATCH', path=self.bucket.path,
                data={self._URL_PATH_ELEM: list(acl)},
                query_params={'projection': 'full'})
            self.entities.clear()
            for entry in result[self._URL_PATH_ELEM]:
                self.add_entity(self.entity_from_dict(entry))
            self.loaded = True

        return self

    def clear(self):
        """Remove all ACL entries.

        Note that this won't actually remove *ALL* the rules, but it
        will remove all the non-default rules.  In short, you'll still
        have access to a bucket that you created even after you clear
        ACL rules with this method.

        For example, imagine that you granted access to this bucket to a
        bunch of coworkers::

          >>> acl.user('coworker1@example.org').grant_read()
          >>> acl.user('coworker2@example.org').grant_read()
          >>> acl.save()

        Now they work in another part of the company and you want to
        'start fresh' on who has access::

          >>> acl.clear()

        At this point all the custom rules you created have been removed.

        :rtype: :class:`gcloud.storage.acl.BucketACL`
        :returns: The current ACL.
        """
        return self.save([])

class DefaultObjectACL(BucketACL):
    """A class representing the default object ACL for a bucket."""

    _URL_PATH_ELEM = 'defaultObjectAcl'

class ObjectACL(ACL):
    """An ACL specifically for a key."""

    def __init__(self, key):
        """
        :type key: :class:`gcloud.storage.key.Key`
        :param key: The key that this ACL corresponds to.
        """
        super(ObjectACL, self).__init__()
        self.key = key

    def reload(self):
        """Reload the ACL data from Cloud Storage.

        :rtype: :class:`ObjectACL`
        :returns: The current ACL.
        """
        self.entities.clear()

        url_path = '%s/acl' % self.key.path
        found = self.key.connection.api_request(method='GET', path=url_path)
        for entry in found['items']:
            self.add_entity(self.entity_from_dict(entry))

        # Even if we fetch no entries, the ACL is still loaded.
        self.loaded = True

        return self

    def save(self, acl=None):
        """Save the ACL data for this key.

        :type acl: :class:`gcloud.storage.acl.ACL`
        :param acl: The ACL object to save.  If left blank, this will
                    save the entries set locally on the ACL.
        """
        if acl is None:
            acl = self
            save_to_backend = acl.loaded
        else:
            save_to_backend = True

        if save_to_backend:
            result = self.key.connection.api_request(
                method='PATCH', path=self.key.path, data={'acl': list(acl)},
                query_params={'projection': 'full'})
            self.entities.clear()
            for entry in result['acl']:
                self.add_entity(self.entity_from_dict(entry))
            self.loaded = True

        return self

    def clear(self):
        """Remove all ACL rules from the key.

        Note that this won't actually remove *ALL* the rules, but it
        will remove all the non-default rules.  In short, you'll still
        have access to a key that you created even after you clear ACL
        rules with this method.
        """
        return self.save([])
<|endoftext|>