<commit_before>from django.db import models, IntegrityError
from django.db.models import get_model

#########################################
# Auxiliar functions
#########################################

def get_fk_tuple(obj, fk_field):
    return getattr(obj, fk_field)

def get_tuple(app_label, module_name, tuple_id):
    model = get_model(app_label, module_name)
    return model.objects.get(id=tuple_id)

def get_related_tuples(app_label, module_name, field_name, tuple_id, lookup={}):
    model = get_model(app_label, module_name)
    lookup[field_name] = tuple_id
    return model.objects.filter(**lookup)

#################################################
## Logic classes
#################################################

class FKCollection:

    def __init__(self):
        self.table_tuple = {}

    def get_fks(self, table_name, old_id):
        if (table_name, old_id) in self.table_tuple:
            return self.table_tuple[(table_name, old_id)]

        return []

    def update_fks(self, cloned_tuple, old_pk):
        table_name = cloned_tuple._meta.object_name
        pending_fks = self.get_fks(table_name, old_pk)

        for linker_app, linker_table, linker_field, linker_tuple_id in pending_fks:
            model = get_model(linker_app, linker_table)

            linker_tuple = model.objects.get(id=linker_tuple_id)

            setattr(linker_tuple, linker_field, cloned_tuple)

            linker_tuple.save()

    def add_fk(self, linker_app, linker_table, linker_field, linker_tuple_id, referenced_table, referenced_tuple):
        if not (referenced_table, referenced_tuple) in self.table_tuple:
            self.table_tuple[(referenced_table, referenced_tuple)] = []

        fks = self.table_tuple[(referenced_table, referenced_tuple)]

        if not (linker_table, linker_field, linker_tuple_id) in fks:
            fks.append((linker_app, linker_table, linker_field, linker_tuple_id))

class Many2ManyCollection:

    def __init__(self):
        self.many2many_list = []

    def get_m2m_info(self, old_from_id, from_table, from_field, old_to_id, to_table):
        for m2m_info in self.many2many_list:
            if (m2m_info['old_to_id'] == old_to_id and m2m_info['to_table'] == to_table and \
                m2m_info['from_table'] == from_table and m2m_info['old_from_id'] == old_from_id and \
                m2m_info['from_field'] == from_field):
                return m2m_info

        return None

    def add_m2m_info(self, old_from_id, new_from_id, from_table, from_field, old_to_id, new_to_id, to_table):
        m2m_info = self.get_m2m_info(old_from_id, from_table, from_field, old_to_id, to_table)

        if (not m2m_info):
            #Time to create the m2m_info object

            m2m_info = dict()

            m2m_info['old_from_id'] = old_from_id
            m2m_info['new_from_id'] = new_from_id
            m2m_info['from_table'] = from_table
            m2m_info['from_field'] = from_field
            m2m_info['old_to_id'] = old_to_id
            m2m_info['new_to_id'] = new_to_id
            m2m_info['to_table'] = to_table
            m2m_info['completed'] = False

            self.many2many_list.append(m2m_info)

            return

        if not m2m_info['new_from_id'] and new_from_id:
            #Missing info about new from id
            #Updating only new data

            m2m_info['new_from_id'] = new_from_id
            m2m_info['completed'] = True

            self.update_m2m(m2m_info)

            return

        if not m2m_info['new_to_id'] and new_to_id:
            #Missing info about new to id
            #Updating only new data

            m2m_info['new_to_id'] = new_to_id
            m2m_info['completed'] = True

            self.update_m2m(m2m_info)

            return

        pass

    def update_m2m(self, m2m_info):
        if m2m_info['completed']:
            from_model = eval(m2m_info['from_table'])
            from_tuple = from_model.objects.get(id=m2m_info['new_from_id'])

            to_model = eval(m2m_info['to_table'])
            to_tuple = to_model.objects.get(id=m2m_info['new_to_id'])

            getattr(from_tuple, m2m_info['from_field']).add(to_tuple)

            from_tuple.save()

            #Erasing m2m_info after linking
            self.many2many_list.remove(m2m_info)
            del m2m_info

class MappingCollection:

    def __init__(self):
        self.tables = {}

    def get_mapping(self, table_name, old_id):
        if not table_name in self.tables:
            return None

        mapping = self.tables[table_name]

        if not mapping:
            return None

        if not old_id in mapping:
            return None

        new_id = mapping[old_id]

        if not new_id:
            return None

        return new_id

    def add_mapping(self, table_name, old_id, new_id):
        if not table_name in self.tables:
            self.tables[table_name] = {}

        mapping = self.tables[table_name]

        mapping[old_id] = new_id

class PackageCloner:

    extra_models = {
        'WorkSpace': [('workspace', 'tab', 'workspace'), ('connectable', 'inout', 'workspace')],
        'Tab': [('igadget', 'igadget', 'tab')],
        'IGadget': [('igadget', 'variable', 'igadget')],
    }

    unique_variant = {
        'Tab': 'name',
    }

    fields_to_overwrite = {
    }

    def __init__(self):
        self.mapping = MappingCollection()
        self.fks = FKCollection()
        self.m2ms = Many2ManyCollection()
        self.final_tables = ('User', 'VariableDef', 'Gadget', 'PublishedWorkSpace', 'Filter', 'UserWorkSpace', 'RemoteChannel', 'SharedVariableDef', 'SharedVariableValue')

    def is_final_table(self, table_name):
        return table_name in self.final_tables

    def clone_tuple(self, tuple):
        meta = tuple._meta
        table_name = meta.object_name

        # Controlling when a final table is reached!
        if self.is_final_table(table_name):
            return tuple

        # Controlling when a tuple has been previously cloned!
        new_id = self.mapping.get_mapping(table_name, tuple.id)

        if new_id:

            return get_tuple(meta.app_label, meta.module_name, new_id)

        else:

            model = get_model(meta.app_label, table_name)

            cloned_tuple = model()

            fields = meta.fields
            fields_to_overwrite = self.fields_to_overwrite.get(table_name, {})

            # Cloning all object data!
            for field in fields:
                if isinstance(field, models.ForeignKey):
                    # get the id of the foreignKey. It may be optional (None)
                    fkValue = getattr(tuple, field.name)
                    if fkValue:
                        fkValue = self.clone_tuple(fkValue)

                    setattr(cloned_tuple, field.name, fkValue)

                elif field != meta.auto_field:

                    if field.name in fields_to_overwrite:
                        value = fields_to_overwrite[field.name]
                    else:
                        value = getattr(tuple, field.name)

                    setattr(cloned_tuple, field.name, value)

            # Getting an id!
            try:
                cloned_tuple.save()
            except IntegrityError:
                variant_field = self.unique_variant[table_name]
                unique_key = {}

                for unique_field in meta.unique_together[0]:
                    unique_key[unique_field] = getattr(cloned_tuple, unique_field)

                sufix = 2
                duplicated_key = True
                while duplicated_key:
                    unique_key[variant_field] = getattr(cloned_tuple, variant_field) + ' ' + str(sufix)
                    try:
                        model.objects.get(**unique_key)
                    except model.DoesNotExist:
                        duplicated_key = False

                setattr(cloned_tuple, variant_field, unique_key[variant_field])
                cloned_tuple.save()

            self.mapping.add_mapping(table_name, tuple.id, cloned_tuple.id)

            ##########################################################################################################
            #Marking many_to_many relationships to be updated when involved tuples are both cloned!
            # Many to many relationships can be iterated over in two different ways:

            #1. cloned FROM-SIDE table first
            m2m_fields = meta.many_to_many

            for m2m_field in m2m_fields:
                field_name = m2m_field.attname
                m2m_objects = getattr(tuple, field_name).all()
                for m2m_object in m2m_objects:
                    referenced_model = m2m_object._meta.object_name
                    referenced_tuple_id = m2m_object.id

                    self.m2ms.add_m2m_info(old_from_id=tuple.id, new_from_id=cloned_tuple.id, from_table=table_name, from_field=field_name,
                                           old_to_id=referenced_tuple_id, new_to_id=None, to_table=referenced_model)

            #2. cloned TO-SIDE table first
            m2m_related_fields = meta._related_many_to_many_cache

            for m2m_field in m2m_related_fields:
                reverse_rel_name = '%s_set' % m2m_field.var_name

                from_field = m2m_field.field.name
                from_table_name = m2m_field.model._meta.object_name

                m2m_objects = getattr(tuple, reverse_rel_name).all()

                for m2m_object in m2m_objects:
                    old_from_id = m2m_object.id

                    self.m2ms.add_m2m_info(old_from_id=old_from_id, new_from_id=None, from_table=from_table_name, from_field=from_field,
                                           old_to_id=tuple.id, new_to_id=cloned_tuple.id, to_table=table_name)
            ###########################################################################################################

            # Continue iterating over data-model structure
            extra_models = self.extra_models.get(table_name, [])
            for model in extra_models:
                lookup = {}
                if len(model) > 3:
                    lookup = model[3]

                related_tuples = get_related_tuples(model[0],  # app_label
                                                    model[1],  # module_name
                                                    model[2],  # field.name
                                                    tuple.id,
                                                    lookup)
                for related_tuple in related_tuples:
                    self.clone_tuple(related_tuple)

            return cloned_tuple

    def merge_workspaces(self, from_ws, to_ws, user):
        meta = from_ws._meta
        table_name = meta.object_name

        # Registering mapping between tuple and cloning tuple!
        self.mapping.add_mapping(table_name, from_ws.id, to_ws.id)

        self.final_tables = list(self.final_tables)
        self.extra_models['Variable'] = (
            ('workspace', 'variablevalue', 'variable', {'user': from_ws.creator}),
        )
        self.fields_to_overwrite['VariableValue'] = {
            'user': user,
        }

        # Continue iterating over data-model structure
        extra_models = self.extra_models.get(table_name, [])
        for model in extra_models:
            related_tuples = get_related_tuples(model[0],  # app_label
                                                model[1],  # module_name
                                                model[2],  # field.name
                                                from_ws.id)
            for related_tuple in related_tuples:
                self.clone_tuple(related_tuple)

        from commons.get_data import _invalidate_cached_variable_values
        _invalidate_cached_variable_values(to_ws)

        return to_ws
<commit_msg>Fix m2m cloning and improve the list of extra models to clone<commit_after>from django.db import models, IntegrityError
from django.db.models import get_model

#########################################
# Auxiliar functions
#########################################

def get_fk_tuple(obj, fk_field):
    return getattr(obj, fk_field)

def get_tuple(app_label, module_name, tuple_id):
    model = get_model(app_label, module_name)
    return model.objects.get(id=tuple_id)

def get_related_tuples(app_label, module_name, field_name, tuple_id, lookup={}):
    model = get_model(app_label, module_name)
    lookup[field_name] = tuple_id
    return model.objects.filter(**lookup)

#################################################
## Logic classes
#################################################

class FKCollection:

    def __init__(self):
        self.table_tuple = {}

    def get_fks(self, table_name, old_id):
        if (table_name, old_id) in self.table_tuple:
            return self.table_tuple[(table_name, old_id)]

        return []

    def update_fks(self, cloned_tuple, old_pk):
        table_name = cloned_tuple._meta.object_name
        pending_fks = self.get_fks(table_name, old_pk)

        for linker_app, linker_table, linker_field, linker_tuple_id in pending_fks:
            model = get_model(linker_app, linker_table)

            linker_tuple = model.objects.get(id=linker_tuple_id)

            setattr(linker_tuple, linker_field, cloned_tuple)

            linker_tuple.save()

    def add_fk(self, linker_app, linker_table, linker_field, linker_tuple_id, referenced_table, referenced_tuple):
        if not (referenced_table, referenced_tuple) in self.table_tuple:
            self.table_tuple[(referenced_table, referenced_tuple)] = []

        fks = self.table_tuple[(referenced_table, referenced_tuple)]

        if not (linker_table, linker_field, linker_tuple_id) in fks:
            fks.append((linker_app, linker_table, linker_field, linker_tuple_id))

class Many2ManyCollection:

    def __init__(self):
        self.many2many_list = []

    def get_m2m_info(self, old_from_id, from_meta, from_field, old_to_id, to_meta):
        for m2m_info in self.many2many_list:
            if (m2m_info['old_to_id'] == old_to_id and m2m_info['to_meta'] == to_meta and \
                m2m_info['from_meta'] == from_meta and m2m_info['old_from_id'] == old_from_id and \
                m2m_info['from_field'] == from_field):
                return m2m_info

        return None

    def add_m2m_info(self, old_from_id, new_from_id, from_meta, from_field, old_to_id, new_to_id, to_meta):
        m2m_info = self.get_m2m_info(old_from_id, from_meta, from_field, old_to_id, to_meta)

        if not m2m_info:
            #Time to create the m2m_info object

            m2m_info = {
                'old_from_id': old_from_id,
                'new_from_id': new_from_id,
                'from_meta': from_meta,
                'from_field': from_field,
                'old_to_id': old_to_id,
                'new_to_id': new_to_id,
                'to_meta': to_meta,
                'completed': False,
            }

            self.many2many_list.append(m2m_info)

            return

        if not m2m_info['new_from_id'] and new_from_id:
            #Missing info about new from id
            #Updating only new data

            m2m_info['new_from_id'] = new_from_id
            m2m_info['completed'] = True

            self.update_m2m(m2m_info)

            return

        if not m2m_info['new_to_id'] and new_to_id:
            #Missing info about new to id
            #Updating only new data

            m2m_info['new_to_id'] = new_to_id
            m2m_info['completed'] = True

            self.update_m2m(m2m_info)

            return

        pass

    def update_m2m(self, m2m_info):
        if m2m_info['completed']:
            from_meta = m2m_info['from_meta']
            from_tuple = get_tuple(from_meta.app_label, from_meta.module_name, m2m_info['new_from_id'])
            to_meta = m2m_info['to_meta']
            to_tuple = get_tuple(to_meta.app_label, to_meta.module_name, m2m_info['new_to_id'])

            getattr(from_tuple, m2m_info['from_field']).add(to_tuple)

            from_tuple.save()

            # Erasing m2m_info after linking
            self.many2many_list.remove(m2m_info)

class MappingCollection:

    def __init__(self):
        self.tables = {}

    def get_mapping(self, table_name, old_id):
        if not table_name in self.tables:
            return None

        mapping = self.tables[table_name]

        if not mapping:
            return None

        if not old_id in mapping:
            return None

        new_id = mapping[old_id]

        if not new_id:
            return None

        return new_id

    def add_mapping(self, table_name, old_id, new_id):
        if not table_name in self.tables:
            self.tables[table_name] = {}

        mapping = self.tables[table_name]

        mapping[old_id] = new_id

class PackageCloner:

    extra_models = {
        'WorkSpace': (
            ('workspace', 'tab', 'workspace'),
            ('connectable', 'inout', 'workspace'),
            ('preferences', 'workspacepreference', 'workspace'),
        ),
        'Tab': (
            ('igadget', 'igadget', 'tab'),
            ('preferences', 'tabpreference', 'tab'),
        ),
        'IGadget': (
            ('igadget', 'variable', 'igadget'),
        ),
        'Variable': (
            ('connectable', 'in', 'variable'),
            ('connectable', 'out', 'variable'),
        ),
    }

    unique_variant = {
        'Tab': 'name',
    }

    fields_to_overwrite = {
    }

    def __init__(self):
        self.mapping = MappingCollection()
        self.fks = FKCollection()
        self.m2ms = Many2ManyCollection()
        self.final_tables = ('User', 'VariableDef', 'Gadget', 'PublishedWorkSpace', 'Filter', 'UserWorkSpace', 'RemoteChannel', 'SharedVariableDef', 'SharedVariableValue')

    def is_final_table(self, table_name):
        return table_name in self.final_tables

    def clone_tuple(self, tuple):
        meta = tuple._meta
        table_name = meta.object_name

        # Controlling when a final table is reached!
        if self.is_final_table(table_name):
            return tuple

        # Controlling when a tuple has been previously cloned!
        new_id = self.mapping.get_mapping(table_name, tuple.id)

        if new_id:

            return get_tuple(meta.app_label, meta.module_name, new_id)

        else:

            model = get_model(meta.app_label, table_name)

            cloned_tuple = model()

            fields = meta.fields
            fields_to_overwrite = self.fields_to_overwrite.get(table_name, {})

            # Cloning all object data!
            for field in fields:
                if isinstance(field, models.ForeignKey):
                    # get the id of the foreignKey. It may be optional (None)
                    fkValue = getattr(tuple, field.name)
                    if fkValue:
                        fkValue = self.clone_tuple(fkValue)

                    setattr(cloned_tuple, field.name, fkValue)

                elif field != meta.auto_field:

                    if field.name in fields_to_overwrite:
                        value = fields_to_overwrite[field.name]
                    else:
                        value = getattr(tuple, field.name)

                    setattr(cloned_tuple, field.name, value)

            # Getting an id!
            try:
                cloned_tuple.save()
            except IntegrityError:
                variant_field = self.unique_variant[table_name]
                unique_key = {}

                for unique_field in meta.unique_together[0]:
                    unique_key[unique_field] = getattr(cloned_tuple, unique_field)

                sufix = 2
                duplicated_key = True
                while duplicated_key:
                    unique_key[variant_field] = getattr(cloned_tuple, variant_field) + ' ' + str(sufix)
                    try:
                        model.objects.get(**unique_key)
                    except model.DoesNotExist:
                        duplicated_key = False

                setattr(cloned_tuple, variant_field, unique_key[variant_field])
                cloned_tuple.save()

            self.mapping.add_mapping(table_name, tuple.id, cloned_tuple.id)

            ##########################################################################################################
            #Marking many_to_many relationships to be updated when involved tuples are both cloned!
            # Many to many relationships can be iterated over in two different ways:

            #1. cloned FROM-SIDE table first
            m2m_fields = meta.many_to_many

            for m2m_field in m2m_fields:
                field_name = m2m_field.attname
                m2m_objects = getattr(tuple, field_name).all()
                for m2m_object in m2m_objects:
                    referenced_meta = m2m_object._meta
                    referenced_tuple_id = m2m_object.id

                    self.m2ms.add_m2m_info(old_from_id=tuple.id,
                        new_from_id=cloned_tuple.id,
                        from_meta=meta,
                        from_field=field_name,
                        old_to_id=referenced_tuple_id,
                        new_to_id=None,
                        to_meta=referenced_meta)

            #2. cloned TO-SIDE table first
            m2m_related_fields = meta._related_many_to_many_cache

            for m2m_field in m2m_related_fields:
                reverse_rel_name = '%s_set' % m2m_field.var_name

                from_field = m2m_field.field.name

                m2m_objects = getattr(tuple, reverse_rel_name).all()

                for m2m_object in m2m_objects:
                    referenced_meta = m2m_object._meta
                    old_from_id = m2m_object.id

                    self.m2ms.add_m2m_info(old_from_id=old_from_id,
                        new_from_id=None,
                        from_meta=referenced_meta,
                        from_field=from_field,
                        old_to_id=tuple.id,
                        new_to_id=cloned_tuple.id,
                        to_meta=meta)

            ###########################################################################################################

            # Continue iterating over data-model structure
            extra_models = self.extra_models.get(table_name, [])
            for model in extra_models:
                lookup = {}
                if len(model) > 3:
                    lookup = model[3]

                related_tuples = get_related_tuples(model[0],  # app_label
                                                    model[1],  # module_name
                                                    model[2],  # field.name
                                                    tuple.id,
                                                    lookup)
                for related_tuple in related_tuples:
                    self.clone_tuple(related_tuple)

            return cloned_tuple

    def merge_workspaces(self, from_ws, to_ws, user):
        meta = from_ws._meta
        table_name = meta.object_name

        # Registering mapping between tuple and cloning tuple!
        self.mapping.add_mapping(table_name, from_ws.id, to_ws.id)

        self.final_tables = list(self.final_tables)
        self.extra_models['Variable'] += (
            ('workspace', 'variablevalue', 'variable', {'user': from_ws.creator}),
        )
        self.fields_to_overwrite['VariableValue'] = {
            'user': user,
        }

        # Continue iterating over data-model structure
        extra_models = self.extra_models.get(table_name, [])
        for model in extra_models:
            related_tuples = get_related_tuples(model[0],  # app_label
                                                model[1],  # module_name
                                                model[2],  # field.name
                                                from_ws.id)
            for related_tuple in related_tuples:
                self.clone_tuple(related_tuple)

        from commons.get_data import _invalidate_cached_variable_values
        _invalidate_cached_variable_values(to_ws)

        return to_ws
<|endoftext|>


<commit_before># Copyright (c) 2015 Ultimaker B.V.
# Uranium is released under the terms of the AGPLv3 or higher.

from UM.Signal import Signal, SignalEmitter

class OutputDevice(SignalEmitter):
    def __init__(self, device_id):
        super().__init__()

        self._id = device_id
        self._name = "Unknown Device"
        self._short_description = "Unknown Device"
        self._description = "Do something with an unknown device"
        self._icon_name = "generic_device"
        self._priority = 0

    metaDataChanged = Signal()

    def getId(self):
        return self._id

    def getName(self):
        return self._name

    def setName(self, name):
        if name != self._name:
            self._name = name
            self.metaDataChanged.emit(self)

    def getShortDescription(self):
        return self._short_description

    def setShortDescription(self, description):
        if description != self._short_description:
            self._short_description = description
            self.metaDataChanged.emit(self)

    def getDescription(self):
        return self._description

    def setDescription(self, description):
        if description != self._description:
            self._description = description
            self.metaDataChanged.emit(self)

    def getIconName(self):
        return self._icon_name

    def setIconName(self, name):
        if name != self._icon_name:
            self._icon_name = name
            self.metaDataChanged.emit(self)

    def getPriority(self):
        return self._priority

    def setPriority(self, priority):
        if priority != self._priority:
            self._priority = priority
            self.metaDataChanged.emit(self)

    def requestWrite(self, node):
        raise NotImplementedError("requestWrite needs to be implemented")

    writeStarted = Signal()
    writeProgress = Signal()
    writeFinished = Signal()
    writeError = Signal()
    writeSuccess = Signal()
<commit_msg>Document OutputDevice<commit_after># Copyright (c) 2015 Ultimaker B.V.
# Uranium is released under the terms of the AGPLv3 or higher.

from UM.Signal import Signal, SignalEmitter

##  Base class for output devices.
#
#   This class provides a base class for output devices. An output device can be
#   anything we want to output to, like a local file, an USB connected printer but
#   also an HTTP web service.
#
#   Each subclass must implement requestWrite(). requestWrite() is expected to raise
#   errors from OutputDeviceError when certain conditions occur, like insufficient
#   permissions. For the rest, output device subclasses are completely free to implement
#   writing however they want, though you should emit writeStarted and related signals
#   whenever certain events happen related to the write process.
#
#   For example, when implementing a web service as output device, it would be completely
#   acceptable to show a login dialog when calling requestWrite() if there are no saved
#   login credentials.
class OutputDevice(SignalEmitter):
    def __init__(self, device_id):
        super().__init__()

        self._id = device_id
        self._name = "Unknown Device"
        self._short_description = "Unknown Device"
        self._description = "Do something with an unknown device"
        self._icon_name = "generic_device"
        self._priority = 0

    metaDataChanged = Signal()

    ##  Get the device id
    def getId(self):
        return self._id

    ##  Get a human-readable name for this device.
    def getName(self):
        return self._name

    ##  Set the human-readable name of this device.
    #
    #   \param name The new name of this device.
    def setName(self, name):
        if name != self._name:
            self._name = name
            self.metaDataChanged.emit(self)

    ##  Get a short description for this device.
    #
    #   The short description can be used as a button label or similar
    #   and should thus be only a few words at most. For example,
    #   "Save to File", "Print with USB".
    def getShortDescription(self):
        return self._short_description

    ##  Set the short description for this device.
    #
    #   \param description The new short description to set.
    def setShortDescription(self, description):
        if description != self._short_description:
            self._short_description = description
            self.metaDataChanged.emit(self)

    ##  Get a full description for this device.
    #
    #   The full description describes what would happen when writing
    #   to this device. For example, "Save to Removable Drive /media/sdcard",
    #   "Upload to YouMagine with account User".
    def getDescription(self):
        return self._description

    ##  Set the full description for this device.
    #
    #   \param description The description of this device.
    def setDescription(self, description):
        if description != self._description:
            self._description = description
            self.metaDataChanged.emit(self)

    ##  Get the name of an icon that can be used to identify this device.
    #
    #   This icon should be available in the theme.
    def getIconName(self):
        return self._icon_name

    ##  Set the name of an icon to identify this device.
    #
    #   \param name The name of the icon to use.
    def setIconName(self, name):
        if name != self._icon_name:
            self._icon_name = name
            self.metaDataChanged.emit(self)

    ##  The priority of this device.
    #
    #   Priority indicates which device is most likely to be used as the
    #   default device to write to. It should be a number and higher numbers
    #   indicate that the device should be preferred over devices with
    #   lower numbers.
    def getPriority(self):
        return self._priority

    ##  Set the priority of this device.
    #
    #   \param priority \type{int} The priority to use.
    def setPriority(self, priority):
        if priority != self._priority:
            self._priority = priority
            self.metaDataChanged.emit(self)

    ##  Request performing a write operation on this device.
    #
    #   This method should be implemented by subclasses. It should write the
    #   given SceneNode tree to a destination relevant for the device. It is
    #   recommended to perform the actual writing asynchronously and rely on
    #   the write- signals to properly indicate state.
    #
    #   \param node \type{SceneNode} The root of a tree of scene nodes that should be
    #                                written to the device.
    #
    #   \exception OutputDeviceError.WriteRequestFailedError
    def requestWrite(self, node):
        raise NotImplementedError("requestWrite needs to be implemented")

    writeStarted = Signal()
    writeProgress = Signal()
    writeFinished = Signal()
    writeError = Signal()
    writeSuccess = Signal()
<|endoftext|>


<commit_before><commit_msg>[FIX] mail: when warning about password user, fetch partner as su<commit_after><|endoftext|>


<commit_before>from datetime import datetime
from corehq import Domain
from corehq.apps.app_manager.models import Application
from corehq.apps.reports.datatables import DataTablesHeader, DataTablesColumn
from corehq.apps.reports.dispatcher import AdminReportDispatcher
from corehq.apps.reports.generic import ElasticTabularReport, GenericTabularReport
from django.utils.translation import ugettext as _, ugettext_noop
from corehq.apps.users.models import WebUser
from corehq.elastic import es_query, parse_args_for_es, fill_mapping_with_facets
from corehq.pillows.mappings.app_mapping import APP_INDEX
from corehq.pillows.mappings.user_mapping import USER_INDEX
from corehq.apps.app_manager.commcare_settings import SETTINGS as CC_SETTINGS
from corehq.toggles import IS_DEVELOPER

class AdminReport(GenericTabularReport):
    dispatcher = AdminReportDispatcher
    base_template = "hqadmin/faceted_report.html"

class AdminFacetedReport(AdminReport, ElasticTabularReport):
    default_sort = None
    es_prefix = "es_" # facet keywords in the url will be prefixed with this
    asynchronous = False
    ajax_pagination = True
    exportable = True
    es_queried = False
    es_facet_list = []
    es_facet_mapping = []
    section_name = ugettext_noop("ADMINREPORT")
    es_url = ''

    @property
    def template_context(self):
        ctxt = super(AdminFacetedReport, self).template_context

        self.run_query(0)
        if self.es_params.get('search'):
            ctxt["search_query"] = self.es_params.get('search')[0]
        ctxt.update({
            'layout_flush_content': True,
            'facet_map': self.es_facet_map,
            'query_str': self.request.META['QUERY_STRING'],
            'facet_prefix': self.es_prefix,
            'facet_report': self,
            'grouped_facets': True,
            'startdate': self.request.GET.get('startdate', ''),
            'enddate': self.request.GET.get('enddate', ''),
            'interval': self.request.GET.get('interval', ''),
        })
        return ctxt

    @property
    def total_records(self):
        return int(self.es_results['hits']['total'])

    def is_custom_param(self, param):
        return param.startswith(self.es_prefix)

    @property
    def shared_pagination_GET_params(self):
        ret = super(AdminFacetedReport, self).shared_pagination_GET_params
        for param in self.request.GET.iterlists():
            if self.is_custom_param(param[0]):
                for val in param[1]:
                    ret.append(dict(name=param[0], value=val))
        return ret

    def es_query(self, params=None, size=None):
        if params is None:
            params = {}
        terms = ['search']
        q = {"query": {"match_all":{}}}

        search_query = params.get('search', "")
        if search_query:
            q['query'] = {
                "bool": {
                    "must": {
                        "match" : {
                            "_all" : {
                                "query" : search_query,
                                "operator" : "and", }}}}}

        q["facets"] = {}

        q["sort"] = self.get_sorting_block()
        start_at=self.pagination.start
        size = size if size is not None else self.pagination.count
        return es_query(params, self.es_facet_list, terms, q, self.es_url, start_at, size, facet_size=25)

    @property
    def es_results(self):
        if not self.es_queried:
            self.run_query()
        return self.es_response

    def run_query(self, size=None):
        self.es_params, _ = parse_args_for_es(self.request, prefix=self.es_prefix)
        results = self.es_query(self.es_params, size)
        self.es_facet_map = fill_mapping_with_facets(self.es_facet_mapping, results, self.es_params)
        self.es_response = results
        self.es_queried = True
        return self.es_response

    @property
    def export_table(self):
        self.pagination.count = 1000000 # terrible hack to get the export to return all rows
        self.show_name = True
        return super(AdminFacetedReport, self).export_table

class AdminUserReport(AdminFacetedReport):
    slug = "user_list"
    name = ugettext_noop('User List')
    facet_title = ugettext_noop("User Facets")
    search_for = ugettext_noop("users...")
    default_sort = {'username.exact': 'asc'}
    es_url = USER_INDEX + '/user/_search'

    es_facet_list = [
        "is_active", # this is NOT "has submitted a form in the last 30 days"
        "is_staff",
        "is_superuser",
        "domain",
        "doc_type",
    ]

    es_facet_mapping = [
        ("", True, [
            {"facet": "is_active", "name": "Can Log In?", "expanded": True },
            {"facet": "is_superuser", "name": "SuperUser?", "expanded": True },
            {"facet": "is_staff", "name": "Staff?", "expanded": True },
            {"facet": "domain", "name": "Domain", "expanded": True },
            {"facet": "doc_type", "name": "User Type", "expanded": True },
        ]),
    ]

    @property
    def headers(self):
        headers = DataTablesHeader(
            DataTablesColumn(_("Username"), prop_name="username.exact"),
            DataTablesColumn(_("Project Spaces")),
            DataTablesColumn(_("Date Joined"), prop_name="date_joined"),
            DataTablesColumn(_("Last Login"), prop_name="last_login"),
            DataTablesColumn(_("Type"), prop_name="doc_type"),
            DataTablesColumn(_("SuperUser?"), prop_name="is_superuser"),
        )
        return headers

    @property
    def rows(self):
        users = [res['_source'] for res in self.es_results.get('hits', {}).get('hits', [])]

        def format_date(dstr, default):
            # use [:19] so that only only the 'YYYY-MM-DDTHH:MM:SS' part of the string is parsed
            return datetime.strptime(dstr[:19], '%Y-%m-%dT%H:%M:%S').strftime('%Y/%m/%d %H:%M:%S') if dstr else default

        def get_domains(user):
            if user.get('doc_type') == "WebUser":
                return ", ".join([dm['domain'] for dm in user.get('domain_memberships', [])])
            return user.get('domain_membership', {}).get('domain', _('No Domain Data'))

        for u in users:
            yield [
                u.get('username'),
                get_domains(u),
                format_date(u.get('date_joined'), _('No date')),
                format_date(u.get('last_login'), _('No date')),
                u.get('doc_type'),
                u.get('is_superuser'),
            ]

def create_mapping_from_list(l, name="", expand_outer=False, expand_inner=False, name_change_fn=None):
    name_change_fn = name_change_fn or (lambda x: x)
    facets = [{"facet": item, "name": name_change_fn(item), "expanded": expand_inner } for item in sorted(l)]
    return (name, expand_outer, facets)

class AdminAppReport(AdminFacetedReport):
    slug = "app_list"
    name = ugettext_noop('Application List')
    facet_title = ugettext_noop("App Facets")
    search_for = ugettext_noop("apps...")
    default_sort = {'name.exact': 'asc'}
    es_url = APP_INDEX + '/app/_search'

    excluded_properties = ["_id", "_rev", "_attachments", "admin_password_charset", "short_odk_url", "version",
                           "admin_password", "built_on", ]
    profile_list = ["profile.%s.%s" % (c['type'], c['id']) for c in CC_SETTINGS if c['type'] != 'hq']
    calculated_properties_mapping = ("Calculations", True,
                                     [{"facet": "cp_is_active", "name": "Active", "expanded": True }])

    @property
    def properties(self):
        return filter(lambda p: p and p not in self.excluded_properties, Application.properties().keys())

    @property
    def es_facet_list(self):
        props = self.properties + self.profile_list + ["cp_is_active"]
        return filter(lambda p: p not in self.excluded_properties, props)

    @property
    def es_facet_mapping(self):
        def remove_profile(name):
            return name[len("profile."):]
        profile_mapping = create_mapping_from_list(self.profile_list, "Profile", True, True, remove_profile)
        other_mapping = create_mapping_from_list(self.properties, "Other")
        return [profile_mapping, self.calculated_properties_mapping, other_mapping]

    @property
    def headers(self):
        headers = DataTablesHeader(
            DataTablesColumn(_("Name"), prop_name="name.exact"),
            DataTablesColumn(_("Project Space"), prop_name="domain"),
            DataTablesColumn(_("Build Comment"), prop_name="build_comment"),
        )
        return headers

    @property
    def rows(self):
        apps = [res['_source'] for res in self.es_results.get('hits', {}).get('hits', [])]

        for app in apps:
            yield [
                app.get('name'),
                app.get('domain'),
                app.get('build_comment'),
            ]

class GlobalAdminReports(AdminReport):
    base_template = "hqadmin/hqadmin_base_report.html"
    section_name = ugettext_noop("ADMINREPORT")  # not sure why ...

class RealProjectSpacesReport(GlobalAdminReports):
    slug = 'real_project_spaces'
    name = ugettext_noop('Real Project Spaces')

    @property
    def template_context(self):
        context = super(AdminReport, self).template_context
        context.update({
            'report_breadcrumbs': '%s' % self.name,
        })
        return context

    @property
    def headers(self):
        return DataTablesHeader(
            DataTablesColumn(_("Project Space")),
            DataTablesColumn(_("Number of Countries")),
        )

    @property
    def rows(self):
        return [
            [domain.name, domain.deployment] for domain in Domain.get_all()
        ]
<commit_msg>move template context to base class<commit_after>from datetime import datetime
from corehq import Domain
from corehq.apps.app_manager.models import Application
from corehq.apps.reports.datatables import DataTablesHeader, DataTablesColumn
from corehq.apps.reports.dispatcher import AdminReportDispatcher
from corehq.apps.reports.generic import ElasticTabularReport, GenericTabularReport
from django.utils.translation import ugettext as _, ugettext_noop
from corehq.apps.users.models import WebUser
from corehq.elastic import es_query, parse_args_for_es, fill_mapping_with_facets
from corehq.pillows.mappings.app_mapping import APP_INDEX
from corehq.pillows.mappings.user_mapping import USER_INDEX
from corehq.apps.app_manager.commcare_settings import SETTINGS as CC_SETTINGS
from corehq.toggles import IS_DEVELOPER

class AdminReport(GenericTabularReport):
    dispatcher = AdminReportDispatcher
    base_template = "hqadmin/faceted_report.html"

class AdminFacetedReport(AdminReport, ElasticTabularReport):
    default_sort = None
    es_prefix = "es_" # facet keywords in the url will be prefixed with this
    asynchronous = False
    ajax_pagination = True
    exportable = True
    es_queried = False
    es_facet_list = []
    es_facet_mapping = []
    section_name = ugettext_noop("ADMINREPORT")
    es_url = ''

    @property
    def template_context(self):
        ctxt = super(AdminFacetedReport, self).template_context

        self.run_query(0)
        if self.es_params.get('search'):
            ctxt["search_query"] = self.es_params.get('search')[0]
        ctxt.update({
            'layout_flush_content': True,
            'facet_map': self.es_facet_map,
            'query_str': self.request.META['QUERY_STRING'],
            'facet_prefix': self.es_prefix,
            'facet_report': self,
            'grouped_facets': True,
            'startdate': self.request.GET.get('startdate', ''),
            'enddate': self.request.GET.get('enddate', ''),
            'interval': self.request.GET.get('interval', ''),
        })
        return ctxt

    @property
    def total_records(self):
        return int(self.es_results['hits']['total'])

    def is_custom_param(self, param):
        return param.startswith(self.es_prefix)

    @property
    def shared_pagination_GET_params(self):
        ret = super(AdminFacetedReport, self).shared_pagination_GET_params
        for param in self.request.GET.iterlists():
            if self.is_custom_param(param[0]):
                for val in param[1]:
                    ret.append(dict(name=param[0], value=val))
        return ret

    def es_query(self, params=None, size=None):
        if params is None:
            params = {}
        terms = ['search']
        q = {"query": {"match_all":{}}}

        search_query = params.get('search', "")
        if search_query:
            q['query'] = {
                "bool": {
                    "must": {
                        "match" : {
                            "_all" : {
                                "query" : search_query,
                                "operator" : "and", }}}}}

        q["facets"] = {}

        q["sort"] = self.get_sorting_block()
        start_at=self.pagination.start
        size = size if size is not None else self.pagination.count
        return es_query(params, self.es_facet_list, terms, q, self.es_url, start_at, size, facet_size=25)

    @property
    def es_results(self):
        if not self.es_queried:
            self.run_query()
        return self.es_response

    def run_query(self, size=None):
        self.es_params, _ = parse_args_for_es(self.request, prefix=self.es_prefix)
        results = self.es_query(self.es_params, size)
        self.es_facet_map = fill_mapping_with_facets(self.es_facet_mapping, results, self.es_params)
        self.es_response = results
        self.es_queried = True
        return self.es_response

    @property
    def export_table(self):
        self.pagination.count = 1000000 # terrible hack to get the export to return all rows
        self.show_name = True
        return super(AdminFacetedReport, self).export_table

class AdminUserReport(AdminFacetedReport):
    slug = "user_list"
    name = ugettext_noop('User List')
    facet_title = ugettext_noop("User Facets")
    search_for = ugettext_noop("users...")
    default_sort = {'username.exact': 'asc'}
    es_url = USER_INDEX + '/user/_search'

    es_facet_list = [
        "is_active", # this is NOT "has submitted a form in the last 30 days"
        "is_staff",
        "is_superuser",
        "domain",
        "doc_type",
    ]

    es_facet_mapping = [
        ("", True, [
            {"facet": "is_active", "name": "Can Log In?", "expanded": True },
            {"facet": "is_superuser", "name": "SuperUser?", "expanded": True },
            {"facet": "is_staff", "name": "Staff?", "expanded": True },
            {"facet": "domain", "name": "Domain", "expanded": True },
            {"facet": "doc_type", "name": "User Type", "expanded": True },
        ]),
    ]

    @property
    def headers(self):
        headers = DataTablesHeader(
            DataTablesColumn(_("Username"), prop_name="username.exact"),
            DataTablesColumn(_("Project Spaces")),
            DataTablesColumn(_("Date Joined"), prop_name="date_joined"),
            DataTablesColumn(_("Last Login"), prop_name="last_login"),
            DataTablesColumn(_("Type"), prop_name="doc_type"),
            DataTablesColumn(_("SuperUser?"), prop_name="is_superuser"),
        )
        return headers

    @property
    def rows(self):
        users = [res['_source'] for res in self.es_results.get('hits', {}).get('hits', [])]

        def format_date(dstr, default):
            # use [:19] so that only only the 'YYYY-MM-DDTHH:MM:SS' part of the string is parsed
            return datetime.strptime(dstr[:19], '%Y-%m-%dT%H:%M:%S').strftime('%Y/%m/%d %H:%M:%S') if dstr else default

        def get_domains(user):
            if user.get('doc_type') == "WebUser":
                return ", ".join([dm['domain'] for dm in user.get('domain_memberships', [])])
            return user.get('domain_membership', {}).get('domain', _('No Domain Data'))

        for u in users:
            yield [
                u.get('username'),
                get_domains(u),
                format_date(u.get('date_joined'), _('No date')),
                format_date(u.get('last_login'), _('No date')),
                u.get('doc_type'),
                u.get('is_superuser'),
            ]

def create_mapping_from_list(l, name="", expand_outer=False, expand_inner=False, name_change_fn=None):
    name_change_fn = name_change_fn or (lambda x: x)
    facets = [{"facet": item, "name": name_change_fn(item), "expanded": expand_inner } for item in sorted(l)]
    return (name, expand_outer, facets)

class AdminAppReport(AdminFacetedReport):
    slug = "app_list"
    name = ugettext_noop('Application List')
    facet_title = ugettext_noop("App Facets")
    search_for = ugettext_noop("apps...")
    default_sort = {'name.exact': 'asc'}
    es_url = APP_INDEX + '/app/_search'

    excluded_properties = ["_id", "_rev", "_attachments", "admin_password_charset", "short_odk_url", "version",
                           "admin_password", "built_on", ]
    profile_list = ["profile.%s.%s" % (c['type'], c['id']) for c in CC_SETTINGS if c['type'] != 'hq']
    calculated_properties_mapping = ("Calculations", True,
                                     [{"facet": "cp_is_active", "name": "Active", "expanded": True }])

    @property
    def properties(self):
        return filter(lambda p: p and p not in self.excluded_properties, Application.properties().keys())

    @property
    def es_facet_list(self):
        props = self.properties + self.profile_list + ["cp_is_active"]
        return filter(lambda p: p not in self.excluded_properties, props)

    @property
    def es_facet_mapping(self):
        def remove_profile(name):
            return name[len("profile."):]
        profile_mapping = create_mapping_from_list(self.profile_list, "Profile", True, True, remove_profile)
        other_mapping = create_mapping_from_list(self.properties, "Other")
        return [profile_mapping, self.calculated_properties_mapping, other_mapping]

    @property
    def headers(self):
        headers = DataTablesHeader(
            DataTablesColumn(_("Name"), prop_name="name.exact"),
            DataTablesColumn(_("Project Space"), prop_name="domain"),
            DataTablesColumn(_("Build Comment"), prop_name="build_comment"),
        )
        return headers

    @property
    def rows(self):
        apps = [res['_source'] for res in self.es_results.get('hits', {}).get('hits', [])]

        for app in apps:
            yield [
                app.get('name'),
                app.get('domain'),
                app.get('build_comment'),
            ]

class GlobalAdminReports(AdminReport):
    base_template = "hqadmin/hqadmin_base_report.html"
    section_name = ugettext_noop("ADMINREPORT")  # not sure why ...

    @property
    def template_context(self):
        context = super(AdminReport, self).template_context
        context.update({
            'report_breadcrumbs': '%s' % self.name,
        })
        return context

class RealProjectSpacesReport(GlobalAdminReports):
    slug = 'real_project_spaces'
    name = ugettext_noop('Real Project Spaces')

    @property
    def headers(self):
        return DataTablesHeader(
            DataTablesColumn(_("Project Space")),
            DataTablesColumn(_("Number of Countries")),
        )

    @property
    def rows(self):
        return [
            [domain.name, domain.deployment] for domain in Domain.get_all()
        ]
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""Logical to Physical Mappers"""

import collections
from cubes.common import get_logger
from cubes.errors import *
from cubes.mapper import Mapper

__all__ = (
    "SnowflakeMapper",
    "DenormalizedMapper",
    "TableColumnReference",
    "TableJoin",
    "coalesce_physical",
    "DEFAULT_KEY_FIELD"
)

DEFAULT_KEY_FIELD = "id"

"""Physical reference to a table column. Note that the table might be an
aliased table name as specified in relevant join."""
TableColumnReference = collections.namedtuple("TableColumnReference",
                                    ["schema", "table", "column", "extract", "func", "expr", "condition"])

"""Table join specification. `master` and `detail` are TableColumnReference
tuples. `method` denotes which table members should be considered in the join:
*master* – all master members (left outer join), *detail* – all detail members
(right outer join) and *match* – members must match (inner join)."""
TableJoin = collections.namedtuple("TableJoin",
                                    ["master", "detail", "alias", "method"])

_join_method_order = {"detail":0, "master":1, "match": 2}

def coalesce_physical(ref, default_table=None, schema=None):
    """Coalesce physical reference `ref` which might be:

    * a string in form ``"table.column"``
    * a list in form ``(table, column)``
    * a list in form ``(schema, table, column)``
    * a dictionary with keys: ``schema``, ``table``, ``column``, ``extract``, ``func``, ``expr``, ``condition`` where
      ``column`` is required, the rest are optional

    Returns tuple (`schema`, `table`, `column`, `extract`, `func`, `expr`, `condition`), which is a named
    tuple `TableColumnReference`.

    If no table is specified in reference and `default_table` is not
    ``None``, then `default_table` will be used.

    .. note::

        The `table` element might be a table alias specified in list of joins.

    """

    if isinstance(ref, basestring):
        split = ref.split(".")

        if len(split) > 1:
            dim_name = split[0]
            attr_name = ".".join(split[1:])
            return TableColumnReference(schema, dim_name, attr_name, None, None, None, None)
        else:
            return TableColumnReference(schema, default_table, ref, None, None, None, None)
    elif isinstance(ref, dict):
        return TableColumnReference(ref.get("schema", schema),
                                 ref.get("table", default_table),
                                 ref.get("column"),
                                 ref.get("extract"),
                                 ref.get("func"),
                                 ref.get("expr"),
                                 ref.get("condition"))
    else:
        if len(ref) == 2:
            return TableColumnReference(schema, ref[0], ref[1], None, None, None, None)
        elif len(ref) == 3:
            return TableColumnReference(ref[0], ref[1], ref[2], None, None, None, None)
        else:
            raise BackendError("Number of items in table reference should "\
                               "be 2 (table, column) or 3 (schema, table, column)")

class SnowflakeMapper(Mapper):
    """Mapper is core clas for translating logical model to physical
    database schema.
    """
    # WARNING: do not put any SQL/engine/connection related stuff into this
    # class yet. It might be moved to the cubes as one of top-level modules
    # and subclassed here.

    def __init__(self, cube, mappings=None, locale=None, schema=None,
                    fact_name=None, dimension_prefix=None, joins=None,
                    dimension_schema=None, **options):

        """A snowflake schema mapper for a cube. The mapper creates required
        joins, resolves table names and maps logical references to tables and
        respective columns.

        Attributes:

        * `cube` - mapped cube
        * `mappings` – dictionary containing mappings
        * `simplify_dimension_references` – references for flat dimensions
          (with one level and no details) will be just dimension names, no
          attribute name. Might be useful when using single-table schema, for
          example, with couple of one-column dimensions.
        * `dimension_prefix` – default prefix of dimension tables, if
          default table name is used in physical reference construction
        * `fact_name` – fact name, if not specified then `cube.name` is used
        * `schema` – default database schema
        * `dimension_prefix` – prefix for dimension tables
        * `dimension_schema` – schema whre dimension tables are stored (if
          different than fact table schema)

        `mappings` is a dictionary where keys are logical attribute references
        and values are table column references. The keys are mostly in the
        form:

        * ``attribute`` for measures and fact details
        * ``attribute.locale`` for localized fact details
        * ``dimension.attribute`` for dimension attributes
        * ``dimension.attribute.locale`` for localized dimension attributes

        The values might be specified as strings in the form ``table.column``
        (covering most of the cases) or as a dictionary with keys ``schema``,
        ``table`` and ``column`` for more customized references.

        .. In the future it might support automatic join detection.

        """

        super(SnowflakeMapper, self).__init__(cube, locale=locale, **options)

        self.mappings = mappings or cube.mappings
        self.dimension_prefix = dimension_prefix
        self.dimension_schema = dimension_schema

        fact_prefix = options.get("fact_prefix") or ""
        self.fact_name = fact_name or self.cube.fact or fact_prefix+self.cube.name
        self.schema = schema

        self._collect_joins(joins or cube.joins)

    def _collect_joins(self, joins):
        """Collects joins and coalesce physical references. `joins` is a
        dictionary with keys: `master`, `detail` reffering to master and
        detail keys. `alias` is used to give alternative name to a table when
        two tables are being joined."""

        joins = joins or []

        self.joins = []

        for join in joins:
            master = coalesce_physical(join["master"],self.fact_name,schema=self.schema)
            detail = coalesce_physical(join["detail"],schema=self.schema)

            self.logger.debug("collecting join %s -> %s" % (tuple(master), tuple(detail)))
            method = join.get("method", "match").lower()

            self.joins.append(TableJoin(master, detail, join.get("alias"),
                                        method))

    def physical(self, attribute, locale=None):
        """Returns physical reference as tuple for `attribute`, which should
        be an instance of :class:`cubes.model.Attribute`. If there is no
        dimension specified in attribute, then fact table is assumed. The
        returned tuple has structure: (`schema`, `table`, `column`).

        The algorithm to find physicl reference is as follows::

            IF localization is requested:
                IF is attribute is localizable:
                    IF requested locale is one of attribute locales
                        USE requested locale
                    ELSE
                        USE default attribute locale
                ELSE
                    do not localize

            IF mappings exist:
                GET string for logical reference
                IF locale:
                    append '.' and locale to the logical reference

                IF mapping value exists for localized logical reference
                    USE value as reference

            IF no mappings OR no mapping was found:
                column name is attribute name

                IF locale:
                    append '_' and locale to the column name

                IF dimension specified:
                    # Example: 'date.year' -> 'date.year'
                    table name is dimension name

                    IF there is dimension table prefix
                        use the prefix for table name

                ELSE (if no dimension is specified):
                    # Example: 'date' -> 'fact.date'
                    table name is fact table name
        """

        reference = None

        # Fix locale: if attribute is not localized, use none, if it is
        # localized, then use specified if exists otherwise use default
        # locale of the attribute (first one specified in the list)

        locale = locale or self.locale

        if attribute.is_localizable():
            locale = locale if locale in attribute.locales \
                                else attribute.locales[0]
        else:
            locale = None

        # Try to get mapping if exists
        if self.cube.mappings:
            logical = self.logical(attribute, locale)

            # TODO: should default to non-localized reference if no mapping
            # was found?
            mapped_ref = self.cube.mappings.get(logical)

            if mapped_ref:
                reference = coalesce_physical(mapped_ref, self.fact_name, self.schema)

        # No mappings exist or no mapping was found - we are going to create
        # default physical reference
        if not reference:
            column_name = attribute.name

            if locale:
                column_name += "_" + locale

            # TODO: temporarily preserved. it should be attribute.owner
            dimension = attribute.dimension
            if dimension and not (self.simplify_dimension_references \
                                   and (dimension.is_flat
                                        and not dimension.has_details)):
                table_name = str(dimension)
                if self.dimension_prefix:
                    table_name = self.dimension_prefix + table_name

            else:
                table_name = self.fact_name

            schema = self.dimension_schema or self.schema
            reference = TableColumnReference(schema, table_name, column_name, None, None, None, None)

        return reference

    def table_map(self):
        """Return list of references to all tables. Keys are aliased
        tables: (`schema`, `aliased_table_name`) and values are
        real tables: (`schema`, `table_name`). Included is the fact table
        and all tables mentioned in joins.

        To get list of all physical tables where aliased tablesare included
        only once::

            finder = JoinFinder(cube, joins, fact_name)
            tables = set(finder.table_map().keys())
        """

        tables = {
            (self.schema, self.fact_name): (self.schema, self.fact_name)
        }

        for join in self.joins:
            if not join.detail.table or (join.detail.table == self.fact_name and not join.alias):
                raise BackendError("Detail table name should be present and should not be a fact table unless aliased.")

            ref = (join.master.schema, join.master.table)
            tables[ref] = ref

            ref = (join.detail.schema, join.alias or join.detail.table)
            tables[ref] = (join.detail.schema, join.detail.table)

        return tables

    def physical_references(self, attributes, expand_locales=False):
        """Convert `attributes` to physical attributes. If `expand_locales` is
        ``True`` then physical reference for every attribute locale is
        returned."""

        if expand_locales:
            physical_attrs = []

            for attr in attributes:
                if attr.is_localizable():
                    refs = [self.physical(attr, locale) for locale in attr.locales]
                else:
                    refs = [self.physical(attr)]
                physical_attrs += refs
        else:
            physical_attrs = [self.physical(attr) for attr in attributes]

        return physical_attrs

    def relevant_joins(self, attributes, expand_locales=False):
        """Get relevant joins to the attributes - list of joins that
        are required to be able to acces specified attributes. `attributes`
        is a list of three element tuples: (`schema`, `table`, `attribute`).
        """

        # Attribute: (schema, table, column)
        # Join: ((schema, table, column), (schema, table, column), alias)

        self.logger.debug("getting relevant joins for %s attributes" % len(attributes))

        references = self.physical_references(attributes, expand_locales)

        if not self.joins:
            self.logger.debug("no joins to be searched for")

        tables_to_join = set((ref[0], ref[1]) for ref in references)
        joined_tables = set()
        fact_table = (self.schema, self.fact_name)
        joined_tables.add( fact_table )

        joins = []
        self.logger.debug("tables to join: %s" % list(tables_to_join))

        while tables_to_join:
            table = tables_to_join.pop()
            # self.logger.debug("joining table %s" % (table, ))

            joined = False
            for order, join in enumerate(self.joins):
                # self.logger.debug("testing join: %s" % (join,e))
                # print "--- testing join: %s" % (join, )
                master = (join.master.schema, join.master.table)
                detail = (join.detail.schema, join.alias or join.detail.table)

                if table == detail:
                    # self.logger.debug("detail matches")
                    # Preserve join order
                    # TODO: temporary way of ordering according to match
                    method_order = _join_method_order.get(join.method, 99)
                    joins.append( (method_order, order, join) )

                    if master not in joined_tables:
                        self.logger.debug("adding master %s to be joined" % (master, ))
                        tables_to_join.add(master)

                    self.logger.debug("joined detail %s" % (detail, ) )
                    joined_tables.add(detail)
                    joined = True
                    break

            if joins and not joined and table != fact_table:
                self.logger.warn("No table joined for %s" % (table, ))

        self.logger.debug("%s tables joined (of %s joins)" % (len(joins), len(self.joins)) )

        # Sort joins according to original order specified in the model
        joins.sort()
        self.logger.debug("joined tables: %s" % ([join[2].detail.table for join in
                                                                joins], ) )

        # Retrieve actual joins from tuples. Remember? We preserved order.
        joins = [join[2] for join in joins]
        return joins

class DenormalizedMapper(Mapper):
    def __init__(self, cube, locale=None, schema=None,
                    fact_name=None, denormalized_view_prefix=None,
                    denormalized_view_schema=None,
                    **options):

        """Creates a mapper for a cube that has data stored in a denormalized
        view/table.

        Attributes:

        * `denormalized_view_prefix` – default prefix used for constructing
           view name from cube name
        * `fact_name` – fact name, if not specified then `cube.name` is used
        * `schema` – schema where the denormalized view is stored
        * `fact_schema` – database schema for the original fact table
        """

        super(DenormalizedMapper, self).__init__(cube, locale=locale,
                                        schema=schema, fact_name=fact_name)

        dview_prefix = denormalized_view_prefix or ""

        # FIXME: this hides original fact name, we do not want that

        self.fact_name = options.get("denormalized_view") or dview_prefix + self.cube.name
        self.fact_schema = self.schema
        self.schema = self.schema or denormalized_view_schema

    def physical(self, attribute, locale=None):
        """Returns same name as localized logical reference.
        """

        locale = locale or self.locale
        try:
            if attribute.locales:
                locale = locale if locale in attribute.locales \
                                    else attribute.locales[0]
            else:
                locale = None
        except:
            locale = None

        column_name = self.logical(attribute, locale)
        reference = TableColumnReference(self.schema,
                                          self.fact_name,
                                          column_name,
                                          None, None, None, None)

        return reference

    def relevant_joins(self, attributes):
        """Returns an empty list. No joins are necessary for denormalized
        view.
        """

        self.logger.debug("getting relevant joins: not needed for denormalized table")

        return []

<commit_msg>added tables_for_attributes()<commit_after># -*- coding: utf-8 -*-
"""Logical to Physical Mappers"""

import collections
from cubes.common import get_logger
from cubes.errors import *
from cubes.mapper import Mapper

__all__ = (
    "SnowflakeMapper",
    "DenormalizedMapper",
    "TableColumnReference",
    "TableJoin",
    "coalesce_physical",
    "DEFAULT_KEY_FIELD"
)

DEFAULT_KEY_FIELD = "id"

"""Physical reference to a table column. Note that the table might be an
aliased table name as specified in relevant join."""
TableColumnReference = collections.namedtuple("TableColumnReference",
                                    ["schema", "table", "column", "extract", "func", "expr", "condition"])

"""Table join specification. `master` and `detail` are TableColumnReference
tuples. `method` denotes which table members should be considered in the join:
*master* – all master members (left outer join), *detail* – all detail members
(right outer join) and *match* – members must match (inner join)."""
TableJoin = collections.namedtuple("TableJoin",
                                    ["master", "detail", "alias", "method"])

_join_method_order = {"detail":0, "master":1, "match": 2}

def coalesce_physical(ref, default_table=None, schema=None):
    """Coalesce physical reference `ref` which might be:

    * a string in form ``"table.column"``
    * a list in form ``(table, column)``
    * a list in form ``(schema, table, column)``
    * a dictionary with keys: ``schema``, ``table``, ``column``, ``extract``, ``func``, ``expr``, ``condition`` where
      ``column`` is required, the rest are optional

    Returns tuple (`schema`, `table`, `column`, `extract`, `func`, `expr`, `condition`), which is a named
    tuple `TableColumnReference`.

    If no table is specified in reference and `default_table` is not
    ``None``, then `default_table` will be used.

    .. note::

        The `table` element might be a table alias specified in list of joins.

    """

    if isinstance(ref, basestring):
        split = ref.split(".")

        if len(split) > 1:
            dim_name = split[0]
            attr_name = ".".join(split[1:])
            return TableColumnReference(schema, dim_name, attr_name, None, None, None, None)
        else:
            return TableColumnReference(schema, default_table, ref, None, None, None, None)
    elif isinstance(ref, dict):
        return TableColumnReference(ref.get("schema", schema),
                                 ref.get("table", default_table),
                                 ref.get("column"),
                                 ref.get("extract"),
                                 ref.get("func"),
                                 ref.get("expr"),
                                 ref.get("condition"))
    else:
        if len(ref) == 2:
            return TableColumnReference(schema, ref[0], ref[1], None, None, None, None)
        elif len(ref) == 3:
            return TableColumnReference(ref[0], ref[1], ref[2], None, None, None, None)
        else:
            raise BackendError("Number of items in table reference should "\
                               "be 2 (table, column) or 3 (schema, table, column)")

class SnowflakeMapper(Mapper):
    """Mapper is core clas for translating logical model to physical
    database schema.
    """
    # WARNING: do not put any SQL/engine/connection related stuff into this
    # class yet. It might be moved to the cubes as one of top-level modules
    # and subclassed here.

    def __init__(self, cube, mappings=None, locale=None, schema=None,
                    fact_name=None, dimension_prefix=None, joins=None,
                    dimension_schema=None, **options):

        """A snowflake schema mapper for a cube. The mapper creates required
        joins, resolves table names and maps logical references to tables and
        respective columns.

        Attributes:

        * `cube` - mapped cube
        * `mappings` – dictionary containing mappings
        * `simplify_dimension_references` – references for flat dimensions
          (with one level and no details) will be just dimension names, no
          attribute name. Might be useful when using single-table schema, for
          example, with couple of one-column dimensions.
        * `dimension_prefix` – default prefix of dimension tables, if
          default table name is used in physical reference construction
        * `fact_name` – fact name, if not specified then `cube.name` is used
        * `schema` – default database schema
        * `dimension_prefix` – prefix for dimension tables
        * `dimension_schema` – schema whre dimension tables are stored (if
          different than fact table schema)

        `mappings` is a dictionary where keys are logical attribute references
        and values are table column references. The keys are mostly in the
        form:

        * ``attribute`` for measures and fact details
        * ``attribute.locale`` for localized fact details
        * ``dimension.attribute`` for dimension attributes
        * ``dimension.attribute.locale`` for localized dimension attributes

        The values might be specified as strings in the form ``table.column``
        (covering most of the cases) or as a dictionary with keys ``schema``,
        ``table`` and ``column`` for more customized references.

        .. In the future it might support automatic join detection.

        """

        super(SnowflakeMapper, self).__init__(cube, locale=locale, **options)

        self.mappings = mappings or cube.mappings
        self.dimension_prefix = dimension_prefix
        self.dimension_schema = dimension_schema

        fact_prefix = options.get("fact_prefix") or ""
        self.fact_name = fact_name or self.cube.fact or fact_prefix+self.cube.name
        self.schema = schema

        self._collect_joins(joins or cube.joins)

    def _collect_joins(self, joins):
        """Collects joins and coalesce physical references. `joins` is a
        dictionary with keys: `master`, `detail` reffering to master and
        detail keys. `alias` is used to give alternative name to a table when
        two tables are being joined."""

        joins = joins or []

        self.joins = []

        for join in joins:
            master = coalesce_physical(join["master"],self.fact_name,schema=self.schema)
            detail = coalesce_physical(join["detail"],schema=self.schema)

            self.logger.debug("collecting join %s -> %s" % (tuple(master), tuple(detail)))
            method = join.get("method", "match").lower()

            self.joins.append(TableJoin(master, detail, join.get("alias"),
                                        method))

    def physical(self, attribute, locale=None):
        """Returns physical reference as tuple for `attribute`, which should
        be an instance of :class:`cubes.model.Attribute`. If there is no
        dimension specified in attribute, then fact table is assumed. The
        returned tuple has structure: (`schema`, `table`, `column`).

        The algorithm to find physicl reference is as follows::

            IF localization is requested:
                IF is attribute is localizable:
                    IF requested locale is one of attribute locales
                        USE requested locale
                    ELSE
                        USE default attribute locale
                ELSE
                    do not localize

            IF mappings exist:
                GET string for logical reference
                IF locale:
                    append '.' and locale to the logical reference

                IF mapping value exists for localized logical reference
                    USE value as reference

            IF no mappings OR no mapping was found:
                column name is attribute name

                IF locale:
                    append '_' and locale to the column name

                IF dimension specified:
                    # Example: 'date.year' -> 'date.year'
                    table name is dimension name

                    IF there is dimension table prefix
                        use the prefix for table name

                ELSE (if no dimension is specified):
                    # Example: 'date' -> 'fact.date'
                    table name is fact table name
        """

        reference = None

        # Fix locale: if attribute is not localized, use none, if it is
        # localized, then use specified if exists otherwise use default
        # locale of the attribute (first one specified in the list)

        locale = locale or self.locale

        if attribute.is_localizable():
            locale = locale if locale in attribute.locales \
                                else attribute.locales[0]
        else:
            locale = None

        # Try to get mapping if exists
        if self.cube.mappings:
            logical = self.logical(attribute, locale)

            # TODO: should default to non-localized reference if no mapping
            # was found?
            mapped_ref = self.cube.mappings.get(logical)

            if mapped_ref:
                reference = coalesce_physical(mapped_ref, self.fact_name, self.schema)

        # No mappings exist or no mapping was found - we are going to create
        # default physical reference
        if not reference:
            column_name = attribute.name

            if locale:
                column_name += "_" + locale

            # TODO: temporarily preserved. it should be attribute.owner
            dimension = attribute.dimension
            if dimension and not (self.simplify_dimension_references \
                                   and (dimension.is_flat
                                        and not dimension.has_details)):
                table_name = str(dimension)
                if self.dimension_prefix:
                    table_name = self.dimension_prefix + table_name

            else:
                table_name = self.fact_name

            schema = self.dimension_schema or self.schema
            reference = TableColumnReference(schema, table_name, column_name, None, None, None, None)

        return reference

    def table_map(self):
        """Return list of references to all tables. Keys are aliased
        tables: (`schema`, `aliased_table_name`) and values are
        real tables: (`schema`, `table_name`). Included is the fact table
        and all tables mentioned in joins.

        To get list of all physical tables where aliased tablesare included
        only once::

            finder = JoinFinder(cube, joins, fact_name)
            tables = set(finder.table_map().keys())
        """

        tables = {
            (self.schema, self.fact_name): (self.schema, self.fact_name)
        }

        for join in self.joins:
            if not join.detail.table or (join.detail.table == self.fact_name and not join.alias):
                raise BackendError("Detail table name should be present and should not be a fact table unless aliased.")

            ref = (join.master.schema, join.master.table)
            tables[ref] = ref

            ref = (join.detail.schema, join.alias or join.detail.table)
            tables[ref] = (join.detail.schema, join.detail.table)

        return tables

    def physical_references(self, attributes, expand_locales=False):
        """Convert `attributes` to physical attributes. If `expand_locales` is
        ``True`` then physical reference for every attribute locale is
        returned."""

        if expand_locales:
            physical_attrs = []

            for attr in attributes:
                if attr.is_localizable():
                    refs = [self.physical(attr, locale) for locale in attr.locales]
                else:
                    refs = [self.physical(attr)]
                physical_attrs += refs
        else:
            physical_attrs = [self.physical(attr) for attr in attributes]

        return physical_attrs

    def tables_for_attributes(self, attributes, expand_locales=False):
        """Returns a list of tables – tuples (`schema`, `table`) that contain
        `attributes`."""

        references = self.physical_references(attributes, expand_locales)
        tables = [(ref[0], ref[1]) for ref in references]
        return tables

    def relevant_joins(self, attributes, expand_locales=False):
        """Get relevant joins to the attributes - list of joins that
        are required to be able to acces specified attributes. `attributes`
        is a list of three element tuples: (`schema`, `table`, `attribute`).
        """

        # Attribute: (schema, table, column)
        # Join: ((schema, table, column), (schema, table, column), alias)

        self.logger.debug("getting relevant joins for %s attributes" % len(attributes))

        if not self.joins:
            self.logger.debug("no joins to be searched for")

        tables_to_join = set(self.tables_for_attributes(attributes,
                                                        expand_locales))
        joined_tables = set()
        fact_table = (self.schema, self.fact_name)
        joined_tables.add( fact_table )

        joins = []
        self.logger.debug("tables to join: %s" % list(tables_to_join))

        while tables_to_join:
            table = tables_to_join.pop()
            # self.logger.debug("joining table %s" % (table, ))

            joined = False
            for order, join in enumerate(self.joins):
                # self.logger.debug("testing join: %s" % (join,e))
                # print "--- testing join: %s" % (join, )
                master = (join.master.schema, join.master.table)
                detail = (join.detail.schema, join.alias or join.detail.table)

                if table == detail:
                    # self.logger.debug("detail matches")
                    # Preserve join order
                    # TODO: temporary way of ordering according to match
                    method_order = _join_method_order.get(join.method, 99)
                    joins.append( (method_order, order, join) )

                    if master not in joined_tables:
                        self.logger.debug("adding master %s to be joined" % (master, ))
                        tables_to_join.add(master)

                    self.logger.debug("joined detail %s" % (detail, ) )
                    joined_tables.add(detail)
                    joined = True
                    break

            if joins and not joined and table != fact_table:
                self.logger.warn("No table joined for %s" % (table, ))

        self.logger.debug("%s tables joined (of %s joins)" % (len(joins), len(self.joins)) )

        # Sort joins according to original order specified in the model
        joins.sort()
        self.logger.debug("joined tables: %s" % ([join[2].detail.table for join in
                                                                joins], ) )

        # Retrieve actual joins from tuples. Remember? We preserved order.
        joins = [join[2] for join in joins]
        return joins

class DenormalizedMapper(Mapper):
    def __init__(self, cube, locale=None, schema=None,
                    fact_name=None, denormalized_view_prefix=None,
                    denormalized_view_schema=None,
                    **options):

        """Creates a mapper for a cube that has data stored in a denormalized
        view/table.

        Attributes:

        * `denormalized_view_prefix` – default prefix used for constructing
           view name from cube name
        * `fact_name` – fact name, if not specified then `cube.name` is used
        * `schema` – schema where the denormalized view is stored
        * `fact_schema` – database schema for the original fact table
        """

        super(DenormalizedMapper, self).__init__(cube, locale=locale,
                                        schema=schema, fact_name=fact_name)

        dview_prefix = denormalized_view_prefix or ""

        # FIXME: this hides original fact name, we do not want that

        self.fact_name = options.get("denormalized_view") or dview_prefix + self.cube.name
        self.fact_schema = self.schema
        self.schema = self.schema or denormalized_view_schema

    def physical(self, attribute, locale=None):
        """Returns same name as localized logical reference.
        """

        locale = locale or self.locale
        try:
            if attribute.locales:
                locale = locale if locale in attribute.locales \
                                    else attribute.locales[0]
            else:
                locale = None
        except:
            locale = None

        column_name = self.logical(attribute, locale)
        reference = TableColumnReference(self.schema,
                                          self.fact_name,
                                          column_name,
                                          None, None, None, None)

        return reference

    def relevant_joins(self, attributes):
        """Returns an empty list. No joins are necessary for denormalized
        view.
        """

        self.logger.debug("getting relevant joins: not needed for denormalized table")

        return []

<|endoftext|>