<commit_before># -*- coding: utf-8 -*-
from base import BaseTopology

class BaseRedis(BaseTopology):
    def deploy_first_steps(self):
        return (
            'workflow.steps.redis.deploy.build_databaseinfra.BuildDatabaseInfra',
            'workflow.steps.redis.deploy.create_virtualmachines.CreateVirtualMachine',
            'workflow.steps.redis.deploy.create_dns.CreateDns',
            'workflow.steps.util.deploy.create_nfs.CreateNfs',
            'workflow.steps.redis.deploy.init_database.InitDatabaseRedis',
            'workflow.steps.util.deploy.config_backup_log.ConfigBackupLog',
            'workflow.steps.util.deploy.check_database_connection.CheckDatabaseConnection',
            'workflow.steps.util.deploy.check_dns.CheckDns',
            'workflow.steps.util.deploy.start_monit.StartMonit',
        )

    def deploy_last_steps(self):
        return (
            'workflow.steps.util.deploy.build_database.BuildDatabase',
            'workflow.steps.util.deploy.check_database_binds.CheckDatabaseBinds',
        )

    def get_clone_steps(self):
        return self.deploy_first_steps() + self.deploy_last_steps() + (
            'workflow.steps.redis.clone.clone_database.CloneDatabase',
        ) + self.monitoring_steps()

    def get_upgrade_steps_extra(self):
        return (
            'workflow.steps.util.plan.InitializationForUpgrade',
            'workflow.steps.util.plan.ConfigureForUpgrade',
            'workflow.steps.util.pack.Configure',
        )

    def get_resize_extra_steps(self):
        return super(BaseRedis, self).get_resize_extra_steps() + (
            'workflow.steps.util.infra.Memory',
        )

    def add_database_instances_first_steps(self):
        return ()

    def add_database_instances_last_steps(self):
        return ()

class RedisSingle(BaseRedis):

    @property
    def driver_name(self):
        return 'redis_single'

class RedisSentinel(BaseRedis):

    def get_upgrade_steps_final(self):
        return [{
            'Resetting Sentinel': (
                'workflow.steps.redis.upgrade.sentinel.Reset',
                'workflow.steps.util.database.SetSlave',
            ),
        }] + super(RedisSentinel, self).get_upgrade_steps_final()

    def get_reinstallvm_steps_final(self):
        return [{
            'Resetting Sentinel': (
                'workflow.steps.redis.upgrade.sentinel.ResetAllSentinel',
                'workflow.steps.util.database.SetSlave',
            ),
        }] + super(RedisSentinel, self).get_reinstallvm_steps_final()

    def get_add_database_instances_middle_steps(self):
        return (
            'workflow.steps.util.plan.Initialization',
            'workflow.steps.util.plan.Configure',
            'workflow.steps.util.pack.Configure',
            'workflow.steps.util.database.Start',
            'workflow.steps.redis.horizontal_elasticity.database.AddInstanceToRedisCluster',
        )

    @property
    def driver_name(self):
        return 'redis_sentinel'

    def get_restore_snapshot_steps(self):
        return (
            'workflow.steps.util.restore_snapshot.restore_snapshot.RestoreSnapshot',
            'workflow.steps.util.restore_snapshot.grant_nfs_access.GrantNFSAccess',
            'workflow.steps.util.restore_snapshot.stop_database.StopDatabase',
            'workflow.steps.util.restore_snapshot.umount_data_volume.UmountDataVolume',
            'workflow.steps.util.restore_snapshot.update_fstab.UpdateFstab',
            'workflow.steps.util.restore_snapshot.mount_data_volume.MountDataVolume',
            'workflow.steps.redis.restore_snapshot.start_database.StartDatabase',
            'workflow.steps.util.restore_snapshot.make_export_snapshot.MakeExportSnapshot',
            'workflow.steps.util.restore_snapshot.update_dbaas_metadata.UpdateDbaaSMetadata',
            'workflow.steps.util.restore_snapshot.clean_old_volumes.CleanOldVolumes',
        )

class RedisNoPersistence(RedisSingle):
    pass

class RedisSentinelNoPersistence(RedisSentinel):
    pass

class RedisCluster(BaseRedis):

    def deploy_quantity_of_instances(self):
        return 6

    def get_deploy_steps(self):
        return [{
            'Creating virtual machine': (
                'workflow.steps.util.vm.CreateVirtualMachineNewInfra',
            )}, {
            'Creating dns': (
                'workflow.steps.util.dns.CreateDNS',
            )}, {
            'Creating disk': (
                'workflow.steps.util.disk.CreateExport',
            )}, {
            'Waiting VMs': (
                'workflow.steps.util.vm.WaitingBeReady',
                'workflow.steps.util.vm.UpdateOSDescription'
            )}, {
            'Configuring database': (
                'workflow.steps.util.plan.InitializationForNewInfra',
                'workflow.steps.util.plan.ConfigureForNewInfra',
                'workflow.steps.util.database.Start',
                'workflow.steps.util.database.CheckIsUp',
            )}, {
            'Configuring Cluster': (
                'workflow.steps.redis.cluster.CreateCluster',
                'workflow.steps.redis.cluster.CheckClusterStatus',
            )}, {
            'Check DNS': (
                'workflow.steps.util.dns.CheckIsReady',
            )}, {
            'Creating Database': (
                'workflow.steps.util.database.Create',
            )}, {
            'Creating monitoring and alarms': (
                'workflow.steps.util.zabbix.CreateAlarms',
            )
        }]

    def get_destroy_steps(self):
        return super(RedisCluster, self).get_destroy_steps()

    def get_restore_snapshot_steps(self):
        return [{
            'Disable monitoring': (
                'workflow.steps.util.zabbix.DisableAlarms',
            )}, {
            'Restoring': (
                'workflow.steps.util.disk.RestoreSnapshot',
            )}, {
            'Stopping datbase': (
                'workflow.steps.util.database.Stop',
                'workflow.steps.util.database.CheckIsDown',
                'workflow.steps.redis.cluster.SaveNodeConfig'
            )}, {
            'Configuring': (
                'workflow.steps.util.disk.AddDiskPermissionsRestoredDisk',
                'workflow.steps.util.disk.UnmountOldestExportRestore',
                'workflow.steps.util.disk.MountNewerExportRestore',
                'workflow.steps.util.disk.ConfigureFstabRestore',
                'workflow.steps.util.disk.CleanData',
                'workflow.steps.util.plan.InitializationRestore',
                'workflow.steps.util.plan.ConfigureRestore',
                'workflow.steps.redis.cluster.RestoreNodeConfig'
            )}, {
            'Starting database': (
                'workflow.steps.util.database.Start',
                'workflow.steps.util.database.CheckIsUp',
            )}, {
            'Old data': (
                'workflow.steps.util.disk.BackupRestore',
                'workflow.steps.util.disk.UpdateRestore',
            )}, {
            'Enabling monitoring': (
                'workflow.steps.util.zabbix.EnableAlarms',
            )
        }]

    @property
    def driver_name(self):
        return 'redis_cluster'
<commit_msg>steps for redis cluster<commit_after># -*- coding: utf-8 -*-
from base import BaseTopology

class BaseRedis(BaseTopology):
    def deploy_first_steps(self):
        return (
            'workflow.steps.redis.deploy.build_databaseinfra.BuildDatabaseInfra',
            'workflow.steps.redis.deploy.create_virtualmachines.CreateVirtualMachine',
            'workflow.steps.redis.deploy.create_dns.CreateDns',
            'workflow.steps.util.deploy.create_nfs.CreateNfs',
            'workflow.steps.redis.deploy.init_database.InitDatabaseRedis',
            'workflow.steps.util.deploy.config_backup_log.ConfigBackupLog',
            'workflow.steps.util.deploy.check_database_connection.CheckDatabaseConnection',
            'workflow.steps.util.deploy.check_dns.CheckDns',
            'workflow.steps.util.deploy.start_monit.StartMonit',
        )

    def deploy_last_steps(self):
        return (
            'workflow.steps.util.deploy.build_database.BuildDatabase',
            'workflow.steps.util.deploy.check_database_binds.CheckDatabaseBinds',
        )

    def get_clone_steps(self):
        return self.deploy_first_steps() + self.deploy_last_steps() + (
            'workflow.steps.redis.clone.clone_database.CloneDatabase',
        ) + self.monitoring_steps()

    def get_upgrade_steps_extra(self):
        return (
            'workflow.steps.util.plan.InitializationForUpgrade',
            'workflow.steps.util.plan.ConfigureForUpgrade',
            'workflow.steps.util.pack.Configure',
        )

    def get_resize_extra_steps(self):
        return super(BaseRedis, self).get_resize_extra_steps() + (
            'workflow.steps.util.infra.Memory',
        )

    def add_database_instances_first_steps(self):
        return ()

    def add_database_instances_last_steps(self):
        return ()

class RedisSingle(BaseRedis):

    @property
    def driver_name(self):
        return 'redis_single'

class RedisSentinel(BaseRedis):

    def get_upgrade_steps_final(self):
        return [{
            'Resetting Sentinel': (
                'workflow.steps.redis.upgrade.sentinel.Reset',
                'workflow.steps.util.database.SetSlave',
            ),
        }] + super(RedisSentinel, self).get_upgrade_steps_final()

    def get_reinstallvm_steps_final(self):
        return [{
            'Resetting Sentinel': (
                'workflow.steps.redis.upgrade.sentinel.ResetAllSentinel',
                'workflow.steps.util.database.SetSlave',
            ),
        }] + super(RedisSentinel, self).get_reinstallvm_steps_final()

    def get_add_database_instances_middle_steps(self):
        return (
            'workflow.steps.util.plan.Initialization',
            'workflow.steps.util.plan.Configure',
            'workflow.steps.util.pack.Configure',
            'workflow.steps.util.database.Start',
            'workflow.steps.redis.horizontal_elasticity.database.AddInstanceToRedisCluster',
        )

    @property
    def driver_name(self):
        return 'redis_sentinel'

    def get_restore_snapshot_steps(self):
        return (
            'workflow.steps.util.restore_snapshot.restore_snapshot.RestoreSnapshot',
            'workflow.steps.util.restore_snapshot.grant_nfs_access.GrantNFSAccess',
            'workflow.steps.util.restore_snapshot.stop_database.StopDatabase',
            'workflow.steps.util.restore_snapshot.umount_data_volume.UmountDataVolume',
            'workflow.steps.util.restore_snapshot.update_fstab.UpdateFstab',
            'workflow.steps.util.restore_snapshot.mount_data_volume.MountDataVolume',
            'workflow.steps.redis.restore_snapshot.start_database.StartDatabase',
            'workflow.steps.util.restore_snapshot.make_export_snapshot.MakeExportSnapshot',
            'workflow.steps.util.restore_snapshot.update_dbaas_metadata.UpdateDbaaSMetadata',
            'workflow.steps.util.restore_snapshot.clean_old_volumes.CleanOldVolumes',
        )

class RedisNoPersistence(RedisSingle):
    pass

class RedisSentinelNoPersistence(RedisSentinel):
    pass

class RedisCluster(BaseRedis):

    def deploy_quantity_of_instances(self):
        return 6

    def get_deploy_steps(self):
        return [{
            'Creating virtual machine': (
                'workflow.steps.util.vm.CreateVirtualMachineNewInfra',
            )}, {
            'Creating dns': (
                'workflow.steps.util.dns.CreateDNS',
            )}, {
            'Creating disk': (
                'workflow.steps.util.disk.CreateExport',
            )}, {
            'Waiting VMs': (
                'workflow.steps.util.vm.WaitingBeReady',
                'workflow.steps.util.vm.UpdateOSDescription'
            )}, {
            'Configuring database': (
                'workflow.steps.util.plan.InitializationForNewInfra',
                'workflow.steps.util.plan.ConfigureForNewInfra',
                'workflow.steps.util.database.Start',
                'workflow.steps.util.database.CheckIsUp',
            )}, {
            'Configuring Cluster': (
                'workflow.steps.redis.cluster.CreateCluster',
                'workflow.steps.redis.cluster.CheckClusterStatus',
                'workflow.steps.redis.cluster.SetInstanceShardTag',
            )}, {
            'Check DNS': (
                'workflow.steps.util.dns.CheckIsReady',
            )}, {
            'Creating Database': (
                'workflow.steps.util.database.Create',
            )}, {
            'Creating monitoring and alarms': (
                'workflow.steps.util.zabbix.CreateAlarms',
                'workflow.steps.util.db_monitor.CreateInfraMonitoring',
            )
        }]

    def get_destroy_steps(self):
        return super(RedisCluster, self).get_destroy_steps()

    def get_restore_snapshot_steps(self):
        return [{
            'Disable monitoring': (
                'workflow.steps.util.zabbix.DisableAlarms',
            )}, {
            'Restoring': (
                'workflow.steps.util.disk.RestoreSnapshot',
            )}, {
            'Stopping datbase': (
                'workflow.steps.util.database.Stop',
                'workflow.steps.util.database.CheckIsDown',
                'workflow.steps.redis.cluster.SaveNodeConfig'
            )}, {
            'Configuring': (
                'workflow.steps.util.disk.AddDiskPermissionsRestoredDisk',
                'workflow.steps.util.disk.UnmountOldestExportRestore',
                'workflow.steps.util.disk.MountNewerExportRestore',
                'workflow.steps.util.disk.ConfigureFstabRestore',
                'workflow.steps.util.disk.CleanData',
                'workflow.steps.util.plan.InitializationRestore',
                'workflow.steps.util.plan.ConfigureRestore',
                'workflow.steps.redis.cluster.RestoreNodeConfig'
            )}, {
            'Starting database': (
                'workflow.steps.util.database.Start',
                'workflow.steps.util.database.CheckIsUp',
            )}, {
            'Old data': (
                'workflow.steps.util.disk.BackupRestore',
                'workflow.steps.util.disk.UpdateRestore',
            )}, {
            'Enabling monitoring': (
                'workflow.steps.util.zabbix.EnableAlarms',
            )
        }]

    @property
    def driver_name(self):
        return 'redis_cluster'
<|endoftext|>


<commit_before>import weakref

class NotificationCenter(object):

    def __init__(self):
        self._observables = {}
        self._holds = {}
        self._disabled = {}

    def addObserver(self, observer, methodName, notification, observable):
        """
        Add an observer to this notification dispatcher.

        * **observer** An object that can be referenced with weakref.
        * **methodName** A string epresenting the method to be called
          when the notification is posted.
        * **notification** The notification that teh observer should
          be notified of.
        * **observable** The object to observe.

        The method that will be called as a result of the action
        must accept a single *notification* argument. This will
        be a :class:`Notification` object.
        """
        if observable is not None:
            observable = weakref.ref(observable)
        observer = weakref.ref(observer)
        if notification not in self._observables:
            self._observables[notification] = {}
        observableDict = self._observables[notification]
        if observable not in observableDict:
            observableDict[observable] = {}
        observerDict = observableDict[observable]
        observerDict[observer] = methodName

    def removeObserver(self, observer, notification, observable):
        """
        Remove an observer from this notification dispatcher.

        * **observer** A registered object.
        * **notification** The notification that the observer was registered
          to be notified of.
        * **observable** The object being observed.
        """
        if observable is not None:
            observable = weakref.ref(observable)
        observer = weakref.ref(observer)
        del self._observables[notification][observable][observer]

    def hasObserver(self, observer, notification, observable):
        """
        Returns a boolean indicating is the **observer** is registered
        for **notification** posted by **observable**.
        """
        if observable is not None:
            observable = weakref.ref(observable)
        observer = weakref.ref(observer)
        if notification not in self._observables:
            return False
        if observable not in self._observables[notification]:
            return False
        if observer not in self._observables[notification][observable]:
            return False
        return True

    def postNotification(self, notification, observable, data=None):
        """
        Post a notification to all objects observing **notification**
        in **observable**.

        * **notification** The name of the notification.
        * **observable** The object that the notification belongs to.
        * **data** Arbitrary data that will be stored in the :class:`Notification` object.

        This will create a :class:`Notification` object and post it to
        all relevant observers.
        """
        observableObj = observable
        if observable is not None:
            observable = weakref.ref(observable)
        # disabled
        if (observable, notification) in self._disabled:
            return
        if observable in self._disabled:
            return
        # held
        if (observable, notification) in self._holds:
            n = (notification, observable, data)
            if self._holds[observable, notification]["notifications"] and self._holds[observable, notification]["notifications"][-1] != n:
                self._holds[observable, notification]["notifications"].append(n)
            return
        if observable in self._holds:
            n = (notification, observable, data)
            if self._holds[observable]["notifications"] and self._holds[observable]["notifications"][-1] != n:
                self._holds[observable]["notifications"].append(n)
            return
        # post
        if notification in self._observables:
            for observableRef, observerDict in self._observables[notification].items():
                if observable == observableRef:
                    for observerRef, methodName in observerDict.items():
                        observer = observerRef()
                        callback = getattr(observer, methodName)
                        n = Notification(notification, observableRef, data)
                        callback(n)

    def holdNotificationsForObservable(self, observable, notification=None):
        """
        Hold all notifications posted to all objects observing
        **notification** in **observable**.

        * **observable** The object that the notification belongs to.
        * **notification** The name of the notification. This is optional.
          If no *notification* is given, *all* notifications for *observable*
          will be held.

        Held notifications will be posted after the a matching *notification*
        and *observable* have been passed to :meth:`Notification.releaseHeldNotificationsForObservable`.
        This object will retain a count of how many times it has been told to
        hold notifications for *notification* and *observable*. It will not
        post the notifications until the *notification* and *observable*
        have been released the same number of times.
        """
        observable = weakref.ref(observable)
        if notification is not None:
            key = (observable, notification)
        else:
            key = observable
        if key not in self._holds:
            self._holds[key] = dict(holdCount=0, notifications=set())
        self._holds[key]["holdCount"] += 1

    def releaseHeldNotificationsForObservable(self, observable, notification=None):
        """
        Release all held notifications posted to all objects observing
        **notification** in **observable**.

        * **observable** The object that the notification belongs to.
        * **notification** The name of the notification. This is optional.
        """
        observableObj = observable
        observable = weakref.ref(observable)
        if notification is not None:
            key = (observable, notification)
        else:
            key = observable
        self._holds[key]["holdCount"] -= 1
        if self._holds[key]["holdCount"] == 0:
            notifications = self._holds[key]["notifications"]
            del self._holds[key]
            for notification, o, data in notifications:
                self.postNotification(notification, observableObj, data)

    def areNotificationsHeldForObservable(self, observable, notification=None):
        """
        Returns a boolean indicating if notifications posted to all objects observing
        **notification** in **observable** are being held.

        * **observable** The object that the notification belongs to.
        * **notification** The name of the notification. This is optional.
        """
        if (observable, notification) in self._holds:
            return True
        return weakref.ref(observable) in self._holds

    def disableNotificationsForObservable(self, observable, notification=None):
        """
        Disable all notifications posted to all objects observing
        **notification** in **observable**.

        * **observable** The object that the notification belongs to.
        * **notification** The name of the notification. This is optional.
          If no *notification* is given, *all* notifications for *observable*
          will be disabled.

        This object will retain a count of how many times it has been told to
        disable notifications for *notification* and *observable*. It will not
        enable new notifications until the *notification* and *observable*
        have been released the same number of times.
        """
        observable = weakref.ref(observable)
        if notification is not None:
            key = (observable, notification)
        else:
            key = observable
        if key not in self._disabled:
            self._disabled[key] = 0
        self._disabled[key] += 1

    def enableNotificationsForObservable(self, observable, notification=None):
        """
        Enable notifications posted to all objects observing
        **notification** in **observable**.

        * **observable** The object that the notification belongs to.
        * **notification** The name of the notification. This is optional.
        """
        observableObj = observable
        observable = weakref.ref(observable)
        if notification is not None:
            key = (observable, notification)
        else:
            key = observable
        self._disabled[key] -= 1
        if self._disabled[key] == 0:
            del self._disabled[key]

    def areNotificationsDisabledForObservable(self, observable, notification=None):
        """
        Returns a boolean indicating if notifications posted to all objects observing
        **notification** in **observable** are disabled.

        * **observable** The object that the notification belongs to.
        * **notification** The name of the notification. This is optional.
        """
        if (observable, notification) in self._disabled:
            return True
        return weakref.ref(observable) in self._disabled

class Notification(object):

    """An object that wraps notification data."""

    def __init__(self, name, objRef, data):
        self._name = name
        self._objRef = objRef
        self._data = data

    def __repr__(self):
        return "<Notification: %s %s>" % (self.name, repr(self.object))

    def _get_name(self):
        return self._name

    name = property(_get_name, doc="The notification name. A string.")

    def _get_object(self):
        if self._objRef is not None:
            return self._objRef()
        return None

    object = property(_get_object, doc="The observable object the notification belongs to.")

    def _get_data(self):
        return self._data

    data = property(_get_data, doc="Arbitrary data passed along with the notification. There is no set format for this data and there is not requirement that any data be present. Refer to the documentation for methods that are responsible for generating notifications for information about this data.")
<commit_msg>- Fixed a bug in the notification hold method. (Thanks to Frederik for pointing this out.) - Added the ability to disable notifications for a specific observer. (Maybe it should be possible to hold notifications for a specific object as well?)<commit_after>import weakref

class NotificationCenter(object):

    def __init__(self):
        self._observables = {}
        self._holds = {}
        self._disabled = {}

    def addObserver(self, observer, methodName, notification, observable):
        """
        Add an observer to this notification dispatcher.

        * **observer** An object that can be referenced with weakref.
        * **methodName** A string epresenting the method to be called
          when the notification is posted.
        * **notification** The notification that teh observer should
          be notified of.
        * **observable** The object to observe.

        The method that will be called as a result of the action
        must accept a single *notification* argument. This will
        be a :class:`Notification` object.
        """
        if observable is not None:
            observable = weakref.ref(observable)
        observer = weakref.ref(observer)
        if notification not in self._observables:
            self._observables[notification] = {}
        observableDict = self._observables[notification]
        if observable not in observableDict:
            observableDict[observable] = {}
        observerDict = observableDict[observable]
        observerDict[observer] = methodName

    def removeObserver(self, observer, notification, observable):
        """
        Remove an observer from this notification dispatcher.

        * **observer** A registered object.
        * **notification** The notification that the observer was registered
          to be notified of.
        * **observable** The object being observed.
        """
        if observable is not None:
            observable = weakref.ref(observable)
        observer = weakref.ref(observer)
        del self._observables[notification][observable][observer]

    def hasObserver(self, observer, notification, observable):
        """
        Returns a boolean indicating is the **observer** is registered
        for **notification** posted by **observable**.
        """
        if observable is not None:
            observable = weakref.ref(observable)
        observer = weakref.ref(observer)
        if notification not in self._observables:
            return False
        if observable not in self._observables[notification]:
            return False
        if observer not in self._observables[notification][observable]:
            return False
        return True

    def postNotification(self, notification, observable, data=None):
        """
        Post a notification to all objects observing **notification**
        in **observable**.

        * **notification** The name of the notification.
        * **observable** The object that the notification belongs to.
        * **data** Arbitrary data that will be stored in the :class:`Notification` object.

        This will create a :class:`Notification` object and post it to
        all relevant observers.
        """
        observableObj = observable
        if observable is not None:
            observable = weakref.ref(observable)
        # disabled
        if (observable, notification) in self._disabled:
            return
        if observable in self._disabled:
            return
        # held
        if (observable, notification) in self._holds:
            n = (notification, observable, data)
            if not self._holds[observable, notification]["notifications"] or self._holds[observable, notification]["notifications"][-1] != n:
                self._holds[observable, notification]["notifications"].append(n)
            return
        if observable in self._holds:
            n = (notification, observable, data)
            if not self._holds[observable]["notifications"] or self._holds[observable]["notifications"][-1] != n:
                self._holds[observable]["notifications"].append(n)
            return
        # post
        if notification in self._observables:
            for observableRef, observerDict in self._observables[notification].items():
                if observable == observableRef:
                    for observerRef, methodName in observerDict.items():
                        observer = observerRef()
                        # disabled for observer
                        if self._disabled:
                            if (observableRef, notification, observerRef) in self._disabled:
                                continue
                            elif (observableRef, observerRef) in self._disabled:
                                continue
                            elif (notification, observerRef) in self._disabled:
                                continue
                        callback = getattr(observer, methodName)
                        n = Notification(notification, observableRef, data)
                        callback(n)

    def holdNotificationsForObservable(self, observable, notification=None):
        """
        Hold all notifications posted to all objects observing
        **notification** in **observable**.

        * **observable** The object that the notification belongs to.
        * **notification** The name of the notification. This is optional.
          If no *notification* is given, *all* notifications for *observable*
          will be held.

        Held notifications will be posted after the a matching *notification*
        and *observable* have been passed to :meth:`Notification.releaseHeldNotificationsForObservable`.
        This object will retain a count of how many times it has been told to
        hold notifications for *notification* and *observable*. It will not
        post the notifications until the *notification* and *observable*
        have been released the same number of times.
        """
        observable = weakref.ref(observable)
        if notification is not None:
            key = (observable, notification)
        else:
            key = observable
        if key not in self._holds:
            self._holds[key] = dict(holdCount=0, notifications=[])
        self._holds[key]["holdCount"] += 1

    def releaseHeldNotificationsForObservable(self, observable, notification=None):
        """
        Release all held notifications posted to all objects observing
        **notification** in **observable**.

        * **observable** The object that the notification belongs to.
        * **notification** The name of the notification. This is optional.
        """
        observableObj = observable
        observable = weakref.ref(observable)
        if notification is not None:
            key = (observable, notification)
        else:
            key = observable
        self._holds[key]["holdCount"] -= 1
        if self._holds[key]["holdCount"] == 0:
            notifications = self._holds[key]["notifications"]
            del self._holds[key]
            for notification, o, data in notifications:
                self.postNotification(notification, observableObj, data)

    def areNotificationsHeldForObservable(self, observable, notification=None):
        """
        Returns a boolean indicating if notifications posted to all objects observing
        **notification** in **observable** are being held.

        * **observable** The object that the notification belongs to.
        * **notification** The name of the notification. This is optional.
        """
        if (observable, notification) in self._holds:
            return True
        return weakref.ref(observable) in self._holds

    def disableNotificationsForObservable(self, observable, notification=None, observer=None):
        """
        Disable all posts of **notification** from **observable** posted
        to **observer** observing.

        * **observable** The object that the notification belongs to.
        * **notification** The name of the notification. This is optional.
          If no *notification* is given, *all* notifications for *observable*
          will be disabled for *observer*.
        * **observer** The specific observer to not send posts to. If no
          *observer* is given, the appropriate notifications will not
          be posted to any observers.

        This object will retain a count of how many times it has been told to
        disable notifications for *notification* and *observable*. It will not
        enable new notifications until the *notification* and *observable*
        have been released the same number of times.
        """
        observable = weakref.ref(observable)
        if observer is not None:
            observer = weakref.ref(observer)
        key = self._makeDisableKey(observable, notification=notification, observer=observer)
        if key not in self._disabled:
            self._disabled[key] = 0
        self._disabled[key] += 1

    def enableNotificationsForObservable(self, observable, notification=None, observer=None):
        """
        Enable notifications posted to all objects observing
        **notification** in **observable**.

        * **observable** The object that the notification belongs to.
        * **notification** The name of the notification. This is optional.
        * **observer** The observer. This is optional.
        """
        observableObj = observable
        observable = weakref.ref(observable)
        if observer is not None:
            observer = weakref.ref(observer)
        key = self._makeDisableKey(observable, notification=notification, observer=observer)
        self._disabled[key] -= 1
        if self._disabled[key] == 0:
            del self._disabled[key]

    def areNotificationsDisabledForObservable(self, observable, notification=None, observer=None):
        """
        Returns a boolean indicating if notifications posted to all objects observing
        **notification** in **observable** are disabled.

        * **observable** The object that the notification belongs to.
        * **notification** The name of the notification. This is optional.
        * **observer** The observer. This is optional.
        """
        observable = weakref.ref(observable)
        if observer is not None:
            observer = weakref.ref(observer)
        key = self._makeDisableKey(observable, notification=notification, observer=observer)
        return key in self._disabled

    def _makeDisableKey(self, observable, notification=None, observer=None):
        key = [observable]
        if notification is not None:
            key.append(notification)
        if observer is not None:
            key.append(observer)
        if len(key) == 1:
            key = key[0]
        else:
            key = tuple(key)
        return key

class Notification(object):

    """An object that wraps notification data."""

    def __init__(self, name, objRef, data):
        self._name = name
        self._objRef = objRef
        self._data = data

    def __repr__(self):
        return "<Notification: %s %s>" % (self.name, repr(self.object))

    def _get_name(self):
        return self._name

    name = property(_get_name, doc="The notification name. A string.")

    def _get_object(self):
        if self._objRef is not None:
            return self._objRef()
        return None

    object = property(_get_object, doc="The observable object the notification belongs to.")

    def _get_data(self):
        return self._data

    data = property(_get_data, doc="Arbitrary data passed along with the notification. There is no set format for this data and there is not requirement that any data be present. Refer to the documentation for methods that are responsible for generating notifications for information about this data.")
<|endoftext|>


<commit_before>import os
import glob
import time
import pipes
import shutil
import random
import tempfile
import hashlib
import logging
import collections
try:
    from collections import OrderedDict
except ImportError:
    from ordereddict import OrderedDict

import envoy
import jinjatagext

from utils import is_updated

logger = logging.getLogger('jinjastatic')

extensions = {
    'text/css': 'css',
    'text/javascript': 'js',
    }

pre_compilers = {
                # compiler, new mime type, new ext, on-demand compilation
    'text/less': ('lessc %(input)s', 'text/css', 'css', False),
    'text/coffeescript': ('coffee --join %(output)s -c %(input)s', 'text/javascript', 'js', True),
    }

ext_mime = {
    'less': 'text/less',
    'coffee': 'text/coffeescript',
    'js': 'text/javascript',
    'css': 'text/css',
}

compilers = {
    'text/javascript': './runyui.sh --nomunge %(input)s',
    'text/css': 'uglifycss %(input)s',
    }

g = {
    'debug': False,
    ('text/css', False): {},
    ('text/javascript', False): {},
    ('text/css', True): {},
    ('text/javascript', True): {},
    'temp_dir': tempfile.mkdtemp(prefix='jinjastatic'),
    'config': {},
    'compiled': {},
    'compsheets': {},
    'minified': {},
}

def _handle_tag(type_, ctx, src, debug=False, head=False, **kwargs):
    kwargs.setdefault('type', type_)
    format = style_formats[type_]
    if g['debug']:
        if type_ in pre_compilers and pre_compilers[type_][3]:
            src = rename_ext(src, pre_compilers[type_][2])
            kwargs['type'] = pre_compilers[type_][1]
        return format.format(
            src,
            u' '.join('{0}={1!r}'.format(k, _force_str(v)) for k, v in kwargs.items()))
    elif debug:
        return u''
    key = (type_, head)
    compiled_key = (pre_compilers.get(type_, (None, type_))[1], head)
    format = style_formats[compiled_key[0]]
    ctxname = ctx.name
    min_dict = g['minified'].setdefault(ctxname, {})

    if min_dict.get(compiled_key):
        return ''
    elif ctxname in g.get(compiled_key, {}) and g['compiled']:
        files = OrderedDict([(g['compiled'][orig], 1)
                             for orig in g[compiled_key][ctxname]]).keys()
        min_dict[compiled_key] = True
        return u'\n'.join(format.format(src, 'type="{0}"'.format(compiled_key[0]))
                          for src in files)
    elif type_ in pre_compilers:
        pre_compile(src, type_, head, ctxname)
        return u''
    else:
        g[key].setdefault(ctxname, []).append(src.lstrip('/'))
    return '**FIRSTPASS**'

style_formats = {
    'text/javascript': u'<script src="{0}" {1}></script>',
    'text/css': u'<link rel="stylesheet" href="{0}" {1}>',
    'text/less': u'<link rel="stylesheet/less" href="{0}" {1}>',
    'text/coffeescript': u'<script src="{0}" {1}></script>',
}

@jinjatagext.simple_context_tag
def script(ctx, src, **kwargs):
    return _handle_tag(u'text/javascript', ctx,
                       src, **kwargs)

@jinjatagext.simple_context_tag
def style(ctx, href, **kwargs):
    return _handle_tag(u'text/css', ctx, href, **kwargs)

@jinjatagext.simple_context_tag
def less(ctx, href, **kwargs):
    return _handle_tag(u'text/less', ctx, href, **kwargs)

@jinjatagext.simple_context_tag
def coffee(ctx, src, **kwargs):
    return _handle_tag(u'text/coffeescript', ctx,
                       src, **kwargs)

def clear_data():
    try:
        shutil.rmtree(g['temp_dir'])
    except:
        pass
    g.update({
            ('text/css', False): {},
            ('text/javascript', False): {},
            ('text/css', True): {},
            ('text/javascript', True): {},
            'compiled': {},
            'temp_dir': tempfile.mkdtemp(prefix='jinjastatic'),
            'minified': {},
            })

def set_config(debug, config, base_dir):
    g['debug'] = debug
    g['base_dir'] = base_dir
    mapper = config.get('map', {})
    config['map'] = {}
    for k, v in mapper.items():
        for f in v:
            config['map'][f] = k
    g['config'] = config

def compile(base_dir, output_dir, dest_dir):
    config = g['config']
    _remove_old_files(output_dir)

    rel_output = '/' + output_dir[len(dest_dir):].lstrip('/')

    static_dirs = set()

    for key in g:
        if not isinstance(key, tuple):
            continue
        unique_key = _gen_key()
        filemap = g[key]
        ext = extensions[key[0]]
        files = OrderedDict((filename, 1) for template in filemap for filename in filemap[template]).keys()
        file_comp = collections.defaultdict(list)
        for filename in files:
            aggregate = _decorate_key(config['map'].get(filename, 'maincompiled.' + ext), unique_key)
            file_comp[aggregate].append(filename)
        for target, filelist in file_comp.items():
            abstarget = os.path.join(output_dir, target)
            absfilelist = [os.path.join(base_dir, filename) for filename in filelist]
            logger.info('Compiling {0}'.format(abstarget))
            combined_file_obj = None
            if len(absfilelist) == 1:
                combined_file = absfilelist[0]
            else:
                combined_file_obj = _combine_files(absfilelist, ext)
                combined_file = combined_file_obj.name

            compiler_fmt = compilers[key[0]]
            if '%(input)s' in compiler_fmt:
                data = None
                cmd = compiler_fmt % {'input': combined_file}
            else:
                data = read_file_data([combined_file])
                cmd = compiler_fmt
            output = envoy.run(cmd, data=data)
            if output.status_code:
                raise RuntimeError(output.std_err)
            with open(abstarget, 'wb+') as f:
                f.write(output.std_out)
            target = os.path.join(rel_output, target)
            g['compiled'].update(dict((filename, target) for filename in filelist))

            if key[0] == 'text/css':
                static_dirs.update(os.path.dirname(filename) for filename in absfilelist)

    for d in static_dirs:
        for dirpath, dirnames, filenames in os.walk(d, followlinks=True):
            reldir = dirpath[len(d):].lstrip('/')
            target_dir = os.path.join(output_dir, reldir)
            for filepath in filenames:
                target_path = os.path.join(target_dir, filepath)
                filepath = os.path.join(dirpath, filepath)
                if is_updated(filepath, target_path):
                    new_dir = os.path.dirname(target_path)
                    if not os.path.exists(new_dir):
                        os.makedirs(new_dir)
                    shutil.copyfile(filepath, target_path)

g = {
    'debug': False,
    ('text/css', False): {},
    ('text/javascript', False): {},
    ('text/css', True): {},
    ('text/javascript', True): {},
    'temp_dir': tempfile.mkdtemp(prefix='jinjastatic'),
    'config': {},
    'compiled': {},
    'compsheets': {},
    'minified': {},
}

def pre_compile(src, type_, head, ctxname):
    compiler, type_, ext = pre_compilers[type_][:3]
    key = (type_, head)
    script_list = g[key].setdefault(ctxname, [])
    old_file = os.path.join(g['base_dir'], src.lstrip('/'))
    new_name = os.path.join(os.path.dirname(src).strip('/'), 'compiled-' + hashlib.md5(src).hexdigest()) + "." + ext

    if new_name in script_list:
        return

    script_list.append(new_name)

    new_name = os.path.join(g['base_dir'], new_name.lstrip('/'))

    if not is_updated(old_file, new_name):
        return

    _run_precompile(old_file, new_name, compiler)

def _run_precompile(old_file, new_file, compiler):
    logger.info('Pre-compiling {0} -> {1}'.format(old_file, new_file))

    params = {'input': pipes.quote(old_file)}
    use_stdout = True

    if '%(output)s' in compiler:
        use_stdout = False
        params['output'] = pipes.quote(new_file)
    output = envoy.run(compiler % params)
    if output.status_code:
        raise RuntimeError(output.std_err)

    if not use_stdout:
        return
    with open(new_file, 'wb+') as f:
        f.write(output.std_out)
    return

def handle_precompile_file(source, dest, incremental=False):
    if '.' not in os.path.basename(source):
        return False
    ext = source.rsplit('.', 1)[1]
    if not (ext in ext_mime and ext_mime[ext] in pre_compilers):
        return False
    compiler, type_, new_ext, incremental = pre_compilers[ext_mime[ext]][:4]
    if not incremental:
        return
    dest = rename_ext(dest, new_ext)
    if incremental and not is_updated(source, dest):
        return False
    _run_precompile(source, dest, compiler)
    return True

def read_file_data(filelist):
    result = []
    for fname in filelist:
        with open(fname) as f:
            result.append(f.read())
    return ''.join(result)

def _remove_old_files(output_dir):
    for fname in glob.glob(os.path.join(output_dir, '*_min*')):
        os.unlink(fname)

def _combine_files(filelist, ext):
    tmp = tempfile.NamedTemporaryFile(suffix='.' + ext)
    for filename in filelist:
        with open(filename, 'rb') as f:
            shutil.copyfileobj(f, tmp)
        if ext.lower() == 'js':
            tmp.write(';')
    tmp.flush()
    return tmp

def _decorate_key(filename, key):
    first, second = filename.rsplit('.', 1)
    return first + '-' + key + '.' + second

def _gen_key():
    return hashlib.md5(str(time.time() + random.random())).hexdigest()[:10] + '_min'

def _force_str(obj):
    if isinstance(obj, unicode):
        return obj.encode('utf8')
    return str(obj)

def rename_ext(filename, new_ext):
    if '.' not in os.path.basename(filename):
        return filename + '.' + new_ext
    else:
        return filename.rsplit('.', 1)[0] + '.' + new_ext
<commit_msg>Refs #1. Better error messages for missing compilers.<commit_after>import os
import glob
import time
import pipes
import shutil
import random
import tempfile
import hashlib
import logging
import collections
try:
    from collections import OrderedDict
except ImportError:
    from ordereddict import OrderedDict

import envoy
import jinjatagext

from utils import is_updated

logger = logging.getLogger('jinjastatic')

extensions = {
    'text/css': 'css',
    'text/javascript': 'js',
    }

pre_compilers = {
                # compiler, new mime type, new ext, on-demand compilation
    'text/less': ('lessc %(input)s', 'text/css', 'css', False),
    'text/coffeescript': ('coffee --join %(output)s -c %(input)s', 'text/javascript', 'js', True),
    }

ext_mime = {
    'less': 'text/less',
    'coffee': 'text/coffeescript',
    'js': 'text/javascript',
    'css': 'text/css',
}

compilers = {
    'text/javascript': './runyui.sh --nomunge %(input)s',
    'text/css': 'uglifycss %(input)s',
    }

g = {
    'debug': False,
    ('text/css', False): {},
    ('text/javascript', False): {},
    ('text/css', True): {},
    ('text/javascript', True): {},
    'temp_dir': tempfile.mkdtemp(prefix='jinjastatic'),
    'config': {},
    'compiled': {},
    'compsheets': {},
    'minified': {},
}

def _handle_tag(type_, ctx, src, debug=False, head=False, **kwargs):
    kwargs.setdefault('type', type_)
    format = style_formats[type_]
    if g['debug']:
        if type_ in pre_compilers and pre_compilers[type_][3]:
            src = rename_ext(src, pre_compilers[type_][2])
            kwargs['type'] = pre_compilers[type_][1]
        return format.format(
            src,
            u' '.join('{0}={1!r}'.format(k, _force_str(v)) for k, v in kwargs.items()))
    elif debug:
        return u''
    key = (type_, head)
    compiled_key = (pre_compilers.get(type_, (None, type_))[1], head)
    format = style_formats[compiled_key[0]]
    ctxname = ctx.name
    min_dict = g['minified'].setdefault(ctxname, {})

    if min_dict.get(compiled_key):
        return ''
    elif ctxname in g.get(compiled_key, {}) and g['compiled']:
        files = OrderedDict([(g['compiled'][orig], 1)
                             for orig in g[compiled_key][ctxname]]).keys()
        min_dict[compiled_key] = True
        return u'\n'.join(format.format(src, 'type="{0}"'.format(compiled_key[0]))
                          for src in files)
    elif type_ in pre_compilers:
        pre_compile(src, type_, head, ctxname)
        return u''
    else:
        g[key].setdefault(ctxname, []).append(src.lstrip('/'))
    return '**FIRSTPASS**'

style_formats = {
    'text/javascript': u'<script src="{0}" {1}></script>',
    'text/css': u'<link rel="stylesheet" href="{0}" {1}>',
    'text/less': u'<link rel="stylesheet/less" href="{0}" {1}>',
    'text/coffeescript': u'<script src="{0}" {1}></script>',
}

@jinjatagext.simple_context_tag
def script(ctx, src, **kwargs):
    return _handle_tag(u'text/javascript', ctx,
                       src, **kwargs)

@jinjatagext.simple_context_tag
def style(ctx, href, **kwargs):
    return _handle_tag(u'text/css', ctx, href, **kwargs)

@jinjatagext.simple_context_tag
def less(ctx, href, **kwargs):
    return _handle_tag(u'text/less', ctx, href, **kwargs)

@jinjatagext.simple_context_tag
def coffee(ctx, src, **kwargs):
    return _handle_tag(u'text/coffeescript', ctx,
                       src, **kwargs)

def clear_data():
    try:
        shutil.rmtree(g['temp_dir'])
    except:
        pass
    g.update({
            ('text/css', False): {},
            ('text/javascript', False): {},
            ('text/css', True): {},
            ('text/javascript', True): {},
            'compiled': {},
            'temp_dir': tempfile.mkdtemp(prefix='jinjastatic'),
            'minified': {},
            })

def set_config(debug, config, base_dir):
    g['debug'] = debug
    g['base_dir'] = base_dir
    mapper = config.get('map', {})
    config['map'] = {}
    for k, v in mapper.items():
        for f in v:
            config['map'][f] = k
    g['config'] = config

def compile(base_dir, output_dir, dest_dir):
    config = g['config']
    _remove_old_files(output_dir)

    rel_output = '/' + output_dir[len(dest_dir):].lstrip('/')

    static_dirs = set()

    for key in g:
        if not isinstance(key, tuple):
            continue
        unique_key = _gen_key()
        filemap = g[key]
        ext = extensions[key[0]]
        files = OrderedDict((filename, 1) for template in filemap for filename in filemap[template]).keys()
        file_comp = collections.defaultdict(list)
        for filename in files:
            aggregate = _decorate_key(config['map'].get(filename, 'maincompiled.' + ext), unique_key)
            file_comp[aggregate].append(filename)
        for target, filelist in file_comp.items():
            abstarget = os.path.join(output_dir, target)
            absfilelist = [os.path.join(base_dir, filename) for filename in filelist]
            logger.info('Compiling {0}'.format(abstarget))
            combined_file_obj = None
            if len(absfilelist) == 1:
                combined_file = absfilelist[0]
            else:
                combined_file_obj = _combine_files(absfilelist, ext)
                combined_file = combined_file_obj.name

            compiler_fmt = compilers[key[0]]
            if '%(input)s' in compiler_fmt:
                data = None
                cmd = compiler_fmt % {'input': combined_file}
            else:
                data = read_file_data([combined_file])
                cmd = compiler_fmt
            output = envoy.run(cmd, data=data)
            if output.status_code:
                sys.stderr.write("Error while running command: {0}\n".format(cmd))
                raise RuntimeError(output.std_err)
            with open(abstarget, 'wb+') as f:
                f.write(output.std_out)
            target = os.path.join(rel_output, target)
            g['compiled'].update(dict((filename, target) for filename in filelist))

            if key[0] == 'text/css':
                static_dirs.update(os.path.dirname(filename) for filename in absfilelist)

    for d in static_dirs:
        for dirpath, dirnames, filenames in os.walk(d, followlinks=True):
            reldir = dirpath[len(d):].lstrip('/')
            target_dir = os.path.join(output_dir, reldir)
            for filepath in filenames:
                target_path = os.path.join(target_dir, filepath)
                filepath = os.path.join(dirpath, filepath)
                if is_updated(filepath, target_path):
                    new_dir = os.path.dirname(target_path)
                    if not os.path.exists(new_dir):
                        os.makedirs(new_dir)
                    shutil.copyfile(filepath, target_path)

g = {
    'debug': False,
    ('text/css', False): {},
    ('text/javascript', False): {},
    ('text/css', True): {},
    ('text/javascript', True): {},
    'temp_dir': tempfile.mkdtemp(prefix='jinjastatic'),
    'config': {},
    'compiled': {},
    'compsheets': {},
    'minified': {},
}

def pre_compile(src, type_, head, ctxname):
    compiler, type_, ext = pre_compilers[type_][:3]
    key = (type_, head)
    script_list = g[key].setdefault(ctxname, [])
    old_file = os.path.join(g['base_dir'], src.lstrip('/'))
    new_name = os.path.join(os.path.dirname(src).strip('/'), 'compiled-' + hashlib.md5(src).hexdigest()) + "." + ext

    if new_name in script_list:
        return

    script_list.append(new_name)

    new_name = os.path.join(g['base_dir'], new_name.lstrip('/'))

    if not is_updated(old_file, new_name):
        return

    _run_precompile(old_file, new_name, compiler)

def _run_precompile(old_file, new_file, compiler):
    logger.info('Pre-compiling {0} -> {1}'.format(old_file, new_file))

    params = {'input': pipes.quote(old_file)}
    use_stdout = True

    if '%(output)s' in compiler:
        use_stdout = False
        params['output'] = pipes.quote(new_file)
    output = envoy.run(compiler % params)
    if output.status_code:
        sys.stderr.write("Error while running command: {0}\n".format(compiler % params))
        raise RuntimeError(output.std_err)

    if not use_stdout:
        return
    with open(new_file, 'wb+') as f:
        f.write(output.std_out)
    return

def handle_precompile_file(source, dest, incremental=False):
    if '.' not in os.path.basename(source):
        return False
    ext = source.rsplit('.', 1)[1]
    if not (ext in ext_mime and ext_mime[ext] in pre_compilers):
        return False
    compiler, type_, new_ext, incremental = pre_compilers[ext_mime[ext]][:4]
    if not incremental:
        return
    dest = rename_ext(dest, new_ext)
    if incremental and not is_updated(source, dest):
        return False
    _run_precompile(source, dest, compiler)
    return True

def read_file_data(filelist):
    result = []
    for fname in filelist:
        with open(fname) as f:
            result.append(f.read())
    return ''.join(result)

def _remove_old_files(output_dir):
    for fname in glob.glob(os.path.join(output_dir, '*_min*')):
        os.unlink(fname)

def _combine_files(filelist, ext):
    tmp = tempfile.NamedTemporaryFile(suffix='.' + ext)
    for filename in filelist:
        with open(filename, 'rb') as f:
            shutil.copyfileobj(f, tmp)
        if ext.lower() == 'js':
            tmp.write(';')
    tmp.flush()
    return tmp

def _decorate_key(filename, key):
    first, second = filename.rsplit('.', 1)
    return first + '-' + key + '.' + second

def _gen_key():
    return hashlib.md5(str(time.time() + random.random())).hexdigest()[:10] + '_min'

def _force_str(obj):
    if isinstance(obj, unicode):
        return obj.encode('utf8')
    return str(obj)

def rename_ext(filename, new_ext):
    if '.' not in os.path.basename(filename):
        return filename + '.' + new_ext
    else:
        return filename.rsplit('.', 1)[0] + '.' + new_ext
<|endoftext|>


<commit_before>#!/usr/bin/env python

#
#   BSD LICENSE
#
#   Copyright(c) 2010-2014 Intel Corporation. All rights reserved.
#   All rights reserved.
#
#   Redistribution and use in source and binary forms, with or without
#   modification, are permitted provided that the following conditions
#   are met:
#
#     * Redistributions of source code must retain the above copyright
#       notice, this list of conditions and the following disclaimer.
#     * Redistributions in binary form must reproduce the above copyright
#       notice, this list of conditions and the following disclaimer in
#       the documentation and/or other materials provided with the
#       distribution.
#     * Neither the name of Intel Corporation nor the names of its
#       contributors may be used to endorse or promote products derived
#       from this software without specific prior written permission.
#
#   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
#   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
#   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
#   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
#   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
#   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
#   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
#   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
#   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
#   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
#   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#
from __future__ import print_function
import sys

sockets = []
cores = []
core_map = {}

fd = open("/proc/cpuinfo")
lines = fd.readlines()
fd.close()

core_details = []
core_lines = {}
for line in lines:
    if len(line.strip()) != 0:
        name, value = line.split(":", 1)
        core_lines[name.strip()] = value.strip()
    else:
        core_details.append(core_lines)
        core_lines = {}

for core in core_details:
    for field in ["processor", "core id", "physical id"]:
        if field not in core:
            print("Error getting '%s' value from /proc/cpuinfo" % field)
            sys.exit(1)
        core[field] = int(core[field])

    if core["core id"] not in cores:
        cores.append(core["core id"])
    if core["physical id"] not in sockets:
        sockets.append(core["physical id"])
    key = (core["physical id"], core["core id"])
    if key not in core_map:
        core_map[key] = []
    core_map[key].append(core["processor"])

print("============================================================")
print("Core and Socket Information (as reported by '/proc/cpuinfo')")
print("============================================================\n")
print("cores = ", cores)
print("sockets = ", sockets)
print("")

max_processor_len = len(str(len(cores) * len(sockets) * 2 - 1))
max_core_map_len = max_processor_len * 2 + len('[, ]') + len('Socket ')
max_core_id_len = len(str(max(cores)))

output = " ".ljust(max_core_id_len + len('Core '))
for s in sockets:
    output += " Socket %s" % str(s).ljust(max_core_map_len - len('Socket '))
print(output)

output = " ".ljust(max_core_id_len + len('Core '))
for s in sockets:
    output += " --------".ljust(max_core_map_len)
    output += " "
print(output)

for c in cores:
    output = "Core %s" % str(c).ljust(max_core_id_len)
    for s in sockets:
        output += " " + str(core_map[(s, c)]).ljust(max_core_map_len)
    print(output)
<commit_msg>usertools: use sysfs for CPU layout<commit_after>#!/usr/bin/env python

#
#   BSD LICENSE
#
#   Copyright(c) 2010-2014 Intel Corporation. All rights reserved.
#   Copyright(c) 2017 Cavium Networks Ltd. All rights reserved.
#   All rights reserved.
#
#   Redistribution and use in source and binary forms, with or without
#   modification, are permitted provided that the following conditions
#   are met:
#
#     * Redistributions of source code must retain the above copyright
#       notice, this list of conditions and the following disclaimer.
#     * Redistributions in binary form must reproduce the above copyright
#       notice, this list of conditions and the following disclaimer in
#       the documentation and/or other materials provided with the
#       distribution.
#     * Neither the name of Intel Corporation nor the names of its
#       contributors may be used to endorse or promote products derived
#       from this software without specific prior written permission.
#
#   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
#   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
#   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
#   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
#   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
#   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
#   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
#   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
#   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
#   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
#   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#
from __future__ import print_function
import sys

sockets = []
cores = []
core_map = {}
base_path = "/sys/devices/system/cpu"
fd = open("{}/kernel_max".format(base_path))
max_cpus = int(fd.read())
fd.close()
for cpu in xrange(max_cpus + 1):
    try:
        fd = open("{}/cpu{}/topology/core_id".format(base_path, cpu))
    except:
        break
    core = int(fd.read())
    fd.close()
    fd = open("{}/cpu{}/topology/physical_package_id".format(base_path, cpu))
    socket = int(fd.read())
    fd.close()
    if core not in cores:
        cores.append(core)
    if socket not in sockets:
        sockets.append(socket)
    key = (socket, core)
    if key not in core_map:
        core_map[key] = []
    core_map[key].append(cpu)

print(format("=" * (47 + len(base_path))))
print("Core and Socket Information (as reported by '{}')".format(base_path))
print("{}\n".format("=" * (47 + len(base_path))))
print("cores = ", cores)
print("sockets = ", sockets)
print("")

max_processor_len = len(str(len(cores) * len(sockets) * 2 - 1))
max_core_map_len = max_processor_len * 2 + len('[, ]') + len('Socket ')
max_core_id_len = len(str(max(cores)))

output = " ".ljust(max_core_id_len + len('Core '))
for s in sockets:
    output += " Socket %s" % str(s).ljust(max_core_map_len - len('Socket '))
print(output)

output = " ".ljust(max_core_id_len + len('Core '))
for s in sockets:
    output += " --------".ljust(max_core_map_len)
    output += " "
print(output)

for c in cores:
    output = "Core %s" % str(c).ljust(max_core_id_len)
    for s in sockets:
        output += " " + str(core_map[(s, c)]).ljust(max_core_map_len)
    print(output)
<|endoftext|>


<commit_before>#!/usr/bin/env python
import optparse
from itertools import chain
import json
from urllib import quote_plus

from restkit import Resource, BasicAuth, request

# Using REST is pretty simple. The vast majority of this code is about the "other stuff": dealing with
# command line options, formatting graphviz, calling Google Charts, etc. The actual JIRA REST-specific code
# is only about 5 lines.

GOOGLE_CHART_URL = 'http://chart.apis.google.com/chart?'

def fetcher_factory(url, auth):
    """ This factory will create the actual method used to fetch issues from JIRA. This is really just a closure that saves us having
        to pass a bunch of parameters all over the place all the time. """
    def get_issue(key):
        """ Given an issue key (i.e. JRA-9) return the JSON representation of it. This is the only place where we deal
            with JIRA's REST API. """
        print('Fetching ' + key)
        # we need to expand subtasks and links since that's what we care about here.
        resource = Resource(url + ('/rest/api/latest/issue/%s' % key), filters=[auth])
        response = resource.get(headers = {'Content-Type' : 'application/json'})
        if response.status_int == 200:
            # Not all resources will return 200 on success. There are other success status codes. Like 204. We've read
            # the documentation for though and know what to expect here.
            issue = json.loads(response.body_string())
            return issue
        else:
            return None
    return get_issue

def build_graph_data(start_issue_key, get_issue):
    """ Given a starting image key and the issue-fetching function build up the GraphViz data representing relationships
        between issues. This will consider both subtasks and issue links.
    """
    def get_key(issue):
        return issue['key']

    # since the graph can be cyclic we need to prevent infinite recursion
    seen = []

    def walk(issue_key, graph):
        """ issue is the JSON representation of the issue """
        issue = get_issue(issue_key)
        seen.append(issue_key)
        children = []
        fields = issue['fields']
        if fields.has_key('subtasks'):
            for other_issue in fields['subtasks']:
                node = '"%s"->"%s"[color=blue][penwidth=2.0]' % (issue_key, get_key(other_issue))
                graph.append(node)
                children.append(get_key(other_issue))
        if fields.has_key('issuelinks'):
            for other_link in fields['issuelinks']:
                # Only add graphviz data for outbound links since that will also draw the corresponding inbound link
                if other_link.has_key('outwardIssue'):
                    other_issue = other_link['outwardIssue']
                    other_issue_key = get_key(other_issue)
                    link_type = other_link['type']['outward']
                    print(issue_key + ' => ' + link_type + ' => ' + other_issue_key)
                    node = '"%s"->"%s"[arrowhead=empty][label="%s"]' % (issue_key, other_issue_key, quote_plus(link_type))
                    graph.append(node)
                    children.append(other_issue_key)
                if other_link.has_key('inwardIssue'):
                    other_issue = other_link['inwardIssue']
                    other_issue_key = get_key(other_issue)
                    link_type = other_link['type']['inward']
                    print(issue_key + ' <= ' + link_type + ' <= ' + other_issue_key)
                    node = '"%s"->"%s"[arrowhead=empty][label="%s"]' % (issue_key, other_issue_key, quote_plus(link_type))
                    graph.append(node)
                    children.append(other_issue_key)
        # now construct graph data for all subtasks and links of this issue
        for child in (x for x in children if x not in seen):
            walk(child, graph)
        return graph

    graph = walk(start_issue_key, [])
    return graph

def create_graph_image(graph_data, image_file):
    """ Given a formatted blob of graphviz chart data[1], make the actual request to Google
        and store the resulting image to disk.

        [1]: http://code.google.com/apis/chart/docs/gallery/graphviz.html
    """
    chart_url = GOOGLE_CHART_URL + 'cht=gv&chl=digraph{%s}' % ';'.join(graph_data)

    print('Google Chart request:')
    print(chart_url)

    g = request(chart_url)

    print('Writing to ' + image_file)
    image = open(image_file, 'w+')
    image.write(g.body_stream().read())
    image.close()
    return image_file

def parse_args():
    parser = optparse.OptionParser()
    parser.add_option('-u', '--user', dest='user', default='admin', help='Username to access JIRA')
    parser.add_option('-p', '--password', dest='password', default='admin', help='Password to access JIRA')
    parser.add_option('-j', '--jira', dest='jira_url', default='http://jira.example.com', help='JIRA Base URL')
    parser.add_option('-f', '--file', dest='image_file', default='issue_graph.png', help='Filename to write image to')

    return parser.parse_args()

if __name__ == '__main__':
    (options, args) = parse_args()

    # Basic Auth is usually easier for scripts like this to deal with than Cookies.
    auth = BasicAuth(options.user, options.password)
    issue_fetcher = fetcher_factory(options.jira_url, auth)

    if len(args) != 1:
        print('Must specify exactly one issue key. (e.g. JRADEV-1107, JRADEV-1391)')
        import sys; sys.exit(1)
    start_issue_key = args[0]

    graph = build_graph_data(start_issue_key, issue_fetcher)
    create_graph_image(graph, options.image_file)

<commit_msg>little refactoring for links<commit_after>#!/usr/bin/env python
import optparse
from itertools import chain
import json
from urllib import quote_plus

from restkit import Resource, BasicAuth, request

# Using REST is pretty simple. The vast majority of this code is about the "other stuff": dealing with
# command line options, formatting graphviz, calling Google Charts, etc. The actual JIRA REST-specific code
# is only about 5 lines.

GOOGLE_CHART_URL = 'http://chart.apis.google.com/chart?'

def fetcher_factory(url, auth):
    """ This factory will create the actual method used to fetch issues from JIRA. This is really just a closure that saves us having
        to pass a bunch of parameters all over the place all the time. """
    def get_issue(key):
        """ Given an issue key (i.e. JRA-9) return the JSON representation of it. This is the only place where we deal
            with JIRA's REST API. """
        print('Fetching ' + key)
        # we need to expand subtasks and links since that's what we care about here.
        resource = Resource(url + ('/rest/api/latest/issue/%s' % key), filters=[auth])
        response = resource.get(headers = {'Content-Type' : 'application/json'})
        if response.status_int == 200:
            # Not all resources will return 200 on success. There are other success status codes. Like 204. We've read
            # the documentation for though and know what to expect here.
            issue = json.loads(response.body_string())
            return issue
        else:
            return None
    return get_issue

def build_graph_data(start_issue_key, get_issue):
    """ Given a starting image key and the issue-fetching function build up the GraphViz data representing relationships
        between issues. This will consider both subtasks and issue links.
    """
    def get_key(issue):
        return issue['key']

    def process_link(issue_key, link):
        if link.has_key('outwardIssue'):
            direction = 'outward'
        elif link.has_key('inwardIssue'):
            direction = 'inward'
        else:
            return

        linked_issue = link[direction + 'Issue']
        linked_issue_key = get_key(linked_issue)
        link_type = link['type'][direction]

        if direction == 'outward':
            print(issue_key + ' => ' + link_type + ' => ' + linked_issue_key)
        else:
            print(issue_key + ' <= ' + link_type + ' <= ' + linked_issue_key)

        node = '"%s"->"%s"[arrowhead=empty][label="%s"]' % (issue_key, linked_issue_key, quote_plus(link_type))
        return linked_issue_key, node

    # since the graph can be cyclic we need to prevent infinite recursion
    seen = []

    def walk(issue_key, graph):
        """ issue is the JSON representation of the issue """
        issue = get_issue(issue_key)
        seen.append(issue_key)
        children = []
        fields = issue['fields']
        if fields.has_key('subtasks'):
            for other_issue in fields['subtasks']:
                node = '"%s"->"%s"[color=blue][penwidth=2.0]' % (issue_key, get_key(other_issue))
                graph.append(node)
                children.append(get_key(other_issue))
        if fields.has_key('issuelinks'):
            for other_link in fields['issuelinks']:
                result = process_link(issue_key, other_link)
                if result is not None:
                    children.append(result[0])
                    graph.append(result[1])
        # now construct graph data for all subtasks and links of this issue
        for child in (x for x in children if x not in seen):
            walk(child, graph)
        return graph

    graph = walk(start_issue_key, [])
    return graph

def create_graph_image(graph_data, image_file):
    """ Given a formatted blob of graphviz chart data[1], make the actual request to Google
        and store the resulting image to disk.

        [1]: http://code.google.com/apis/chart/docs/gallery/graphviz.html
    """
    chart_url = GOOGLE_CHART_URL + 'cht=gv&chl=digraph{%s}' % ';'.join(graph_data)

    print('Google Chart request:')
    print(chart_url)

    g = request(chart_url)

    print('Writing to ' + image_file)
    image = open(image_file, 'w+')
    image.write(g.body_stream().read())
    image.close()
    return image_file

def parse_args():
    parser = optparse.OptionParser()
    parser.add_option('-u', '--user', dest='user', default='admin', help='Username to access JIRA')
    parser.add_option('-p', '--password', dest='password', default='admin', help='Password to access JIRA')
    parser.add_option('-j', '--jira', dest='jira_url', default='http://jira.example.com', help='JIRA Base URL')
    parser.add_option('-f', '--file', dest='image_file', default='issue_graph.png', help='Filename to write image to')

    return parser.parse_args()

if __name__ == '__main__':
    (options, args) = parse_args()

    # Basic Auth is usually easier for scripts like this to deal with than Cookies.
    auth = BasicAuth(options.user, options.password)
    issue_fetcher = fetcher_factory(options.jira_url, auth)

    if len(args) != 1:
        print('Must specify exactly one issue key. (e.g. JRADEV-1107, JRADEV-1391)')
        import sys; sys.exit(1)
    start_issue_key = args[0]

    graph = build_graph_data(start_issue_key, issue_fetcher)
    create_graph_image(graph, options.image_file)

<|endoftext|>


<commit_before># Copyright 2017 AT&T Intellectual Property.  All other rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

substitution_schema = {
    'type': 'object',
    'properties': {
        'dest': {'type': 'string'},
        'src': {
            'type': 'object',
            'properties': {
                'apiVersion': {
                    'type': 'string',
                    'choices': ['deckhand/v1']
                },
                'kind': {'type': 'string'},
                'name': {'type': 'string'}
            },
            'additionalProperties': False,
            'required': ['apiVersion', 'kind', 'name']
        }
    },
    'additionalProperties': False,
    'required': ['dest', 'src']
}

schema = {
    'type': 'object',
    'properties': {
        'apiVersion': {
            'type': 'string',
            'pattern': '^([A-Za-z]+\/v[0-9]{1})$'
        },
        'kind': {
            'type': 'string',
            'pattern': '^([A-Za-z]+$'
        },
        'metadata': {
            'type': 'object',
            'properties': {
                'name': {'type': 'string'},
                'storage': {'type': 'string'},
                'substitutions': {
                    'type': 'array',
                    'items': substitution_schema
                }
            },
            'additionalProperties': False,
            'required': ['name', 'storage', 'substitutions']
        },
        'data': {
            'type': 'object'
        }
    },
    'additionalProperties': False,
    'required': ['apiVersion', 'kind', 'metadata', 'data']
}
<commit_msg>Trivial fix to default_schema<commit_after># Copyright 2017 AT&T Intellectual Property.  All other rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

substitution_schema = {
    'type': 'object',
    'properties': {
        'dest': {'type': 'string'},
        'src': {
            'type': 'object',
            'properties': {
                'apiVersion': {
                    'type': 'string',
                    'choices': ['deckhand/v1']
                },
                'kind': {'type': 'string'},
                'name': {'type': 'string'}
            },
            'additionalProperties': False,
            'required': ['apiVersion', 'kind', 'name']
        }
    },
    'additionalProperties': False,
    'required': ['dest', 'src']
}

schema = {
    'type': 'object',
    'properties': {
        'apiVersion': {
            'type': 'string',
            'pattern': '^([A-Za-z]+\/v[0-9]{1})$'
        },
        'kind': {
            'type': 'string',
            'pattern': '^([A-Za-z]+)$'
        },
        'metadata': {
            'type': 'object',
            'properties': {
                'name': {'type': 'string'},
                'storage': {'type': 'string'},
                'substitutions': {
                    'type': 'array',
                    'items': substitution_schema
                }
            },
            'additionalProperties': False,
            'required': ['name', 'storage', 'substitutions']
        },
        'data': {
            'type': 'object'
        }
    },
    'additionalProperties': False,
    'required': ['apiVersion', 'kind', 'metadata', 'data']
}
<|endoftext|>


<commit_before>### Generates en/de/fr/ja translations for a given fflogs report.

import argparse
from collections import defaultdict
from datetime import datetime
import fflogs
import json
import re
import sys

# 'en' here is 'www' which we consider the "base" and do automatically.
# 'cn' exists on fflogs but does not have proper translations, sorry.
languages = ['en', 'de', 'fr', 'ja']
prefixes = {
  'en': 'www',
  'de': 'de',
  'fr': 'fr',
  'ja': 'ja',
}
default_language = 'en'
ignore_abilities = ['Attack']
ignore_effects = [
  'Brink Of Death',
  'Sprint',
  'Regen',
  'Vulnerability Up',
  'Weakness',
]

def find_start_end_time(report, args):
  # Get the start and end timestamps for the specific fight
  start_time = 0
  end_time = 0
  fight_id = 0
  for fight in report['fights']:
    if args.fight and fight['id'] == args.fight:
      start_time = fight['start_time']
      end_time = fight['end_time']
      fight_id = fight['id']
      break
    elif fight['end_time'] - fight['start_time'] > end_time - start_time:
      start_time = fight['start_time']
      end_time = fight['end_time']
      fight_id = fight['id']

  if args.fight and not fight_id:
      raise Exception('Fight ID not found in report')

  return start_time, end_time, fight_id

def add_default_ability_mappings(ability_replace):
  # Many timelines have this, so just include it by default.
  ability_replace['de']['Engage!'] = 'Start!'
  ability_replace['fr']['Engage!'] = "À l'attaque"
  ability_replace['ja']['Engage!'] = '戦闘開始！'

# generate a mapping of lang => { default_name => name }
def build_mapping(translations, ignore_list=[]):
  replace = defaultdict(dict)
  for _, item in translations.items():
    for lang, name in item.items():
      if lang == default_language:
        continue
      default_name = item[default_language]
      if default_name in ignore_list:
        continue
      if default_name.startswith('Unknown_'):
        continue
      if default_name in replace[lang]:
        existing_name = replace[lang][default_name]
        if existing_name != name:
          raise Exception('Conflict on %s: "%s" and "%s"' % (default_name, existing_name, name))
      else:
        replace[lang][default_name] = name
  return replace

def main(args):
    # Get overall reports and enemies.
    enemies = defaultdict(dict)
    options = {
      'api_key': args.key,
      'translate': 'true',
    }
    fight_id = 0

    for lang in languages:
      report = fflogs.api('fights', args.report, prefixes[lang], options)
      if fight_id == 0:
        start_time, end_time, fight_id = find_start_end_time(report, args)
      for enemy in report['enemies']:
        # Because the overall report contains all enemies from any report,
        # filter them by the fight_id that we care about.
        for fight in enemy['fights']:
          if fight['id'] == fight_id:
            enemies[enemy['id']][lang] = enemy['name']
            break

    # Get abilities for the chosen fight.
    options = {
        'api_key': args.key,
        'start': start_time,
        'end': end_time,
        # filtering for disposition='enemy' drops neutral abilities like Encumber.
        # including death also gets you Wroth Ghosts that don't show up otherwise.
        'filter': 'source.type="NPC" and (type="cast" or type="death")',
        'translate': 'true',
    }
    abilities = defaultdict(dict)
    for lang in languages:
      events = fflogs.api('events', args.report, prefixes[lang], options)['events']
      for event in events:
        actor = 0
        if 'targetID' in event:
          actor = event['targetID']

        if 'ability' not in event:
          # Some mobs don't show up in the fight enemy list, but are things that get death events.
          # e.g. {'sourceIsFriendly': False, 'type': 'death', 'target': {'guid': 58, 'name': 'Wroth Ghost', 'type': 'NPC' } }
          if event['type'] == 'death' and not actor and 'target' in event:
            target = event['target']
            enemies[target['guid']][lang] = target['name']
          continue
        ability = event['ability']
        abilities[(actor, ability['guid'])][lang] = ability['name']

    # Finally, get boss debuff names.  These aren't used directly in timeline replacement
    # but are helpful to have listed when writing other triggers.
    options = {
        'api_key': args.key,
        'start': start_time,
        'end': end_time,
        'filter': 'source.type="NPC" and (type="applydebuffstack" or type="applydebuff" or type="refreshdebuff")',
        'translate': 'true',
    }
    effects = defaultdict(dict)
    for lang in languages:
      events = fflogs.api('events', args.report, prefixes[lang], options)['events']
      for event in events:
        actor = 0
        if 'targetID' in event:
          actor = event['targetID']
        ability = event['ability']
        effects[(actor, ability['guid'])][lang] = ability['name']

    # Generate mappings of english => locale names.
    mob_replace = build_mapping(enemies)
    ability_replace = build_mapping(abilities, ignore_abilities)
    effect_replace = build_mapping(effects, ignore_effects)

    add_default_ability_mappings(ability_replace)

    # Build some JSON similar to what cactbot expects in trigger files.
    timeline_replace = []
    for lang in languages:
      if lang == default_language:
        continue
      timeline_replace.append({
        'locale': lang,
        'replaceText': ability_replace[lang],
        'replaceSync': mob_replace[lang],
        # sort this last <_<
        '~effectNames': effect_replace[lang],
      })
    output = { 'timelineReplace': timeline_replace }
    output_str = json.dumps(output, ensure_ascii=False, indent=2, sort_keys=True)

    # hacky reformatting: single quotes, and remove quotes
    lines = []
    headers = ['timelineReplace', 'locale', 'replaceSync', 'replaceText', 'locale']
    for line in output_str.splitlines():
      # add trailing commas
      line = re.sub(r"\"\s*$", "\",", line)
      line = re.sub(r"]\s*$", "],", line)
      line = re.sub(r"}\s*$", "},", line)

      # replace all quotes on headers
      for header in headers:
        if line.find('"' + header + '":') != -1:
          line = line.replace('"', '', 2)
      # replace double with single quotes on any line without apostrophes.
      if line.find("'") == -1:
        line = line.replace('"', "'")
      lines.append(line)

    # Write that out to the user.
    if args.output_file:
      with open(args.output_file, 'w', encoding='utf-8') as fp:
        fp.write('\n'.join(lines))
    else:
      try:
        print(output_str)
      except UnicodeEncodeError as e:
        print("Warning: unable to print to console, use --output-file", file=sys.stderr)
        raise

if __name__ == "__main__":
    example_usage = """
    example:
        translate_fight.py -r WfrP3jgTqCbk4Ady --output-file=output.json

        Finds translations of this fight on fflogs and writes a json
        file suitable for including in a cactbot trigger file"""

    parser = argparse.ArgumentParser(
        description="Creates a translation from an fflogs report",
        epilog=example_usage,
        formatter_class=argparse.RawDescriptionHelpFormatter)

    parser.add_argument('-r', '--report', help="The ID of an FFLogs report")
    parser.add_argument('-of', '--output-file', help="The file to write output in")

    parser.add_argument('-k', '--key', help="The FFLogs API key to use, from https://www.fflogs.com/accounts/changeuser")
    parser.add_argument('-rf', '--fight', type=int, help="Fight ID of the report to use. Defaults to longest in the report")

    args = parser.parse_args()

    if args.report and not args.key:
        raise parser.error("FFlogs parsing requires an API key. Visit https://www.fflogs.com/accounts/changeuser and use the Public key")

    main(args)
<commit_msg>util: add more automatic translations<commit_after>### Generates en/de/fr/ja translations for a given fflogs report.

import argparse
from collections import defaultdict
from datetime import datetime
import fflogs
import json
import re
import sys

# 'en' here is 'www' which we consider the "base" and do automatically.
# 'cn' exists on fflogs but does not have proper translations, sorry.
languages = ['en', 'de', 'fr', 'ja']
prefixes = {
  'en': 'www',
  'de': 'de',
  'fr': 'fr',
  'ja': 'ja',
}
default_language = 'en'
ignore_abilities = ['Attack']
ignore_effects = [
  'Brink Of Death',
  'Sprint',
  'Regen',
  'Vulnerability Up',
  'Weakness',
]

def find_start_end_time(report, args):
  # Get the start and end timestamps for the specific fight
  start_time = 0
  end_time = 0
  fight_id = 0
  for fight in report['fights']:
    if args.fight and fight['id'] == args.fight:
      start_time = fight['start_time']
      end_time = fight['end_time']
      fight_id = fight['id']
      break
    elif fight['end_time'] - fight['start_time'] > end_time - start_time:
      start_time = fight['start_time']
      end_time = fight['end_time']
      fight_id = fight['id']

  if args.fight and not fight_id:
      raise Exception('Fight ID not found in report')

  return start_time, end_time, fight_id

def add_default_ability_mappings(ability_replace):
  # Many timelines have this, so just include it by default.
  ability_replace['de']['Engage!'] = 'Start!'
  ability_replace['fr']['Engage!'] = "À l'attaque"
  ability_replace['ja']['Engage!'] = '戦闘開始！'

  # FIXME: add fr/ja translations here.
  ability_replace['de']['--targetable--'] = '--anvisierbar--'
  ability_replace['de']['--untargetable--'] = '--nich anvisierbar--'
  ability_replace['de']['Enrage'] = 'Finalangriff'

# generate a mapping of lang => { default_name => name }
def build_mapping(translations, ignore_list=[]):
  replace = defaultdict(dict)
  for _, item in translations.items():
    for lang, name in item.items():
      if lang == default_language:
        continue
      default_name = item[default_language]
      if default_name in ignore_list:
        continue
      if default_name.startswith('Unknown_'):
        continue
      if default_name in replace[lang]:
        existing_name = replace[lang][default_name]
        if existing_name != name:
          raise Exception('Conflict on %s: "%s" and "%s"' % (default_name, existing_name, name))
      else:
        replace[lang][default_name] = name
  return replace

def main(args):
    # Get overall reports and enemies.
    enemies = defaultdict(dict)
    options = {
      'api_key': args.key,
      'translate': 'true',
    }
    fight_id = 0

    for lang in languages:
      report = fflogs.api('fights', args.report, prefixes[lang], options)
      if fight_id == 0:
        start_time, end_time, fight_id = find_start_end_time(report, args)
      for enemy in report['enemies']:
        # Because the overall report contains all enemies from any report,
        # filter them by the fight_id that we care about.
        for fight in enemy['fights']:
          if fight['id'] == fight_id:
            enemies[enemy['id']][lang] = enemy['name']
            break

    # Get abilities for the chosen fight.
    options = {
        'api_key': args.key,
        'start': start_time,
        'end': end_time,
        # filtering for disposition='enemy' drops neutral abilities like Encumber.
        # including death also gets you Wroth Ghosts that don't show up otherwise.
        'filter': 'source.type="NPC" and (type="cast" or type="death")',
        'translate': 'true',
    }
    abilities = defaultdict(dict)
    for lang in languages:
      events = fflogs.api('events', args.report, prefixes[lang], options)['events']
      for event in events:
        actor = 0
        if 'targetID' in event:
          actor = event['targetID']

        if 'ability' not in event:
          # Some mobs don't show up in the fight enemy list, but are things that get death events.
          # e.g. {'sourceIsFriendly': False, 'type': 'death', 'target': {'guid': 58, 'name': 'Wroth Ghost', 'type': 'NPC' } }
          if event['type'] == 'death' and not actor and 'target' in event:
            target = event['target']
            enemies[target['guid']][lang] = target['name']
          continue
        ability = event['ability']
        abilities[(actor, ability['guid'])][lang] = ability['name']

    # Finally, get boss debuff names.  These aren't used directly in timeline replacement
    # but are helpful to have listed when writing other triggers.
    options = {
        'api_key': args.key,
        'start': start_time,
        'end': end_time,
        'filter': 'source.type="NPC" and (type="applydebuffstack" or type="applydebuff" or type="refreshdebuff")',
        'translate': 'true',
    }
    effects = defaultdict(dict)
    for lang in languages:
      events = fflogs.api('events', args.report, prefixes[lang], options)['events']
      for event in events:
        actor = 0
        if 'targetID' in event:
          actor = event['targetID']
        ability = event['ability']
        effects[(actor, ability['guid'])][lang] = ability['name']

    # Generate mappings of english => locale names.
    mob_replace = build_mapping(enemies)
    ability_replace = build_mapping(abilities, ignore_abilities)
    effect_replace = build_mapping(effects, ignore_effects)

    add_default_ability_mappings(ability_replace)

    # Build some JSON similar to what cactbot expects in trigger files.
    timeline_replace = []
    for lang in languages:
      if lang == default_language:
        continue
      timeline_replace.append({
        'locale': lang,
        'replaceText': ability_replace[lang],
        'replaceSync': mob_replace[lang],
        # sort this last <_<
        '~effectNames': effect_replace[lang],
      })
    output = { 'timelineReplace': timeline_replace }
    output_str = json.dumps(output, ensure_ascii=False, indent=2, sort_keys=True)

    # hacky reformatting: single quotes, and remove quotes
    lines = []
    headers = ['timelineReplace', 'locale', 'replaceSync', 'replaceText', 'locale']
    for line in output_str.splitlines():
      # add trailing commas
      line = re.sub(r"\"\s*$", "\",", line)
      line = re.sub(r"]\s*$", "],", line)
      line = re.sub(r"}\s*$", "},", line)

      # replace all quotes on headers
      for header in headers:
        if line.find('"' + header + '":') != -1:
          line = line.replace('"', '', 2)
      # replace double with single quotes on any line without apostrophes.
      if line.find("'") == -1:
        line = line.replace('"', "'")
      lines.append(line)

    # Write that out to the user.
    if args.output_file:
      with open(args.output_file, 'w', encoding='utf-8') as fp:
        fp.write('\n'.join(lines))
    else:
      try:
        print(output_str)
      except UnicodeEncodeError as e:
        print("Warning: unable to print to console, use --output-file", file=sys.stderr)
        raise

if __name__ == "__main__":
    example_usage = """
    example:
        translate_fight.py -r WfrP3jgTqCbk4Ady --output-file=output.json

        Finds translations of this fight on fflogs and writes a json
        file suitable for including in a cactbot trigger file"""

    parser = argparse.ArgumentParser(
        description="Creates a translation from an fflogs report",
        epilog=example_usage,
        formatter_class=argparse.RawDescriptionHelpFormatter)

    parser.add_argument('-r', '--report', help="The ID of an FFLogs report")
    parser.add_argument('-of', '--output-file', help="The file to write output in")

    parser.add_argument('-k', '--key', help="The FFLogs API key to use, from https://www.fflogs.com/accounts/changeuser")
    parser.add_argument('-rf', '--fight', type=int, help="Fight ID of the report to use. Defaults to longest in the report")

    args = parser.parse_args()

    if args.report and not args.key:
        raise parser.error("FFlogs parsing requires an API key. Visit https://www.fflogs.com/accounts/changeuser and use the Public key")

    main(args)
<|endoftext|>