<commit_before>ficsClient.factory("Proxy", function(Console, User, Chat, Observe) {
  var socket = this.socket = new SockJS("/socket");
  var socketOpen = false;

  socket.onopen = function() {
    socketOpen = true;
  };

  socket.onclose = function() {
    socketOpen = false;
  };

  socket.onmessage = function(e) {
    if (e.type !== "message") {
      return;
    }

    var message = JSON.parse(e.data);

    switch(message.operation) {
    case "login":
      User.setUsername(message.data.username);

      userList();
      channelList();
      gameList();

      break;
    case "raw":
      Console.append(message.data);
      break;
    case "channelList":
      Chat.setChannels(message.data);
      break;
    case "gameList":
      Observe.setGames(message.data);
      break;
    case "userList":
      Chat.setUsers(message.data);
      break;
    }
  };

  this.login = function(userData) {
    sendMessage("login", userData);
  };

  function sendMessage(operation, params) {
    ensureSocket(function() {
      socket.send(JSON.stringify({ operation: operation, params: params }));
    });
  }

  function ensureSocket(cb) {
    await(function() { return socketOpen; }, cb)
  }

  function await(condition, cb) {
    (function wait() {
      if (condition()) cb();
      else setTimeout(wait, 50);
    })();
  }

  function channelList() {
    sendMessage("channelList", {});
  };

  function userList() {
    sendMessage("userList", {});
  };

  function gameList() {
    sendMessage("gameList", {});
  };

  return this;
});
<commit_msg>simplify socket open wait<commit_after>ficsClient.factory("Proxy", function(Console, User, Chat, Observe) {
  var socket = this.socket = new SockJS("/socket");
  var socketOpen = false;

  socket.onopen = function() {
    socketOpen = true;
  };

  socket.onclose = function() {
    socketOpen = false;
  };

  socket.onmessage = function(e) {
    if (e.type !== "message") {
      return;
    }

    var message = JSON.parse(e.data);

    switch(message.operation) {
    case "login":
      User.setUsername(message.data.username);

      userList();
      channelList();
      gameList();

      break;
    case "raw":
      Console.append(message.data);
      break;
    case "channelList":
      Chat.setChannels(message.data);
      break;
    case "gameList":
      Observe.setGames(message.data);
      break;
    case "userList":
      Chat.setUsers(message.data);
      break;
    }
  };

  this.login = function(userData) {
    sendMessage("login", userData);
  };

  function sendMessage(operation, params) {
    ensureSocket(function() {
      socket.send(JSON.stringify({ operation: operation, params: params }));
    });
  }

  function ensureSocket(cb) {
    if (socketOpen) {
      cb();
    } else {
      socket.onopen(cb);
    }
  }

  function channelList() {
    sendMessage("channelList", {});
  };

  function userList() {
    sendMessage("userList", {});
  };

  function gameList() {
    sendMessage("gameList", {});
  };

  return this;
});
<|endoftext|>


<commit_before>'use strict'

const v8Util = process.atomBinding('v8_util')

const getOwnerKey = (webContents, contextId) => {
  return `${webContents.id}-${contextId}`
}

class ObjectsRegistry {
  constructor () {
    this.nextId = 0

    // Stores all objects by ref-counting.
    // (id) => {object, count}
    this.storage = {}

    // Stores the IDs of objects referenced by WebContents.
    // (ownerKey) => [id]
    this.owners = {}
  }

  // Register a new object and return its assigned ID. If the object is already
  // registered then the already assigned ID would be returned.
  add (webContents, contextId, obj) {
    // Get or assign an ID to the object.
    const id = this.saveToStorage(obj)

    // Add object to the set of referenced objects.
    const ownerKey = getOwnerKey(webContents, contextId)
    let owner = this.owners[ownerKey]
    if (!owner) {
      owner = this.owners[ownerKey] = new Set()
      this.registerDeleteListener(webContents, contextId)
    }
    if (!owner.has(id)) {
      owner.add(id)
      // Increase reference count if not referenced before.
      this.storage[id].count++
    }
    return id
  }

  // Get an object according to its ID.
  get (id) {
    const pointer = this.storage[id]
    if (pointer != null) return pointer.object
  }

  // Dereference an object according to its ID.
  // Note that an object may be double-freed (cleared when page is reloaded, and
  // then garbage collected in old page).
  remove (webContents, contextId, id) {
    const ownerKey = getOwnerKey(webContents, contextId)
    let owner = this.owners[ownerKey]
    if (owner) {
      // Remove the reference in owner.
      owner.delete(id)
      // Dereference from the storage.
      this.dereference(id)
    }
  }

  // Clear all references to objects refrenced by the WebContents.
  clear (webContents, contextId) {
    const ownerKey = getOwnerKey(webContents, contextId)
    let owner = this.owners[ownerKey]
    if (!owner) return

    for (let id of owner) this.dereference(id)

    delete this.owners[ownerKey]
  }

  // Private: Saves the object into storage and assigns an ID for it.
  saveToStorage (object) {
    let id = v8Util.getHiddenValue(object, 'atomId')
    if (!id) {
      id = ++this.nextId
      this.storage[id] = {
        count: 0,
        object: object
      }
      v8Util.setHiddenValue(object, 'atomId', id)
    }
    return id
  }

  // Private: Dereference the object from store.
  dereference (id) {
    let pointer = this.storage[id]
    if (pointer == null) {
      return
    }
    pointer.count -= 1
    if (pointer.count === 0) {
      v8Util.deleteHiddenValue(pointer.object, 'atomId')
      delete this.storage[id]
    }
  }

  // Private: Clear the storage when renderer process is destroyed.
  registerDeleteListener (webContents, contextId) {
    const processId = webContents.getProcessId()
    const listener = (event, deletedProcessId) => {
      if (deletedProcessId === processId) {
        webContents.removeListener('render-view-deleted', listener)
        this.clear(webContents, contextId)
      }
    }
    webContents.on('render-view-deleted', listener)
  }
}

module.exports = new ObjectsRegistry()
<commit_msg>fix: add a hidden option to disable remote dereferencing (#14102)<commit_after>'use strict'

const v8Util = process.atomBinding('v8_util')

const getOwnerKey = (webContents, contextId) => {
  return `${webContents.id}-${contextId}`
}

class ObjectsRegistry {
  constructor () {
    this.nextId = 0

    // Stores all objects by ref-counting.
    // (id) => {object, count}
    this.storage = {}

    // Stores the IDs of objects referenced by WebContents.
    // (ownerKey) => [id]
    this.owners = {}
  }

  // Register a new object and return its assigned ID. If the object is already
  // registered then the already assigned ID would be returned.
  add (webContents, contextId, obj) {
    // Get or assign an ID to the object.
    const id = this.saveToStorage(obj)

    // Add object to the set of referenced objects.
    const ownerKey = getOwnerKey(webContents, contextId)
    let owner = this.owners[ownerKey]
    if (!owner) {
      owner = this.owners[ownerKey] = new Set()
      this.registerDeleteListener(webContents, contextId)
    }
    if (!owner.has(id)) {
      owner.add(id)
      // Increase reference count if not referenced before.
      this.storage[id].count++
    }
    return id
  }

  // Get an object according to its ID.
  get (id) {
    const pointer = this.storage[id]
    if (pointer != null) return pointer.object
  }

  // Dereference an object according to its ID.
  // Note that an object may be double-freed (cleared when page is reloaded, and
  // then garbage collected in old page).
  remove (webContents, contextId, id) {
    const ownerKey = getOwnerKey(webContents, contextId)
    let owner = this.owners[ownerKey]
    if (owner) {
      // Remove the reference in owner.
      owner.delete(id)
      // Dereference from the storage.
      this.dereference(id)
    }
  }

  // Clear all references to objects refrenced by the WebContents.
  clear (webContents, contextId) {
    const ownerKey = getOwnerKey(webContents, contextId)
    let owner = this.owners[ownerKey]
    if (!owner) return

    for (let id of owner) this.dereference(id)

    delete this.owners[ownerKey]
  }

  // Private: Saves the object into storage and assigns an ID for it.
  saveToStorage (object) {
    let id = v8Util.getHiddenValue(object, 'atomId')
    if (!id) {
      id = ++this.nextId
      this.storage[id] = {
        count: 0,
        object: object
      }
      v8Util.setHiddenValue(object, 'atomId', id)
    }
    return id
  }

  // Private: Dereference the object from store.
  dereference (id) {
    if (process.env.ELECTRON_DISABLE_REMOTE_DEREFERENCING) return

    let pointer = this.storage[id]
    if (pointer == null) {
      return
    }
    pointer.count -= 1
    if (pointer.count === 0) {
      v8Util.deleteHiddenValue(pointer.object, 'atomId')
      delete this.storage[id]
    }
  }

  // Private: Clear the storage when renderer process is destroyed.
  registerDeleteListener (webContents, contextId) {
    const processId = webContents.getProcessId()
    const listener = (event, deletedProcessId) => {
      if (deletedProcessId === processId) {
        webContents.removeListener('render-view-deleted', listener)
        this.clear(webContents, contextId)
      }
    }
    webContents.on('render-view-deleted', listener)
  }
}

module.exports = new ObjectsRegistry()
<|endoftext|>


<commit_before>'use strict';

const PSQL = require('cartodb-psql');
const dbParamsFromReqParams = require('../utils/database-params');
const debug = require('debug')('backend:cluster');
const AggregationMapConfig = require('../models/aggregation/aggregation-mapconfig');

module.exports = class ClusterBackend {
    // TODO: reduce complexity
    // jshint maxcomplexity: 10
    getClusterFeatures (mapConfigProvider, params, callback) {
        mapConfigProvider.getMapConfig((err, _mapConfig) => {
            if (err) {
                return callback(err);
            }

            let pg;
            try {
                pg = new PSQL(dbParamsFromReqParams(params));
            } catch (error) {
                return callback(error);
            }

            const { user, token, layer: layerIndex } = params;
            const mapConfig = new AggregationMapConfig(user, _mapConfig.obj(), pg);

            const layer = mapConfig.getLayer(layerIndex);

            if (!hasAggregationLayer(mapConfig, layerIndex)) {
                const error = new Error(`Map ${token} has no aggregation defined for layer ${layerIndex}`);
                error.http_status = 400;
                error.type = 'layer';
                error.subtype = 'aggregation';
                error.layer = {
                    index: layerIndex,
                    type: layer.type
                };

                debug(error);

                return callback(error);
            }

            let { aggregation } = params;

            if (aggregation !== undefined) {
                try {
                    aggregation = JSON.parse(aggregation);
                } catch (err) {
                    const error = new Error(`Invalid aggregation input, should be a a valid JSON`);
                    error.http_status = 400;
                    error.type = 'layer';
                    error.subtype = 'aggregation';
                    error.layer = {
                        index: layerIndex,
                        type: layer.type
                    };

                    return callback(error);
                }

                const { columns, expressions } = aggregation;

                if (!hasColumns(columns)) {
                    const error = new Error(
                        `Invalid aggregation input, columns should be and array of column names`
                    );
                    error.http_status = 400;
                    error.type = 'layer';
                    error.subtype = 'aggregation';
                    error.layer = {
                        index: layerIndex,
                        type: layer.type
                    };

                    return callback(error);
                }

                if (expressions !== undefined) {
                    try {
                        validateExpressions(expressions);
                    } catch (error) {
                        error.http_status = 400;
                        error.type = 'layer';
                        error.subtype = 'aggregation';
                        error.layer = {
                            index: layerIndex,
                            type: layer.type
                        };

                        return callback(error);
                    }
                }
            }

            const query = layer.options.sql_raw;
            const resolution = layer.options.aggregation.resolution || 1;

            getColumnsName(pg, query, (err, columns) => {
                if (err) {
                    return callback(err);
                }

                const { zoom, clusterId } = params;

                getClusterFeatures(pg, zoom, clusterId, columns, query, resolution, aggregation, (err, features) => {
                    if (err) {
                        return callback(err);
                    }

                    return callback(null, features);
                });
            });
        });
    }
};

const SKIP_COLUMNS = {
    'the_geom': true,
    'the_geom_webmercator': true
};

function getColumnsName (pg, query, callback) {
    const sql = limitedQuery({
        query: query
    });

    debug('> getColumnsName:', sql);

    pg.query(sql, function (err, resultSet) {
        if (err) {
            return callback(err);
        }

        const fields = resultSet.fields || [];
        const columnNames = fields.map(field => field.name)
            .filter(columnName => !SKIP_COLUMNS[columnName]);

        return callback(null, columnNames);
    }, true);
}

function getClusterFeatures (pg, zoom, clusterId, columns, query, resolution, aggregation, callback) {
    let sql = clusterFeaturesQuery({
        zoom: zoom,
        id: clusterId,
        query: query,
        res: 256/resolution,
        columns: columns
    });

    if (aggregation !== undefined) {
        let { columns = [], expressions = [] } = aggregation;

        if (expressions) {
            expressions = Object.entries(expressions)
                .map(([columnName, exp]) => `${exp.aggregate_function}(${exp.aggregated_column}) AS ${columnName}`);
        }

        sql = aggregationQuery({
            columns,
            expressions,
            query: sql
        });
    }

    debug('> getClusterFeatures:', sql);

    pg.query(sql, (err, data) => {
        if (err) {
            return callback(err);
        }

        return callback(null, data);
    } , true); // use read-only transaction
}

const limitedQuery = ctx => `SELECT * FROM (${ctx.query}) __cdb_schema LIMIT 0`;
const clusterFeaturesQuery = ctx => `
    WITH
    _cdb_params AS (
        SELECT
        ${gridResolution(ctx)} AS res
    ),
    _cell AS (
        SELECT
        ST_MakeEnvelope(
            Floor(ST_X(_cdb_query.the_geom_webmercator)/_cdb_params.res)*_cdb_params.res,
            Floor(ST_Y(_cdb_query.the_geom_webmercator)/_cdb_params.res)*_cdb_params.res,
            Floor(ST_X(_cdb_query.the_geom_webmercator)/_cdb_params.res + 1)*_cdb_params.res,
            Floor(ST_Y(_cdb_query.the_geom_webmercator)/_cdb_params.res + 1)*_cdb_params.res,
            3857
        ) AS bbox
        FROM (${ctx.query}) _cdb_query, _cdb_params
        WHERE _cdb_query.cartodb_id = ${ctx.id}
    )
    SELECT
        ${ctx.columns.join(', ')}
    FROM (
        SELECT _cdb_query.*
        FROM _cell, (${ctx.query}) _cdb_query
        WHERE ST_Intersects(_cdb_query.the_geom_webmercator, _cell.bbox)
    ) __cdb_non_geoms_query
`;

const gridResolution = ctx => {
    const minimumResolution = 2*Math.PI*6378137/Math.pow(2,38);
    const pixelSize = `CDB_XYZ_Resolution(${ctx.zoom})`;
    return `GREATEST(${256/ctx.res}*${pixelSize}, ${minimumResolution})::double precision`;
};

const aggregationQuery = ctx => `
    SELECT
        count(1) as _cdb_feature_count
        ${ctx.columns.length ? `,${ctx.columns.join(', ')}` : ''}
        ${ctx.expressions.length ? `,${ctx.expressions.join(', ')}` : ''}
    FROM (${ctx.query}) __cdb_aggregation
    ${ctx.columns.length ? `GROUP BY ${ctx.columns.join(', ')}` : ''}
`;

// TODO: update when https://github.com/CartoDB/Windshaft-cartodb/pull/1082 is merged
function hasAggregationLayer (mapConfig, layerIndex) {
    const layer = mapConfig.getLayer(layerIndex);

    if (typeof layer.options.aggregation === 'boolean') {
        return layer.options.aggregation;
    }

    return mapConfig.isAggregationLayer(layerIndex);
}

function hasColumns (columns) {
    return Array.isArray(columns) && columns.length;
}

function validateExpressions (expressions) {
    if (!isValidExpression(expressions)) {
        throw new Error(`Invalid aggregation input, expressions should be and object with valid functions`);
    }

    for (const { aggregate_function, aggregated_column } of Object.values(expressions)) {
        if (typeof aggregated_column !== 'string') {
            throw new Error(`Invalid aggregation input, aggregated column should be an string`);
        }

        if (typeof aggregate_function !== 'string') {
            throw new Error(`Invalid aggregation input, aggregate function should be an string`);
        }
    }
}

function isValidExpression (expressions) {
    const invalidTypes = ['string', 'number', 'boolean'];

    return expressions !== null && !Array.isArray(expressions) && !invalidTypes.includes(typeof expressions);
}
<commit_msg>Reduce complexity<commit_after>'use strict';

const PSQL = require('cartodb-psql');
const dbParamsFromReqParams = require('../utils/database-params');
const debug = require('debug')('backend:cluster');
const AggregationMapConfig = require('../models/aggregation/aggregation-mapconfig');

module.exports = class ClusterBackend {
    getClusterFeatures (mapConfigProvider, params, callback) {
        mapConfigProvider.getMapConfig((err, _mapConfig) => {
            if (err) {
                return callback(err);
            }

            let pg;
            try {
                pg = new PSQL(dbParamsFromReqParams(params));
            } catch (error) {
                return callback(error);
            }

            const { user, token, layer: layerIndex } = params;
            const mapConfig = new AggregationMapConfig(user, _mapConfig.obj(), pg);

            const layer = mapConfig.getLayer(layerIndex);

            if (!hasAggregationLayer(mapConfig, layerIndex)) {
                const error = new Error(`Map ${token} has no aggregation defined for layer ${layerIndex}`);
                error.http_status = 400;
                error.type = 'layer';
                error.subtype = 'aggregation';
                error.layer = {
                    index: layerIndex,
                    type: layer.type
                };

                debug(error);

                return callback(error);
            }

            let { aggregation } = params;

            try {
                aggregation = parseAggregation(aggregation);
                validateAggregation(aggregation);
            } catch (error) {
                error.http_status = 400;
                error.type = 'layer';
                error.subtype = 'aggregation';
                error.layer = {
                    index: layerIndex,
                    type: layer.type
                };

                return callback(error);
            }

            const query = layer.options.sql_raw;
            const resolution = layer.options.aggregation.resolution || 1;

            getColumnsName(pg, query, (err, columns) => {
                if (err) {
                    return callback(err);
                }

                const { zoom, clusterId } = params;

                getClusterFeatures(pg, zoom, clusterId, columns, query, resolution, aggregation, (err, features) => {
                    if (err) {
                        return callback(err);
                    }

                    return callback(null, features);
                });
            });
        });
    }
};

const SKIP_COLUMNS = {
    'the_geom': true,
    'the_geom_webmercator': true
};

function getColumnsName (pg, query, callback) {
    const sql = limitedQuery({
        query: query
    });

    debug('> getColumnsName:', sql);

    pg.query(sql, function (err, resultSet) {
        if (err) {
            return callback(err);
        }

        const fields = resultSet.fields || [];
        const columnNames = fields.map(field => field.name)
            .filter(columnName => !SKIP_COLUMNS[columnName]);

        return callback(null, columnNames);
    }, true);
}

function getClusterFeatures (pg, zoom, clusterId, columns, query, resolution, aggregation, callback) {
    let sql = clusterFeaturesQuery({
        zoom: zoom,
        id: clusterId,
        query: query,
        res: 256/resolution,
        columns: columns
    });

    if (aggregation !== undefined) {
        let { columns = [], expressions = [] } = aggregation;

        if (expressions) {
            expressions = Object.entries(expressions)
                .map(([columnName, exp]) => `${exp.aggregate_function}(${exp.aggregated_column}) AS ${columnName}`);
        }

        sql = aggregationQuery({
            columns,
            expressions,
            query: sql
        });
    }

    debug('> getClusterFeatures:', sql);

    pg.query(sql, (err, data) => {
        if (err) {
            return callback(err);
        }

        return callback(null, data);
    } , true); // use read-only transaction
}

const limitedQuery = ctx => `SELECT * FROM (${ctx.query}) __cdb_schema LIMIT 0`;
const clusterFeaturesQuery = ctx => `
    WITH
    _cdb_params AS (
        SELECT
        ${gridResolution(ctx)} AS res
    ),
    _cell AS (
        SELECT
        ST_MakeEnvelope(
            Floor(ST_X(_cdb_query.the_geom_webmercator)/_cdb_params.res)*_cdb_params.res,
            Floor(ST_Y(_cdb_query.the_geom_webmercator)/_cdb_params.res)*_cdb_params.res,
            Floor(ST_X(_cdb_query.the_geom_webmercator)/_cdb_params.res + 1)*_cdb_params.res,
            Floor(ST_Y(_cdb_query.the_geom_webmercator)/_cdb_params.res + 1)*_cdb_params.res,
            3857
        ) AS bbox
        FROM (${ctx.query}) _cdb_query, _cdb_params
        WHERE _cdb_query.cartodb_id = ${ctx.id}
    )
    SELECT
        ${ctx.columns.join(', ')}
    FROM (
        SELECT _cdb_query.*
        FROM _cell, (${ctx.query}) _cdb_query
        WHERE ST_Intersects(_cdb_query.the_geom_webmercator, _cell.bbox)
    ) __cdb_non_geoms_query
`;

const gridResolution = ctx => {
    const minimumResolution = 2*Math.PI*6378137/Math.pow(2,38);
    const pixelSize = `CDB_XYZ_Resolution(${ctx.zoom})`;
    return `GREATEST(${256/ctx.res}*${pixelSize}, ${minimumResolution})::double precision`;
};

const aggregationQuery = ctx => `
    SELECT
        count(1) as _cdb_feature_count
        ${ctx.columns.length ? `,${ctx.columns.join(', ')}` : ''}
        ${ctx.expressions.length ? `,${ctx.expressions.join(', ')}` : ''}
    FROM (${ctx.query}) __cdb_aggregation
    ${ctx.columns.length ? `GROUP BY ${ctx.columns.join(', ')}` : ''}
`;

// TODO: update when https://github.com/CartoDB/Windshaft-cartodb/pull/1082 is merged
function hasAggregationLayer (mapConfig, layerIndex) {
    const layer = mapConfig.getLayer(layerIndex);

    if (typeof layer.options.aggregation === 'boolean') {
        return layer.options.aggregation;
    }

    return mapConfig.isAggregationLayer(layerIndex);
}

function parseAggregation (aggregation) {
    if (aggregation !== undefined) {
        try {
            aggregation = JSON.parse(aggregation);
        } catch (err) {
            throw new Error(`Invalid aggregation input, should be a a valid JSON`);
        }

    }

    return aggregation;
}

function validateAggregation (aggregation) {
    if (aggregation !== undefined) {
        const { columns, expressions } = aggregation;

        if (!hasColumns(columns)) {
            throw new Error(`Invalid aggregation input, columns should be and array of column names`);
        }

        validateExpressions(expressions);
    }
}

function hasColumns (columns) {
    return Array.isArray(columns) && columns.length;
}

function validateExpressions (expressions) {
    if (expressions !== undefined) {
        if (!isValidExpression(expressions)) {
            throw new Error(`Invalid aggregation input, expressions should be and object with valid functions`);
        }

        for (const { aggregate_function, aggregated_column } of Object.values(expressions)) {
            if (typeof aggregated_column !== 'string') {
                throw new Error(`Invalid aggregation input, aggregated column should be an string`);
            }

            if (typeof aggregate_function !== 'string') {
                throw new Error(`Invalid aggregation input, aggregate function should be an string`);
            }
        }
    }
}

function isValidExpression (expressions) {
    const invalidTypes = ['string', 'number', 'boolean'];

    return expressions !== null && !Array.isArray(expressions) && !invalidTypes.includes(typeof expressions);
}
<|endoftext|>


<commit_before>'use strict';

var _ = require('lodash');
var Promise = require('bluebird');
var event = require('../../utils/events');
var shortid = require('shortid');
var Queue = require('../../utils/queue');

module.exports = function(context, server) {
    var io = require('socket.io')(server);
    var logger = context.logger;
    var configTimeout = context.sysconfig.teraslice.cluster.timeout;
    var pendingWorkerRequests = new Queue();

    var cluster_state = {};

    io.on('connection', function(socket) {

        socket.on('node_master:node_online', function(data) {
            logger.info('node ', data.node_id, ' has connected to cluster master');
            socket.join(data.node_id);
            socket.node_id = data.node_id;
            cluster_state[data.node_id] = data;
            //if new node comes online, check if jobs need more workers
            event.emit('cluster:available_workers')

        });

        socket.on('node_master:node_state', function(data) {
            cluster_state[data.node_id] = data;
            event.emit('cluster:available_workers')
        });

        socket.on('node_master:message_processed', function(data) {
            //emitting the unique msg id to allow easier listener cleanup
            event.emit(data._msgID, data);
            //TODO check if i need to emit more things
        });

        socket.on('node_master:over_allocated_workers', function(requestData) {
            //TODO check that job is still running
            pendingWorkerRequests.enqueue(requestData);
        });

        socket.on('slicer:job_finished', function(data) {
            //remove any pending worker request on completed jobs
            pendingWorkerRequests.remove(data);

            io.emit('cluster_service:stop_job', data);
            //if errors in slices, emit error event
            if (data.errorCount > 0) {
                var message = 'job: ' + data.job_id + ' had ' + data.errorCount + ' slice failures during processing.';
                data.error = message;

                logger.warn(message);
                event.emit('cluster:job_failure', data)
            }
            else {
                event.emit('cluster:job_finished', data);
            }
        });

        socket.on('slicer:failed_recovery', function(data) {
            //remove any pending worker request
            pendingWorkerRequests.remove(data.job_id);
            io.emit('cluster_service:stop_job', {job_id: data.job_id});
            event.emit('cluster:job_failure', data)

        });

        socket.on('slicer:error', function(data) {
            pendingWorkerRequests.remove(data.job_id);
            io.emit('cluster_service:stop_job', {job_id: data.job_id});
            event.emit('cluster:job_failure', data)
        });

        socket.on('slicer:processing_error', function(data) {
            event.emit('slicer:processing_error', data)
        });

        socket.on('slicer:slicer_initialized', function(data) {
            event.emit('slicer:slicer_initialized', data)
        });

        socket.on('disconnect', function() {
            logger.info('node ' + socket.node_id + ' has disconnected');
            delete cluster_state[socket.node_id];
        });

        socket.on('error', function(ev) {
            logger.error(ev)
        })

    });

    setInterval(function() {
        io.emit('cluster_service:get_node_state')
    }, 5000);

    function getClusterState() {
        return _.cloneDeep(cluster_state);
    }

    function checkForSlicer(node) {
        var obj = {hasSlicer: false, num: 0, id: node.id, available: node.available};

        return node.active.reduce(function(prev, curr) {
            if (curr.assignment === 'slicer') {
                prev.hasSlicer = true;
                prev.num++;
            }
            return prev;
        }, obj);

    }

    //TODO duplicate nodes may still be chosen on multiple slicer allocation attempts
    function findNodeForSlicer(stateArray, errorNodes) {
        var slicerNode = null;
        for (var i = 0; i < stateArray.length; i++) {

            if (stateArray[i].available > 0 && !errorNodes[stateArray[i].node_id]) {

                var node = checkForSlicer(stateArray[i]);

                if (!node.hasSlicer) {
                    slicerNode = stateArray[i].node_id;
                    break;
                }
            }
        }

        //if all nodes have a slicer
        if (!slicerNode) {
            //list is already sorted by num available since stateArray is sorted
            slicerNode = stateArray[0].node_id;
        }

        return slicerNode;
    }

    function findNodesForJob(job_id, slicer_only) {
        var nodes = [];

        _.forOwn(cluster_state, function(node) {
            var hasJob = node.active.filter(function(worker) {
                if (slicer_only) {
                    return worker.job_id === job_id && worker.assignment === 'slicer';
                }
                else {
                    return worker.job_id === job_id;
                }
            });

            if (hasJob.length >= 1) {
                nodes.push({node_id: node.node_id, job_id: job_id, hostname: node.hostname, workers: hasJob})
            }
        });

        return nodes
    }

    //TODO need guard here is node is actually up and available
    function availableWorkers(all_workers, forceCheck) {
        var num = 0;
        //need to prevent job service from starting more jobs while workers are still needed for existing jobs
        if (pendingWorkerRequests.size() === 0 || forceCheck) {

            var key = all_workers ? 'total' : 'available';

            _.forOwn(cluster_state, function(node) {
                num += node[key];
            });
        }

        return num;
    }

    function allWorkers() {
        //total number of workers function
        return availableWorkers(true)
    }

    function allNodes() {
        return Object.keys(cluster_state);
    }

    function findPort(node_id) {
        return notifyNode(node_id, 'cluster_service:get_open_port', {});
    }

    function removeFromQueue(id) {
        pendingWorkerRequests.remove(id)
    }

    function makeDispatch() {
        var methods = {};
        var dispatch = {};

        methods.set = function(node_id, numOfWorkers) {
            if (dispatch[node_id]) {
                dispatch[node_id] += numOfWorkers;
            }
            else {
                dispatch[node_id] = numOfWorkers;
            }
        };
        methods.getDispatch = function() {
            return dispatch;
        };

        return methods;
    }

    function notifyNode(node_id, msg, msgData) {
        return new Promise(function(resolve, reject) {
            //setting an unique id to know which message belongs to which
            msgData._msgID = shortid.generate();

            //set up the listener, since there is no clean way to have a dynamically named function or variable to
            //identify the correct function to remove on cleanup, we are listening on and emitting the unique id as key
            event.on(msgData._msgID, function(nodeMasterData) {
                //remove listener to prevent memory leaks
                event.removeAllListeners(msgData._msgID);

                if (nodeMasterData.error) {
                    reject('Error: ' + nodeMasterData.error + ' occured on node: ' + nodeMasterData.node_id)
                }
                else {
                    resolve(nodeMasterData)
                }
            });
            //send message
            io.sockets.in(node_id).emit(msg, msgData);

            //reject if timeout has been reached
            setTimeout(function() {
                //remove listener to prevent memory leaks
                event.removeAllListeners(msgData._msgID);
                reject("Error communicating with node: " + node_id + " .\n Could not send msg: " + msg + "\n  data: " + JSON.stringify(msgData))
            }, configTimeout)
        })
    }

    function allocateWorker(job) {
        var job_id = job.job_id;
        var jobStr = JSON.stringify(job);
        var sortedNodes = _.sortByOrder(cluster_state, 'available', 'desc');

        //since its sorted each time, the first should always have the most available
        var workerNodeID = sortedNodes[0].node_id;

        var data = {job: jobStr, job_id: job_id, workers: 1, node_id: workerNodeID, assignment: 'worker'};

        return notifyNode(workerNodeID, 'cluster_service:create_workers', data);
    }

    //designed to allocate additional workers, not any future slicers
    function allocateWorkers(job, numOfWorkersRequested) {
        var job_id = job.job_id;
        var jobStr = JSON.stringify(job);
        var sortedNodes = _.orderBy(cluster_state, 'available', 'desc');

        var availWorkers = availableWorkers(false, true);

        var dispatch = makeDispatch();

        while (numOfWorkersRequested > 0 && availWorkers > 0) {

            for (var i = 0; i < sortedNodes.length; i++) {
                //each iteration check if it can allocate
                if (numOfWorkersRequested > 0 && availWorkers > 0) {
                    if (sortedNodes[i].available >= 1) {
                        dispatch.set(sortedNodes[i].node_id, 1);
                        availWorkers -= 1;
                        numOfWorkersRequested -= 1;
                    }
                }
                else {
                    break;
                }
            }
        }
        //if left over worker requests, enqueue them, queue works based off of id, so it redundantly references job_id
        var workerData = {job: jobStr, id: job_id, job_id: job_id, workers: 1, assignment: 'worker'};

        while (numOfWorkersRequested > 0) {
            pendingWorkerRequests.enqueue(workerData);
            numOfWorkersRequested -= 1;
        }
        var results = [];

        _.forOwn(dispatch.getDispatch(), function(workerRequested, node_id) {
            var requestedWorkersData = {
                job: jobStr,
                id: job_id,
                job_id: job_id,
                node_id: node_id,
                workers: workerRequested,
                assignment: 'worker'
            };

            results.push(
                notifyNode(node_id, 'cluster_service:create_workers', requestedWorkersData)
                    .catch(function(ev) {
                        //TODO if a worker allocation failed, need to check if job is still running before enqueueing
                        logger.error('An error has occurred in allocating : ' + workerRequested + ' workers to node : ' + +node_id + ' , the worker request has been enqueued');
                        pendingWorkerRequests.enqueue(requestedWorkersData);
                    })
            )
        });

        //this will resolve successfully if one worker was actually allocated
        return Promise.all(results);
    }

    function createSlicer(job, recover_job, errorNodes) {
        var sortedNodes = _.orderBy(cluster_state, 'available', 'desc');
        var slicerNodeID = findNodeForSlicer(sortedNodes, errorNodes);

        //need to mutate job so that workers will know the specific port and hostname of the created slicer
        return findPort(slicerNodeID).then(function(portObj) {
            job.slicer_port = portObj.port;
            job.slicer_hostname = cluster_state[slicerNodeID].hostname;

            var job_id = job.job_id;
            var jobStr = JSON.stringify(job);

            var data = {
                job: jobStr,
                job_id: job_id,
                workers: 1,
                slicer_port: portObj.port,
                node_id: slicerNodeID,
                assignment: 'slicer',
                recover_job: recover_job
            };

            return notifyNode(slicerNodeID, 'cluster_service:create_slicer', data);
        })
    }

    function allocateSlicer(job, recover_job) {
        var retryCount = 0;
        var errorObj = {};
        return new Promise(function(resolve, reject) {

            function retry(errorNodes) {
                createSlicer(job, recover_job, errorNodes)
                    .then(function(results) {
                        resolve(results)
                    }).catch(function(err) {
                    retryCount += 1;

                    var nodeErr = err.split(':');
                    var nodeID = nodeErr[nodeErr.length - 1].trim();
                    errorNodes[nodeID] = nodeID;

                    if (retryCount >= 3) {
                        reject(err);
                    }
                    else {
                        retry(errorNodes)
                    }
                })
            }

            retry(errorObj);
        })
    }

    //TODO verify timing of dequeue vs job service allocation && race allocations of job finished but workers are enqueued afterwards
    var schedulePendingRequests = _.debounce(function() {
        if (pendingWorkerRequests.size() && availableWorkers(false, true) >= 1) {
            var requestedWorker = pendingWorkerRequests.dequeue();
            var job = JSON.parse(requestedWorker.job);

            allocateWorkers(job, requestedWorker.workers)
                .catch(function(err) {
                    logger.error("Error processing pending requests. " + err.stack);
                })

        }
    }, 500, {leading: false, trailing: true});

    event.on('cluster:available_workers', schedulePendingRequests);

    function removePendingRequests(job_id) {
        pendingWorkerRequests.remove(job_id);
    }

    function shutdown() {
        logger.info("ClusterService: shutting down.");
        // TODO: this needs to only resolve once the cluster and service state
        // is actually ready to shutdown.
        return Promise.resolve(true);
    }

    function iterate_state(cb) {
        return _.chain(cluster_state)
        /*.filter(function(node) {
         return node.state === 'connected'
         })*/
            .map(function(node) {
                var workers = node.active.filter(cb);

                return workers.map(function(worker) {
                    worker.node_id = node.node_id;
                    worker.hostname = node.hostname;
                    return worker;
                })
            })
            .flatten()
            .value();
    }

    function findAllWorkers() {
        return iterate_state(_.identity);
    }

    function findAllSlicers() {
        var allSlicers = function(worker) {
            return worker.assignment === 'slicer';
        };

        return iterate_state(allSlicers)
    }

    function findAllByJobID(job_id) {
        var allByID = function(worker) {
            return worker.job_id === job_id;
        };

        return iterate_state(allByID)
    }

    function findWorkersByJobID(job_id) {
        var workersByJobID = function(worker) {
            return worker.assignment === 'worker' && worker.job_id === job_id;
        };

        return iterate_state(workersByJobID)
    }

    function findSlicersByJobID(job_id) {
        var slicerByJobID = function(worker) {
            return worker.assignment === 'slicer' && worker.job_id === job_id;
        };

        return iterate_state(slicerByJobID)
    }

    var api = {
        getClusterState: getClusterState,
        availableWorkers: availableWorkers,
        allWorkers: allWorkers,
        allNodes: allNodes,
        allocateWorker: allocateWorker,
        allocateWorkers: allocateWorkers,
        allocateSlicer: allocateSlicer,
        findNodesForJob: findNodesForJob,
        notifyNode: notifyNode,
        removePendingRequests: removePendingRequests,
        findAllSlicers: findAllSlicers,
        findAllWorkers: findAllWorkers,
        findAllByJobID: findAllByJobID,
        findWorkersByJobID: findWorkersByJobID,
        findSlicersByJobID: findSlicersByJobID,
        removeFromQueue: removeFromQueue,
        shutdown: shutdown
    };

    function _initialize() {
        // TODO: handle the initialization of the cluster and only
        // actually resolve the promise once everything is up and running.
        // Should be delays here as we wait for nodes to join and share their
        // state.
        logger.info("ClusterService: Initializing");
        return Promise.resolve(api);
    }

    return _initialize();
};

<commit_msg>fixed worker allocation bug resolves #247 (#248)<commit_after>'use strict';

var _ = require('lodash');
var Promise = require('bluebird');
var event = require('../../utils/events');
var shortid = require('shortid');
var Queue = require('../../utils/queue');

module.exports = function(context, server) {
    var io = require('socket.io')(server);
    var logger = context.logger;
    var configTimeout = context.sysconfig.teraslice.cluster.timeout;
    var pendingWorkerRequests = new Queue();

    var cluster_state = {};

    io.on('connection', function(socket) {

        socket.on('node_master:node_online', function(data) {
            logger.info('node ', data.node_id, ' has connected to cluster master');
            socket.join(data.node_id);
            socket.node_id = data.node_id;
            cluster_state[data.node_id] = data;
            //if new node comes online, check if jobs need more workers
            event.emit('cluster:available_workers')

        });

        socket.on('node_master:node_state', function(data) {
            cluster_state[data.node_id] = data;
            event.emit('cluster:available_workers')
        });

        socket.on('node_master:message_processed', function(data) {
            //emitting the unique msg id to allow easier listener cleanup
            event.emit(data._msgID, data);
            //TODO check if i need to emit more things
        });

        socket.on('node_master:over_allocated_workers', function(requestData) {
            //TODO check that job is still running
            pendingWorkerRequests.enqueue(requestData);
        });

        socket.on('slicer:job_finished', function(data) {
            //remove any pending worker request on completed jobs
            pendingWorkerRequests.remove(data.job_id);

            io.emit('cluster_service:stop_job', data);
            //if errors in slices, emit error event
            if (data.errorCount > 0) {
                var message = 'job: ' + data.job_id + ' had ' + data.errorCount + ' slice failures during processing.';
                data.error = message;

                logger.warn(message);
                event.emit('cluster:job_failure', data)
            }
            else {
                event.emit('cluster:job_finished', data);
            }
        });

        socket.on('slicer:failed_recovery', function(data) {
            //remove any pending worker request
            pendingWorkerRequests.remove(data.job_id);
            io.emit('cluster_service:stop_job', {job_id: data.job_id});
            event.emit('cluster:job_failure', data)

        });

        socket.on('slicer:error', function(data) {
            pendingWorkerRequests.remove(data.job_id);
            io.emit('cluster_service:stop_job', {job_id: data.job_id});
            event.emit('cluster:job_failure', data)
        });

        socket.on('slicer:processing_error', function(data) {
            event.emit('slicer:processing_error', data)
        });

        socket.on('slicer:slicer_initialized', function(data) {
            event.emit('slicer:slicer_initialized', data)
        });

        socket.on('disconnect', function() {
            logger.info('node ' + socket.node_id + ' has disconnected');
            delete cluster_state[socket.node_id];
        });

        socket.on('error', function(ev) {
            logger.error(ev)
        })

    });

    setInterval(function() {
        io.emit('cluster_service:get_node_state')
    }, 5000);

    function getClusterState() {
        return _.cloneDeep(cluster_state);
    }

    function checkForSlicer(node) {
        var obj = {hasSlicer: false, num: 0, id: node.id, available: node.available};

        return node.active.reduce(function(prev, curr) {
            if (curr.assignment === 'slicer') {
                prev.hasSlicer = true;
                prev.num++;
            }
            return prev;
        }, obj);

    }

    //TODO duplicate nodes may still be chosen on multiple slicer allocation attempts
    function findNodeForSlicer(stateArray, errorNodes) {
        var slicerNode = null;
        for (var i = 0; i < stateArray.length; i++) {

            if (stateArray[i].available > 0 && !errorNodes[stateArray[i].node_id]) {

                var node = checkForSlicer(stateArray[i]);

                if (!node.hasSlicer) {
                    slicerNode = stateArray[i].node_id;
                    break;
                }
            }
        }

        //if all nodes have a slicer
        if (!slicerNode) {
            //list is already sorted by num available since stateArray is sorted
            slicerNode = stateArray[0].node_id;
        }

        return slicerNode;
    }

    function findNodesForJob(job_id, slicer_only) {
        var nodes = [];

        _.forOwn(cluster_state, function(node) {
            var hasJob = node.active.filter(function(worker) {
                if (slicer_only) {
                    return worker.job_id === job_id && worker.assignment === 'slicer';
                }
                else {
                    return worker.job_id === job_id;
                }
            });

            if (hasJob.length >= 1) {
                nodes.push({node_id: node.node_id, job_id: job_id, hostname: node.hostname, workers: hasJob})
            }
        });

        return nodes
    }

    //TODO need guard here is node is actually up and available
    function availableWorkers(all_workers, forceCheck) {
        var num = 0;
        //need to prevent job service from starting more jobs while workers are still needed for existing jobs
        if (pendingWorkerRequests.size() === 0 || forceCheck) {

            var key = all_workers ? 'total' : 'available';

            _.forOwn(cluster_state, function(node) {
                num += node[key];
            });
        }

        return num;
    }

    function allWorkers() {
        //total number of workers function
        return availableWorkers(true)
    }

    function allNodes() {
        return Object.keys(cluster_state);
    }

    function findPort(node_id) {
        return notifyNode(node_id, 'cluster_service:get_open_port', {});
    }

    function removeFromQueue(id) {
        pendingWorkerRequests.remove(id)
    }

    function makeDispatch() {
        var methods = {};
        var dispatch = {};

        methods.set = function(node_id, numOfWorkers) {
            if (dispatch[node_id]) {
                dispatch[node_id] += numOfWorkers;
            }
            else {
                dispatch[node_id] = numOfWorkers;
            }
        };
        methods.getDispatch = function() {
            return dispatch;
        };

        return methods;
    }

    function notifyNode(node_id, msg, msgData) {
        return new Promise(function(resolve, reject) {
            //setting an unique id to know which message belongs to which
            msgData._msgID = shortid.generate();

            //set up the listener, since there is no clean way to have a dynamically named function or variable to
            //identify the correct function to remove on cleanup, we are listening on and emitting the unique id as key
            event.on(msgData._msgID, function(nodeMasterData) {
                //remove listener to prevent memory leaks
                event.removeAllListeners(msgData._msgID);

                if (nodeMasterData.error) {
                    reject('Error: ' + nodeMasterData.error + ' occured on node: ' + nodeMasterData.node_id)
                }
                else {
                    resolve(nodeMasterData)
                }
            });
            //send message
            io.sockets.in(node_id).emit(msg, msgData);

            //reject if timeout has been reached
            setTimeout(function() {
                //remove listener to prevent memory leaks
                event.removeAllListeners(msgData._msgID);
                reject("Error communicating with node: " + node_id + " .\n Could not send msg: " + msg + "\n  data: " + JSON.stringify(msgData))
            }, configTimeout)
        })
    }

    function allocateWorker(job) {
        var job_id = job.job_id;
        var jobStr = JSON.stringify(job);
        var sortedNodes = _.sortByOrder(cluster_state, 'available', 'desc');

        //since its sorted each time, the first should always have the most available
        var workerNodeID = sortedNodes[0].node_id;

        var data = {job: jobStr, job_id: job_id, workers: 1, node_id: workerNodeID, assignment: 'worker'};

        return notifyNode(workerNodeID, 'cluster_service:create_workers', data);
    }

    //designed to allocate additional workers, not any future slicers
    function allocateWorkers(job, numOfWorkersRequested) {
        var job_id = job.job_id;
        var jobStr = JSON.stringify(job);
        var sortedNodes = _.orderBy(cluster_state, 'available', 'desc');

        var availWorkers = availableWorkers(false, true);

        var dispatch = makeDispatch();

        while (numOfWorkersRequested > 0 && availWorkers > 0) {

            for (var i = 0; i < sortedNodes.length; i++) {
                //each iteration check if it can allocate
                if (numOfWorkersRequested > 0 && availWorkers > 0) {
                    if (sortedNodes[i].available >= 1) {
                        dispatch.set(sortedNodes[i].node_id, 1);
                        availWorkers -= 1;
                        numOfWorkersRequested -= 1;
                    }
                }
                else {
                    break;
                }
            }
        }
        //if left over worker requests, enqueue them, queue works based off of id, so it redundantly references job_id
        var workerData = {job: jobStr, id: job_id, job_id: job_id, workers: 1, assignment: 'worker'};

        while (numOfWorkersRequested > 0) {
            pendingWorkerRequests.enqueue(workerData);
            numOfWorkersRequested -= 1;
        }
        var results = [];

        _.forOwn(dispatch.getDispatch(), function(workerRequested, node_id) {
            var requestedWorkersData = {
                job: jobStr,
                id: job_id,
                job_id: job_id,
                node_id: node_id,
                workers: workerRequested,
                assignment: 'worker'
            };

            results.push(
                notifyNode(node_id, 'cluster_service:create_workers', requestedWorkersData)
                    .catch(function(ev) {
                        //TODO if a worker allocation failed, need to check if job is still running before enqueueing
                        logger.error('An error has occurred in allocating : ' + workerRequested + ' workers to node : ' + +node_id + ' , the worker request has been enqueued');
                        pendingWorkerRequests.enqueue(requestedWorkersData);
                    })
            )
        });

        //this will resolve successfully if one worker was actually allocated
        return Promise.all(results);
    }

    function createSlicer(job, recover_job, errorNodes) {
        var sortedNodes = _.orderBy(cluster_state, 'available', 'desc');
        var slicerNodeID = findNodeForSlicer(sortedNodes, errorNodes);

        //need to mutate job so that workers will know the specific port and hostname of the created slicer
        return findPort(slicerNodeID).then(function(portObj) {
            job.slicer_port = portObj.port;
            job.slicer_hostname = cluster_state[slicerNodeID].hostname;

            var job_id = job.job_id;
            var jobStr = JSON.stringify(job);

            var data = {
                job: jobStr,
                job_id: job_id,
                workers: 1,
                slicer_port: portObj.port,
                node_id: slicerNodeID,
                assignment: 'slicer',
                recover_job: recover_job
            };

            return notifyNode(slicerNodeID, 'cluster_service:create_slicer', data);
        })
    }

    function allocateSlicer(job, recover_job) {
        var retryCount = 0;
        var errorObj = {};
        return new Promise(function(resolve, reject) {

            function retry(errorNodes) {
                createSlicer(job, recover_job, errorNodes)
                    .then(function(results) {
                        resolve(results)
                    }).catch(function(err) {
                    retryCount += 1;

                    var nodeErr = err.split(':');
                    var nodeID = nodeErr[nodeErr.length - 1].trim();
                    errorNodes[nodeID] = nodeID;

                    if (retryCount >= 3) {
                        reject(err);
                    }
                    else {
                        retry(errorNodes)
                    }
                })
            }

            retry(errorObj);
        })
    }

    //TODO verify timing of dequeue vs job service allocation && race allocations of job finished but workers are enqueued afterwards
    var schedulePendingRequests = _.debounce(function() {
        if (pendingWorkerRequests.size() && availableWorkers(false, true) >= 1) {
            var requestedWorker = pendingWorkerRequests.dequeue();
            var job = JSON.parse(requestedWorker.job);

            allocateWorkers(job, requestedWorker.workers)
                .catch(function(err) {
                    logger.error("Error processing pending requests. " + err.stack);
                })

        }
    }, 500, {leading: false, trailing: true});

    event.on('cluster:available_workers', schedulePendingRequests);

    function removePendingRequests(job_id) {
        pendingWorkerRequests.remove(job_id);
    }

    function shutdown() {
        logger.info("ClusterService: shutting down.");
        // TODO: this needs to only resolve once the cluster and service state
        // is actually ready to shutdown.
        return Promise.resolve(true);
    }

    function iterate_state(cb) {
        return _.chain(cluster_state)
        /*.filter(function(node) {
         return node.state === 'connected'
         })*/
            .map(function(node) {
                var workers = node.active.filter(cb);

                return workers.map(function(worker) {
                    worker.node_id = node.node_id;
                    worker.hostname = node.hostname;
                    return worker;
                })
            })
            .flatten()
            .value();
    }

    function findAllWorkers() {
        return iterate_state(_.identity);
    }

    function findAllSlicers() {
        var allSlicers = function(worker) {
            return worker.assignment === 'slicer';
        };

        return iterate_state(allSlicers)
    }

    function findAllByJobID(job_id) {
        var allByID = function(worker) {
            return worker.job_id === job_id;
        };

        return iterate_state(allByID)
    }

    function findWorkersByJobID(job_id) {
        var workersByJobID = function(worker) {
            return worker.assignment === 'worker' && worker.job_id === job_id;
        };

        return iterate_state(workersByJobID)
    }

    function findSlicersByJobID(job_id) {
        var slicerByJobID = function(worker) {
            return worker.assignment === 'slicer' && worker.job_id === job_id;
        };

        return iterate_state(slicerByJobID)
    }

    var api = {
        getClusterState: getClusterState,
        availableWorkers: availableWorkers,
        allWorkers: allWorkers,
        allNodes: allNodes,
        allocateWorker: allocateWorker,
        allocateWorkers: allocateWorkers,
        allocateSlicer: allocateSlicer,
        findNodesForJob: findNodesForJob,
        notifyNode: notifyNode,
        removePendingRequests: removePendingRequests,
        findAllSlicers: findAllSlicers,
        findAllWorkers: findAllWorkers,
        findAllByJobID: findAllByJobID,
        findWorkersByJobID: findWorkersByJobID,
        findSlicersByJobID: findSlicersByJobID,
        removeFromQueue: removeFromQueue,
        shutdown: shutdown
    };

    function _initialize() {
        // TODO: handle the initialization of the cluster and only
        // actually resolve the promise once everything is up and running.
        // Should be delays here as we wait for nodes to join and share their
        // state.
        logger.info("ClusterService: Initializing");
        return Promise.resolve(api);
    }

    return _initialize();
};

<|endoftext|>


<commit_before>'use strict';

module.exports = {
    name: 'ember-cli-jsdoc',

    run: function() {
        var exec = require( 'child_process' ).exec;
        var rsvp = require( 'rsvp' );
        var path = require( 'path' );
        var cmd = path.resolve( 'node_modules', 'ember-cli-jsdoc', 'node_modules', '.bin', 'jsdoc' );
        cmd = cmd + ' -c jsdoc.json';
        var chalk = require( 'chalk' );

        return new rsvp.Promise( function( resolve, reject ) {
            exec( cmd, function( error, stdout, stderr ) {
                console.log( stderr );

                if ( error ) {
                    console.log( chalk.red( 'AN ERROR HAS OCCURRED PREVENTING DOCUMENTATION GENERATION' ) );
                    reject();
                } else {
                    console.log( chalk.yellow( 'ONLY WARNINGS OCCURRED' ) );
                    resolve();
                }
            });
        });
    }
}
<commit_msg>Closes softlayer/ember-cli-jsdoc#16<commit_after>'use strict';

module.exports = {
    name: 'ember-cli-jsdoc',

    run: function() {
        var exec = require( 'child_process' ).exec;
        var rsvp = require( 'rsvp' );
        var path = require( 'path' );
        var cwd = require( 'process' ).cwd;
        var cmdPath = path.join( 'node_modules', 'ember-cli-jsdoc', 'node_modules', '.bin', 'jsdoc' );
        var chalk = require( 'chalk' );

        return new rsvp.Promise( function( resolve, reject ) {
            exec( cmdPath + ' -c jsdoc.json', { cwd: cwd() }, function( error, stdout, stderr ) {
                console.log( stderr );

                if ( error ) {
                    console.log( chalk.red( 'AN ERROR HAS OCCURRED PREVENTING DOCUMENTATION GENERATION' ) );
                    reject();
                } else {
                    console.log( chalk.yellow( 'ONLY WARNINGS OCCURRED' ) );
                    resolve();
                }
            });
        });
    }
}
<|endoftext|>


<commit_before>import { isArray } from 'substance'
import { toString as cellStatusToString } from '../../src/engine/CellStates'
import { parseSymbol } from '../../src/shared/expressionHelpers'
import { getIndexesFromRange, getRangeFromMatrix } from '../../src/shared/cellHelpers'

export function getValue(cell) {
  if (cell.value) {
    return cell.value.data
  }
}

export function getValues(cells) {
  return cells.map(rowOrCell => {
    if (isArray(rowOrCell)) {
      return rowOrCell.map(getValue)
    } else {
      return getValue(rowOrCell)
    }
  })
}

export function getSource(cell) {
  return cell.source
}

export function getSources(cells) {
  return cells.map(rowOrCell => {
    if (isArray(rowOrCell)) {
      return rowOrCell.map(getValue)
    } else {
      return getSource(rowOrCell)
    }
  })
}

export function getErrors(cells) {
  return cells.map(cell => {
    return cell.errors.map(err => {
      return err.name || 'unknown'
    })
  })
}

export function getStates(cells) {
  return cells.map(cell => {
    return cellStatusToString(cell.status)
  })
}

export function queryValues(engine, expr) {
  let symbol = parseSymbol(expr)
  if (!symbol.scope) throw new Error('query must use fully qualified identifiers')
  let docId = engine._lookupDocumentId(symbol.scope)
  if (!docId) throw new Error('Unknown resource:', symbol.scope)
  switch (symbol.type) {
    case 'var':
    case 'cell': {
      return getValue(_resolveCell(engine, docId, symbol.name))
    }
    case 'range': {
      let sheet = engine._docs[docId]
      const { startRow, startCol, endRow, endCol } = getIndexesFromRange(symbol.anchor, symbol.focus)
      let cells = getRangeFromMatrix(sheet.getCells(), startRow, startCol, endRow, endCol)
      return getValues(cells)
    }
    default:
      //
  }
}

function _resolveCell(engine, docId, name) {
  const graph = engine._graph
  let cellId = graph._out[docId+'!'+name]
  if (cellId) {
    return graph.getCell(cellId)
  }
}

/*
  Waits for all actions to be finished.
  This is the slowest kind of scheduling, as every cycle
  takes as long as the longest evaluation.
  In a real environment, the Engine should be triggered as often as possible,
  but still with a little delay, so that all 'simultanous' actions can be
  done at once.
*/
export function cycle(engine) {
  let actions = engine.cycle()
  return Promise.all(actions)
}

/*
  Triggers a cycle as long as next actions are coming in.
*/
export function play(engine) {
  return new Promise((resolve) => {
    function step() {
      if (_needsUpdate(engine)) {
        cycle(engine).then(step)
      } else {
        resolve()
      }
    }
    step()
  })
}

function _needsUpdate(engine) {
  const graph = engine._graph
  if (graph.needsUpdate()) return true
  const nextActions = engine._nextActions
  if (nextActions.size === 0) return false
  // update is required if there is an action that has not been suspended
  for (let [, a] of nextActions) {
    if (!a.suspended) return true
  }
  return false
}
<commit_msg>Fix regressions.<commit_after>import { isArray } from 'substance'
import { toString as cellStatusToString } from '../../src/engine/CellStates'
import { parseSymbol } from '../../src/shared/expressionHelpers'
import { getIndexesFromRange, getRangeFromMatrix, getRowCol } from '../../src/shared/cellHelpers'

export function getValue(cell) {
  if (cell.value) {
    return cell.value.data
  }
}

export function getValues(cells) {
  return cells.map(rowOrCell => {
    if (isArray(rowOrCell)) {
      return rowOrCell.map(getValue)
    } else {
      return getValue(rowOrCell)
    }
  })
}

export function getSource(cell) {
  return cell.source
}

export function getSources(cells) {
  return cells.map(rowOrCell => {
    if (isArray(rowOrCell)) {
      return rowOrCell.map(getValue)
    } else {
      return getSource(rowOrCell)
    }
  })
}

export function getErrors(cells) {
  return cells.map(cell => {
    return cell.errors.map(err => {
      return err.name || 'unknown'
    })
  })
}

export function getStates(cells) {
  return cells.map(cell => {
    return cellStatusToString(cell.status)
  })
}

export function queryValues(engine, expr) {
  let symbol = parseSymbol(expr)
  if (!symbol.scope) throw new Error('query must use fully qualified identifiers')
  let docId = engine._lookupDocumentId(symbol.scope)
  if (!docId) throw new Error('Unknown resource:', symbol.scope)
  switch (symbol.type) {
    case 'var': {
      return engine._graph.getValue(expr)
    }
    case 'cell': {
      let sheet = engine._docs[docId]
      let [row, col] = getRowCol(symbol.name)
      return getValue(sheet.cells[row][col])
    }
    case 'range': {
      let sheet = engine._docs[docId]
      const { startRow, startCol, endRow, endCol } = getIndexesFromRange(symbol.anchor, symbol.focus)
      let cells = getRangeFromMatrix(sheet.getCells(), startRow, startCol, endRow, endCol)
      return getValues(cells)
    }
    default:
      //
  }
}

/*
  Waits for all actions to be finished.
  This is the slowest kind of scheduling, as every cycle
  takes as long as the longest evaluation.
  In a real environment, the Engine should be triggered as often as possible,
  but still with a little delay, so that all 'simultanous' actions can be
  done at once.
*/
export function cycle(engine) {
  let actions = engine.cycle()
  return Promise.all(actions)
}

/*
  Triggers a cycle as long as next actions are coming in.
*/
export function play(engine) {
  return new Promise((resolve) => {
    function step() {
      if (_needsUpdate(engine)) {
        cycle(engine).then(step)
      } else {
        resolve()
      }
    }
    step()
  })
}

function _needsUpdate(engine) {
  const graph = engine._graph
  if (graph.needsUpdate()) return true
  const nextActions = engine._nextActions
  if (nextActions.size === 0) return false
  // update is required if there is an action that has not been suspended
  for (let [, a] of nextActions) {
    if (!a.suspended) return true
  }
  return false
}
<|endoftext|>


<commit_before>"use strict";

const DiscordieError = require("./DiscordieError");
const Constants = require("../Constants");
const EventTypes = Object.keys(Constants.Events);

const events = require("events");

let lastEvent = null;

function validateEvent(eventType) {
  if (EventTypes.indexOf(eventType) < 0)
    throw new DiscordieError(`Invalid event '${eventType}'`);
}

class DiscordieDispatcher extends events.EventEmitter {
  constructor() {
    super();
    this._anyeventlisteners = [];
  }
  static _getLastEvent() { return lastEvent; }
  on(eventType, listener) {
    validateEvent(eventType);
    return super.on.apply(this, arguments);
  }
  onAny(fn) {
    if (typeof fn !== "function")
      return this;
    this._anyeventlisteners.push(fn);
    return this;
  }
  emit(eventType) {
    validateEvent(eventType);

    lastEvent = [].slice.call(arguments);
    super.emit.apply(this, arguments);
    const _arguments = arguments;
    this._anyeventlisteners.forEach((fn) => {
      fn.apply(this, _arguments);
    });
  }
  hasListeners(eventType) {
    validateEvent(eventType);

    if (this._anyeventlisteners.length || this.listenerCount(eventType))
      return true;
    return false;
  }
}

module.exports = DiscordieDispatcher;
<commit_msg>Increase max dispatcher listeners<commit_after>"use strict";

const DiscordieError = require("./DiscordieError");
const Constants = require("../Constants");
const EventTypes = Object.keys(Constants.Events);

const events = require("events");

let lastEvent = null;

function validateEvent(eventType) {
  if (EventTypes.indexOf(eventType) < 0)
    throw new DiscordieError(`Invalid event '${eventType}'`);
}

class DiscordieDispatcher extends events.EventEmitter {
  constructor() {
    super();
    this._anyeventlisteners = [];
    this.setMaxListeners(12);
  }
  static _getLastEvent() { return lastEvent; }
  on(eventType, listener) {
    validateEvent(eventType);
    return super.on.apply(this, arguments);
  }
  onAny(fn) {
    if (typeof fn !== "function")
      return this;
    this._anyeventlisteners.push(fn);
    return this;
  }
  emit(eventType) {
    validateEvent(eventType);

    lastEvent = [].slice.call(arguments);
    super.emit.apply(this, arguments);
    const _arguments = arguments;
    this._anyeventlisteners.forEach((fn) => {
      fn.apply(this, _arguments);
    });
  }
  hasListeners(eventType) {
    validateEvent(eventType);

    if (this._anyeventlisteners.length || this.listenerCount(eventType))
      return true;
    return false;
  }
}

module.exports = DiscordieDispatcher;
<|endoftext|>


<commit_before>/* See license.txt for terms of usage */

"use strict";

var self = require("sdk/self");

const { Cu, Ci } = require("chrome");
const { Trace, TraceError } = require("../core/trace.js").get(module.id);
const { loadSheet, removeSheet } = require("sdk/stylesheet/utils");
const { EventTarget } = require("sdk/event/target");
const { Class } = require("sdk/core/heritage");
const { BaseOverlay } = require("../chrome/baseOverlay.js");
const { Win } = require("../core/window.js");

/**
 * This object is responsible for {@Debugger} panel customization.
 * Part of the customization is loading Firebug styles into CodeMirror
 * script editor.
 */
const DebuggerOverlay = Class(
/** @lends DebuggerOverlay */
{
  extends: BaseOverlay,

  // Initialization
  initialize: function(options) {
    BaseOverlay.prototype.initialize.apply(this, arguments);
    Trace.sysout("debuggerOverlay.initialize;", options);
  },

  onReady: function(options) {
    BaseOverlay.prototype.onReady.apply(this, arguments);

    Trace.sysout("debuggerOverlay.onReady;", options);
  },

  destroy: function() {
  },

  onApplyTheme: function(iframeWin, oldTheme) {
    Trace.sysout("debuggerOverlay.onApplyTheme; old theme: " +
      oldTheme, iframeWin);

    // Apply Firebug specific attributes on CodeMirror instance
    getCodeMirrorDoc(iframeWin, (doc) => {
      let body = doc.querySelector(".CodeMirror");
      let html = doc.getElementsByTagName("html")[0];

      html.classList.add("theme-firebug");
      html.classList.add("cm-s-firebug");
      html.classList.remove("theme-light");
      body.classList.remove("cm-s-mozilla");
    });

    // Load Debugger panel stylesheets.
    loadSheet(iframeWin, "chrome://firebug/skin/jsdebugger.css", "author");
  },

  onUnapplyTheme: function(iframeWin, newTheme) {
    Trace.sysout("debuggerOverlay.onUnapplyTheme; new theme: " +
      newTheme, iframeWin);

    getCodeMirrorDoc(iframeWin, doc => {
      let html = doc.getElementsByTagName("html")[0];

      html.classList.remove("theme-firebug");
      html.classList.remove("cm-s-firebug");
    });

    removeSheet(iframeWin, "chrome://firebug/skin/jsdebugger.css", "author");
  },
});

// Helpers

/**
 * Returns CodeMirror's document as soon as available.
 */
function getCodeMirrorDoc(iframeWin, callback) {
  // Wait till panel iframe is loaded
  Win.loaded(iframeWin).then(doc => {
    let editorBox = doc.getElementById("editor");

    // Wait till the CM iframe is appended into the editor box.
    iframeAdded(editorBox).then(cmFrame => {
      // Wait till the CM iframe is loaded.
      Win.loaded(cmFrame.contentWindow).then(callback);
    });
  });
}

/**
 * Wait till an iframe element is added into the specified element
 * xxxHonza: could be generalized into a shared function?
 *
 * @param {Element} parentNode The parent element where the iframe will be
 * appended to.
 */
function iframeAdded(parentNode) {
  let iframe = parentNode.querySelector("iframe");
  if (iframe)
    return new Promise(resolve => resolve(iframe));

  return new Promise(resolve => {
    const { MutationObserver } = parentNode.ownerDocument.defaultView;
    const observer = new MutationObserver(mutations => {
      for (let mutation of mutations) {
        for (let node of mutation.addedNodes || []) {
          if (node.tagName == "iframe") {
            observer.disconnect();
            resolve(node);
          }
        }
      }
    });
    observer.observe(parentNode, {childList: true});
  });
};

// Exports from this module
exports.DebuggerOverlay = DebuggerOverlay;
<commit_msg>Clean up module deps<commit_after>/* See license.txt for terms of usage */

"use strict";

var self = require("sdk/self");

const { Cu, Ci } = require("chrome");
const { Trace, TraceError } = require("../core/trace.js").get(module.id);
const { loadSheet, removeSheet } = require("sdk/stylesheet/utils");
const { Class } = require("sdk/core/heritage");
const { BaseOverlay } = require("../chrome/baseOverlay.js");
const { Win } = require("../core/window.js");

/**
 * This object is responsible for {@Debugger} panel customization.
 * Part of the customization is loading Firebug styles into CodeMirror
 * script editor.
 */
const DebuggerOverlay = Class(
/** @lends DebuggerOverlay */
{
  extends: BaseOverlay,

  // Initialization
  initialize: function(options) {
    BaseOverlay.prototype.initialize.apply(this, arguments);
    Trace.sysout("debuggerOverlay.initialize;", options);
  },

  onReady: function(options) {
    BaseOverlay.prototype.onReady.apply(this, arguments);

    Trace.sysout("debuggerOverlay.onReady;", options);
  },

  destroy: function() {
  },

  onApplyTheme: function(iframeWin, oldTheme) {
    Trace.sysout("debuggerOverlay.onApplyTheme; old theme: " +
      oldTheme, iframeWin);

    // Apply Firebug specific attributes on CodeMirror instance
    getCodeMirrorDoc(iframeWin, (doc) => {
      let body = doc.querySelector(".CodeMirror");
      let html = doc.getElementsByTagName("html")[0];

      html.classList.add("theme-firebug");
      html.classList.add("cm-s-firebug");
      html.classList.remove("theme-light");
      body.classList.remove("cm-s-mozilla");
    });

    // Load Debugger panel stylesheets.
    loadSheet(iframeWin, "chrome://firebug/skin/jsdebugger.css", "author");
  },

  onUnapplyTheme: function(iframeWin, newTheme) {
    Trace.sysout("debuggerOverlay.onUnapplyTheme; new theme: " +
      newTheme, iframeWin);

    getCodeMirrorDoc(iframeWin, doc => {
      let html = doc.getElementsByTagName("html")[0];

      html.classList.remove("theme-firebug");
      html.classList.remove("cm-s-firebug");
    });

    removeSheet(iframeWin, "chrome://firebug/skin/jsdebugger.css", "author");
  },
});

// Helpers

/**
 * Returns CodeMirror's document as soon as available.
 */
function getCodeMirrorDoc(iframeWin, callback) {
  // Wait till panel iframe is loaded
  Win.loaded(iframeWin).then(doc => {
    let editorBox = doc.getElementById("editor");

    // Wait till the CM iframe is appended into the editor box.
    iframeAdded(editorBox).then(cmFrame => {
      // Wait till the CM iframe is loaded.
      Win.loaded(cmFrame.contentWindow).then(callback);
    });
  });
}

/**
 * Wait till an iframe element is added into the specified element
 * xxxHonza: could be generalized into a shared function?
 *
 * @param {Element} parentNode The parent element where the iframe will be
 * appended to.
 */
function iframeAdded(parentNode) {
  let iframe = parentNode.querySelector("iframe");
  if (iframe)
    return new Promise(resolve => resolve(iframe));

  return new Promise(resolve => {
    const { MutationObserver } = parentNode.ownerDocument.defaultView;
    const observer = new MutationObserver(mutations => {
      for (let mutation of mutations) {
        for (let node of mutation.addedNodes || []) {
          if (node.tagName == "iframe") {
            observer.disconnect();
            resolve(node);
          }
        }
      }
    });
    observer.observe(parentNode, {childList: true});
  });
};

// Exports from this module
exports.DebuggerOverlay = DebuggerOverlay;
<|endoftext|>


<commit_before>'use strict';

const dot = require('dot');
const fs = require('graceful-fs');
const fsx = require('../helpers/fs-additions');
const path = require('path');
const Promise = require('bluebird');
const _ = require('lodash');

function loadTemplate(context) {
    return new Promise(function(resolve) {
        // Result
        const stackCF = fsx.readJSONFileSync(path.join(context.directories.root, 'templates', 'cf.json'));
        const customCF = path.join(context.directories.cwd, 'cf.json');

        // If file exists, load that in
        if (fsx.fileExists(customCF)) {
            const customStackCF = fsx.readJSONFileSync(customCF);
            _.merge(stackCF, customStackCF);
        }

        // Project name parameter need to be set (as they include an allowedValues listing)
        stackCF['Parameters']['aaProjectName'] = {
            Type: 'String',
            Default: context.project.name,
            AllowedValues: [
                context.project.name
            ]
        };

        resolve({
            context: context,
            stack: stackCF
        });
    });
}

function loadLambdas(result) {
    const context = result.context;
    const stack = result.stack;

    // If there are additional Lamdba policies, add those
    const policyFile = path.join(context.directories.cwd, 'lambda_policies.json');
    if (fsx.fileExists(policyFile)) {
        const policies = [].concat(fsx.readJSONFileSync(policyFile));
        const current = stack['Resources']['IamPolicyLambda']['Properties']['PolicyDocument']['Statement'];
        stack['Resources']['IamPolicyLambda']['Properties']['PolicyDocument']['Statement'] = current.concat(policies);
    }

    // Templates
    const lambdaOutputTemplate = dot.template(fs.readFileSync(path.join(context.directories.root, 'templates/lambda.resource.dot'), 'utf8'));
    const lambdaVersionTemplate = dot.template(fs.readFileSync(path.join(context.directories.root, 'templates/lambda.version.dot'), 'utf8'));

    // Alias name is the timestamp of the current deployment
    const timestamp = context.project.timestamp;

    // Add Lambdas to CF stack
    context.lambdas.forEach(function(lambda) {
        let camelName = _.camelCase(lambda.name);
        camelName = camelName.charAt(0).toUpperCase() + camelName.substring(1);

        const outputName = 'l' + camelName;
        stack['Resources'][camelName] = lambda.config;

        // Version & Alias
        const versionName = camelName + 'Version';

        stack['Resources'][versionName] = JSON.parse(lambdaVersionTemplate({
            resource: 'lambda-tools-lambda-version-resource-beta',
            lambda: camelName,
            description: timestamp
        }));

        // Final output (used by API Gateway)
        stack['Outputs'][outputName] = JSON.parse(lambdaOutputTemplate({ name: versionName }));
    });

    return {
        context: context,
        stack: stack
    };
}

function loadAPI(result) {
    const context = result.context;
    const stack = result.stack;

    if (context.api.skip) {
        return result;
    }

    const template = dot.template(fs.readFileSync(path.join(context.directories.root, 'templates/api.cf.dot'), 'utf8'));

    // Variables in our case are a list of resources for which we want to grab ARNs
    const variables = [];
    const dependencies = [];

    context.lambdas.forEach(function(lambda) {
        const camelName = _.camelCase(lambda.name);
        const name = camelName.charAt(0).toUpperCase() + camelName.substring(1);
        const version = name + 'Version';

        variables.push({
            key: 'l' + name,
            value: version,
            lambda: true
        });

        dependencies.push(version);
    });

    // Carry over two of the IAM roles as well
    variables.push({
        key: 'IamRoleArnLambda',
        value: 'IamRoleLambda'
    });

    dependencies.push('IamRoleLambda');

    variables.push({
        key: 'IamRoleArnApiGateway',
        value: 'IamRoleApiGateway'
    });

    dependencies.push('IamRoleApiGateway');

    // Populate a template that will make our custom API Gateway resource
    const parsedTemplate = template({
        lambda: {
            name: 'lambda-tools-api-gateway-resource-beta'
        },
        dependencies: dependencies,
        stageName: _.snakeCase(context.project.stage),
        s3: {
            bucket: context.project.bucket,
            key: context.project.timestamp + '/' + path.basename(context.api.configuration)
        },
        variables: variables
    });

    stack['Resources']['LambdaToolsAPIGateway'] = JSON.parse(parsedTemplate);

    return {
        context: context,
        stack: stack
    };
}

function saveStackConfiguration(result) {
    const context = result.context;
    const stack = result.stack;

    const deploymentCF = path.join(context.directories.staging, 'deployment.cf.json');
    fs.writeFileSync(deploymentCF, JSON.stringify(stack));

    const ctx = _.clone(context);
    ctx.stack = _.assign({}, ctx.stack, {
        configuration: deploymentCF
    });

    return ctx;
}

//
//  Step that derives the final CF stack configuration, based on
//  the information available in the context and combining it
//  with various templates found on disk
//
module.exports = function(context) {
    return context.logger.task('Deriving stack configuration', function(resolve, reject) {
        loadTemplate(context)
            .then(loadLambdas)
            .then(loadAPI)
            .then(saveStackConfiguration)
            .then(resolve, reject);
    });
};
<commit_msg>Remove reference to alias<commit_after>'use strict';

const dot = require('dot');
const fs = require('graceful-fs');
const fsx = require('../helpers/fs-additions');
const path = require('path');
const Promise = require('bluebird');
const _ = require('lodash');

function loadTemplate(context) {
    return new Promise(function(resolve) {
        // Result
        const stackCF = fsx.readJSONFileSync(path.join(context.directories.root, 'templates', 'cf.json'));
        const customCF = path.join(context.directories.cwd, 'cf.json');

        // If file exists, load that in
        if (fsx.fileExists(customCF)) {
            const customStackCF = fsx.readJSONFileSync(customCF);
            _.merge(stackCF, customStackCF);
        }

        // Project name parameter need to be set (as they include an allowedValues listing)
        stackCF['Parameters']['aaProjectName'] = {
            Type: 'String',
            Default: context.project.name,
            AllowedValues: [
                context.project.name
            ]
        };

        resolve({
            context: context,
            stack: stackCF
        });
    });
}

function loadLambdas(result) {
    const context = result.context;
    const stack = result.stack;

    // If there are additional Lamdba policies, add those
    const policyFile = path.join(context.directories.cwd, 'lambda_policies.json');
    if (fsx.fileExists(policyFile)) {
        const policies = [].concat(fsx.readJSONFileSync(policyFile));
        const current = stack['Resources']['IamPolicyLambda']['Properties']['PolicyDocument']['Statement'];
        stack['Resources']['IamPolicyLambda']['Properties']['PolicyDocument']['Statement'] = current.concat(policies);
    }

    // Templates
    const lambdaOutputTemplate = dot.template(fs.readFileSync(path.join(context.directories.root, 'templates/lambda.resource.dot'), 'utf8'));
    const lambdaVersionTemplate = dot.template(fs.readFileSync(path.join(context.directories.root, 'templates/lambda.version.dot'), 'utf8'));

    // Alias name is the timestamp of the current deployment
    const timestamp = context.project.timestamp;

    // Add Lambdas to CF stack
    context.lambdas.forEach(function(lambda) {
        let camelName = _.camelCase(lambda.name);
        camelName = camelName.charAt(0).toUpperCase() + camelName.substring(1);

        const outputName = 'l' + camelName;
        stack['Resources'][camelName] = lambda.config;

        // Version
        const versionName = camelName + 'Version';

        stack['Resources'][versionName] = JSON.parse(lambdaVersionTemplate({
            resource: 'lambda-tools-lambda-version-resource-beta',
            lambda: camelName,
            description: timestamp
        }));

        // Final output (used by API Gateway)
        stack['Outputs'][outputName] = JSON.parse(lambdaOutputTemplate({ name: versionName }));
    });

    return {
        context: context,
        stack: stack
    };
}

function loadAPI(result) {
    const context = result.context;
    const stack = result.stack;

    if (context.api.skip) {
        return result;
    }

    const template = dot.template(fs.readFileSync(path.join(context.directories.root, 'templates/api.cf.dot'), 'utf8'));

    // Variables in our case are a list of resources for which we want to grab ARNs
    const variables = [];
    const dependencies = [];

    context.lambdas.forEach(function(lambda) {
        const camelName = _.camelCase(lambda.name);
        const name = camelName.charAt(0).toUpperCase() + camelName.substring(1);
        const version = name + 'Version';

        variables.push({
            key: 'l' + name,
            value: version,
            lambda: true
        });

        dependencies.push(version);
    });

    // Carry over two of the IAM roles as well
    variables.push({
        key: 'IamRoleArnLambda',
        value: 'IamRoleLambda'
    });

    dependencies.push('IamRoleLambda');

    variables.push({
        key: 'IamRoleArnApiGateway',
        value: 'IamRoleApiGateway'
    });

    dependencies.push('IamRoleApiGateway');

    // Populate a template that will make our custom API Gateway resource
    const parsedTemplate = template({
        lambda: {
            name: 'lambda-tools-api-gateway-resource-beta'
        },
        dependencies: dependencies,
        stageName: _.snakeCase(context.project.stage),
        s3: {
            bucket: context.project.bucket,
            key: context.project.timestamp + '/' + path.basename(context.api.configuration)
        },
        variables: variables
    });

    stack['Resources']['LambdaToolsAPIGateway'] = JSON.parse(parsedTemplate);

    return {
        context: context,
        stack: stack
    };
}

function saveStackConfiguration(result) {
    const context = result.context;
    const stack = result.stack;

    const deploymentCF = path.join(context.directories.staging, 'deployment.cf.json');
    fs.writeFileSync(deploymentCF, JSON.stringify(stack));

    const ctx = _.clone(context);
    ctx.stack = _.assign({}, ctx.stack, {
        configuration: deploymentCF
    });

    return ctx;
}

//
//  Step that derives the final CF stack configuration, based on
//  the information available in the context and combining it
//  with various templates found on disk
//
module.exports = function(context) {
    return context.logger.task('Deriving stack configuration', function(resolve, reject) {
        loadTemplate(context)
            .then(loadLambdas)
            .then(loadAPI)
            .then(saveStackConfiguration)
            .then(resolve, reject);
    });
};
<|endoftext|>