<commit_before>'use strict';

var Promise = require('bluebird');
var moment = require('moment');
var processInterval = require('./../utils/elastic_utils').processInterval;
var getClient = require('../utils/config').getClient;
var buildQuery = require('./../utils/elastic_utils').buildQuery;
var buildRangeQuery = require('./../utils/elastic_utils').buildRangeQuery;
var dateFormat = require('./../utils/elastic_utils').dateFormat;
var parser = require('datemath-parser');
var _ = require('lodash');

function newReader(context, opConfig, jobConfig) {
    var client = getClient(context, opConfig, 'elasticsearch');

    if (opConfig.full_response) {

        return function(msg) {
            var query = buildQuery(opConfig, msg);
            return client.search(query);
        }
    }
    else {
        return function(msg) {
            return new Promise(function(resolve, reject) {
                var query = buildQuery(opConfig, msg);
                client.search(query).then(function(data) {
                    resolve(data.hits.hits.map(function(data) {
                        return data._source
                    }))
                });
            })
        }
    }
}

function checkElasticsearch(client, opConfig, logger) {
    return client.cluster.stats({}).then(function(data) {
        var version = data.nodes.versions[0];

        if (checkVersion(version)) {
            return client.indices.getSettings({}).then(function(results) {
                //TODO needs error handling if index doesn't exist
                var data = results[opConfig.index].settings.index.max_result_window;
                var window = data ? data : 10000;

                logger.info(' max_result_window for index: ' + opConfig.index + ' is set at ' + window + '. On very' +
                    ' large indices it is possible that a slice can not be divided to stay below this limit. ' +
                    'If that occurs an error will be thrown by Elasticsearch and the slice can not be processed. ' +
                    'Increasing max_result_window in the Elasticsearch index settings will resolve the problem. ');
            }).catch(function() {
                logger.error('index specified in reader does not exist')
            })
        }
    });
}

function checkVersion(str) {
    var num = Number(str.replace(/\./g, ''));
    return num >= 210;
}

function parseDate(date) {
    var result;

    if (moment(new Date(date)).isValid()) {
        result = moment(new Date(date));
    }
    else {
        try {
            var ms = parser.parse(date);
            result = moment(date)
        }
        catch (e) {
            throw new Error('elasticsearch_reader start and/or end dates are invalid')
        }
    }

    return result;
}

function getIndexDate(opConfig, client, date, order) {
    //if an actual date is given, no need to query
    if (date) {
        return parseDate(date)
    }
    var sortObj = {};
    var sortOrder = order === 'start' ? 'asc' : 'desc';

    sortObj[opConfig.date_field_name] = {order: sortOrder};

    var query = {
        index: opConfig.index,
        size: 1,
        body: {
            sort: [
                sortObj
            ]
        }
    };

    if (opConfig.query) {
        query.q = opConfig.query;
    }

    return client.search(query).then(function(data) {
        if (order === 'start') {
            return parseDate(data.hits.hits[0]._source[opConfig.date_field_name])
        }
        else {
            //end date is non-inclusive, adding 1s so range will cover it
            var date = data.hits.hits[0]._source[opConfig.date_field_name];
            var time = moment(date).add(1, 's');
            return parseDate(time.format(dateFormat));
        }
    }).catch(function(e) {
        throw new Error('Could not retrieve ' + order + ' date for index: ' + e)
    });

}

function getDates(context, opConfig) {
    var client = getClient(context, opConfig, 'elasticsearch');
    return Promise.resolve(getIndexDate(opConfig, client, opConfig.start, 'start'))
        .then(function(startDate) {
            return Promise.resolve(getIndexDate(opConfig, client, opConfig.end, 'end'))
                .then(function(endDate) {
                    return {start: startDate, limit: endDate};
                });
        });
}

function determineSlice(client, config, dateParams, isExpandedSlice) {
    return client.count({
        index: config.index,
        body: buildRangeQuery(config, {
            start: dateParams.start.format(dateFormat),
            end: dateParams.end.format(dateFormat)
        })
    })
        .then(function(data) {
            if (data.count > dateParams.size) {
                //if size is to big after increasing slice, use alternative division behavior
                if (isExpandedSlice) {
                    //recurse down to the appropriate size
                    var clonedParams = _.assign({}, dateParams);
                    clonedParams.start = moment(dateParams.end).subtract(dateParams.interval[0], dateParams.interval[1]);

                    //get diff from new start
                    var diff = Math.floor(dateParams.end.diff(clonedParams.start) / 2);
                    clonedParams.end = moment(clonedParams.start).add(diff, 'ms');

                    //return the zero range start with the correct end
                    return Promise.resolve(determineSlice(client, config, clonedParams, false))
                        .then(function(recursedData) {
                            return {start: dateParams.start, end: recursedData.end, count: recursedData.count};
                        });
                }
                else {
                    //find difference in milliseconds and divide in half
                    var diff = Math.floor(dateParams.end.diff(dateParams.start) / 2);
                    var newEnd = moment(dateParams.start).add(diff, 'ms');

                    //prevent recursive call if difference is one millisecond
                    if (diff < 2) {
                        return {start: dateParams.start, end: newEnd, count: data.count};
                    }
                    else {
                        //recurse to find smaller chunk
                        dateParams.end = newEnd;
                        return determineSlice(client, config, dateParams, isExpandedSlice)
                    }
                }
            }
            else {
                //interval is only passed in with once mode, expand slice to prevent slices of count 0
                if (data.count === 0 && dateParams.interval) {

                    //if end is same or after limit return limit as end
                    if (dateParams.end.isSameOrAfter(dateParams.limit)) {
                        return {start: dateParams.start, end: dateParams.limit, count: data.count};
                    }
                    else {
                        //increase the slice range to find documents
                        var newEnd = moment(dateParams.end).add(dateParams.interval[0], dateParams.interval[1]);
                        dateParams.end = newEnd;
                        return determineSlice(client, config, dateParams, true)
                    }
                }
                else {
                    return {start: dateParams.start, end: dateParams.end, count: data.count};
                }
            }
        })
        .catch(function(err) {
            console.log('index: "' + config.index + '" does not exist \n', err.stack);

            //TODO flesh this out
            process.exit();
        })
}

function nextChunk(opConfig, client, jobConfig, dates, retryData) {
    var dateParams = {};
    dateParams.size = opConfig.size;
    dateParams.interval = processInterval(opConfig.interval);
    dateParams.start = moment(dates.start);

    if (retryData && retryData.lastSlice && retryData.lastSlice.end) {
        dateParams.start = moment(retryData.lastSlice.end);
    }

    dateParams.limit = moment(dates.end);
    dateParams.end = moment(dateParams.start.format(dateFormat)).add(dateParams.interval[0], dateParams.interval[1]);

    return function(msg) {
        if (dateParams.start.isSameOrAfter(dateParams.limit)) {
            return null;
        }
        else {
            return determineSlice(client, opConfig, dateParams, false)
                .then(function(data) {
                    dateParams.start = data.end;
                    if (moment(data.end).add(dateParams.interval[0], dateParams.interval[1]) > dateParams.limit) {
                        dateParams.end = moment(data.end).add(dateParams.limit - data.end);
                    }
                    else {
                        dateParams.end = moment(data.end).add(dateParams.interval[0], dateParams.interval[1]);
                    }
                    return {
                        start: data.start.format(dateFormat),
                        end: data.end.format(dateFormat),
                        count: data.count
                    }
                }
            );
        }
    };

}

function getMilliseconds(interval) {
    var times = {d: 86400000, h: 3600000, m: 60000, s: 1000, ms: 1};

    return interval[0] * times[interval[1]]
}

function divideRange(start, end, job) {
    var numOfSlicers = job.jobConfig.slicers;
    var results = [];
    var startNum = Number(moment(start).format('x'));
    var endNum = Number(moment(end).format('x'));
    var range = (endNum - startNum) / numOfSlicers;

    var step = moment(start);

    for (var i = 0; i < numOfSlicers; i++) {
        var rangeObj = {start: step.format(dateFormat), end: step.add(range).format(dateFormat)};
        results.push(rangeObj);
    }

    return results;
}

function getTimes(opConfig, jobConfig) {
    var end = processInterval(opConfig.end);
    var delayInterval = processInterval(opConfig.delay);

    var delayTime = getMilliseconds(end);

    var delayedEnd = moment().subtract(delayInterval[0], delayInterval[1]).format(dateFormat);
    var delayedStart = moment(delayedEnd).subtract(end[0], end[1]).format(dateFormat);

    var dateArray = divideRange(delayedStart, delayedEnd, {jobConfig: jobConfig});

    return dateArray.map(function(dates) {
        dates.delayTime = delayTime;
        dates.interval = end;
        return dates;
    });
}

function awaitChunk(opConfig, client, jobConfig, slicerDates) {
    var dateParams = {};
    dateParams.size = opConfig.size;
    dateParams.start = moment(slicerDates.start);
    dateParams.end = moment(slicerDates.end);

    var delayTime = slicerDates.delayTime;
    var startPoint = moment(slicerDates.start);
    var limit = moment(slicerDates.end);
    var interval = slicerDates.interval;
    var dateArray = [];

    //set a timer to add the next set it should process
    setInterval(function() {
        //keep a list of next batches in cases current batch is still running
        dateArray.push({
            startPoint: moment(startPoint).add(interval[0], interval[1]),
            limit: moment(limit).add(interval[0], interval[1])
        });
    }, delayTime);

    return function(msg) {
        if (dateParams.start.isSameOrAfter(limit)) {
            //all done processing current chunk range, check for next range
            if (dateArray.length > 0) {
                var newRange = dateArray.shift();
                startPoint = newRange.startPoint;
                limit = newRange.limit;
                //make separate references to prevent mutating both at same time
                dateParams.start = moment(newRange.startPoint);
                dateParams.end = moment(newRange.limit);
            }
            return null;
        }
        else {
            return determineSlice(client, opConfig, dateParams, false)
                .then(function(data) {
                    dateParams.start = data.end;

                    if (moment(data.end).add(interval[0], interval[1]).isAfter(limit)) {
                        dateParams.end = moment(data.end).add(limit - data.end);

                    }
                    else {
                        dateParams.end = moment(data.end).add(interval[0], interval[1]);
                    }

                    return {start: data.start.format(dateFormat), end: data.end.format(dateFormat), count: data.count}
                }
            );
        }
    };
}

function newSlicer(context, job, retryData) {
    var opConfig = job.readerConfig;
    var jobConfig = job.jobConfig;
    var client = getClient(context, opConfig, 'elasticsearch');
    var isPersistent = jobConfig.lifecycle === 'persistent';
    var slicers = [];

    if (isPersistent) {
        var dataIntervals = getTimes(opConfig, jobConfig);

        dataIntervals.forEach(function(dates) {
            slicers.push(awaitChunk(opConfig, client, jobConfig, dates));
        });

        return Promise.resolve(slicers);
    }
    else {
        return checkElasticsearch(client, opConfig, context.logger)
            .then(function() {
                return getDates(context, opConfig)
            })
            .then(function(esDates) {
                var dateRange = divideRange(esDates.start, esDates.limit, job);

                dateRange.forEach(function(dates, index) {
                    slicers.push(nextChunk(opConfig, client, jobConfig, dates, retryData[index]));
                });

                return slicers;
            })
    }
}

function schema() {
    return {
        index: {
            doc: 'Which index to read from',
            default: '',
            format: 'required_String'

        },
        size: {
            doc: 'The limit to the number of docs pulled in a chunk, if the number of docs retrieved ' +
            'by the interval exceeds this number, it will cause the function to recurse to provide a smaller batch',
            default: 5000,
            format: 'integer'
        },
        start: {
            doc: 'The start date (ISOstring or in ms) to which it will read from ',
            default: null,
            format: 'optional_String'
        },
        end: {
            doc: 'The end date (ISOstring or in ms) to which it will read to',
            default: null,
            format: 'optional_String'
        },
        interval: {
            doc: 'The time interval in which it will read from, the number must be separated from the unit of time ' +
            'by an underscore. The unit of time may be months, weeks, days, hours, minutes, seconds, millesconds ' +
            'or their appropriate abbreviations',
            default: '5m',
            format: String
        },
        full_response: {
            doc: 'Set to true to receive the full Elasticsearch query response including index metadata.',
            default: false,
            format: Boolean
        },
        date_field_name: {
            doc: 'field name where the date of the doc is located',
            default: '',
            format: 'required_String'
        },
        delay: {
            doc: 'used for persistent',
            default: '30s'
        }
    };
}

var parallelSlicers = true;

module.exports = {
    newReader: newReader,
    newSlicer: newSlicer,
    schema: schema,
    parallelSlicers: parallelSlicers
};
<commit_msg>improved query, data_field_name error handling<commit_after>'use strict';

var Promise = require('bluebird');
var moment = require('moment');
var processInterval = require('./../utils/elastic_utils').processInterval;
var getClient = require('../utils/config').getClient;
var buildQuery = require('./../utils/elastic_utils').buildQuery;
var buildRangeQuery = require('./../utils/elastic_utils').buildRangeQuery;
var dateFormat = require('./../utils/elastic_utils').dateFormat;
var parser = require('datemath-parser');
var _ = require('lodash');

function newReader(context, opConfig, jobConfig) {
    var client = getClient(context, opConfig, 'elasticsearch');

    if (opConfig.full_response) {

        return function(msg) {
            var query = buildQuery(opConfig, msg);
            return client.search(query);
        }
    }
    else {
        return function(msg) {
            return new Promise(function(resolve, reject) {
                var query = buildQuery(opConfig, msg);
                client.search(query).then(function(data) {
                    resolve(data.hits.hits.map(function(data) {
                        return data._source
                    }))
                });
            })
        }
    }
}

function checkElasticsearch(client, opConfig, logger) {
    return client.cluster.stats({}).then(function(data) {
        var version = data.nodes.versions[0];

        if (checkVersion(version)) {
            return client.indices.getSettings({}).then(function(results) {
                //TODO needs error handling if index doesn't exist
                var data = results[opConfig.index].settings.index.max_result_window;
                var window = data ? data : 10000;

                logger.info(' max_result_window for index: ' + opConfig.index + ' is set at ' + window + '. On very' +
                    ' large indices it is possible that a slice can not be divided to stay below this limit. ' +
                    'If that occurs an error will be thrown by Elasticsearch and the slice can not be processed. ' +
                    'Increasing max_result_window in the Elasticsearch index settings will resolve the problem. ');
            }).catch(function() {
                throw new Error('index specified in reader does not exist');
            })
        }
    });
}

function checkVersion(str) {
    var num = Number(str.replace(/\./g, ''));
    return num >= 210;
}

function parseDate(date) {
    var result;

    if (moment(new Date(date)).isValid()) {
        result = moment(new Date(date));
    }
    else {
        try {
            var ms = parser.parse(date);
            result = moment(date)
        }
        catch (e) {
            throw new Error('elasticsearch_reader start and/or end dates are invalid')
        }
    }

    return result;
}

function getIndexDate(opConfig, client, date, order) {

    var sortObj = {};
    var givenDate = null;
    var query = null;

    if (date) {
        givenDate = parseDate(date);
        query = buildQuery(opConfig, {count: 1, start: opConfig.start, end: opConfig.end})
    }

    else {
        var sortOrder = order === 'start' ? 'asc' : 'desc';

        sortObj[opConfig.date_field_name] = {order: sortOrder};

        query = {
            index: opConfig.index,
            size: 1,
            body: {
                sort: [
                    sortObj
                ]
            }
        };

        if (opConfig.query) {
            query.q = opConfig.query;
        }
    }

    //using this query to catch potential errors even if a date is given already
    return client.search(query).then(function(data) {
        console.log('whats the data', data);
        if (data.hits.hits.length === 0) {
            throw new Error('No data found, please check the query parameter, ' +
                'start/end dates and/or date-field_name in elasticsearch_reader config');
        }

        if (data.hits.hits[0]._source[opConfig.date_field_name] === undefined) {
            throw new Error('date_field_name: "' + opConfig.date_field_name + '" for index: ' +
                '' + opConfig.index + ' does not exist');
        }

        if (givenDate) {
            return givenDate;
        }

        if (order === 'start') {
            return parseDate(data.hits.hits[0]._source[opConfig.date_field_name])
        }
        else {
            //end date is non-inclusive, adding 1s so range will cover it
            var date = data.hits.hits[0]._source[opConfig.date_field_name];
            var time = moment(date).add(1, 's');
            return parseDate(time.format(dateFormat));
        }
    }).catch(function(err) {
        throw new Error(err.stack);
    });

}

function getDates(context, opConfig) {
    var client = getClient(context, opConfig, 'elasticsearch');
    return Promise.resolve(getIndexDate(opConfig, client, opConfig.start, 'start'))
        .then(function(startDate) {
            return Promise.resolve(getIndexDate(opConfig, client, opConfig.end, 'end'))
                .then(function(endDate) {
                    return {start: startDate, limit: endDate};
                });
        });
}

function determineSlice(client, config, dateParams, isExpandedSlice) {
    return client.count({
        index: config.index,
        body: buildRangeQuery(config, {
            start: dateParams.start.format(dateFormat),
            end: dateParams.end.format(dateFormat)
        })
    })
        .then(function(data) {
            if (data.count > dateParams.size) {
                //if size is to big after increasing slice, use alternative division behavior
                if (isExpandedSlice) {
                    //recurse down to the appropriate size
                    var clonedParams = _.assign({}, dateParams);
                    clonedParams.start = moment(dateParams.end).subtract(dateParams.interval[0], dateParams.interval[1]);

                    //get diff from new start
                    var diff = Math.floor(dateParams.end.diff(clonedParams.start) / 2);
                    clonedParams.end = moment(clonedParams.start).add(diff, 'ms');

                    //return the zero range start with the correct end
                    return Promise.resolve(determineSlice(client, config, clonedParams, false))
                        .then(function(recursedData) {
                            return {start: dateParams.start, end: recursedData.end, count: recursedData.count};
                        });
                }
                else {
                    //find difference in milliseconds and divide in half
                    var diff = Math.floor(dateParams.end.diff(dateParams.start) / 2);
                    var newEnd = moment(dateParams.start).add(diff, 'ms');

                    //prevent recursive call if difference is one millisecond
                    if (diff < 2) {
                        return {start: dateParams.start, end: newEnd, count: data.count};
                    }
                    else {
                        //recurse to find smaller chunk
                        dateParams.end = newEnd;
                        return determineSlice(client, config, dateParams, isExpandedSlice)
                    }
                }
            }
            else {
                //interval is only passed in with once mode, expand slice to prevent slices of count 0
                if (data.count === 0 && dateParams.interval) {

                    //if end is same or after limit return limit as end
                    if (dateParams.end.isSameOrAfter(dateParams.limit)) {
                        return {start: dateParams.start, end: dateParams.limit, count: data.count};
                    }
                    else {
                        //increase the slice range to find documents
                        var newEnd = moment(dateParams.end).add(dateParams.interval[0], dateParams.interval[1]);
                        dateParams.end = newEnd;
                        return determineSlice(client, config, dateParams, true)
                    }
                }
                else {
                    return {start: dateParams.start, end: dateParams.end, count: data.count};
                }
            }
        })
        .catch(function(err) {
            console.log('index: "' + config.index + '" does not exist \n', err.stack);

            //TODO flesh this out
            process.exit();
        })
}

function nextChunk(opConfig, client, jobConfig, dates, retryData) {
    var dateParams = {};
    dateParams.size = opConfig.size;
    dateParams.interval = processInterval(opConfig.interval);
    dateParams.start = moment(dates.start);

    if (retryData && retryData.lastSlice && retryData.lastSlice.end) {
        dateParams.start = moment(retryData.lastSlice.end);
    }

    dateParams.limit = moment(dates.end);
    dateParams.end = moment(dateParams.start.format(dateFormat)).add(dateParams.interval[0], dateParams.interval[1]);

    return function(msg) {
        if (dateParams.start.isSameOrAfter(dateParams.limit)) {
            return null;
        }
        else {
            return determineSlice(client, opConfig, dateParams, false)
                .then(function(data) {
                    dateParams.start = data.end;
                    if (moment(data.end).add(dateParams.interval[0], dateParams.interval[1]) > dateParams.limit) {
                        dateParams.end = moment(data.end).add(dateParams.limit - data.end);
                    }
                    else {
                        dateParams.end = moment(data.end).add(dateParams.interval[0], dateParams.interval[1]);
                    }
                    return {
                        start: data.start.format(dateFormat),
                        end: data.end.format(dateFormat),
                        count: data.count
                    }
                }
            );
        }
    };

}

function getMilliseconds(interval) {
    var times = {d: 86400000, h: 3600000, m: 60000, s: 1000, ms: 1};

    return interval[0] * times[interval[1]]
}

function divideRange(start, end, job) {
    var numOfSlicers = job.jobConfig.slicers;
    var results = [];
    var startNum = Number(moment(start).format('x'));
    var endNum = Number(moment(end).format('x'));
    var range = (endNum - startNum) / numOfSlicers;

    var step = moment(start);

    for (var i = 0; i < numOfSlicers; i++) {
        var rangeObj = {start: step.format(dateFormat), end: step.add(range).format(dateFormat)};
        results.push(rangeObj);
    }

    return results;
}

function getTimes(opConfig, jobConfig) {
    var end = processInterval(opConfig.end);
    var delayInterval = processInterval(opConfig.delay);

    var delayTime = getMilliseconds(end);

    var delayedEnd = moment().subtract(delayInterval[0], delayInterval[1]).format(dateFormat);
    var delayedStart = moment(delayedEnd).subtract(end[0], end[1]).format(dateFormat);

    var dateArray = divideRange(delayedStart, delayedEnd, {jobConfig: jobConfig});

    return dateArray.map(function(dates) {
        dates.delayTime = delayTime;
        dates.interval = end;
        return dates;
    });
}

function awaitChunk(opConfig, client, jobConfig, slicerDates) {
    var dateParams = {};
    dateParams.size = opConfig.size;
    dateParams.start = moment(slicerDates.start);
    dateParams.end = moment(slicerDates.end);

    var delayTime = slicerDates.delayTime;
    var startPoint = moment(slicerDates.start);
    var limit = moment(slicerDates.end);
    var interval = slicerDates.interval;
    var dateArray = [];

    //set a timer to add the next set it should process
    setInterval(function() {
        //keep a list of next batches in cases current batch is still running
        dateArray.push({
            startPoint: moment(startPoint).add(interval[0], interval[1]),
            limit: moment(limit).add(interval[0], interval[1])
        });
    }, delayTime);

    return function(msg) {
        if (dateParams.start.isSameOrAfter(limit)) {
            //all done processing current chunk range, check for next range
            if (dateArray.length > 0) {
                var newRange = dateArray.shift();
                startPoint = newRange.startPoint;
                limit = newRange.limit;
                //make separate references to prevent mutating both at same time
                dateParams.start = moment(newRange.startPoint);
                dateParams.end = moment(newRange.limit);
            }
            return null;
        }
        else {
            return determineSlice(client, opConfig, dateParams, false)
                .then(function(data) {
                    dateParams.start = data.end;

                    if (moment(data.end).add(interval[0], interval[1]).isAfter(limit)) {
                        dateParams.end = moment(data.end).add(limit - data.end);

                    }
                    else {
                        dateParams.end = moment(data.end).add(interval[0], interval[1]);
                    }

                    return {start: data.start.format(dateFormat), end: data.end.format(dateFormat), count: data.count}
                }
            );
        }
    };
}

function newSlicer(context, job, retryData) {
    var opConfig = job.readerConfig;
    var jobConfig = job.jobConfig;
    var client = getClient(context, opConfig, 'elasticsearch');
    var isPersistent = jobConfig.lifecycle === 'persistent';
    var slicers = [];

    if (isPersistent) {
        var dataIntervals = getTimes(opConfig, jobConfig);

        dataIntervals.forEach(function(dates) {
            slicers.push(awaitChunk(opConfig, client, jobConfig, dates));
        });

        return Promise.resolve(slicers);
    }
    else {
        return checkElasticsearch(client, opConfig, context.logger)
            .then(function() {
                return getDates(context, opConfig)
            })
            .then(function(esDates) {
                var dateRange = divideRange(esDates.start, esDates.limit, job);

                dateRange.forEach(function(dates, index) {
                    slicers.push(nextChunk(opConfig, client, jobConfig, dates, retryData[index]));
                });

                return slicers;
            })
    }
}

function schema() {
    return {
        index: {
            doc: 'Which index to read from',
            default: '',
            format: 'required_String'

        },
        size: {
            doc: 'The limit to the number of docs pulled in a chunk, if the number of docs retrieved ' +
            'by the interval exceeds this number, it will cause the function to recurse to provide a smaller batch',
            default: 5000,
            format: 'integer'
        },
        start: {
            doc: 'The start date (ISOstring or in ms) to which it will read from ',
            default: null,
            format: 'optional_String'
        },
        end: {
            doc: 'The end date (ISOstring or in ms) to which it will read to',
            default: null,
            format: 'optional_String'
        },
        interval: {
            doc: 'The time interval in which it will read from, the number must be separated from the unit of time ' +
            'by an underscore. The unit of time may be months, weeks, days, hours, minutes, seconds, millesconds ' +
            'or their appropriate abbreviations',
            default: '5m',
            format: String
        },
        full_response: {
            doc: 'Set to true to receive the full Elasticsearch query response including index metadata.',
            default: false,
            format: Boolean
        },
        date_field_name: {
            doc: 'field name where the date of the doc is located',
            default: '',
            format: 'required_String'
        },
        delay: {
            doc: 'used for persistent',
            default: '30s'
        }
    };
}

var parallelSlicers = true;

module.exports = {
    newReader: newReader,
    newSlicer: newSlicer,
    schema: schema,
    parallelSlicers: parallelSlicers
};
<|endoftext|>


<commit_before>/* @flow */

import {combineReducers} from 'redux'
import login from './login'
import devices from './devices'
import search from './search'
import profile from './profile'
import config from './config'
import tabbedRouter from './tabbed-router'
import {List} from 'immutable'
import type {State} from '../constants/reducer'
import serialize from './serialize'
import tracker from './tracker'
import pinentry from './pinentry'
import favorite from './favorite'
import updateConfirm from './update-confirm'
import updatePaused from './update-paused'
import signup from './signup'

import {resetStore} from '../constants/common.js'

import devEdit from './dev-edit'

import * as Constants from '../constants/dev'

let history = List()
let index = 0

function timeTravel (state: State, action: any): State {
  if (action.type !== Constants.timeTravel) {
    history = history.slice(0, index + 1).push(state)
    index = history.size - 1
    return state
  } else {
    const {direction} = action.payload

    if (direction === Constants.timeTravelBack) {
      return history.get(--index, state)
    }
    return history.get(++index, state)
  }
}

const combinedReducer = combineReducers({
  login,
  devices,
  tabbedRouter,
  search,
  profile,
  config,
  tracker,
  pinentry,
  favorite,
  updateConfirm,
  updatePaused,
  signup
})

let reducer
if (__DEV__) {
  reducer = function (state: State, action: any): State {
    return (
      devEdit(
        serialize(
          timeTravel(
            combinedReducer(state, action),
            action),
          action),
        action)
    )
  }
} else {
  reducer = combinedReducer
}

export default function (state: State, action: any): State {
  // Warn if any keys did not change after a resetStore action
  if (action.type === resetStore) {
    const nextState = reducer(state, action)
    Object.keys(nextState).forEach(k => nextState[k] === state[k] && console.warn('Key %s did not change after resetStore action', k))
    return nextState
  }
  return reducer(state, action)
}
<commit_msg>Wrap warnings in __DEV__<commit_after>/* @flow */

import {combineReducers} from 'redux'
import login from './login'
import devices from './devices'
import search from './search'
import profile from './profile'
import config from './config'
import tabbedRouter from './tabbed-router'
import {List} from 'immutable'
import type {State} from '../constants/reducer'
import serialize from './serialize'
import tracker from './tracker'
import pinentry from './pinentry'
import favorite from './favorite'
import updateConfirm from './update-confirm'
import updatePaused from './update-paused'
import signup from './signup'

import {resetStore} from '../constants/common.js'

import devEdit from './dev-edit'

import * as Constants from '../constants/dev'

let history = List()
let index = 0

function timeTravel (state: State, action: any): State {
  if (action.type !== Constants.timeTravel) {
    history = history.slice(0, index + 1).push(state)
    index = history.size - 1
    return state
  } else {
    const {direction} = action.payload

    if (direction === Constants.timeTravelBack) {
      return history.get(--index, state)
    }
    return history.get(++index, state)
  }
}

const combinedReducer = combineReducers({
  login,
  devices,
  tabbedRouter,
  search,
  profile,
  config,
  tracker,
  pinentry,
  favorite,
  updateConfirm,
  updatePaused,
  signup
})

let reducer
if (__DEV__) {
  reducer = function (state: State, action: any): State {
    return (
      devEdit(
        serialize(
          timeTravel(
            combinedReducer(state, action),
            action),
          action),
        action)
    )
  }
} else {
  reducer = combinedReducer
}

export default function (state: State, action: any): State {
  // Warn if any keys did not change after a resetStore action
  if (__DEV__ && action.type === resetStore) {
    const nextState = reducer(state, action)
    Object.keys(nextState).forEach(k => nextState[k] === state[k] && console.warn('Key %s did not change after resetStore action', k))
    return nextState
  }
  return reducer(state, action)
}
<|endoftext|>


<commit_before>/**
 * @author Christian Schulze
 */

(function() {
    'use strict';

    angular
        .module('ActivityLogger')
        .controller('MainCtrl',
            function($scope, $state, $filter, $ionicModal, MockDataService, Competition, Activity, User) {
                // for referencing this later in anonymous functions
                var thisCtrl = this;

                // this is going to be our user
                var currentUserID = MockDataService.getCurrentUserId();

                // set up the selectable options
                this.possibleActivityTypes = [{
                    id: 1,
                    label: 'Run'
                }, {
                    id: 2,
                    label: 'Bike'
                }];

                this.selectedActivityType = this.possibleActivityTypes[0];
                this.comment = '';

                // set up data for competitions
                var competitions = MockDataService.getAllCompetitions();

                // filter the competitions by selecting the ones that have at least one
                // activity that has not been absolved yet (and therefore is null)
                var openCompetitions = competitions.filter(function(el) {
                    return el.activity_id2 === null;
                });

                this.hasOpenCompetitions = openCompetitions.length > 0 ? true : false;
                this.openCompetitionCount = openCompetitions.length;

                // set up competition types (open or new)
                var defaultCompetitionTypes = [{
                    id: 1,
                    label: 'New Competition'
                }];

                // if another user started a competition against the current user
                // this very option is added as a possible competition type
                if (this.hasOpenCompetitions) {
                    defaultCompetitionTypes.push({
                        id: 2,
                        label: 'Open competitions'
                    });
                }

                // set defaults
                this.possibleCompetitionTypes = defaultCompetitionTypes;

                // set up possible opponents
                var allPossibleOpponents = MockDataService.getAllUsers();

                // set defaults
                this.possibleOpponents = allPossibleOpponents;

                // set up default activity types against other users
                // if the competition will be newly created
                var defaultCompetitionActivities = [{
                    id: 1,
                    label: '1KM',
                    meters: 1000
                }, {
                    id: 2,
                    label: '2KM',
                    meters: 2000
                }, {
                    id: 3,
                    label: '3KM',
                    meters: 3000
                }, {
                    id: 3,
                    label: '4KM',
                    meters: 4000
                }];

                // set defaults
                this.possibleCompetitionDisciplins = defaultCompetitionActivities;

                // set up possbile competitions against other users
                var allPossibleOpenCompetitions = openCompetitions.map(function(el, i) {
                    var labelStr;

                    if (el.activity_id1 === null && el.activity_id2 === null) {
                        // none of the activites have been completed yet
                        labelStr = 'Empty Competition';
                    } else {
                        // choose the completed activity and take it as reference
                        var actID = (el.activity_id1 !== null) ? el.activity_id1 : el.activity_id2;
                        var act = MockDataService.getActivityByID(actID);
                        labelStr = $filter('normalizeMeterToKMFilter')(act.distance) + 'km (' + (act.duration / 60) + ')';
                    }

                    return {
                        id: i,
                        competitionID: el.id,
                        label: labelStr
                    };
                });

                // set defaults
                this.possibleCompetitions = allPossibleOpenCompetitions;

                // at start there are not enough options that allow the user
                // to start the activity
                this.competitionCanBeStarted = false;

                var allActivities = MockDataService.getAllActivities(currentUserID);

                var possbibleCompetitionActivities = allActivities.map(function(el, i) {
                    var labelStr = $filter('normalizeMeterToKMFilter')(el.distance) + 'km (' + (el.duration / 60) + ')';

                    return {
                        id: i,
                        label: labelStr
                    };
                });

                // regenerate options shown in the popup if an input was changed
                this.refreshCompetitionPopup = function() {
                    // reset to default
                    this.competitionCanBeStarted = false;

                    // if this option is not set, there ain't no adaptions to
                    // be made
                    if (this.selectedCompetitionType === undefined) {
                        return;
                    }

                    if (this.selectedCompetitionType.id == 1) {
                        // new competition
                        if (this.selectedOpponent !== undefined) {
                            if (this.selectedOpponent.id == currentUserID) {
                                // user wants to compete theirself, therefore
                                // show past activities as activites
                                this.possibleCompetitionDisciplins = possbibleCompetitionActivities;
                            } else {
                                // set available competition lengths back to default
                                this.possibleCompetitionDisciplins = defaultCompetitionActivities;
                            }
                        }

                    } else if (this.selectedCompetitionType.id == 2) {
                        // open competition
                        // find all opponents of open competitions
                        this.possibleOpponents = openCompetitions.map(function(el, index) {
                            var userID = el.user_id1 || el.user_id2;
                            return MockDataService.getUserByID(userID);
                        });

                        this.possibleCompetitionDisciplins = this.possibleCompetitions;
                    }

                    if (this.selectedCompetitionType !== undefined &&
                        this.selectedOpponent !== undefined &&
                        this.selectedCompetitionDisciplin !== undefined) {
                        this.competitionCanBeStarted = true;
                        this.competitionID = this.selectedCompetitionDisciplin.competitionID;
                    }
                };

                this.startCompetitionActivity = function() {
                    if (this.competitionID === undefined) {
                        var dist;

                        if (this.selectedOpponent.id == currentUserID) {
                            dist = this.selectedCompetitionType.meters;
                        } else {
                            dist = this.selectedCompetitionDisciplin.distance;
                        }

                        var competition = new Competition(0, currentUserID, this.selectedOpponent.id, null, null, dist);
                        this.competitionID = MockDataService.addCompetition(competition);
                    }

                    console.log(this.competitionID);
                    this.startActivity();
                };

                /**
                 * @description Start the activity by changing the view to
                 * the `activity` tab
                 */
                this.startActivity = function() {
                    console.log('start competition ' + thisCtrl.competitionID);
                    if (thisCtrl.selectedActivityType !== undefined) {
                        $state.go('tab.activity', {
                            'type': thisCtrl.selectedActivityType.label,
                            'comment': thisCtrl.comment,
                            'competitionID': thisCtrl.competitionID
                        });
                    }
                };

                // competition type popup setup
                $ionicModal.fromTemplateUrl('templates/newCompetitionActivityPopup.html', {
                    scope: $scope,
                    animation: 'slide-in-up'
                }).then(function(modal) {
                    $scope.modal = modal;
                });

                $scope.openModal = function() {
                    $scope.modal.show();
                };

                // clean up modal after closing it
                $scope.$on('$destroy', function() {
                    $scope.modal.remove();
                });
            });
})();<commit_msg>improved formatting of time in competition popup<commit_after>/**
 * @author Christian Schulze
 */

(function() {
    'use strict';

    angular
        .module('ActivityLogger')
        .controller('MainCtrl',
            function($scope, $state, $filter, $ionicModal, MockDataService, Competition, Activity, User) {
                // for referencing this later in anonymous functions
                var thisCtrl = this;

                // this is going to be our user
                var currentUserID = MockDataService.getCurrentUserId();

                // set up the selectable options
                this.possibleActivityTypes = [{
                    id: 1,
                    label: 'Run'
                }, {
                    id: 2,
                    label: 'Bike'
                }];

                this.selectedActivityType = this.possibleActivityTypes[0];
                this.comment = '';

                // set up data for competitions
                var competitions = MockDataService.getAllCompetitions();

                // filter the competitions by selecting the ones that have at least one
                // activity that has not been absolved yet (and therefore is null)
                var openCompetitions = competitions.filter(function(el) {
                    return el.activity_id2 === null;
                });

                this.hasOpenCompetitions = openCompetitions.length > 0 ? true : false;
                this.openCompetitionCount = openCompetitions.length;

                // set up competition types (open or new)
                var defaultCompetitionTypes = [{
                    id: 1,
                    label: 'New Competition'
                }];

                // if another user started a competition against the current user
                // this very option is added as a possible competition type
                if (this.hasOpenCompetitions) {
                    defaultCompetitionTypes.push({
                        id: 2,
                        label: 'Open competitions'
                    });
                }

                // set defaults
                this.possibleCompetitionTypes = defaultCompetitionTypes;

                // set up possible opponents
                var allPossibleOpponents = MockDataService.getAllUsers();

                // set defaults
                this.possibleOpponents = allPossibleOpponents;

                // set up default activity types against other users
                // if the competition will be newly created
                var defaultCompetitionActivities = [{
                    id: 1,
                    label: '1KM',
                    meters: 1000
                }, {
                    id: 2,
                    label: '2KM',
                    meters: 2000
                }, {
                    id: 3,
                    label: '3KM',
                    meters: 3000
                }, {
                    id: 3,
                    label: '4KM',
                    meters: 4000
                }];

                // set defaults
                this.possibleCompetitionDisciplins = defaultCompetitionActivities;

                // set up possbile competitions against other users
                var allPossibleOpenCompetitions = openCompetitions.map(function(el, i) {
                    var labelStr;

                    if (el.activity_id1 === null && el.activity_id2 === null) {
                        // none of the activites have been completed yet
                        labelStr = 'Empty Competition';
                    } else {
                        // choose the completed activity and take it as reference
                        var actID = (el.activity_id1 !== null) ? el.activity_id1 : el.activity_id2;
                        var act = MockDataService.getActivityByID(actID);
                        labelStr = $filter('normalizeMeterToKMFilter')(act.distance) + 'km (' + $filter('convertMillisecondsToMinutesFilter')(act.duration) + ')';
                    }

                    return {
                        id: i,
                        competitionID: el.id,
                        label: labelStr
                    };
                });

                // set defaults
                this.possibleCompetitions = allPossibleOpenCompetitions;

                // at start there are not enough options that allow the user
                // to start the activity
                this.competitionCanBeStarted = false;

                var allActivities = MockDataService.getAllActivities(currentUserID);

                var possbibleCompetitionActivities = allActivities.map(function(el, i) {
                    var labelStr = $filter('normalizeMeterToKMFilter')(el.distance) + 'km (' + $filter('convertMillisecondsToMinutesFilter')(el.duration) + ')';

                    return {
                        id: i,
                        label: labelStr
                    };
                });

                // regenerate options shown in the popup if an input was changed
                this.refreshCompetitionPopup = function() {
                    // reset to default
                    this.competitionCanBeStarted = false;

                    // if this option is not set, there ain't no adaptions to
                    // be made
                    if (this.selectedCompetitionType === undefined) {
                        return;
                    }

                    if (this.selectedCompetitionType.id == 1) {
                        // new competition
                        if (this.selectedOpponent !== undefined) {
                            if (this.selectedOpponent.id == currentUserID) {
                                // user wants to compete theirself, therefore
                                // show past activities as activites
                                this.possibleCompetitionDisciplins = possbibleCompetitionActivities;
                            } else {
                                // set available competition lengths back to default
                                this.possibleCompetitionDisciplins = defaultCompetitionActivities;
                            }
                        }

                    } else if (this.selectedCompetitionType.id == 2) {
                        // open competition
                        // find all opponents of open competitions
                        this.possibleOpponents = openCompetitions.map(function(el, index) {
                            var userID = el.user_id1 || el.user_id2;
                            return MockDataService.getUserByID(userID);
                        });

                        this.possibleCompetitionDisciplins = this.possibleCompetitions;
                    }

                    if (this.selectedCompetitionType !== undefined &&
                        this.selectedOpponent !== undefined &&
                        this.selectedCompetitionDisciplin !== undefined) {
                        this.competitionCanBeStarted = true;
                        this.competitionID = this.selectedCompetitionDisciplin.competitionID;
                    }
                };

                this.startCompetitionActivity = function() {
                    if (this.competitionID === undefined) {
                        var dist;

                        if (this.selectedOpponent.id == currentUserID) {
                            dist = this.selectedCompetitionType.meters;
                        } else {
                            dist = this.selectedCompetitionDisciplin.distance;
                        }

                        var competition = new Competition(0, currentUserID, this.selectedOpponent.id, null, null, dist);
                        this.competitionID = MockDataService.addCompetition(competition);
                    }

                    console.log(this.competitionID);
                    this.startActivity();
                };

                /**
                 * @description Start the activity by changing the view to
                 * the `activity` tab
                 */
                this.startActivity = function() {
                    console.log('start competition ' + thisCtrl.competitionID);
                    if (thisCtrl.selectedActivityType !== undefined) {
                        $state.go('tab.activity', {
                            'type': thisCtrl.selectedActivityType.label,
                            'comment': thisCtrl.comment,
                            'competitionID': thisCtrl.competitionID
                        });
                    }
                };

                // competition type popup setup
                $ionicModal.fromTemplateUrl('templates/newCompetitionActivityPopup.html', {
                    scope: $scope,
                    animation: 'slide-in-up'
                }).then(function(modal) {
                    $scope.modal = modal;
                });

                $scope.openModal = function() {
                    $scope.modal.show();
                };

                // clean up modal after closing it
                $scope.$on('$destroy', function() {
                    $scope.modal.remove();
                });
            });
})();<|endoftext|>


<commit_before>import Ember from 'ember';
import AupacTypeahead from './aupac-typeahead';

const {isNone, inject, computed, observer, typeOf} = Ember;

export default AupacTypeahead.extend({

  modelClass : null, //@public
  displayKey : 'displayName', //@public
  params : {}, //@public
  async : true, //@public
  queryKey : 'q', //@public

  //private
  store : inject.service('store'),

  /**
   * @Override
   */
  display : computed(function() {
    return (model) => {
      return model.get(this.get('displayKey'));
    };
  }),

  /**
   * @Override
   */
  setValue : function(selection) {
    if (this.get('_typeahead')) { // Was failing in tests with this probably due to a stray observer
      selection = this.transformSelection(selection);
      if (typeof selection === 'string') {
        this.get('_typeahead').typeahead('val', selection);
      } else {
        const displayKey = this.get('displayKey');
        const modelClass = this.get('modelClass');
        if(selection && selection.get('id')) {
          const item = this.get('store').peekRecord(modelClass, selection.get('id'));
          if (isNone(item)) {
            this.get('store').findRecord(modelClass, selection.get('id')).then((model) => {
              this.get('_typeahead').typeahead('val', model.get(displayKey));
            });
          } else {
            this.get('_typeahead').typeahead('val', item.get(displayKey));
          }
        } else {
          this.get('_typeahead').typeahead('val', '');
        }
      }
    }
  },

  /**
   * @Override
   */
  _commitSelection: function() {
    const model = this.get('selection');

    if (this.get('allowFreeInput')) {
      const displayKey = this.get('displayKey');
      const displayValue = typeOf(model) === 'instance' ? model.get(displayKey) : ''
      const value = this.get('_typeahead').typeahead('val');

      if (displayValue !== value) {
        this.selectionChanged(value);
      }
    } else if (model) {
      this.setValue(model);
    } else {
      this.setValue(null);
    }
  },

  /**
   * @Override
   */
  init : function() {
    this._super(...arguments);

    if(isNone(this.get('modelClass'))) {
      throw new Error('modelClass must be supplied to aupac-typeahead');
    }
  },

  /**
   * @Override
   */
  source : computed(function() {
    const _this = this;
    return function (query, syncResults, asyncResults) {
      const q = {};
      q[_this.get('queryKey')] = query;
      const queryObj = Ember.$.extend(true, {}, q , _this.get('params'));

      _this.get('store').query(_this.get('modelClass'), queryObj).then(function(models) {
        let emberDataModels = [];
        models.get('content').forEach(function(model, i) {
          emberDataModels[i] = model.getRecord();
        });
        asyncResults(emberDataModels);
      });
    };
  }),

  /**
   * @Override
   */
  selectionUpdated : observer('selection.id', '_typeahead', function() {
    const selection = this.get('selection');
    if(isNone(selection)) {
      this.setValue(null);
    } else {
      this.setValue(selection);
    }
  })

});
<commit_msg>Fix called method name<commit_after>import Ember from 'ember';
import AupacTypeahead from './aupac-typeahead';

const {isNone, inject, computed, observer, typeOf} = Ember;

export default AupacTypeahead.extend({

  modelClass : null, //@public
  displayKey : 'displayName', //@public
  params : {}, //@public
  async : true, //@public
  queryKey : 'q', //@public

  //private
  store : inject.service('store'),

  /**
   * @Override
   */
  display : computed(function() {
    return (model) => {
      return model.get(this.get('displayKey'));
    };
  }),

  /**
   * @Override
   */
  setValue : function(selection) {
    if (this.get('_typeahead')) { // Was failing in tests with this probably due to a stray observer
      selection = this.transformSelection(selection);
      if (typeof selection === 'string') {
        this.get('_typeahead').typeahead('val', selection);
      } else {
        const displayKey = this.get('displayKey');
        const modelClass = this.get('modelClass');
        if(selection && selection.get('id')) {
          const item = this.get('store').peekRecord(modelClass, selection.get('id'));
          if (isNone(item)) {
            this.get('store').findRecord(modelClass, selection.get('id')).then((model) => {
              this.get('_typeahead').typeahead('val', model.get(displayKey));
            });
          } else {
            this.get('_typeahead').typeahead('val', item.get(displayKey));
          }
        } else {
          this.get('_typeahead').typeahead('val', '');
        }
      }
    }
  },

  /**
   * @Override
   */
  _commitSelection: function() {
    const model = this.get('selection');

    if (this.get('allowFreeInput')) {
      const displayKey = this.get('displayKey');
      const displayValue = typeOf(model) === 'instance' ? model.get(displayKey) : ''
      const value = this.get('_typeahead').typeahead('val');

      if (displayValue !== value) {
        this.updateSelectionWhenChanged(value);
      }
    } else if (model) {
      this.setValue(model);
    } else {
      this.setValue(null);
    }
  },

  /**
   * @Override
   */
  init : function() {
    this._super(...arguments);

    if(isNone(this.get('modelClass'))) {
      throw new Error('modelClass must be supplied to aupac-typeahead');
    }
  },

  /**
   * @Override
   */
  source : computed(function() {
    const _this = this;
    return function (query, syncResults, asyncResults) {
      const q = {};
      q[_this.get('queryKey')] = query;
      const queryObj = Ember.$.extend(true, {}, q , _this.get('params'));

      _this.get('store').query(_this.get('modelClass'), queryObj).then(function(models) {
        let emberDataModels = [];
        models.get('content').forEach(function(model, i) {
          emberDataModels[i] = model.getRecord();
        });
        asyncResults(emberDataModels);
      });
    };
  }),

  /**
   * @Override
   */
  selectionUpdated : observer('selection.id', '_typeahead', function() {
    const selection = this.get('selection');
    if(isNone(selection)) {
      this.setValue(null);
    } else {
      this.setValue(selection);
    }
  })

});
<|endoftext|>


<commit_before>const getNestedProperty = require('../nested-property');
const getValues = require('./get-values');
const getJSONLD = require('./json-ld');
const typeMapping = require('../type-mapping');
const licenses = require('../../fixtures/licenses');
const usersList = require('../../fixtures/users');
const associationList = require('../../fixtures/associations');
const normalise = require('../../templates/helpers/normalise');
const prettyStatus = require('../helpers/json-to-html-data/pretty-status');
const dataLayer = require('./data-layer');
const getDescription = require('../helpers/json-to-html-data/get-description');

module.exports = (resource) => {
  const title = getValues.getTitle(resource.data);
  const type = getType(resource.data);
  const links = getLinks(resource.data);
  const related = getRelated(resource, type);
  const system = getValues.getSystem(resource.data);
  const description = getDescription(resource.data);
  const displayLocation = getValues.getDisplayLocation(resource.data);
  const level = getValues.getDocumentLevel(resource.data);
  const date = getValues.getDate(resource.data);
  const parent = getParent(resource);
  const fonds = getFonds(resource);
  const images = getImages(resource.data);
  var identifier = getIdentifier(resource.data);
  identifier = identifier ? identifier.value : null;
  const footer = require('../../fixtures/footer');
  const museums = require('../../fixtures/museums');
  const jsonLD = getJSONLD[typeMapping.toInternal(type)](resource);
  const tree = resource.tree;
  const current = getNestedProperty(resource, 'data.attributes.admin.uid');
  const titlePage = normalise(getValues.getTitle(resource.data), {system: {value: system}, type: type}) + ' | Science Museum Group Collection';
  const fact = getFacts(resource, links);
  const details = getDetails(resource, links);
  const location = getDisplayLocation(resource.data);
  const layerData = fact.concat(details, [location]);
  const layer = dataLayer(type, layerData);
  const wikipedia = getWikiLink(resource.data);
  const inProduction = Boolean(resource.inProduction);
  return {
    title,
    titlePage,
    type,
    fact,
    related,
    description,
    details,
    displayLocation,
    system,
    level,
    date,
    links,
    identifier,
    parent,
    fonds,
    images,
    footer,
    museums,
    jsonLD,
    tree,
    current,
    layer,
    wikipedia,
    inProduction
  };
};

function getDisplayLocation (data) {
  const value = getNestedProperty(data, 'attributes.locations.0.value');
  return value ? {key: 'Display Location', value: value, link: value} : null;
}

function getWikiLink (data) {
  var wikiregex = /en\.wikipedia\.org\/(?:wiki\/)(.+)/;

  var note = (data.attributes.description || []).find(el => el.value.indexOf('wiki') > -1);

  if (note && wikiregex.exec(note.value)) {
    return note.value.match(wikiregex)[1];
  } else {
    return null;
  }
}

function getType (data) {
  return data.type;
}

function getLinks (data) {
  return data.links;
}

function getFacts (resource, links) {
  // Need confirmation of what attributes to use as facts
  // TODO: only get attributes relevant to specific resource type
  const occupation = getValues.getOccupation(resource.data, links);
  const nationality = getValues.getNationality(resource.data);
  const born = getValues.getBorn(resource.data, links);
  const makers = getValues.getMakers(resource);
  const made = getMade(resource, links);
  const fonds = getFondsFact(resource);

  return [occupation, nationality, born, made, fonds].concat(makers).filter(el => el);
}

function getRelated (resource, type) {
  if (resource.data.relationships) {
    var related = {};
    Object.keys(resource.data.relationships).forEach(group => {
      related[group] = resource.data.relationships[group].data.map(item => {
        return resource.included.find(inc => inc.id === item.id);
      });
    });

    if (related.people) {
      related.people = related.people.map(e => {
        if (usersList.indexOf(getNestedProperty(e, 'attributes.role.value')) === -1 &&
          associationList.indexOf(getNestedProperty(e, 'attributes.role.value')) === -1) {
          e.attributes.role = null;
        }
        return e;
      });
    }

    return related;
  }
}

function getDetails (resource, links) {
  // Need confirmation of what attributes to use as details
  // TODO: only get attributes relevant to specific resource type
  const website = getWebsite(resource.data);
  const category = getCategory(resource.data, links);
  const accession = getAccession(resource.data);
  const materials = getMaterials(resource.data);
  const measurements = getMeasurements(resource.data);
  const identifier = getIdentifier(resource.data);
  const access = getAccess(resource.data);
  const legal = getLegal(resource.data);
  const arrangement = getArrangement(resource.data);
  const notes = getNotes(resource.data);
  const objectType = getObjectType(resource.data, links);
  const taxonomy = getTaxonomy(resource, links);

  return [website, category, accession, materials, measurements, identifier, access, arrangement, objectType, taxonomy].concat(legal, notes).filter(el => el);
}

function getWebsite (data) {
  const value = getNestedProperty(data, 'attributes.website');
  return value ? {key: 'Website', value: value, link: value} : null;
}

function getMade (resource, links) {
  const date = getValues.getMadeDate(resource.data);
  const places = getMadePlace(resource);
  const placeLinks = places && places.map(function (p) {
    return {
      value: p,
      link: links.root + '/search/places/' + encodeURIComponent(p).toLowerCase()
    };
  });
  const dateLink = createDateLink(date, links);

  if (date || places) {
    return {key: 'Made', place: placeLinks, date: {value: date, link: dateLink}};
  } else {
    return null;
  }
}

function createDateLink (date, links) {
  if (!date || !links) {
    return null;
  }
  var dateFrom = date;
  var dateTo = date;
  if (/[^\d]/.test(date)) {
    var splitDate = date.split('-');
    if (splitDate.length === 2 && splitDate[0].length === 4) {
      dateFrom = splitDate[0];
      if (splitDate[1].length === 4) {
        dateTo = splitDate[1];
      } else {
        dateTo = splitDate[0];
      }
    } else {
      return null;
    }
  }
  return links.root + '/search/date[from]/' + encodeURIComponent(dateFrom) + '/date[to]/' + encodeURIComponent(dateTo);
}

function getMadePlace (resource) {
  var places = [];
  if (resource.included) {
    resource.included.forEach(el => {
      if (el.type === 'place' && getNestedProperty(el, 'attributes.role.value') === 'made') {
        places.push(getNestedProperty(el, 'attributes.summary_title'));
      }
    });
    return places.length ? places : null;
  }
}

function getCategory (data, links) {
  var values = [];
  if (data.attributes.categories) {
    data.attributes.categories.forEach(category => {
      values.push(category.name);
    });
    return {key: 'Category', value: values.toString(), link: links.root + '/search/categories/' + encodeURIComponent(values.toString().toLowerCase())};
  } else {
    return null;
  }
}

function getAccession (data) {
  var value;
  if (data.attributes.identifier) {
    data.attributes.identifier.forEach(el => {
      if (el.type === 'accession number') {
        value = el.value;
      }
    });
    return value ? {key: 'Object Number', value: value} : null;
  } else {
    return null;
  }
}

function getMaterials (data) {
  var values;
  if (data.attributes.materials) {
    values = data.attributes.materials.map(material => {
      return {
        value: material,
        link: `/search/objects/material/${material.toLowerCase()}`
      };
    });
    return {key: 'Materials', value: values};
  } else {
    return null;
  }
}

function getMeasurements (data) {
  var values = [];
  if (data.attributes.component) {
    data.attributes.component.forEach(el => {
      values.push(el.measurements.display + '
');
    });
    return values.length ? {key: 'Measurements', value: values.join('')} : null;
  } else if (data.type === 'documents' && data.attributes.measurements) {
    data.attributes.measurements.dimensions.forEach(el => {
      values.push(el.value + '
');
    });
    return values.length ? {key: 'Extent', value: values.join('')} : null;
  }
}

function getIdentifier (data) {
  var value;
  if (data.attributes.identifier) {
    data.attributes.identifier.forEach(el => {
      if (el.type !== 'accession number' && el.primary) {
        value = el.value;
      }
    });
    return value ? {key: 'Identifier', value: value} : null;
  } else {
    return null;
  }
}

function getAccess (data) {
  const value = getNestedProperty(data, 'attributes.access.note.0.value');
  return value ? {key: 'Access', value: value} : null;
}

function getLegal (data) {
  var legals = [];
  var value;
  var key;
  if (getNestedProperty(data, 'attributes.legal')) {
    Object.keys(data.attributes.legal).forEach((el) => {
      key = el;
      if (typeof data.attributes.legal[el] === 'object' && data.attributes.legal[el].length && el === 'rights') {
        data.attributes.legal[el].forEach(function (r) {
          if (r.type && r.type.toLowerCase() === 'copyright') {
            value = r.holder;
          } else if (r.details) {
            value = r.details;
          }
        });
      } else {
        value = data.attributes.legal[el];
      }
      legals.push(value ? {key: keyMap(key), value: value} : null);
    });
  }
  legals = legals.filter(Boolean);

  return legals.map(function (l) {
    return {
      key: l.key,
      value: prettyStatus(l.value) || l.value
    };
  });
}

function getArrangement (data) {
  const value = getNestedProperty(data, 'attributes.arrangement.system.0');
  return value ? {key: 'System of Arrangement', value: value} : null;
}

function getNotes (data) {
  var notes = [];
  if (data.attributes.note && data.type !== 'people') {
    data.attributes.note.forEach(el => {
      if (el.type) {
        notes.push({key: el.type, value: el.value});
      }
    });
    return notes;
  } else {
    return null;
  }
}

function getObjectType (data, links) {
  var value;
  if (data.type !== 'people' && data.attributes.name) {
    value = data.attributes.name.map((el, i, arr) => {
      var val = el.value.toLowerCase();
      return {
        value: val,
        link: links.root + '/search/objects/object_type/' + encodeURIComponent(val.toLowerCase())
      };
    });
    return value ? {key: 'type', value: value} : null;
  }
}

function getTaxonomy (resource, links) {
  var taxonomy = [];
  if (resource.data.type !== 'people' && resource.included) {
    resource.included.forEach(el => {
      if (el.attributes.hierarchy) {
        el.attributes.hierarchy.forEach(el => {
          taxonomy.push({value: el.name[0].value.toLowerCase(), link: links.root + '/search/objects/object_type/' + encodeURIComponent(el.name[0].value.toLowerCase())});
        });
      }
    });
    return taxonomy.length ? {key: 'taxonomy', value: taxonomy.filter(el => el.value.indexOf('<') === -1).reverse()} : null;
  }
}

function getParent (resource) {
  if (getNestedProperty(resource, 'data.relationships.parent')) {
    return resource.included.find(el => {
      return el.id === resource.data.relationships.parent.data[0].id;
    });
  } else {
    return null;
  }
}

function getFonds (resource) {
  var fondsID;
  var fonds;
  if (getNestedProperty(resource, 'data.relationships.fonds')) {
    fondsID = resource.data.relationships.fonds.data[0].id;
    fonds = resource.included.find(el => el.id === fondsID);
    if (fonds) {
      return {
        id: fondsID,
        link: fonds.links.self,
        summary_title: fonds.attributes.summary_title
      };
    }
  }
  return null;
}

function getFondsFact (resource) {
  var fonds = getFonds(resource);
  if (fonds) {
    return {
      key: 'part of archive',
      value: fonds.summary_title,
      link: '#archive-block'
    };
  } else {
    return null;
  }
}

function getImages (data) {
  var images = [];

  if (data.attributes.multimedia) {
    images = data.attributes.multimedia.map(e => {
      if (e.processed) {
        var img = {
          large: getNestedProperty(e, 'processed.large.location'),
          medium: getNestedProperty(e, 'processed.medium.location'),
          card: getNestedProperty(e, 'processed.medium_thumbnail.location'),
          thumb: getNestedProperty(e, 'processed.small_thumbnail.location'),
          zoom: getNestedProperty(e, 'processed.zoom.location'),
          author: e.author,
          credit: e.credit
        };

        if (e.source && e.source.legal) {
          img.rights = formatLicense(e.source.legal.rights[0]);
        }

        if (data.type === 'documents') {
          img.title = '';
        } else {
          img.title = (e.source && e.source.title) ? e.source.title.find(el => el.type === 'main title').value || e.source.title[0].value : '';
        }

        return img;
      } else {
        return null;
      }
    });
  }
  return images.filter(Boolean);
}

function formatLicense (license) {
  var formatted;
  var cc;
  var image;
  if (license.usage_terms) {
    Object.keys(licenses).forEach(el => {
      if (license.usage_terms.match(el)) {
        formatted = license.usage_terms.replace(el, licenses[el].text);
        image = licenses[el].image;
        cc = true;
      }
    });
  }
  if (license.details === 'unknown') license.details = 'Copyright holder unknown';
  return {details: license.details, usage_terms: formatted, cc: cc, image: image};
}

function keyMap (key) {
  const mappings = {
    'credit_line': 'credit',
    'legal_status': 'status',
    'rights': 'copyright'
  };

  return mappings[key] || key;
}
<commit_msg>move location out of display facts/details<commit_after>const getNestedProperty = require('../nested-property');
const getValues = require('./get-values');
const getJSONLD = require('./json-ld');
const typeMapping = require('../type-mapping');
const licenses = require('../../fixtures/licenses');
const usersList = require('../../fixtures/users');
const associationList = require('../../fixtures/associations');
const normalise = require('../../templates/helpers/normalise');
const prettyStatus = require('../helpers/json-to-html-data/pretty-status');
const dataLayer = require('./data-layer');
const getDescription = require('../helpers/json-to-html-data/get-description');

module.exports = (resource) => {
  const title = getValues.getTitle(resource.data);
  const type = getType(resource.data);
  const links = getLinks(resource.data);
  const related = getRelated(resource, type);
  const system = getValues.getSystem(resource.data);
  const description = getDescription(resource.data);
  const displayLocation = getValues.getDisplayLocation(resource.data);
  const level = getValues.getDocumentLevel(resource.data);
  const date = getValues.getDate(resource.data);
  const parent = getParent(resource);
  const fonds = getFonds(resource);
  const images = getImages(resource.data);
  var identifier = getIdentifier(resource.data);
  identifier = identifier ? identifier.value : null;
  const footer = require('../../fixtures/footer');
  const museums = require('../../fixtures/museums');
  const jsonLD = getJSONLD[typeMapping.toInternal(type)](resource);
  const tree = resource.tree;
  const current = getNestedProperty(resource, 'data.attributes.admin.uid');
  const titlePage = normalise(getValues.getTitle(resource.data), {system: {value: system}, type: type}) + ' | Science Museum Group Collection';
  const fact = getFacts(resource, links);
  const details = getDetails(resource, links);
  const location = getDisplayLocation(resource.data);
  const layerData = fact.concat(details);
  if (location) { layerData.push(location); }
  const layer = dataLayer(type, layerData);
  const wikipedia = getWikiLink(resource.data);
  const inProduction = Boolean(resource.inProduction);
  return {
    title,
    titlePage,
    type,
    fact,
    related,
    description,
    details,
    displayLocation,
    system,
    level,
    date,
    links,
    identifier,
    parent,
    fonds,
    images,
    footer,
    museums,
    jsonLD,
    tree,
    current,
    layer,
    wikipedia,
    inProduction
  };
};

function getDisplayLocation (data) {
  const value = getNestedProperty(data, 'attributes.locations.0.value');
  return value ? {key: 'Display Location', value: value, link: value} : null;
}

function getWikiLink (data) {
  var wikiregex = /en\.wikipedia\.org\/(?:wiki\/)(.+)/;

  var note = (data.attributes.description || []).find(el => el.value.indexOf('wiki') > -1);

  if (note && wikiregex.exec(note.value)) {
    return note.value.match(wikiregex)[1];
  } else {
    return null;
  }
}

function getType (data) {
  return data.type;
}

function getLinks (data) {
  return data.links;
}

function getFacts (resource, links) {
  // Need confirmation of what attributes to use as facts
  // TODO: only get attributes relevant to specific resource type
  const occupation = getValues.getOccupation(resource.data, links);
  const nationality = getValues.getNationality(resource.data);
  const born = getValues.getBorn(resource.data, links);
  const makers = getValues.getMakers(resource);
  const made = getMade(resource, links);
  const fonds = getFondsFact(resource);

  return [occupation, nationality, born, made, fonds].concat(makers).filter(el => el);
}

function getRelated (resource, type) {
  if (resource.data.relationships) {
    var related = {};
    Object.keys(resource.data.relationships).forEach(group => {
      related[group] = resource.data.relationships[group].data.map(item => {
        return resource.included.find(inc => inc.id === item.id);
      });
    });

    if (related.people) {
      related.people = related.people.map(e => {
        if (usersList.indexOf(getNestedProperty(e, 'attributes.role.value')) === -1 &&
          associationList.indexOf(getNestedProperty(e, 'attributes.role.value')) === -1) {
          e.attributes.role = null;
        }
        return e;
      });
    }

    return related;
  }
}

function getDetails (resource, links) {
  // Need confirmation of what attributes to use as details
  // TODO: only get attributes relevant to specific resource type
  const website = getWebsite(resource.data);
  const category = getCategory(resource.data, links);
  const accession = getAccession(resource.data);
  const materials = getMaterials(resource.data);
  const measurements = getMeasurements(resource.data);
  const identifier = getIdentifier(resource.data);
  const access = getAccess(resource.data);
  const legal = getLegal(resource.data);
  const arrangement = getArrangement(resource.data);
  const notes = getNotes(resource.data);
  const objectType = getObjectType(resource.data, links);
  const taxonomy = getTaxonomy(resource, links);

  return [website, category, accession, materials, measurements, identifier, access, arrangement, objectType, taxonomy].concat(legal, notes).filter(el => el);
}

function getWebsite (data) {
  const value = getNestedProperty(data, 'attributes.website');
  return value ? {key: 'Website', value: value, link: value} : null;
}

function getMade (resource, links) {
  const date = getValues.getMadeDate(resource.data);
  const places = getMadePlace(resource);
  const placeLinks = places && places.map(function (p) {
    return {
      value: p,
      link: links.root + '/search/places/' + encodeURIComponent(p).toLowerCase()
    };
  });
  const dateLink = createDateLink(date, links);

  if (date || places) {
    return {key: 'Made', place: placeLinks, date: {value: date, link: dateLink}};
  } else {
    return null;
  }
}

function createDateLink (date, links) {
  if (!date || !links) {
    return null;
  }
  var dateFrom = date;
  var dateTo = date;
  if (/[^\d]/.test(date)) {
    var splitDate = date.split('-');
    if (splitDate.length === 2 && splitDate[0].length === 4) {
      dateFrom = splitDate[0];
      if (splitDate[1].length === 4) {
        dateTo = splitDate[1];
      } else {
        dateTo = splitDate[0];
      }
    } else {
      return null;
    }
  }
  return links.root + '/search/date[from]/' + encodeURIComponent(dateFrom) + '/date[to]/' + encodeURIComponent(dateTo);
}

function getMadePlace (resource) {
  var places = [];
  if (resource.included) {
    resource.included.forEach(el => {
      if (el.type === 'place' && getNestedProperty(el, 'attributes.role.value') === 'made') {
        places.push(getNestedProperty(el, 'attributes.summary_title'));
      }
    });
    return places.length ? places : null;
  }
}

function getCategory (data, links) {
  var values = [];
  if (data.attributes.categories) {
    data.attributes.categories.forEach(category => {
      values.push(category.name);
    });
    return {key: 'Category', value: values.toString(), link: links.root + '/search/categories/' + encodeURIComponent(values.toString().toLowerCase())};
  } else {
    return null;
  }
}

function getAccession (data) {
  var value;
  if (data.attributes.identifier) {
    data.attributes.identifier.forEach(el => {
      if (el.type === 'accession number') {
        value = el.value;
      }
    });
    return value ? {key: 'Object Number', value: value} : null;
  } else {
    return null;
  }
}

function getMaterials (data) {
  var values;
  if (data.attributes.materials) {
    values = data.attributes.materials.map(material => {
      return {
        value: material,
        link: `/search/objects/material/${material.toLowerCase()}`
      };
    });
    return {key: 'Materials', value: values};
  } else {
    return null;
  }
}

function getMeasurements (data) {
  var values = [];
  if (data.attributes.component) {
    data.attributes.component.forEach(el => {
      values.push(el.measurements.display + '
');
    });
    return values.length ? {key: 'Measurements', value: values.join('')} : null;
  } else if (data.type === 'documents' && data.attributes.measurements) {
    data.attributes.measurements.dimensions.forEach(el => {
      values.push(el.value + '
');
    });
    return values.length ? {key: 'Extent', value: values.join('')} : null;
  }
}

function getIdentifier (data) {
  var value;
  if (data.attributes.identifier) {
    data.attributes.identifier.forEach(el => {
      if (el.type !== 'accession number' && el.primary) {
        value = el.value;
      }
    });
    return value ? {key: 'Identifier', value: value} : null;
  } else {
    return null;
  }
}

function getAccess (data) {
  const value = getNestedProperty(data, 'attributes.access.note.0.value');
  return value ? {key: 'Access', value: value} : null;
}

function getLegal (data) {
  var legals = [];
  var value;
  var key;
  if (getNestedProperty(data, 'attributes.legal')) {
    Object.keys(data.attributes.legal).forEach((el) => {
      key = el;
      if (typeof data.attributes.legal[el] === 'object' && data.attributes.legal[el].length && el === 'rights') {
        data.attributes.legal[el].forEach(function (r) {
          if (r.type && r.type.toLowerCase() === 'copyright') {
            value = r.holder;
          } else if (r.details) {
            value = r.details;
          }
        });
      } else {
        value = data.attributes.legal[el];
      }
      legals.push(value ? {key: keyMap(key), value: value} : null);
    });
  }
  legals = legals.filter(Boolean);

  return legals.map(function (l) {
    return {
      key: l.key,
      value: prettyStatus(l.value) || l.value
    };
  });
}

function getArrangement (data) {
  const value = getNestedProperty(data, 'attributes.arrangement.system.0');
  return value ? {key: 'System of Arrangement', value: value} : null;
}

function getNotes (data) {
  var notes = [];
  if (data.attributes.note && data.type !== 'people') {
    data.attributes.note.forEach(el => {
      if (el.type) {
        notes.push({key: el.type, value: el.value});
      }
    });
    return notes;
  } else {
    return null;
  }
}

function getObjectType (data, links) {
  var value;
  if (data.type !== 'people' && data.attributes.name) {
    value = data.attributes.name.map((el, i, arr) => {
      var val = el.value.toLowerCase();
      return {
        value: val,
        link: links.root + '/search/objects/object_type/' + encodeURIComponent(val.toLowerCase())
      };
    });
    return value ? {key: 'type', value: value} : null;
  }
}

function getTaxonomy (resource, links) {
  var taxonomy = [];
  if (resource.data.type !== 'people' && resource.included) {
    resource.included.forEach(el => {
      if (el.attributes.hierarchy) {
        el.attributes.hierarchy.forEach(el => {
          taxonomy.push({value: el.name[0].value.toLowerCase(), link: links.root + '/search/objects/object_type/' + encodeURIComponent(el.name[0].value.toLowerCase())});
        });
      }
    });
    return taxonomy.length ? {key: 'taxonomy', value: taxonomy.filter(el => el.value.indexOf('<') === -1).reverse()} : null;
  }
}

function getParent (resource) {
  if (getNestedProperty(resource, 'data.relationships.parent')) {
    return resource.included.find(el => {
      return el.id === resource.data.relationships.parent.data[0].id;
    });
  } else {
    return null;
  }
}

function getFonds (resource) {
  var fondsID;
  var fonds;
  if (getNestedProperty(resource, 'data.relationships.fonds')) {
    fondsID = resource.data.relationships.fonds.data[0].id;
    fonds = resource.included.find(el => el.id === fondsID);
    if (fonds) {
      return {
        id: fondsID,
        link: fonds.links.self,
        summary_title: fonds.attributes.summary_title
      };
    }
  }
  return null;
}

function getFondsFact (resource) {
  var fonds = getFonds(resource);
  if (fonds) {
    return {
      key: 'part of archive',
      value: fonds.summary_title,
      link: '#archive-block'
    };
  } else {
    return null;
  }
}

function getImages (data) {
  var images = [];

  if (data.attributes.multimedia) {
    images = data.attributes.multimedia.map(e => {
      if (e.processed) {
        var img = {
          large: getNestedProperty(e, 'processed.large.location'),
          medium: getNestedProperty(e, 'processed.medium.location'),
          card: getNestedProperty(e, 'processed.medium_thumbnail.location'),
          thumb: getNestedProperty(e, 'processed.small_thumbnail.location'),
          zoom: getNestedProperty(e, 'processed.zoom.location'),
          author: e.author,
          credit: e.credit
        };

        if (e.source && e.source.legal) {
          img.rights = formatLicense(e.source.legal.rights[0]);
        }

        if (data.type === 'documents') {
          img.title = '';
        } else {
          img.title = (e.source && e.source.title) ? e.source.title.find(el => el.type === 'main title').value || e.source.title[0].value : '';
        }

        return img;
      } else {
        return null;
      }
    });
  }
  return images.filter(Boolean);
}

function formatLicense (license) {
  var formatted;
  var cc;
  var image;
  if (license.usage_terms) {
    Object.keys(licenses).forEach(el => {
      if (license.usage_terms.match(el)) {
        formatted = license.usage_terms.replace(el, licenses[el].text);
        image = licenses[el].image;
        cc = true;
      }
    });
  }
  if (license.details === 'unknown') license.details = 'Copyright holder unknown';
  return {details: license.details, usage_terms: formatted, cc: cc, image: image};
}

function keyMap (key) {
  const mappings = {
    'credit_line': 'credit',
    'legal_status': 'status',
    'rights': 'copyright'
  };

  return mappings[key] || key;
}
<|endoftext|>