<commit_before>define(['../../../js/common'], function () {

    require(['template', 'ztree', 'form', 'select2', 'ue', 'ue.file', 'ue.video'], function (template) {

        var siteId = util.get_query('siteId'),
            id = util.get_query('id');

        document.title = (id ? '编辑文章' : '添加文章') + ' - CMS内容管理系统';

        $('#txt_header').text(id ? '编辑文章' : '添加文章');

        var nav = $('#nav_tools'),
            container = $('#editor_container'),
            editor = {},
            save = function (e, publish, callback) {

                var title = $('#txt_title', container).val(),
                    content = editor.getContent(),
                    category = $('#txt_category', container).val(),
                    summary = $('#txt_summary', container).val(),
                    sort_order = $('#txt_sort_order', container).val(),
                    view_count = $('#txt_view_count', container).val(),
                    date_created = $('#txt_date_created', container).val(),
                    is_top = $('[name=cb_top]', container).val(),
                    props = {};

                if (title.length == 0) {

                    alert('文章的标题不能为空');
                    $('#txt_title', container).focus();

                    e.data('pending', false);
                    return false;
                }

                if (content.length == 0) {

                    alert('文章的内容不能为空');
                    editor.focus(true);

                    e.data('pending', false);
                    return false;
                }

                if (category.length == 0) {

                    alert('文章的栏目不能为空');
                    $('#btn_category', container).trigger('click');

                    e.data('pending', false);
                    return false;
                }

                //处理自定义属性
                var has_error = false;
                $('.prop_key', container).each(function (i, v) {

                    var key = $(this).val();

                    if (key.length == 0) return;

                    if (props[key]) {

                        has_error = true;
                        alert('自定义属性的键不能重复');
                        return false;
                    }

                    props[key] = $(this).next('.prop_value').val();
                });

                if (has_error) {

                    e.data('pending', false);
                    return false;
                }

                $.post(config.host + 'posts/save', {
                    siteId: siteId,
                    id: id,
                    title: title,
                    subTitle: '',
                    content: content,
                    summary: summary,
                    categoryId: category,
                    viewCount: view_count,
                    sortOrder: sort_order,
                    publish: publish,
                    dateCreated: date_created,
                    isTop: is_top,
                    props: JSON.stringify(props)
                }, function (r) {

                    r = handleException(r);

                    if (!r) return false;
                    if (r.code < 0) {

                        alert(r.msg || '发生未知错误，请刷新后尝试');
                        return false;
                    }

                    callback && callback.apply(this, [r]);
                }, 'json');
            },
            init = function (data) {

                var props = JSON.parse(data.post.props || '{}');

                data.post.props = [];

                for (var p in props) {

                    data.post.props.push({
                        key: p,
                        value: props[p]
                    });
                }

                //构造HTML
                container.html(template('post_edit_form', data));

                //获取图片，文件，视频上传配置
                $.ajax({
                    url: config.host + 'config/get',
                    type: 'POST',
                    dataType: 'json',
                    data: {
                        siteId: siteId
                    },
                    async: false
                }).done(function (r) {

                    if (!r || r.code < 0) {
                        alert(r.msg || '发生未知错误，请刷新页面后尝试');

                        window.close();
                        return false;
                    }

                    //init ueditor
                    editor = UE.getEditor('txt_content', {
                        initialFrameWidth: '100%',
                        initialFrameHeight: $(window).height() - 270,
                        image: r.image,
                        file: r.file,
                        video: r.video
                    });
                });

                //select category
                $('#btn_category', container).on('click', function () {

                    var p_tree = $('

'),
                        selected = {
                            title: '',
                            id: ''
                        },
                        dlg = $M({
                            title: '选择栏目信息',
                            content: p_tree[0],
                            lock: true,
                            width: '300px',
                            height: '300px',
                            ok: function () {

                                dlg.close();

                                $('#txt_category', container).val(selected.id);
                                $('#btn_category', container).text(selected.title || '选择栏目');
                            },
                            okVal: '保存',
                            cancel: false,
                            cancelVal: '取消'
                        });

                    p_tree = $.fn.zTree.init(p_tree, {
                        async: {
                            enable: true,
                            url: config.host + 'category/list_with_permission',
                            autoParam: ['id=parentId'],
                            otherParam: {
                                'siteId': util.get_query('siteId')
                            },
                            type: "post"
                        },
                        callback: {
                            onClick: function (event, tId, node) {
                                selected = {
                                    title: node.name,
                                    id: node.id
                                };
                            }
                        }
                    });

                    return false;
                });

                //select is_top
                $('[name=cb_top]', container).select2({
                    minimumResultsForSearch: -1
                }).val($('[name=cb_top]', container).data('selected').toString()).trigger('change');
            };

        //保存为草稿
        $('.btn_save', nav).on('click', function () {

            var _this = $(this);

            if (_this.data('disable') || _this.data('pending')) return false;

            _this.data('pending', true);

            save(_this, false, function (r) {

                _this.data('pending', false);

                if (r.code < 0) {

                    alert(r.msg || '发生未知错误，请刷新页面后尝试');
                    return false;
                }

                //保存草稿后将本文章的ID更新为保存后的ID，防止保存&发布时出现多个相同的文章
                id = r.id;

                alert('已保存为草稿，可继续编辑');
            });
        });

        //保存并发布
        $('.btn_publish', nav).on('click', function () {

            var _this = $(this);

            if (_this.data('pending')) return false;

            _this.data('pending', true);

            save(_this, true, function (r) {

                _this.data('pending', false);

                alert(r.is_pending ? '已保存文章，并提交至审核，待审核通过后可展示' : '已成功保存并发布文章');

                window.history.go(-1);
            });

            return false;
        });

        //添加
        if (!id) {

            init({ post: { category: {} } });

            return false;
        }

        $.ajax({
            url: config.host + 'posts/detail',
            type: 'POST',
            dataType: 'json',
            data: {
                siteId: siteId,
                id: id
            },
            async: false
        }).done(function (r) {

            if (!r || r.code < 0) {
                alert(r.msg || '发生未知错误，请刷新页面后尝试');

                window.close();
                return false;
            }

            //更新时不需要保存草稿
            $('.btn_save').data('disable', true);

            $('.btn_publish').text('更新').prop('title', '更新当前文章内容') && init(r);
        });
    });
});<commit_msg>tiny fix<commit_after>define(['../../../js/common'], function () {

    require(['template', 'ztree', 'form', 'select2', 'ue', 'ue.file', 'ue.video'], function (template) {

        var siteId = util.get_query('siteId'),
            id = util.get_query('id');

        document.title = (id ? '编辑文章' : '添加文章') + ' - CMS内容管理系统';

        $('#txt_header').text(id ? '编辑文章' : '添加文章');

        var nav = $('#nav_tools'),
            container = $('#editor_container'),
            editor = {},
            save = function (e, publish, callback) {

                var title = $('#txt_title', container).val(),
                    content = editor.getContent(),
                    category = $('#txt_category', container).val(),
                    summary = $('#txt_summary', container).val(),
                    sort_order = $('#txt_sort_order', container).val(),
                    view_count = $('#txt_view_count', container).val(),
                    date_created = $('#txt_date_created', container).val(),
                    is_top = $('[name=cb_top]', container).val(),
                    props = {};

                if (title.length == 0) {

                    alert('文章的标题不能为空');
                    $('#txt_title', container).focus();

                    e.data('pending', false);
                    return false;
                }

                if (content.length == 0) {

                    alert('文章的内容不能为空');
                    editor.focus(true);

                    e.data('pending', false);
                    return false;
                }

                if (category.length == 0) {

                    alert('文章的栏目不能为空');
                    $('#btn_category', container).trigger('click');

                    e.data('pending', false);
                    return false;
                }

                //处理自定义属性
                var has_error = false;
                $('.prop_key', container).each(function (i, v) {

                    var key = $(this).val();

                    if (key.length == 0) return;

                    if (props[key]) {

                        has_error = true;
                        alert('自定义属性的键不能重复');
                        return false;
                    }

                    props[key] = $(this).next('.prop_value').val();
                });

                if (has_error) {

                    e.data('pending', false);
                    return false;
                }

                $.post(config.host + 'posts/save', {
                    siteId: siteId,
                    id: id,
                    title: title,
                    subTitle: '',
                    content: content,
                    summary: summary,
                    categoryId: category,
                    viewCount: view_count,
                    sortOrder: sort_order,
                    publish: publish,
                    dateCreated: date_created,
                    isTop: is_top,
                    props: JSON.stringify(props)
                }, function (r) {

                    r = handleException(r);

                    if (!r) return false;
                    if (r.code < 0) {

                        alert(r.msg || '发生未知错误，请刷新后尝试');

                        e.data('pending', false);
                        return false;
                    }

                    callback && callback.apply(this, [r]);
                }, 'json');
            },
            init = function (data) {

                var props = JSON.parse(data.post.props || '{}');

                data.post.props = [];

                for (var p in props) {

                    data.post.props.push({
                        key: p,
                        value: props[p]
                    });
                }

                //构造HTML
                container.html(template('post_edit_form', data));

                //获取图片，文件，视频上传配置
                $.ajax({
                    url: config.host + 'config/get',
                    type: 'POST',
                    dataType: 'json',
                    data: {
                        siteId: siteId
                    },
                    async: false
                }).done(function (r) {

                    if (!r || r.code < 0) {
                        alert(r.msg || '发生未知错误，请刷新页面后尝试');

                        window.close();
                        return false;
                    }

                    //init ueditor
                    editor = UE.getEditor('txt_content', {
                        initialFrameWidth: '100%',
                        initialFrameHeight: $(window).height() - 270,
                        image: r.image,
                        file: r.file,
                        video: r.video
                    });
                });

                //select category
                $('#btn_category', container).on('click', function () {

                    var p_tree = $('

'),
                        selected = {
                            title: '',
                            id: ''
                        },
                        dlg = $M({
                            title: '选择栏目信息',
                            content: p_tree[0],
                            lock: true,
                            width: '300px',
                            height: '300px',
                            ok: function () {

                                dlg.close();

                                $('#txt_category', container).val(selected.id);
                                $('#btn_category', container).text(selected.title || '选择栏目');
                            },
                            okVal: '保存',
                            cancel: false,
                            cancelVal: '取消'
                        });

                    p_tree = $.fn.zTree.init(p_tree, {
                        async: {
                            enable: true,
                            url: config.host + 'category/list_with_permission',
                            autoParam: ['id=parentId'],
                            otherParam: {
                                'siteId': util.get_query('siteId')
                            },
                            type: "post"
                        },
                        callback: {
                            onClick: function (event, tId, node) {
                                selected = {
                                    title: node.name,
                                    id: node.id
                                };
                            }
                        }
                    });

                    return false;
                });

                //select is_top
                $('[name=cb_top]', container).select2({
                    minimumResultsForSearch: -1
                }).val($('[name=cb_top]', container).data('selected').toString()).trigger('change');
            };

        //保存为草稿
        $('.btn_save', nav).on('click', function () {

            var _this = $(this);

            if (_this.data('disable') || _this.data('pending')) return false;

            _this.data('pending', true);

            save(_this, false, function (r) {

                _this.data('pending', false);

                if (r.code < 0) {

                    alert(r.msg || '发生未知错误，请刷新页面后尝试');
                    return false;
                }

                //保存草稿后将本文章的ID更新为保存后的ID，防止保存&发布时出现多个相同的文章
                id = r.id;

                alert('已保存为草稿，可继续编辑');
            });
        });

        //保存并发布
        $('.btn_publish', nav).on('click', function () {

            var _this = $(this);

            if (_this.data('pending')) return false;

            _this.data('pending', true);

            save(_this, true, function (r) {

                _this.data('pending', false);

                alert(r.is_pending ? '已保存文章，并提交至审核，待审核通过后可展示' : '已成功保存并发布文章');

                window.history.go(-1);
            });

            return false;
        });

        //添加
        if (!id) {

            init({ post: { category: {} } });

            return false;
        }

        $.ajax({
            url: config.host + 'posts/detail',
            type: 'POST',
            dataType: 'json',
            data: {
                siteId: siteId,
                id: id
            },
            async: false
        }).done(function (r) {

            if (!r || r.code < 0) {
                alert(r.msg || '发生未知错误，请刷新页面后尝试');

                window.close();
                return false;
            }

            //更新时不需要保存草稿
            $('.btn_save').data('disable', true);

            $('.btn_publish').text('更新').prop('title', '更新当前文章内容') && init(r);
        });
    });
});<|endoftext|>


<commit_before>var nrf = require('nrf')
var reverse = require('reverse-string')
var Bacon = require('baconjs')
var _ = require('lodash')

var config = {
  spiDevice: process.env.NRF24_SPI_DEVICE ? process.env.NRF24_SPI_DEVICE : "/dev/spidev0.1",
  cePin: 25,                     // GPIO_25
  irqPin: 24,                    // GPIO_24
  channel: 76,                   // 76 is the default channel for RF24 Arduino library
  rxAddress: process.env.NRF24_RX_ADDRESS ? process.env.NRF24_RX_ADDRESS : "nrf01",
  dataRate: '250kbps',
  crcBytes: 2,
  txPower: 'PA_MAX'
}

console.log("Starting with configuration:\n", config)

var radio = nrf.connect(config.spiDevice, config.cePin, config.irqPin)
radio
  .dataRate(config.dataRate)
  .channel(config.channel)
  .crcBytes(config.crcBytes)
  .transmitPower(config.txPower)

var sensorStream = Bacon.fromCallback(radio.begin)
  .flatMapLatest(() => Bacon.fromCallback(radio.setStates, {EN_ACK_PAY:false, EN_DYN_ACK:false}))   // Disable ACK payloads & dynamic ack to get auto ACK function with PA+LNA module
  .flatMapLatest(() => {
    var rx = radio.openPipe('rx', new Buffer(reverse(config.rxAddress))) // RF24 on Arduino doesn't send data in LSB order -> reverse to match
    return Bacon.fromEvent(rx, 'data').map(dataReceived).filter(_.identity)
      .merge(Bacon.fromEvent(rx, 'error', Bacon.Error))
  })

function dataReceived(buffer) {
  Array.prototype.reverse.call(buffer)        // RF24 on Arduino doesn't send data in LSB order -> reverse to match
  var data = _.assign(parseTagAndInstance(buffer), { ts: new Date() })

  try {
    switch(data.tag) {
      case 't':
        fillTemperatureData(buffer, data)
        break;
      case 'p':
        fillPressureData(buffer, data)
        break;
      case 'c':
        fillCurrentData(buffer, data)
        break;
      case 'a':
        fillAutopilotRemoteData(buffer, data)
        break;
      case 'w':
        fillTankData(buffer, data)
        break;
      default:
        console.error("Received unknown data!", buffer)
        return
    }
  } catch(err) {
    console.error("Error while decoding received data! Data: ", buffer, "\nError:", err)
    return
  }

  return data
}

function parseTagAndInstance(buffer) {
  return { tag: buffer.toString('utf8', 0, 1), instance: buffer.readUInt8(1) }
}

function fillTemperatureData(buffer, data) {
  data.temperature = buffer.readFloatLE(2)
  data.vcc = buffer.readInt16LE(6)
  data.previousSampleTimeMicros = buffer.readUInt32LE(8)
}

function fillPressureData(buffer, data) {
  data.pressure = buffer.readFloatLE(2)
  data.vcc = buffer.readInt16LE(6)
  data.previousSampleTimeMicros = buffer.readUInt32LE(8)
}

function fillCurrentData(buffer, data) {
  data.rawMeasurement = buffer.readInt16LE(2)
  data.shuntVoltageMilliVolts = buffer.readFloatLE(4)
  data.current = buffer.readFloatLE(8)
  data.vcc = buffer.readInt16LE(12)
  data.previousSampleTimeMicros = buffer.readUInt32LE(14)
}

function fillAutopilotRemoteData(buffer, data) {
  data.buttonId = buffer.readUInt8(2)
  data.isLongPress = buffer.readUInt8(3) !== 0
  data.vcc = buffer.readInt16LE(4)
  data.previousSampleTimeMicros = buffer.readUInt32LE(6)
}

function fillTankData(buffer, data) {
  data.tankLevel = buffer.readUInt8(2)
  data.vcc = buffer.readInt16LE(3)
  data.previousSampleTimeMicros = buffer.readUInt32LE(5)
}

module.exports = sensorStream<commit_msg>Add support for RFM gateway status messages<commit_after>var nrf = require('nrf')
var reverse = require('reverse-string')
var Bacon = require('baconjs')
var _ = require('lodash')

var config = {
  spiDevice: process.env.NRF24_SPI_DEVICE ? process.env.NRF24_SPI_DEVICE : "/dev/spidev0.1",
  cePin: 25,                     // GPIO_25
  irqPin: 24,                    // GPIO_24
  channel: 76,                   // 76 is the default channel for RF24 Arduino library
  rxAddress: process.env.NRF24_RX_ADDRESS ? process.env.NRF24_RX_ADDRESS : "nrf01",
  dataRate: '250kbps',
  crcBytes: 2,
  txPower: 'PA_MAX'
}

console.log("Starting with configuration:\n", config)

var radio = nrf.connect(config.spiDevice, config.cePin, config.irqPin)
radio
  .dataRate(config.dataRate)
  .channel(config.channel)
  .crcBytes(config.crcBytes)
  .transmitPower(config.txPower)

var sensorStream = Bacon.fromCallback(radio.begin)
  .flatMapLatest(() => Bacon.fromCallback(radio.setStates, {EN_ACK_PAY:false, EN_DYN_ACK:false}))   // Disable ACK payloads & dynamic ack to get auto ACK function with PA+LNA module
  .flatMapLatest(() => {
    var rx = radio.openPipe('rx', new Buffer(reverse(config.rxAddress))) // RF24 on Arduino doesn't send data in LSB order -> reverse to match
    return Bacon.fromEvent(rx, 'data').map(dataReceived).filter(_.identity)
      .merge(Bacon.fromEvent(rx, 'error', Bacon.Error))
  })

function dataReceived(buffer) {
  Array.prototype.reverse.call(buffer)        // RF24 on Arduino doesn't send data in LSB order -> reverse to match
  var data = _.assign(parseTagAndInstance(buffer), { ts: new Date() })

  try {
    switch(data.tag) {
      case 't':
        fillTemperatureData(buffer, data)
        break;
      case 'p':
        fillPressureData(buffer, data)
        break;
      case 'c':
        fillCurrentData(buffer, data)
        break;
      case 'a':
        fillAutopilotRemoteData(buffer, data)
        break;
      case 'w':
        fillTankData(buffer, data)
        break;
      case 's':
        fillRFMGatewayData(buffer, data)
        break;
      default:
        console.error("Received unknown data!", buffer)
        return
    }
  } catch(err) {
    console.error("Error while decoding received data! Data: ", buffer, "\nError:", err)
    return
  }

  return data
}

function parseTagAndInstance(buffer) {
  return { tag: buffer.toString('utf8', 0, 1), instance: buffer.readUInt8(1) }
}

function fillTemperatureData(buffer, data) {
  data.temperature = buffer.readFloatLE(2)
  data.vcc = buffer.readInt16LE(6)
  data.previousSampleTimeMicros = buffer.readUInt32LE(8)
}

function fillPressureData(buffer, data) {
  data.pressure = buffer.readFloatLE(2)
  data.vcc = buffer.readInt16LE(6)
  data.previousSampleTimeMicros = buffer.readUInt32LE(8)
}

function fillCurrentData(buffer, data) {
  data.rawMeasurement = buffer.readInt16LE(2)
  data.shuntVoltageMilliVolts = buffer.readFloatLE(4)
  data.current = buffer.readFloatLE(8)
  data.vcc = buffer.readInt16LE(12)
  data.previousSampleTimeMicros = buffer.readUInt32LE(14)
}

function fillAutopilotRemoteData(buffer, data) {
  data.buttonId = buffer.readUInt8(2)
  data.isLongPress = buffer.readUInt8(3) !== 0
  data.vcc = buffer.readInt16LE(4)
  data.previousSampleTimeMicros = buffer.readUInt32LE(6)
}

function fillTankData(buffer, data) {
  data.tankLevel = buffer.readUInt8(2)
  data.vcc = buffer.readInt16LE(3)
  data.previousSampleTimeMicros = buffer.readUInt32LE(5)
}

function fillRFMGatewayData(buffer, data) {
  data.rssi = buffer.readInt16LE(2)
  data.ackSent = buffer.readUInt8(4) !== 0
  data.previousSampleTimeMicros = buffer.readUInt32LE(5)
}

module.exports = sensorStream
<|endoftext|>


<commit_before>

var fs = require('fs');
var Parser = require('./parse.js').StreamParser;

var argv = process.argv.slice(2);
for(var i=0; i<argv.length; i++){
	var arg = argv[i];
	if(arg.slice(0,1)=='-') switch(arg){
		case '-v': case '--version':
			console.log();
			break;
	}
}

var dirpath = 'test/syntax-suite/test_parsing';
var syntaxTestFiles = fs.readdirSync(dirpath).filter(function(n){
	return n[0]=='y' || n[0]=='n';
});

var validationTestFiles = [
	'tests/draft6/additionalItems.json',
	'tests/draft6/additionalProperties.json',
	'tests/draft6/allOf.json',
	//'tests/draft6/anyOf.json',
	'tests/draft6/default.json',
	//'tests/draft6/definitions.json',
	//'tests/draft6/dependencies.json',
	//'tests/draft6/enum.json',
	'tests/draft6/items.json',
	'tests/draft6/maximum.json',
	'tests/draft6/maxItems.json',
	//'tests/draft6/maxLength.json',
	'tests/draft6/maxProperties.json',
	'tests/draft6/minimum.json',
	'tests/draft6/minItems.json',
	//'tests/draft6/minLength.json',
	'tests/draft6/minProperties.json',
	'tests/draft6/multipleOf.json',
	//'tests/draft6/not.json',
	//'tests/draft6/oneOf.json',
	'tests/draft6/pattern.json',
	'tests/draft6/patternProperties.json',
	'tests/draft6/properties.json',
	'tests/draft6/ref.json',
	//'tests/draft6/refRemote.json',
	'tests/draft6/required.json',
	'tests/draft6/type.json',
	//'tests/draft6/uniqueItems.json',
];

function runSyntaxTest(filename, done){
	var res = {
		pass: 0,
		fail: 0,
		messages: [],
		total: 0,
	};
	var filepath = dirpath + '/' + filename;
	var valid = null;
	if(filename[0]=='y') valid = true;
	else if(filename[0]=='n') valid=false;
	else return void done(new Error('Unknown test type'));
	var testError = null;
	var t = fs.createReadStream(filepath);
	var p = new Parser({}, {});
	t.pipe(p);
	p.on('error', function(err){
		if(!err) return;
		testError = err;
		result(err);
	});
	t.on('end', function(){
		if(testError) return;
		result();
	});
	function result(err){
		res.total++;
		if((valid==false && err) || (valid==true && !err)){
			res.pass++;
		}else{
			res.fail++;
			var er = {description:err.stack, valid:valid, error:err, value:p.value};
			res.messages.push(er);
		}
		return void done(null, res);
	}
}

function runValidationTest(filepath, done){
	var res = {
		pass: 0,
		fail: 0,
		messages: [],
		total: 0,
	};
	var schemas = require('./test/schema-suite/'+filepath);
	//console.log(filepath + ' ...');
	schemas.forEach(function(s){
		var schema = s.schema;
		var tests = s.tests;
		tests.forEach(function(t, i){
			var tjson = JSON.stringify(t.data, null, "\t");
			var throws = false;
			try {
				var p = new Parser(schema, {});
				p._transform(tjson);
				p.eof();
			}catch(e){
				throws = e;
			}
			res.total++;
			if(throws){
				res.fail++;
				var er = {description:t.description, i:i, valid:t.valid, errors:throws, schema:schema, value:t.data};
				res.messages.push(er);
			}else if(t.valid!=!!p.errors.length){
				res.pass++;
			}else{
				res.fail++;
				var er = {description:t.description, i:i, valid:t.valid, errors:p.errors, schema:schema, value:t.data};
				res.messages.push(er);
				//console.error('  Error: ('+er.description+') expected '+(er.valid?'valid':'invalid')+' but got '+er.errors.length+' errors');
				//console.log(er.schema, er.value);
				//console.error(er.errors);
			}
		});
	});
	return void done(null, res);
}

function runTests(){
	var totalFail = 0;
	var pass = 0;
	var fail = 0;
	var total = 0;
	var messages = [];
	nextSyntaxTest(0);
	function nextSyntaxTest(i){
		if(i==0) console.log('Running syntax tests...');
		var n = syntaxTestFiles[i];
		if(!n) return void finishSyntaxTests();
		runSyntaxTest(n, function(err, res){
			if(err) throw err;
			res.messages.forEach(function(v){
				console.log('Error: syntaxTest('+n+') '+v.description);
			});
			pass += res.pass;
			fail += res.fail;
			totalFail += res.fail;
			total += res.total;
			nextSyntaxTest(i+1);
		});
	}
	function finishSyntaxTests(){
		if(fail) console.log('Running syntax tests: ' + fail + ' failed');
		console.log('Running syntax tests: ' + pass + '/' + total + ' passed');
		console.log('Running validation tests...');
		pass = 0;
		fail = 0;
		total = 0;
		nextValidationTest(0);
	}
	function nextValidationTest(i){
		var n = validationTestFiles[i];
		if(!n) return void finishValidationTests(0);
		runValidationTest(n, function(err, res){
			if(err) throw err;
			res.messages.forEach(function(v){
				console.log('Error: validationTest('+n+') '+v.description, v);
			});
			pass += res.pass;
			fail += res.fail;
			totalFail += res.fail;
			total += res.total;
			nextValidationTest(i+1);
		});
	}
	function finishValidationTests(){
		if(fail) console.log('Validation tests: ' + fail + ' failed');
		console.log('Validation tests: ' + pass + '/' + total + ' passed');
		//if(totalFail) process.exit(1);
	}
}

runTests();

<commit_msg>Print more descriptive stack traces<commit_after>

var fs = require('fs');
var Parser = require('./parse.js').StreamParser;

Error.stackTraceLimit = 25;

var argv = process.argv.slice(2);
for(var i=0; i<argv.length; i++){
	var arg = argv[i];
	if(arg.slice(0,1)=='-') switch(arg){
		case '-v': case '--version':
			console.log();
			break;
	}
}

var dirpath = 'test/syntax-suite/test_parsing';
var syntaxTestFiles = fs.readdirSync(dirpath).filter(function(n){
	return n[0]=='y' || n[0]=='n';
});

var validationTestFiles = [
	'tests/draft6/additionalItems.json',
	'tests/draft6/additionalProperties.json',
	'tests/draft6/allOf.json',
	//'tests/draft6/anyOf.json',
	'tests/draft6/default.json',
	//'tests/draft6/definitions.json',
	//'tests/draft6/dependencies.json',
	//'tests/draft6/enum.json',
	'tests/draft6/items.json',
	'tests/draft6/maximum.json',
	'tests/draft6/maxItems.json',
	//'tests/draft6/maxLength.json',
	'tests/draft6/maxProperties.json',
	'tests/draft6/minimum.json',
	'tests/draft6/minItems.json',
	//'tests/draft6/minLength.json',
	'tests/draft6/minProperties.json',
	'tests/draft6/multipleOf.json',
	//'tests/draft6/not.json',
	//'tests/draft6/oneOf.json',
	'tests/draft6/pattern.json',
	'tests/draft6/patternProperties.json',
	'tests/draft6/properties.json',
	'tests/draft6/ref.json',
	//'tests/draft6/refRemote.json',
	'tests/draft6/required.json',
	'tests/draft6/type.json',
	//'tests/draft6/uniqueItems.json',
];

function runSyntaxTest(filename, done){
	var res = {
		pass: 0,
		fail: 0,
		messages: [],
		total: 0,
	};
	var filepath = dirpath + '/' + filename;
	var valid = null;
	if(filename[0]=='y') valid = true;
	else if(filename[0]=='n') valid=false;
	else return void done(new Error('Unknown test type'));
	var testError = null;
	var t = fs.createReadStream(filepath);
	var p = new Parser({}, {});
	t.pipe(p);
	p.on('error', function(err){
		if(!err) return;
		testError = err;
		result(err);
	});
	t.on('end', function(){
		if(testError) return;
		result();
	});
	function result(err){
		res.total++;
		if((valid==false && err) || (valid==true && !err)){
			res.pass++;
		}else{
			res.fail++;
			var er = {description:err.stack, valid:valid, error:err, value:p.value};
			res.messages.push(er);
		}
		return void done(null, res);
	}
}

function runValidationTest(filepath, done){
	var res = {
		pass: 0,
		fail: 0,
		messages: [],
		total: 0,
	};
	var schemas = require('./test/schema-suite/'+filepath);
	//console.log(filepath + ' ...');
	schemas.forEach(function(s){
		var schema = s.schema;
		var tests = s.tests;
		tests.forEach(function(t, i){
			var tjson = JSON.stringify(t.data, null, "\t");
			var throws = false;
			try {
				var p = new Parser(schema, {});
				p._transform(tjson);
				p.eof();
			}catch(e){
				throws = e;
			}
			res.total++;
			if(throws){
				res.fail++;
				var er = {description:t.description, i:i, valid:t.valid, errors:throws, schema:schema, value:t.data};
				res.messages.push(er);
			}else if(t.valid!=!!p.errors.length){
				res.pass++;
			}else{
				res.fail++;
				var er = {description:t.description, i:i, valid:t.valid, errors:p.errors, schema:schema, value:t.data};
				res.messages.push(er);
				//console.error('  Error: ('+er.description+') expected '+(er.valid?'valid':'invalid')+' but got '+er.errors.length+' errors');
				//console.log(er.schema, er.value);
				//console.error(er.errors);
			}
		});
	});
	return void done(null, res);
}

function runTests(){
	var totalFail = 0;
	var pass = 0;
	var fail = 0;
	var total = 0;
	var messages = [];
	nextSyntaxTest(0);
	function nextSyntaxTest(i){
		if(i==0) console.log('Running syntax tests...');
		var n = syntaxTestFiles[i];
		if(!n) return void finishSyntaxTests();
		runSyntaxTest(n, function(err, res){
			if(err) throw err;
			res.messages.forEach(function(v){
				console.log('Error: syntaxTest('+n+') '+v.description);
			});
			pass += res.pass;
			fail += res.fail;
			totalFail += res.fail;
			total += res.total;
			nextSyntaxTest(i+1);
		});
	}
	function finishSyntaxTests(){
		if(fail) console.log('Running syntax tests: ' + fail + ' failed');
		console.log('Running syntax tests: ' + pass + '/' + total + ' passed');
		console.log('Running validation tests...');
		pass = 0;
		fail = 0;
		total = 0;
		nextValidationTest(0);
	}
	function nextValidationTest(i){
		var n = validationTestFiles[i];
		if(!n) return void finishValidationTests(0);
		runValidationTest(n, function(err, res){
			if(err) throw err;
			res.messages.forEach(function(v){
				console.log('Error: validationTest('+n+') '+v.description, v);
			});
			pass += res.pass;
			fail += res.fail;
			totalFail += res.fail;
			total += res.total;
			process.nextTick(nextValidationTest.bind(null, i+1));
		});
	}
	function finishValidationTests(){
		if(fail) console.log('Validation tests: ' + fail + ' failed');
		console.log('Validation tests: ' + pass + '/' + total + ' passed');
		//if(totalFail) process.exit(1);
	}
}

runTests();

<|endoftext|>


<commit_before>import Reflux from 'reflux';
import clonedeep from 'lodash.clonedeep';
import {stores} from 'js/stores';
import {actions} from 'js/actions';
import {getRouteAssetUid} from 'js/routerUtils';
import {getSurveyFlatPaths} from 'js/assetUtils';
import {
  QUESTION_TYPES,
  GROUP_TYPES_BEGIN,
  GROUP_TYPES_END,
} from 'js/constants';
import {
  EXCLUDED_COLUMNS,
  VALIDATION_STATUS_ID_PROP,
  SUBMISSION_ACTIONS_ID,
  DATA_TABLE_SETTING,
  DATA_TABLE_SETTINGS,
} from 'js/components/submissions/tableConstants';

/**
 * TODO: The tableStore should be handling all data required by table.es6, but
 * as this would mean a huge refactor, we will stick to providing a one way
 * interface for changing things in asset.
 *
 * NOTE: To simplify code a bit, we assume this will be used only on the table
 * route - i.e. asset and assetUid will be fetched separately.
 *
 * Use actions listeners to be up to date.
 */
const tableStore = Reflux.createStore({
  /**
   * A shortcut method, to be deleted in future.
   */
  getCurrentAsset() {
    return stores.allAssets.getAsset(getRouteAssetUid());
  },

  /**
   * @param {object[]} submissions - list of submissions
   * @returns {string[]} a unique list of columns (keys) that should be displayed to users
   */
  getAllColumns(submissions) {
    const asset = this.getCurrentAsset();
    const flatPaths = getSurveyFlatPaths(asset.content.survey);

    // add all questions from the survey definition
    let output = Object.values(flatPaths);

    // Gather unique columns from all visible submissions and add them to output
    const dataKeys = Object.keys(submissions.reduce(function (result, obj) {
      return Object.assign(result, obj);
    }, {}));
    output = [...new Set([...dataKeys, ...output])];

    // exclude some technical non-data columns
    output = output.filter((key) => EXCLUDED_COLUMNS.includes(key) === false);

    // exclude notes
    output = output.filter((key) => {
      const foundPathKey = Object.keys(flatPaths).find(
        (pathKey) => flatPaths[pathKey] === key
      );

      // no path means this definitely is not a note type
      if (!foundPathKey) {
        return true;
      }

      const foundNoteRow = asset.content.survey.find(
        (row) =>
          typeof foundPathKey !== 'undefined' &&
          (foundPathKey === row.name || foundPathKey === row.$autoname) &&
          row.type === QUESTION_TYPES.note.id
      );

      if (typeof foundNoteRow !== 'undefined') {
        // filter out this row as this is a note type
        return false;
      }

      return true;
    });

    // exclude kobomatrix rows as data is not directly tied to them, but
    // to rows user answered to, thus making these columns always empty
    const excludedMatrixKeys = [];
    let isInsideKoboMatrix = false;
    asset.content.survey.forEach((row) => {
      if (row.type === GROUP_TYPES_BEGIN.begin_kobomatrix) {
        isInsideKoboMatrix = true;
      } else if (row.type === GROUP_TYPES_END.end_kobomatrix) {
        isInsideKoboMatrix = false;
      } else if (isInsideKoboMatrix) {
        const rowPath = flatPaths[row.name] || flatPaths[row.$autoname];
        excludedMatrixKeys.push(rowPath);
      }
    });
    output = output.filter((key) => excludedMatrixKeys.includes(key) === false);

    return output;
  },

  /**
   * @param {object[]} submissions - list of submissions
   * @returns {string[]} a list of columns that user can hide
   */
  getHideableColumns(submissions) {
    const columns = this.getAllColumns(submissions);
    columns.push(VALIDATION_STATUS_ID_PROP);
    return columns;
  },

  /**
   * @returns {string[]|null} a list of selected columns from table settings,
   * `null` means no selection, i.e. all columns
   */
  getSelectedColumns() {
    const tableSettings = this.getTableSettings();
    if (Array.isArray(tableSettings[DATA_TABLE_SETTINGS.SELECTED_COLUMNS])) {
      return tableSettings[DATA_TABLE_SETTINGS.SELECTED_COLUMNS];
    }
    return null;
  },

  /**
   * @returns {object} settings or empty object if no settings exist
   */
  getTableSettings() {
    const asset = this.getCurrentAsset();
    if (
      asset?.settings &&
      asset?.settings[DATA_TABLE_SETTING]
    ) {
      return asset.settings[DATA_TABLE_SETTING];
    }
    return {};
  },

  /**
   * @returns {string|null} the current frozen column
   */
  getFrozenColumn() {
    let frozenColumn = null;
    const tableSettings = this.getTableSettings();
    if (tableSettings && tableSettings[DATA_TABLE_SETTINGS.FROZEN_COLUMN]) {
      frozenColumn = tableSettings[DATA_TABLE_SETTINGS.FROZEN_COLUMN];
    }
    return frozenColumn;
  },

  /**
   * @returns {object|null} the current sort by value
   */
  getSortBy() {
    let sortBy = null;
    const tableSettings = this.getTableSettings();
    if (tableSettings && tableSettings[DATA_TABLE_SETTINGS.SORT_BY]) {
      sortBy = tableSettings[DATA_TABLE_SETTINGS.SORT_BY];
    }
    return sortBy;
  },

  /**
   * @param {string} fieldId
   * @returns {boolean}
   */
  isFieldVisible(fieldId) {
    // frozen column is never hidden
    if (this.isFieldFrozen(fieldId)) {
      return true;
    }

    // submission actions is never hidden
    if (fieldId === SUBMISSION_ACTIONS_ID) {
      return true;
    }

    const selectedColumns = this.getSelectedColumns();
    // nothing is selected, so all columns are visible
    if (selectedColumns === null) {
      return true;
    }

    if (Array.isArray(selectedColumns)) {
      return selectedColumns.includes(fieldId);
    }

    return true;
  },

  /**
   * @param {object} asset
   * @param {string} fieldId
   * @returns {boolean}
   */
  isFieldFrozen(fieldId) {
    return this.getFrozenColumn() === fieldId;
  },

  /**
   * @param {string} fieldId
   * @param {boolean} isFrozen
   */
  setFrozenColumn(fieldId, isFrozen) {
    // NOTE: Currently we only support one frozen column at a time, so that is
    // why making column not-frozen means we just null-ify the value, without
    // checking what column is frozen now.
    let newVal = null;
    if (isFrozen) {
      newVal = fieldId;
    }
    const settingsObj = {};
    settingsObj[DATA_TABLE_SETTINGS.FROZEN_COLUMN] = newVal;
    this.saveTableSettings(settingsObj);
  },

  /**
   * @param {string} fieldId
   * @returns {string|null} null for no option, or one of SORT_VALUES
   */
  getFieldSortValue(fieldId) {
    const sortBy = this.getSortBy();
    if (sortBy === null) {
      return null;
    }

    if (sortBy?.fieldId === fieldId) {
      return sortBy.value;
    }

    return null;
  },

  /**
   * @param {string} fieldId
   * @param {string|null} sortValue one of SORT_VALUES or null for clear value
   */
  setSortBy(assetUid, fieldId, sortValue) {
    let newSortBy = null;
    if (sortValue !== null) {
      newSortBy = {
        fieldId: fieldId,
        value: sortValue,
      };
    }

    const settingsObj = {};
    settingsObj[DATA_TABLE_SETTINGS.SORT_BY] = newSortBy;
    this.saveTableSettings(settingsObj);
  },

  /**
   * @param {object} newTableSettings - will be merged into current settings, overwriting any DATA_TABLE_SETTING properties
   */
  saveTableSettings(newTableSettings) {
    const asset = this.getCurrentAsset();

    // get whole asset settings as clone to avoid bugs
    const newSettings = clonedeep(asset.settings);

    if (!newSettings[DATA_TABLE_SETTING]) {
      newSettings[DATA_TABLE_SETTING] = newTableSettings;
    } else {
      newSettings[DATA_TABLE_SETTING] = Object.assign(
        newSettings[DATA_TABLE_SETTING],
        newTableSettings
      );
    }

    if (asset && this.userCan('change_asset', asset)) {
      actions.table.updateSettings(asset.uid, newSettings);
    }
  },

  /**
   * Show single column - shortcut method for setFieldsVisibility
   * @param {string[]} hideableColumns
   * @param {string} fieldId
   */
  showField(hideableColumns, fieldId) {
    const selectedColumns = this.getSelectedColumns();

    // We start with `null` just to be safe, but the case when selectedColumns
    // is `null` already (i.e. all columns visible) and we show column should
    // never happen.
    let newSelectedColumns = null;

    // Some fields are selected and we show one column
    if (Array.isArray(selectedColumns)) {
      newSelectedColumns = [...selectedColumns];
      newSelectedColumns.push(fieldId);
    }

    this.setFieldsVisibility(hideableColumns, newSelectedColumns);
  },

  /**
   * Hide single column - a shortcut method for setFieldsVisibility
   * @param {string[]} hideableColumns
   * @param {string} fieldId
   */
  hideField(hideableColumns, fieldId) {
    const selectedColumns = this.getSelectedColumns();

    let newSelectedColumns = [];

    // Case 1: nothing selected and we hide one column, i.e. we need to select all but one
    if (selectedColumns === null) {
      newSelectedColumns = [...hideableColumns];
      newSelectedColumns.splice(newSelectedColumns.indexOf(fieldId), 1);
    }

    // Case 2: some fields selected and we hide one column
    if (Array.isArray(selectedColumns)) {
      newSelectedColumns = [...selectedColumns];
      newSelectedColumns.splice(newSelectedColumns.indexOf(fieldId), 1);
    }

    this.setFieldsVisibility(hideableColumns, newSelectedColumns);
  },

  /**
   * @param {string[]} hideableColumns - needs it for now, as tableStore doesn't handle all data by itself and hideableColumns is built with data from two endpoints: asset and submissions
   * @param {string[]} columnsToBeVisible
   */
  setFieldsVisibility(hideableColumns, columnsToBeVisible) {
    let newSelectedColumns = columnsToBeVisible;

    // If we make all possible columns visible, we save `null` value
    if (newSelectedColumns.length === hideableColumns.length) {
      newSelectedColumns = null;
    }

    const settingsObj = {};
    settingsObj[DATA_TABLE_SETTINGS.SELECTED_COLUMNS] = newSelectedColumns;

    // If current frozen column is not in the newSelectedColumns, we need to
    // unfreeze it.
    const frozenColumn = this.getFrozenColumn();
    if (frozenColumn !== null && !newSelectedColumns.includes(frozenColumn)) {
      // Currently we allow only one frozen column, so we just set it to `null`.
      settingsObj[DATA_TABLE_SETTINGS.FROZEN_COLUMN] = null;
    }

    // If we are hiding the column that data is sorted by, we need to unsort it.
    const sortBy = this.getSortBy();
    if (
      sortBy !== null &&
      typeof sortBy === 'object' &&
      !newSelectedColumns.includes(sortBy.fieldId)
    ) {
      // Currently we allow only one sort column, so we just set it to `null`.
      settingsObj[DATA_TABLE_SETTINGS.SORT_BY] = null;
    }

    this.saveTableSettings(settingsObj);
  },
});

export default tableStore;
<commit_msg>cleanup comment<commit_after>import Reflux from 'reflux';
import clonedeep from 'lodash.clonedeep';
import {stores} from 'js/stores';
import {actions} from 'js/actions';
import {getRouteAssetUid} from 'js/routerUtils';
import {getSurveyFlatPaths} from 'js/assetUtils';
import {
  QUESTION_TYPES,
  GROUP_TYPES_BEGIN,
  GROUP_TYPES_END,
} from 'js/constants';
import {
  EXCLUDED_COLUMNS,
  VALIDATION_STATUS_ID_PROP,
  SUBMISSION_ACTIONS_ID,
  DATA_TABLE_SETTING,
  DATA_TABLE_SETTINGS,
} from 'js/components/submissions/tableConstants';

/**
 * TODO: tableStore should be handling all data required by table.es6, but as
 * this would mean a huge refactor, we will stick to providing a one way
 * interface for changing things in asset plus some utility functions.
 *
 * NOTE: To simplify code a bit, we assume this will be used only on the table
 * route - i.e. assetUid comes from url, and asset from other stores.
 *
 * Use these actions listeners to be up to date:
 * - `actions.table.updateSettings`
 */
const tableStore = Reflux.createStore({
  /**
   * A shortcut method, to be deleted in future.
   */
  getCurrentAsset() {
    return stores.allAssets.getAsset(getRouteAssetUid());
  },

  /**
   * @param {object[]} submissions - list of submissions
   * @returns {string[]} a unique list of columns (keys) that should be displayed to users
   */
  getAllColumns(submissions) {
    const asset = this.getCurrentAsset();
    const flatPaths = getSurveyFlatPaths(asset.content.survey);

    // add all questions from the survey definition
    let output = Object.values(flatPaths);

    // Gather unique columns from all visible submissions and add them to output
    const dataKeys = Object.keys(submissions.reduce(function (result, obj) {
      return Object.assign(result, obj);
    }, {}));
    output = [...new Set([...dataKeys, ...output])];

    // exclude some technical non-data columns
    output = output.filter((key) => EXCLUDED_COLUMNS.includes(key) === false);

    // exclude notes
    output = output.filter((key) => {
      const foundPathKey = Object.keys(flatPaths).find(
        (pathKey) => flatPaths[pathKey] === key
      );

      // no path means this definitely is not a note type
      if (!foundPathKey) {
        return true;
      }

      const foundNoteRow = asset.content.survey.find(
        (row) =>
          typeof foundPathKey !== 'undefined' &&
          (foundPathKey === row.name || foundPathKey === row.$autoname) &&
          row.type === QUESTION_TYPES.note.id
      );

      if (typeof foundNoteRow !== 'undefined') {
        // filter out this row as this is a note type
        return false;
      }

      return true;
    });

    // exclude kobomatrix rows as data is not directly tied to them, but
    // to rows user answered to, thus making these columns always empty
    const excludedMatrixKeys = [];
    let isInsideKoboMatrix = false;
    asset.content.survey.forEach((row) => {
      if (row.type === GROUP_TYPES_BEGIN.begin_kobomatrix) {
        isInsideKoboMatrix = true;
      } else if (row.type === GROUP_TYPES_END.end_kobomatrix) {
        isInsideKoboMatrix = false;
      } else if (isInsideKoboMatrix) {
        const rowPath = flatPaths[row.name] || flatPaths[row.$autoname];
        excludedMatrixKeys.push(rowPath);
      }
    });
    output = output.filter((key) => excludedMatrixKeys.includes(key) === false);

    return output;
  },

  /**
   * @param {object[]} submissions - list of submissions
   * @returns {string[]} a list of columns that user can hide
   */
  getHideableColumns(submissions) {
    const columns = this.getAllColumns(submissions);
    columns.push(VALIDATION_STATUS_ID_PROP);
    return columns;
  },

  /**
   * @returns {string[]|null} a list of selected columns from table settings,
   * `null` means no selection, i.e. all columns
   */
  getSelectedColumns() {
    const tableSettings = this.getTableSettings();
    if (Array.isArray(tableSettings[DATA_TABLE_SETTINGS.SELECTED_COLUMNS])) {
      return tableSettings[DATA_TABLE_SETTINGS.SELECTED_COLUMNS];
    }
    return null;
  },

  /**
   * @returns {object} settings or empty object if no settings exist
   */
  getTableSettings() {
    const asset = this.getCurrentAsset();
    if (
      asset?.settings &&
      asset?.settings[DATA_TABLE_SETTING]
    ) {
      return asset.settings[DATA_TABLE_SETTING];
    }
    return {};
  },

  /**
   * @returns {string|null} the current frozen column
   */
  getFrozenColumn() {
    let frozenColumn = null;
    const tableSettings = this.getTableSettings();
    if (tableSettings && tableSettings[DATA_TABLE_SETTINGS.FROZEN_COLUMN]) {
      frozenColumn = tableSettings[DATA_TABLE_SETTINGS.FROZEN_COLUMN];
    }
    return frozenColumn;
  },

  /**
   * @returns {object|null} the current sort by value
   */
  getSortBy() {
    let sortBy = null;
    const tableSettings = this.getTableSettings();
    if (tableSettings && tableSettings[DATA_TABLE_SETTINGS.SORT_BY]) {
      sortBy = tableSettings[DATA_TABLE_SETTINGS.SORT_BY];
    }
    return sortBy;
  },

  /**
   * @param {string} fieldId
   * @returns {boolean}
   */
  isFieldVisible(fieldId) {
    // frozen column is never hidden
    if (this.isFieldFrozen(fieldId)) {
      return true;
    }

    // submission actions is never hidden
    if (fieldId === SUBMISSION_ACTIONS_ID) {
      return true;
    }

    const selectedColumns = this.getSelectedColumns();
    // nothing is selected, so all columns are visible
    if (selectedColumns === null) {
      return true;
    }

    if (Array.isArray(selectedColumns)) {
      return selectedColumns.includes(fieldId);
    }

    return true;
  },

  /**
   * @param {object} asset
   * @param {string} fieldId
   * @returns {boolean}
   */
  isFieldFrozen(fieldId) {
    return this.getFrozenColumn() === fieldId;
  },

  /**
   * @param {string} fieldId
   * @param {boolean} isFrozen
   */
  setFrozenColumn(fieldId, isFrozen) {
    // NOTE: Currently we only support one frozen column at a time, so that is
    // why making column not-frozen means we just null-ify the value, without
    // checking what column is frozen now.
    let newVal = null;
    if (isFrozen) {
      newVal = fieldId;
    }
    const settingsObj = {};
    settingsObj[DATA_TABLE_SETTINGS.FROZEN_COLUMN] = newVal;
    this.saveTableSettings(settingsObj);
  },

  /**
   * @param {string} fieldId
   * @returns {string|null} null for no option, or one of SORT_VALUES
   */
  getFieldSortValue(fieldId) {
    const sortBy = this.getSortBy();
    if (sortBy === null) {
      return null;
    }

    if (sortBy?.fieldId === fieldId) {
      return sortBy.value;
    }

    return null;
  },

  /**
   * @param {string} fieldId
   * @param {string|null} sortValue one of SORT_VALUES or null for clear value
   */
  setSortBy(assetUid, fieldId, sortValue) {
    let newSortBy = null;
    if (sortValue !== null) {
      newSortBy = {
        fieldId: fieldId,
        value: sortValue,
      };
    }

    const settingsObj = {};
    settingsObj[DATA_TABLE_SETTINGS.SORT_BY] = newSortBy;
    this.saveTableSettings(settingsObj);
  },

  /**
   * @param {object} newTableSettings - will be merged into current settings, overwriting any DATA_TABLE_SETTING properties
   */
  saveTableSettings(newTableSettings) {
    const asset = this.getCurrentAsset();

    // get whole asset settings as clone to avoid bugs
    const newSettings = clonedeep(asset.settings);

    if (!newSettings[DATA_TABLE_SETTING]) {
      newSettings[DATA_TABLE_SETTING] = newTableSettings;
    } else {
      newSettings[DATA_TABLE_SETTING] = Object.assign(
        newSettings[DATA_TABLE_SETTING],
        newTableSettings
      );
    }

    if (asset && this.userCan('change_asset', asset)) {
      actions.table.updateSettings(asset.uid, newSettings);
    }
  },

  /**
   * Show single column - shortcut method for setFieldsVisibility
   * @param {string[]} hideableColumns
   * @param {string} fieldId
   */
  showField(hideableColumns, fieldId) {
    const selectedColumns = this.getSelectedColumns();

    // We start with `null` just to be safe, but the case when selectedColumns
    // is `null` already (i.e. all columns visible) and we show column should
    // never happen.
    let newSelectedColumns = null;

    // Some fields are selected and we show one column
    if (Array.isArray(selectedColumns)) {
      newSelectedColumns = [...selectedColumns];
      newSelectedColumns.push(fieldId);
    }

    this.setFieldsVisibility(hideableColumns, newSelectedColumns);
  },

  /**
   * Hide single column - a shortcut method for setFieldsVisibility
   * @param {string[]} hideableColumns
   * @param {string} fieldId
   */
  hideField(hideableColumns, fieldId) {
    const selectedColumns = this.getSelectedColumns();

    let newSelectedColumns = [];

    // Case 1: nothing selected and we hide one column, i.e. we need to select all but one
    if (selectedColumns === null) {
      newSelectedColumns = [...hideableColumns];
      newSelectedColumns.splice(newSelectedColumns.indexOf(fieldId), 1);
    }

    // Case 2: some fields selected and we hide one column
    if (Array.isArray(selectedColumns)) {
      newSelectedColumns = [...selectedColumns];
      newSelectedColumns.splice(newSelectedColumns.indexOf(fieldId), 1);
    }

    this.setFieldsVisibility(hideableColumns, newSelectedColumns);
  },

  /**
   * @param {string[]} hideableColumns - needs it for now, as tableStore doesn't handle all data by itself and hideableColumns is built with data from two endpoints: asset and submissions
   * @param {string[]} columnsToBeVisible
   */
  setFieldsVisibility(hideableColumns, columnsToBeVisible) {
    let newSelectedColumns = columnsToBeVisible;

    // If we make all possible columns visible, we save `null` value
    if (newSelectedColumns.length === hideableColumns.length) {
      newSelectedColumns = null;
    }

    const settingsObj = {};
    settingsObj[DATA_TABLE_SETTINGS.SELECTED_COLUMNS] = newSelectedColumns;

    // If current frozen column is not in the newSelectedColumns, we need to
    // unfreeze it.
    const frozenColumn = this.getFrozenColumn();
    if (frozenColumn !== null && !newSelectedColumns.includes(frozenColumn)) {
      // Currently we allow only one frozen column, so we just set it to `null`.
      settingsObj[DATA_TABLE_SETTINGS.FROZEN_COLUMN] = null;
    }

    // If we are hiding the column that data is sorted by, we need to unsort it.
    const sortBy = this.getSortBy();
    if (
      sortBy !== null &&
      typeof sortBy === 'object' &&
      !newSelectedColumns.includes(sortBy.fieldId)
    ) {
      // Currently we allow only one sort column, so we just set it to `null`.
      settingsObj[DATA_TABLE_SETTINGS.SORT_BY] = null;
    }

    this.saveTableSettings(settingsObj);
  },
});

export default tableStore;
<|endoftext|>


<commit_before>Meteor.publish('bets', function(){
  return Bets.find();
});

Meteor.publish('myBets', function(username) {
  return Bets.find({
    bettors: username
  });
});

Meteor.publish('betNotifications', function(username){
  return BetNotifications.find({ toNotify: username });
});

Meteor.publish('messages', function(){
  return Messages.find();
});

Meteor.publish('allUsernames', function(){
  return Meteor.users.find();
});

Meteor.publish('images', function(){
  return Images.find();

});

Meteor.publish("friendsList", function() {
  return Friends.find();
});

 Meteor.publish("myFriends", function(userId) {
  return Friends.find({ user: userId })
 })

<commit_msg>Publications: publish all betnotifications<commit_after>Meteor.publish('bets', function(){
  return Bets.find();
});

Meteor.publish('myBets', function(username) {
  return Bets.find({
    bettors: username
  });
});

Meteor.publish('betNotifications', function(){
  return BetNotifications.find();
});

Meteor.publish('messages', function(){
  return Messages.find();
});

Meteor.publish('allUsernames', function(){
  return Meteor.users.find();
});

Meteor.publish('images', function(){
  return Images.find();

});

Meteor.publish("friendsList", function() {
  return Friends.find();
});

 Meteor.publish("myFriends", function(userId) {
  return Friends.find({ user: userId })
 })

<|endoftext|>


<commit_before>/*global DS*/
var get = Ember.get,
  set = Ember.set;

DS.FLOWRESTAdapter = DS.RESTAdapter.extend({
  serializer: DS.RESTSerializer.extend({
    primaryKey: function (type) {
      return "keyId";
    },
    keyForAttributeName: function (type, name) {
      return name;
    }
  }),

  sideload: function (store, type, json, root) {
    var msg, status, metaObj;
    this._super(store, type, json, root);
    // only change metaControl info if there is actual meta info in the server response
    // and if it does not come from a delete action. We detect this by looking if num == null
    metaObj = this.extractMeta(json);
    if (metaObj && !Ember.none(metaObj.message)) {

      if (type == FLOW.SurveyInstance
          || type == FLOW.SurveyedLocale
          && !Ember.none(this.extractMeta(json).num)) {
        FLOW.metaControl.set(type == FLOW.SurveyInstance ? 'numSILoaded' : 'numSLLoaded', this.extractMeta(json).num);
        FLOW.metaControl.set('since', this.extractMeta(json).since);
        FLOW.metaControl.set('num', this.extractMeta(json).num);
        FLOW.metaControl.set('cursorType', type);
      }
      msg = this.extractMeta(json).message;
      status = this.extractMeta(json).status;
      keyId = this.extractMeta(json).keyId;

      if (msg.indexOf('_') === 0) { // Response is a translatable message
        msg = Ember.String.loc(msg);
      }
      FLOW.metaControl.set('message', msg);
      FLOW.metaControl.set('status', status);
      FLOW.metaControl.set('keyId', keyId);
      FLOW.savingMessageControl.numLoadingChange(-1);
      FLOW.savingMessageControl.set('areSavingBool', false);

      if (status === 'preflight-delete-question') {
        if (msg === 'can_delete') {
          // do deletion
          FLOW.questionControl.deleteQuestion(keyId);
        } else {
          FLOW.dialogControl.set('activeAction', 'ignore');
          FLOW.dialogControl.set('header', Ember.String.loc('_cannot_delete_question'));
          FLOW.dialogControl.set('message', Ember.String.loc('_cannot_delete_question_text'));
          FLOW.dialogControl.set('showCANCEL', false);
          FLOW.dialogControl.set('showDialog', true);
        }
        return;
      }

      if (status === 'preflight-delete-questiongroup') {
        if (msg === 'can_delete') {
          // do deletion
          FLOW.questionGroupControl.deleteQuestionGroup(keyId);
        } else {
          FLOW.dialogControl.set('activeAction', 'ignore');
          FLOW.dialogControl.set('header', Ember.String.loc('_cannot_delete_questiongroup'));
          FLOW.dialogControl.set('message', Ember.String.loc('_cannot_delete_questiongroup_text'));
          FLOW.dialogControl.set('showCANCEL', false);
          FLOW.dialogControl.set('showDialog', true);
        }
        return;
      }

      if (status === 'preflight-delete-survey') {
        if (msg === 'can_delete') {
          // do deletion
          FLOW.surveyControl.deleteSurvey(keyId);
        } else {
          FLOW.dialogControl.set('activeAction', 'ignore');
          FLOW.dialogControl.set('header', Ember.String.loc('_cannot_delete_survey'));
          FLOW.dialogControl.set('message', Ember.String.loc('_cannot_delete_survey_text'));
          FLOW.dialogControl.set('showCANCEL', false);
          FLOW.dialogControl.set('showDialog', true);
        }
        return;
      }

      if (status === 'preflight-delete-surveygroup') {
        if (msg === 'can_delete') {
          // do deletion
          FLOW.surveyGroupControl.deleteSurveyGroup(keyId);
        } else {
          FLOW.dialogControl.set('activeAction', 'ignore');
          FLOW.dialogControl.set('header', Ember.String.loc('_cannot_delete_surveygroup'));
          FLOW.dialogControl.set('message', Ember.String.loc('_cannot_delete_surveygroup_text'));
          FLOW.dialogControl.set('showCANCEL', false);
          FLOW.dialogControl.set('showDialog', true);
        }
        return;
      }

      if (this.extractMeta(json).status === 'failed' || FLOW.metaControl.get('message') !== '') {
        FLOW.dialogControl.set('activeAction', 'ignore');
        FLOW.dialogControl.set('header', '' /*Ember.String.loc('_action_failed')*/ ); //FIXME
        FLOW.dialogControl.set('message', FLOW.metaControl.get('message'));
        FLOW.dialogControl.set('showCANCEL', false);
        FLOW.dialogControl.set('showDialog', true);
      }
    }
  },

  ajax: function (url, type, hash) {
    if (type === 'GET' && url.indexOf('rest/survey_groups/0') >= 0) {
      // Don't fetch the root folder. It doesn't exist.
      return;
    }

    this._super(url, type, hash);
    if (type == "GET") {
      if (url.indexOf('rest/survey_groups') >= 0) {
        FLOW.projectControl.set('isLoading', true);
      }
      FLOW.savingMessageControl.numLoadingChange(1);
    }
  },

  didFindRecord: function (store, type, json, id) {
    this._super(store, type, json, id);
    if (type === FLOW.SurveyGroup) {
      FLOW.projectControl.set('isLoading', false);
    }
    FLOW.savingMessageControl.numLoadingChange(-1);
  },

  didFindAll: function (store, type, json) {
    if (type === FLOW.SurveyGroup) {
      FLOW.projectControl.set('isLoading', false);
    }
    FLOW.savingMessageControl.numLoadingChange(-1);
    this._super(store, type, json);
  },

  didFindQuery: function (store, type, json, recordArray) {
    this._super(store, type, json, recordArray);
    if (type === FLOW.SurveyGroup) {
      FLOW.projectControl.set('isLoading', false);
    }
    FLOW.savingMessageControl.numLoadingChange(-1);
  },

  // adapted from standard ember rest_adapter
  // includes 'bulk' in the POST call, to allign
  // with updateRecords and deleteRecords behaviour.
  createRecords: function (store, type, records) {
    if (get(this, 'bulkCommit') === false) {
      return this._super(store, type, records);
    }

    var root = this.rootForType(type),
      plural = this.pluralize(root);

    var data = {};
    data[plural] = [];
    records.forEach(function (record) {
      data[plural].push(this.serialize(record, {
        includeId: true
      }));
    }, this);

    this.ajax(this.buildURL(root, 'bulk'), "POST", {
      data: data,
      context: this,
      success: function (json) {
        this.didCreateRecords(store, type, records, json);
      }
    });
  }
});
<commit_msg>[#1299] Store latest cursor by type when processing incoming meta data<commit_after>/*global DS*/
var get = Ember.get,
  set = Ember.set;

DS.FLOWRESTAdapter = DS.RESTAdapter.extend({
  serializer: DS.RESTSerializer.extend({
    primaryKey: function (type) {
      return "keyId";
    },
    keyForAttributeName: function (type, name) {
      return name;
    }
  }),

  sideload: function (store, type, json, root) {
    var msg, status, metaObj;
    this._super(store, type, json, root);

    this.setQueryCursor(type, json);

    // only change metaControl info if there is actual meta info in the server response
    // and if it does not come from a delete action. We detect this by looking if num == null
    metaObj = this.extractMeta(json);
    if (metaObj && !Ember.none(metaObj.message)) {

      if (type == FLOW.SurveyInstance
          || type == FLOW.SurveyedLocale
          && !Ember.none(this.extractMeta(json).num)) {
        FLOW.metaControl.set(type == FLOW.SurveyInstance ? 'numSILoaded' : 'numSLLoaded', this.extractMeta(json).num);
        FLOW.metaControl.set('since', this.extractMeta(json).since);
        FLOW.metaControl.set('num', this.extractMeta(json).num);
        FLOW.metaControl.set('cursorType', type);
      }
      msg = this.extractMeta(json).message;
      status = this.extractMeta(json).status;
      keyId = this.extractMeta(json).keyId;

      if (msg.indexOf('_') === 0) { // Response is a translatable message
        msg = Ember.String.loc(msg);
      }
      FLOW.metaControl.set('message', msg);
      FLOW.metaControl.set('status', status);
      FLOW.metaControl.set('keyId', keyId);
      FLOW.savingMessageControl.numLoadingChange(-1);
      FLOW.savingMessageControl.set('areSavingBool', false);

      if (status === 'preflight-delete-question') {
        if (msg === 'can_delete') {
          // do deletion
          FLOW.questionControl.deleteQuestion(keyId);
        } else {
          FLOW.dialogControl.set('activeAction', 'ignore');
          FLOW.dialogControl.set('header', Ember.String.loc('_cannot_delete_question'));
          FLOW.dialogControl.set('message', Ember.String.loc('_cannot_delete_question_text'));
          FLOW.dialogControl.set('showCANCEL', false);
          FLOW.dialogControl.set('showDialog', true);
        }
        return;
      }

      if (status === 'preflight-delete-questiongroup') {
        if (msg === 'can_delete') {
          // do deletion
          FLOW.questionGroupControl.deleteQuestionGroup(keyId);
        } else {
          FLOW.dialogControl.set('activeAction', 'ignore');
          FLOW.dialogControl.set('header', Ember.String.loc('_cannot_delete_questiongroup'));
          FLOW.dialogControl.set('message', Ember.String.loc('_cannot_delete_questiongroup_text'));
          FLOW.dialogControl.set('showCANCEL', false);
          FLOW.dialogControl.set('showDialog', true);
        }
        return;
      }

      if (status === 'preflight-delete-survey') {
        if (msg === 'can_delete') {
          // do deletion
          FLOW.surveyControl.deleteSurvey(keyId);
        } else {
          FLOW.dialogControl.set('activeAction', 'ignore');
          FLOW.dialogControl.set('header', Ember.String.loc('_cannot_delete_survey'));
          FLOW.dialogControl.set('message', Ember.String.loc('_cannot_delete_survey_text'));
          FLOW.dialogControl.set('showCANCEL', false);
          FLOW.dialogControl.set('showDialog', true);
        }
        return;
      }

      if (status === 'preflight-delete-surveygroup') {
        if (msg === 'can_delete') {
          // do deletion
          FLOW.surveyGroupControl.deleteSurveyGroup(keyId);
        } else {
          FLOW.dialogControl.set('activeAction', 'ignore');
          FLOW.dialogControl.set('header', Ember.String.loc('_cannot_delete_surveygroup'));
          FLOW.dialogControl.set('message', Ember.String.loc('_cannot_delete_surveygroup_text'));
          FLOW.dialogControl.set('showCANCEL', false);
          FLOW.dialogControl.set('showDialog', true);
        }
        return;
      }

      if (this.extractMeta(json).status === 'failed' || FLOW.metaControl.get('message') !== '') {
        FLOW.dialogControl.set('activeAction', 'ignore');
        FLOW.dialogControl.set('header', '' /*Ember.String.loc('_action_failed')*/ ); //FIXME
        FLOW.dialogControl.set('message', FLOW.metaControl.get('message'));
        FLOW.dialogControl.set('showCANCEL', false);
        FLOW.dialogControl.set('showDialog', true);
      }
    }
  },

  /*  Process the cursor returned by the query. The cursor is used for pagination requests
      and is based on the type of entities queried */
  setQueryCursor: function(type, json) {
    var cursorArray, cursorStart, cursorIndex;
    if (type === FLOW.SurveyedLocale) {
      cursorArray = FLOW.surveyedLocaleControl.get('sinceArray');
    } if (type === FLOW.SurveyInstance) {
      cursorArray = FLOW.surveyInstanceControl.get('sinceArray');
    }

    cursorStart = this.extractSince(json);
    if (!cursorStart) {
      return;
    }

    cursorIndex = cursorArray.indexOf(cursorStart);
    if (cursorIndex === -1) {
      cursorArray.pushObject(cursorStart);
    } else {
      // drop all cursors after the current one
      cursorArray.splice(cursorIndex + 1, cursorArray.length);
    }

    if (type === FLOW.SurveyedLocale) {
      FLOW.surveyedLocaleControl.set('sinceArray', cursorArray);
    } if (type === FLOW.SurveyInstance) {
      FLOW.surveyInstanceControl.set('sinceArray', cursorArray);
    }
  },

  ajax: function (url, type, hash) {
    if (type === 'GET' && url.indexOf('rest/survey_groups/0') >= 0) {
      // Don't fetch the root folder. It doesn't exist.
      return;
    }

    this._super(url, type, hash);
    if (type == "GET") {
      if (url.indexOf('rest/survey_groups') >= 0) {
        FLOW.projectControl.set('isLoading', true);
      }
      FLOW.savingMessageControl.numLoadingChange(1);
    }
  },

  didFindRecord: function (store, type, json, id) {
    this._super(store, type, json, id);
    if (type === FLOW.SurveyGroup) {
      FLOW.projectControl.set('isLoading', false);
    }
    FLOW.savingMessageControl.numLoadingChange(-1);
  },

  didFindAll: function (store, type, json) {
    if (type === FLOW.SurveyGroup) {
      FLOW.projectControl.set('isLoading', false);
    }
    FLOW.savingMessageControl.numLoadingChange(-1);
    this._super(store, type, json);
  },

  didFindQuery: function (store, type, json, recordArray) {
    this._super(store, type, json, recordArray);
    if (type === FLOW.SurveyGroup) {
      FLOW.projectControl.set('isLoading', false);
    }
    FLOW.savingMessageControl.numLoadingChange(-1);
  },

  // adapted from standard ember rest_adapter
  // includes 'bulk' in the POST call, to allign
  // with updateRecords and deleteRecords behaviour.
  createRecords: function (store, type, records) {
    if (get(this, 'bulkCommit') === false) {
      return this._super(store, type, records);
    }

    var root = this.rootForType(type),
      plural = this.pluralize(root);

    var data = {};
    data[plural] = [];
    records.forEach(function (record) {
      data[plural].push(this.serialize(record, {
        includeId: true
      }));
    }, this);

    this.ajax(this.buildURL(root, 'bulk'), "POST", {
      data: data,
      context: this,
      success: function (json) {
        this.didCreateRecords(store, type, records, json);
      }
    });
  }
});
<|endoftext|>


<commit_before>'use strict';

// Dependencies
// ------------

var fs = require('fs');
var rimraf = require('rimraf');
var path = require('path');
var request = require("request");
var argv = require('minimist')(process.argv.slice(2));
var chalk = require('chalk');
var cheerio = require('cheerio');
var he = require('he');
var moment = require('moment');

// Helpers
// -------

function printUsage () {
    console.log(
`
Usage:

node runner.js --user=[fotolog_username]

Options:

  --skipcomments

    Don't fetch post comments
`
    );
}

function printLogLevel (str, level) {
    var indentation = Array(level + 1).join('  ');
    console.log(chalk[levelColors[level]](`${indentation}${str}`));
}

/**
 * Returns the numeric part of the URL.
 */
function getUrlNumericPart (url) {
    return parseInt(url.match(/\/(\d+)(?:\/|$)/)[1]);
}

/**
 * Synchronously iterates through all links and incrementally get results,
 * using async callbacks. Returns a Promise which is resolved when all links are
 * iterated.
 */
function iterateLinks (links, index, memory, callback, message) {
    if (message) {
        printLogLevel(`- ${message} - ${index + 1}/${links.length}`, 1);
    }

    return new Promise((resolve, reject) => {
        callback(links[index]).then((results) => {
            memory = memory.concat(results);

            if (index === links.length - 1) {
                resolve(memory);
            } else {
                iterateLinks(links, index + 1, memory, callback, message).then((results) => {
                    resolve(results);
                });
            }
        });
    });
}

// API
// ---

if (argv.help || argv.h) {
    printUsage();
    process.exit(0);
}

if (!argv.user) {
    console.error(chalk.red('You must specify a Fotolog username.\n'));
    printUsage();
    process.exit(1);
}

// Program main execution
// ----------------------

var username = argv.user;
var shouldSkipComments = argv.skipcomments;
var postsPerMosaicPage = 30;
var postsLinks = [];
var posts = [];
var mosaicLink = `http://www.fotolog.com/${username}/mosaic`;
var dirName = `fotolog_${username}_data`;
var localisedDateFormats = {
    it: 'DD MMMM YYYY',
    pt: 'DD MMMM YYYY',
    fr: 'DD MMMM YYYY',
    es: 'DD MMMM YYYY',
    en: 'MMMM DD YYYY'
};
var levelColors = ['green', 'white', 'gray'];

function saveUserFotolog () {
    console.log(chalk.yellow(`Getting data from ${username}...`));

    request(mosaicLink, (error, response, body) => {
        var $ = cheerio.load(body);

        getAllPosts(getMosaicPagesLinks($, username))
            .then(getPostsData)
            .then(savePostsDataToDisk)
            .then((postsData) => {
                printLogLevel(`Done! Saved ${postsData.length} post(s) to the ${dirName} folder.`, 0);
            });
    });
}

// Traversing - Gettings Mosaic Pages
// ----------------------------------

/**
 * Look up for the mosaic pagination links on the DOM and returns an array with
 * the link for each page.
 */
function getMosaicPagesLinks ($, username) {
    var $pagination = $('#pagination');
    var lastLink = $pagination.find('a:last-of-type').attr('href');
    var maxOffset;

    printLogLevel('Getting all mosaic pagination links...', 0);

    // There's only one page
    if ($pagination.find('a').length === 1) {
        printLogLevel('- User has only one page of photos', 1);

        maxOffset = 0;
    // The last page is within the first 6 pages, so the last is the link before
    } else if (lastLink === `${mosaicLink}/30`) {
        printLogLevel('- User has up to 6 pages of photos', 1);

        maxOffset = getUrlNumericPart($pagination.find('a:nth-last-of-type(2)').attr('href'));
    // User has a lot of pages, so the last page is the last link
    } else {
        printLogLevel('- User has more than 6 pages of photos', 1);

        maxOffset = getUrlNumericPart(lastLink);
    }

    return buildMosaicPagesLinks(maxOffset);
}

/**
 * Builds an array of all mosaic links, based on the pagination's last offset.
 */
function buildMosaicPagesLinks (maxOffset) {
    var pagesQty = (maxOffset / postsPerMosaicPage) + 1;
    var links = [];

    for (var i = 0; i < pagesQty; i++) {
        let offset = i * postsPerMosaicPage;

        if (offset === 0) {
            links.push(mosaicLink);
        } else {
            links.push(`${mosaicLink}/${offset}`);
        }
    }

    printLogLevel(`- Built links for ${pagesQty} pages`, 1);

    return links;
}

// Traversing - Gettings Posts
// ---------------------------

/**
 * Gets all posts from all mosaic page links. Returns a Promise which is
 * resolved when all posts are retrived.
 */
function getAllPosts (mosaicLinks) {
    printLogLevel('Getting all posts links...', 0);

    return new Promise((resolve, reject) => {
        iterateLinks(mosaicLinks, 0, [], getPostsFromMosaicLink).then(resolve);
    });
}

/**
 * Gets all posts from a single mosaic page link. Returns a Promise which is
 * resolved when all posts are retrived.
 */
function getPostsFromMosaicLink (link) {
    return new Promise((resolve, reject) => {
        request(link, (error, response, body) => {
            var $ = cheerio.load(body);
            var postLinks = [];

            $('#list_photos_mosaic').find('a').each((index, element) => {
                postLinks.push($(element).attr('href'));
            });

            printLogLevel(`- Retrieved ${postLinks.length} post links from ${link}`, 1);

            resolve(postLinks);
        });
    });
}

// Traversing - Gettings photos and data
// -------------------------------------

/**
 * Gets photos and data from all posts. Returns a Promise which is resolved
 * when all data is retrieved.
 */
function getPostsData (postsLinks) {
    printLogLevel(`Retrieving data from ${postsLinks.length} posts...`, 0);

    return new Promise((resolve, reject) => {
        iterateLinks(postsLinks, 0, [], getPostData, 'Retrieving post data').then(resolve);
    });
}

/**
 * Gets a post's main photo and additional data. Returns a Promise which is
 * resolved when the post's data is retrieved.
 */
function getPostData (link) {
    return new Promise((resolve, reject) => {
        request(link, (error, response, body) => {
            var $ = cheerio.load(body);
            var data = {};
            var lang = $('html').attr('lang') || 'en';
            var dateRegex = /[^\s]+\s+[^\s]+\s+[^\s]+\s+\d+\s+(?:Views|Vistas|Vues|Visualizações)/;

            data.id = getUrlNumericPart(link);
            data.imageUrl = $('#flog_img_holder').find('img').attr('src');
            data.description = $('#description_photo').text();
            // Fotolog doesn't give us an easy way to get the date so... yeah.
            moment.locale(lang);
            data.date = moment(data.description.match(dateRegex)[0].split(/\s+/).slice(0, 3).join(' '), localisedDateFormats[lang]);

            if (shouldSkipComments) {
                printLogLevel(`Retrieved post data from ${link}`, 2);

                resolve(data);
                return;
            }

            getPostComments($, data.id).then((comments) => {
                data.comments = comments;

                printLogLevel(`Retrieved post data from ${link}`, 2);

                resolve(data);
            });
        });
    });
}

/**
 * Gets a post's comments. The first 44 are retrieved straight from the DOM.
 * The others are retrieved through POST requests on the comments endpoint.
 * Returns a Promise which is resolved when all comments are retrived.
 */
function getPostComments ($, id) {
    var comments = [];

    $('.flog_img_comments').each((index, element) => {
        // The first element is the login form link :S
        if (index === 0) { return; }

        comments.push({
            avatarImgUrl: $(element).find('.comment_avatar').attr('src'),
            username: $(element).find('p b').text(),
            // this gets the first dd/mm/yyyy occurrence
            date: $(element).children('p').text().match(/\d{2}\/\d{2}\/\d{4}/)[0],
            // since the whole comment block is a single paragraph, we get
            // everything after the first two breaklines
            message: $(element).children('p').html().split('

').slice(1).join('

')
        });
    });

    return new Promise((resolve, reject) => {
        iterateCommentRequests(id, 45, comments).then(resolve);
    });
}

/**
 * Synchronously request comments and incrementally get results,
 * using async callbacks. Returns a Promise which is resolved when the request
 * returns 0 results.
 */
function iterateCommentRequests (id, offset, memory) {
    return new Promise((resolve, reject) => {
        fetchComments(id, offset).then((results) => {
            memory = memory.concat(results);

            if (results.length === 0) {
                resolve(memory);
            } else {
                iterateCommentRequests(id, offset + 45, memory).then((results) => {
                    resolve(results);
                });
            }
        });
    });
}

/**
 * Fetches comments from a given post id and offset. Returns a Promise which
 * is resolved when the comments are fetched.
 */
function fetchComments (id, offset) {
    return new Promise((resolve, reject) => {
        request.post('http://fotolog.com/ajax/load_more_comments', { form: {
            user_name: username,
            identifier: id,
            offset: offset
        } }, (error, response, body) => {
            var response = JSON.parse(body);
            var comments = response.comments;

            if (!response.success || !comments || (!Array.isArray(comments) && Object.keys(comments) === 0)) {
                resolve([]);
                return;
            }

            if (!Array.isArray(comments)) {
                comments = Object.keys(comments).map((index) => {
                    return comments[index];
                });
            }

            resolve(comments.map((comment) => {
                return {
                    avatarImgUrl : comment.avatar,
                    username: comment.poster_user_name,
                    date: comment.posted.split(/\s+/)[1],
                    message: comment.message
                }
            }));
        });
    });
}

// Saving data to the disk
// -----------------------

function savePostsDataToDisk (postsData) {
    printLogLevel(`Saving all data from ${username}`, 0);

    rimraf.sync(dirName);

    printLogLevel(`- Removed any previous data`, 1);

    fs.mkdirSync(dirName);

    printLogLevel(`- Created output directory`, 1);

    return new Promise((resolve, reject) => {
        iterateLinks(postsData, 0, [], savePostDataToDisk, 'Saving post data').then(resolve);
    });
}

function savePostDataToDisk (postData) {
    return new Promise((resolve, reject) => {
        var postYear = postData.date.format('YYYY');
        var postMonth = postData.date.format('MM');
        var postDay = postData.date.format('DD');
        var filePath = `${dirName}/${postYear}/${postMonth}`;
        var filePrefix = `${postYear}${postMonth}${postDay}_`;

        try {
            fs.mkdirSync(`${dirName}/${postYear}`);
        } catch(e) {}

        try {
            fs.mkdirSync(`${dirName}/${postYear}/${postMonth}`);
        } catch(e) {}

        fs.writeFileSync(`${filePath}/${filePrefix}${postData.id}.txt`,
            he.decode(postData.description) +
            (shouldSkipComments ?
                '' :
                '\n\n' +
                '========================================\n' +
                '                COMMENTS\n' +
                '========================================\n\n' +
                he.decode(postData.comments.map((comment) => {
                    return `${comment.username} on ${comment.date}:\n\n${comment.message}`;
                }).join('\n\n-------------------\n\n')))
        );

        request(postData.imageUrl).pipe(fs.createWriteStream(`${filePath}/${filePrefix}${postData.id}.jpg`)).on('close', resolve);

        printLogLevel(`Saved photo and post data from post ${postData.id}`, 2);
        printLogLevel(`Post date: ${postDay}/${postMonth}/${postYear}`, 2);
        printLogLevel(`Comments: ${postData.comments.length}`, 2);
    });
}

saveUserFotolog();<commit_msg>Fix script for profiles without pagination<commit_after>'use strict';

// Dependencies
// ------------

var fs = require('fs');
var rimraf = require('rimraf');
var path = require('path');
var request = require("request");
var argv = require('minimist')(process.argv.slice(2));
var chalk = require('chalk');
var cheerio = require('cheerio');
var he = require('he');
var moment = require('moment');

// Helpers
// -------

function printUsage () {
    console.log(
`
Usage:

node runner.js --user=[fotolog_username]

Options:

  --skipcomments

    Don't fetch post comments
`
    );
}

function printLogLevel (str, level) {
    var indentation = Array(level + 1).join('  ');
    console.log(chalk[levelColors[level]](`${indentation}${str}`));
}

/**
 * Returns the numeric part of the URL.
 */
function getUrlNumericPart (url) {
    return parseInt(url.match(/\/(\d+)(?:\/|$)/)[1]);
}

/**
 * Synchronously iterates through all links and incrementally get results,
 * using async callbacks. Returns a Promise which is resolved when all links are
 * iterated.
 */
function iterateLinks (links, index, memory, callback, message) {
    if (message) {
        printLogLevel(`- ${message} - ${index + 1}/${links.length}`, 1);
    }

    return new Promise((resolve, reject) => {
        callback(links[index]).then((results) => {
            memory = memory.concat(results);

            if (index === links.length - 1) {
                resolve(memory);
            } else {
                iterateLinks(links, index + 1, memory, callback, message).then((results) => {
                    resolve(results);
                });
            }
        });
    });
}

// API
// ---

if (argv.help || argv.h) {
    printUsage();
    process.exit(0);
}

if (!argv.user) {
    console.error(chalk.red('You must specify a Fotolog username.\n'));
    printUsage();
    process.exit(1);
}

// Program main execution
// ----------------------

var username = argv.user;
var shouldSkipComments = argv.skipcomments;
var postsPerMosaicPage = 30;
var postsLinks = [];
var posts = [];
var mosaicLink = `http://www.fotolog.com/${username}/mosaic`;
var dirName = `fotolog_${username}_data`;
var localisedDateFormats = {
    it: 'DD MMMM YYYY',
    pt: 'DD MMMM YYYY',
    fr: 'DD MMMM YYYY',
    es: 'DD MMMM YYYY',
    en: 'MMMM DD YYYY'
};
var levelColors = ['green', 'white', 'gray'];

function saveUserFotolog () {
    console.log(chalk.yellow(`Getting data from ${username}...`));

    request(mosaicLink, (error, response, body) => {
        var $ = cheerio.load(body);

        getAllPosts(getMosaicPagesLinks($, username))
            .then(getPostsData)
            .then(savePostsDataToDisk)
            .then((postsData) => {
                printLogLevel(`Done! Saved ${postsData.length} post(s) to the ${dirName} folder.`, 0);
            });
    });
}

// Traversing - Gettings Mosaic Pages
// ----------------------------------

/**
 * Look up for the mosaic pagination links on the DOM and returns an array with
 * the link for each page.
 */
function getMosaicPagesLinks ($, username) {
    var $pagination = $('#pagination');
    var lastLink = $pagination.find('a:last-of-type').attr('href');
    var maxOffset;

    printLogLevel('Getting all mosaic pagination links...', 0);

    // There's no pagination
    if ($pagination.length === 0 || $pagination.find('a').length === 0) {
        printLogLevel('- User has only one page of photos, no pagination', 1);

        maxOffset = 0;
    // There's only one page
    } else if ($pagination.find('a').length === 1) {
        printLogLevel('- User has only one page of photos', 1);

        maxOffset = 0;
    // The last page is within the first 6 pages, so the last is the link before
    } else if (lastLink === `${mosaicLink}/30`) {
        printLogLevel('- User has up to 6 pages of photos', 1);

        maxOffset = getUrlNumericPart($pagination.find('a:nth-last-of-type(2)').attr('href'));
    // User has a lot of pages, so the last page is the last link
    } else {
        printLogLevel('- User has more than 6 pages of photos', 1);

        maxOffset = getUrlNumericPart(lastLink);
    }

    return buildMosaicPagesLinks(maxOffset);
}

/**
 * Builds an array of all mosaic links, based on the pagination's last offset.
 */
function buildMosaicPagesLinks (maxOffset) {
    var pagesQty = (maxOffset / postsPerMosaicPage) + 1;
    var links = [];

    for (var i = 0; i < pagesQty; i++) {
        let offset = i * postsPerMosaicPage;

        if (offset === 0) {
            links.push(mosaicLink);
        } else {
            links.push(`${mosaicLink}/${offset}`);
        }
    }

    printLogLevel(`- Built links for ${pagesQty} pages`, 1);

    return links;
}

// Traversing - Gettings Posts
// ---------------------------

/**
 * Gets all posts from all mosaic page links. Returns a Promise which is
 * resolved when all posts are retrived.
 */
function getAllPosts (mosaicLinks) {
    printLogLevel('Getting all posts links...', 0);

    return new Promise((resolve, reject) => {
        iterateLinks(mosaicLinks, 0, [], getPostsFromMosaicLink).then(resolve);
    });
}

/**
 * Gets all posts from a single mosaic page link. Returns a Promise which is
 * resolved when all posts are retrived.
 */
function getPostsFromMosaicLink (link) {
    return new Promise((resolve, reject) => {
        request(link, (error, response, body) => {
            var $ = cheerio.load(body);
            var postLinks = [];

            $('#list_photos_mosaic').find('a').each((index, element) => {
                postLinks.push($(element).attr('href'));
            });

            printLogLevel(`- Retrieved ${postLinks.length} post links from ${link}`, 1);

            resolve(postLinks);
        });
    });
}

// Traversing - Gettings photos and data
// -------------------------------------

/**
 * Gets photos and data from all posts. Returns a Promise which is resolved
 * when all data is retrieved.
 */
function getPostsData (postsLinks) {
    printLogLevel(`Retrieving data from ${postsLinks.length} posts...`, 0);

    return new Promise((resolve, reject) => {
        iterateLinks(postsLinks, 0, [], getPostData, 'Retrieving post data').then(resolve);
    });
}

/**
 * Gets a post's main photo and additional data. Returns a Promise which is
 * resolved when the post's data is retrieved.
 */
function getPostData (link) {
    return new Promise((resolve, reject) => {
        request(link, (error, response, body) => {
            var $ = cheerio.load(body);
            var data = {};
            var lang = $('html').attr('lang') || 'en';
            var dateRegex = /[^\s]+\s+[^\s]+\s+[^\s]+\s+\d+\s+(?:Views|Vistas|Vues|Visualizações)/;

            data.id = getUrlNumericPart(link);
            data.imageUrl = $('#flog_img_holder').find('img').attr('src');
            data.description = $('#description_photo').text();
            // Fotolog doesn't give us an easy way to get the date so... yeah.
            moment.locale(lang);
            data.date = moment(data.description.match(dateRegex)[0].split(/\s+/).slice(0, 3).join(' '), localisedDateFormats[lang]);

            if (shouldSkipComments) {
                printLogLevel(`Retrieved post data from ${link}`, 2);

                resolve(data);
                return;
            }

            getPostComments($, data.id).then((comments) => {
                data.comments = comments;

                printLogLevel(`Retrieved post data from ${link}`, 2);

                resolve(data);
            });
        });
    });
}

/**
 * Gets a post's comments. The first 44 are retrieved straight from the DOM.
 * The others are retrieved through POST requests on the comments endpoint.
 * Returns a Promise which is resolved when all comments are retrived.
 */
function getPostComments ($, id) {
    var comments = [];

    $('.flog_img_comments').each((index, element) => {
        // The first element is the login form link :S
        if (index === 0) { return; }

        comments.push({
            avatarImgUrl: $(element).find('.comment_avatar').attr('src'),
            username: $(element).find('p b').text(),
            // this gets the first dd/mm/yyyy occurrence
            date: $(element).children('p').text().match(/\d{2}\/\d{2}\/\d{4}/)[0],
            // since the whole comment block is a single paragraph, we get
            // everything after the first two breaklines
            message: $(element).children('p').html().split('

').slice(1).join('

')
        });
    });

    return new Promise((resolve, reject) => {
        iterateCommentRequests(id, 45, comments).then(resolve);
    });
}

/**
 * Synchronously request comments and incrementally get results,
 * using async callbacks. Returns a Promise which is resolved when the request
 * returns 0 results.
 */
function iterateCommentRequests (id, offset, memory) {
    return new Promise((resolve, reject) => {
        fetchComments(id, offset).then((results) => {
            memory = memory.concat(results);

            if (results.length === 0) {
                resolve(memory);
            } else {
                iterateCommentRequests(id, offset + 45, memory).then((results) => {
                    resolve(results);
                });
            }
        });
    });
}

/**
 * Fetches comments from a given post id and offset. Returns a Promise which
 * is resolved when the comments are fetched.
 */
function fetchComments (id, offset) {
    return new Promise((resolve, reject) => {
        request.post('http://fotolog.com/ajax/load_more_comments', { form: {
            user_name: username,
            identifier: id,
            offset: offset
        } }, (error, response, body) => {
            var response = JSON.parse(body);
            var comments = response.comments;

            if (!response.success || !comments || (!Array.isArray(comments) && Object.keys(comments) === 0)) {
                resolve([]);
                return;
            }

            if (!Array.isArray(comments)) {
                comments = Object.keys(comments).map((index) => {
                    return comments[index];
                });
            }

            resolve(comments.map((comment) => {
                return {
                    avatarImgUrl : comment.avatar,
                    username: comment.poster_user_name,
                    date: comment.posted.split(/\s+/)[1],
                    message: comment.message
                }
            }));
        });
    });
}

// Saving data to the disk
// -----------------------

function savePostsDataToDisk (postsData) {
    printLogLevel(`Saving all data from ${username}`, 0);

    rimraf.sync(dirName);

    printLogLevel(`- Removed any previous data`, 1);

    fs.mkdirSync(dirName);

    printLogLevel(`- Created output directory`, 1);

    return new Promise((resolve, reject) => {
        iterateLinks(postsData, 0, [], savePostDataToDisk, 'Saving post data').then(resolve);
    });
}

function savePostDataToDisk (postData) {
    return new Promise((resolve, reject) => {
        var postYear = postData.date.format('YYYY');
        var postMonth = postData.date.format('MM');
        var postDay = postData.date.format('DD');
        var filePath = `${dirName}/${postYear}/${postMonth}`;
        var filePrefix = `${postYear}${postMonth}${postDay}_`;

        try {
            fs.mkdirSync(`${dirName}/${postYear}`);
        } catch(e) {}

        try {
            fs.mkdirSync(`${dirName}/${postYear}/${postMonth}`);
        } catch(e) {}

        fs.writeFileSync(`${filePath}/${filePrefix}${postData.id}.txt`,
            he.decode(postData.description) +
            (shouldSkipComments ?
                '' :
                '\n\n' +
                '========================================\n' +
                '                COMMENTS\n' +
                '========================================\n\n' +
                he.decode(postData.comments.map((comment) => {
                    return `${comment.username} on ${comment.date}:\n\n${comment.message}`;
                }).join('\n\n-------------------\n\n')))
        );

        request(postData.imageUrl).pipe(fs.createWriteStream(`${filePath}/${filePrefix}${postData.id}.jpg`)).on('close', resolve);

        printLogLevel(`Saved photo and post data from post ${postData.id}`, 2);
        printLogLevel(`Post date: ${postDay}/${postMonth}/${postYear}`, 2);
        printLogLevel(`Comments: ${postData.comments.length}`, 2);
    });
}

saveUserFotolog();<|endoftext|>