<commit_before>/**
 * analyze-css CommonJS module
 */
'use strict';

var cssParser = require('css-parse'),
	debug = require('debug')('analyze-css'),
	fs = require('fs'),
	preprocessors = new (require('./preprocessors'))(),
	slickParse = require('slick').parse,
	VERSION = require('./../package').version;

function analyzer(css, options, callback) {
	var res;

	// options can be omitted
	if (typeof options === 'function') {
		callback = options;
		options = {};
	}

	this.options = options;
	debug('opts: %j', this.options);

	if (typeof css !== 'string') {
		callback(new Error('css parameter passed is not a string!'), null);
		return;
	}

	// preprocess the CSS (issue #3)
	if (typeof options.preprocessor === 'string') {
		debug('Using "%s" preprocessor', options.preprocessor);

		var preprocessor = preprocessors.get(options.preprocessor);

		try {
			css = preprocessor.process(css, options);
		}
		catch(ex) {
			throw new Error('Preprocessing failed: ' + ex);
		}

		debug('Preprocessing completed');
	}

	res = this.analyze(css);

	if (res !== true) {
		callback(res, null);
		return;
	}

	// return the results
	res = {
		generator: 'analyze-css v' + VERSION,
		metrics: this.metrics,
		offenders: this.offenders
	};

	callback(null, res);
}

analyzer.version = VERSION;

analyzer.prototype = {
	emitter: false,
	tree: false,

	metrics: {},
	offenders: {},

	// emit given event
	emit: function(/* eventName, arg1, arg2, ... */) {
		//debug('Event %s emitted', arguments[0]);
		this.emitter.emit.apply(this.emitter, arguments);
	},

	// bind to a given event
	on: function(ev, fn) {
		this.emitter.on(ev, fn);
	},

	setMetric: function(name, value) {
		value = value || 0;

		//debug('setMetric(%s) = %d', name, value);
		this.metrics[name] = value;
	},

	// increements given metric by given number (default is one)
	incrMetric: function(name, incr /* =1 */) {
		var currVal = this.metrics[name] || 0;
		incr = incr || 1;

		//debug('incrMetric(%s) += %d', name, incr);
		this.setMetric(name, currVal + incr);
	},

	addOffender: function(metricName, msg) {
		if (typeof this.offenders[metricName] === 'undefined') {
			this.offenders[metricName] = [];
		}

		this.offenders[metricName].push(msg);
	},

	initRules: function() {
		var debug = require('debug')('analyze-css:rules'),
			re = /\.js$/,
			rules = [];

		// init events emitter
		this.emitter = new (require('events').EventEmitter)();
		this.emitter.setMaxListeners(200);

		// load all rules
		rules = fs.readdirSync(fs.realpathSync(__dirname + '/../rules/'))
			// filter out all non *.js files
			.filter(function(file) {
				return re.test(file);
			})
			// remove file extensions to get just names
			.map(function(file) {
				return file.replace(re, '');
			});

		debug('Rules to be loaded: %s', rules.join(', '));

		rules.forEach(function(name) {
			var rule = require('./../rules/' + name);
			rule(this);

			debug('"%s" loaded: %s', name, rule.description || '-');
		}, this);
	},

	parseCss: function(css) {
		var debug = require('debug')('analyze-css:parser');

		debug('Going to parse %s kB of CSS', (css.length / 1024).toFixed(2));

		try {
			if (css.trim() === '') {
				throw 'Empty CSS was provided';
			}

			this.tree = new cssParser(css);
		}
		catch(ex) {
			var errMsg = 'CSS parsing failed: ' + ex.toString();

			debug(errMsg);
			debug('Offending line: ' + css.split('\n')[ex.line-1]);

			return new Error(errMsg);
		}

		debug('CSS parsed');
		return true;
	},

	parseRules: function(rules) {
		rules.forEach(function(rule, idx) {
			debug('%j', rule);

			switch(rule.type) {
				// {
				//  "type":"media"
				//  "media":"screen and (min-width: 1370px)",
				//  "rules":[{"type":"rule","selectors":["#foo"],"declarations":[]}]
				// }
				case 'media':
					this.emit('media', rule.media, rule.rules);

					// now run recursively to parse rules within the media query
					if (rule.rules) {
						this.parseRules(rule.rules);
					}

					this.emit('mediaEnd', rule.media, rule.rules);
					break;

				// {
				//  "type":"rule",
				//  "selectors":[".ui-header .ui-btn-up-a",".ui-header .ui-btn-hover-a"],
				//  "declarations":[{"type":"declaration","property":"border","value":"0"},{"type":"declaration","property":"background","value":"none"}]
				// }
				case 'rule':
					this.emit('rule', rule);

					if (!rule.selectors || !rule.declarations) {
						return;
					}

					// analyze each selector and declaration
					rule.selectors.forEach(function(selector) {
						var parsedSelector,
							expressions = [],
							i, len;

						// "#features > div:first-child" will become two expressions:
						//  {"combinator":" ","tag":"*","id":"features"}
						//  {"combinator":">","tag":"div","pseudos":[{"key":"first-child","value":null}]}
						parsedSelector = slickParse(selector)[0];

						// convert object with keys to array with numeric index
						for (i=0, len=parsedSelector.length; i<len; i++) {
							expressions.push(parsedSelector[i]);
						}

						this.emit('selector', rule, selector, expressions);

						expressions.forEach(function(expression) {
							this.emit('expression', selector, expression);
						}, this);
					}, this);

					rule.declarations.forEach(function(declaration) {
						switch(declaration.type) {
							case 'declaration':
								this.emit('declaration', rule, declaration.property, declaration.value);
								break;

							case 'comment':
								this.emit('comment', declaration.comment);
								break;
						}
					}, this);
					break;

				// {"type":"comment","comment":" Cached as static-css-r518-9b0f5ab4632defb55d67a1d672aa31bd120f4414 "}
				case 'comment':
					this.emit('comment', rule.comment);
					break;

				// {"type":"import","import":"url('/css/styles.css')"}
				case 'import':
					this.emit('import', rule.import);
					break;
			}
		}, this);
	},

	run: function() {
		var rules = this.tree && this.tree.stylesheet.rules;
		this.parseRules(rules);
	},

	analyze: function(css) {
		var res,
			then = Date.now();

		this.metrics = {};
		this.offenders = {};

		// load and init all rules
		this.initRules();

		// parse CSS
		res = this.parseCss(css);

		if (res !== true) {
			return res;
		}

		this.emit('css', css);

		// now go through parsed CSS tree and emit events for rules
		this.run();

		this.emit('report');

		debug('Completed in %d ms', Date.now() - then);
		return true;
	}
};

module.exports = analyzer;
<commit_msg>Better errors catching<commit_after>/**
 * analyze-css CommonJS module
 */
'use strict';

var cssParser = require('css-parse'),
	debug = require('debug')('analyze-css'),
	fs = require('fs'),
	preprocessors = new (require('./preprocessors'))(),
	slickParse = require('slick').parse,
	VERSION = require('./../package').version;

function analyzer(css, options, callback) {
	var res;

	// options can be omitted
	if (typeof options === 'function') {
		callback = options;
		options = {};
	}

	this.options = options;
	debug('opts: %j', this.options);

	if (typeof css !== 'string') {
		callback(new Error('css parameter passed is not a string!'), null);
		return;
	}

	// preprocess the CSS (issue #3)
	if (typeof options.preprocessor === 'string') {
		debug('Using "%s" preprocessor', options.preprocessor);

		var preprocessor = preprocessors.get(options.preprocessor);

		try {
			css = preprocessor.process(css, options);
		}
		catch(ex) {
			throw new Error('Preprocessing failed: ' + ex);
		}

		debug('Preprocessing completed');
	}

	res = this.analyze(css);

	if (res !== true) {
		callback(res, null);
		return;
	}

	// return the results
	res = {
		generator: 'analyze-css v' + VERSION,
		metrics: this.metrics,
		offenders: this.offenders
	};

	callback(null, res);
}

analyzer.version = VERSION;

analyzer.prototype = {
	emitter: false,
	tree: false,

	metrics: {},
	offenders: {},

	// emit given event
	emit: function(/* eventName, arg1, arg2, ... */) {
		//debug('Event %s emitted', arguments[0]);
		this.emitter.emit.apply(this.emitter, arguments);
	},

	// bind to a given event
	on: function(ev, fn) {
		this.emitter.on(ev, fn);
	},

	setMetric: function(name, value) {
		value = value || 0;

		//debug('setMetric(%s) = %d', name, value);
		this.metrics[name] = value;
	},

	// increements given metric by given number (default is one)
	incrMetric: function(name, incr /* =1 */) {
		var currVal = this.metrics[name] || 0;
		incr = incr || 1;

		//debug('incrMetric(%s) += %d', name, incr);
		this.setMetric(name, currVal + incr);
	},

	addOffender: function(metricName, msg) {
		if (typeof this.offenders[metricName] === 'undefined') {
			this.offenders[metricName] = [];
		}

		this.offenders[metricName].push(msg);
	},

	initRules: function() {
		var debug = require('debug')('analyze-css:rules'),
			re = /\.js$/,
			rules = [];

		// init events emitter
		this.emitter = new (require('events').EventEmitter)();
		this.emitter.setMaxListeners(200);

		// load all rules
		rules = fs.readdirSync(fs.realpathSync(__dirname + '/../rules/'))
			// filter out all non *.js files
			.filter(function(file) {
				return re.test(file);
			})
			// remove file extensions to get just names
			.map(function(file) {
				return file.replace(re, '');
			});

		debug('Rules to be loaded: %s', rules.join(', '));

		rules.forEach(function(name) {
			var rule = require('./../rules/' + name);
			rule(this);

			debug('"%s" loaded: %s', name, rule.description || '-');
		}, this);
	},

	parseCss: function(css) {
		var debug = require('debug')('analyze-css:parser');

		debug('Going to parse %s kB of CSS', (css.length / 1024).toFixed(2));

		try {
			if (css.trim() === '') {
				throw 'Empty CSS was provided';
			}

			this.tree = new cssParser(css);
		}
		catch(ex) {
			var errMsg = 'CSS parsing failed: ' + ex.toString();

			debug(errMsg);
			debug('Offending line: ' + css.split('\n')[ex.line-1]);

			return new Error(errMsg);
		}

		debug('CSS parsed');
		return true;
	},

	parseRules: function(rules) {
		rules.forEach(function(rule, idx) {
			debug('%j', rule);

			switch(rule.type) {
				// {
				//  "type":"media"
				//  "media":"screen and (min-width: 1370px)",
				//  "rules":[{"type":"rule","selectors":["#foo"],"declarations":[]}]
				// }
				case 'media':
					this.emit('media', rule.media, rule.rules);

					// now run recursively to parse rules within the media query
					if (rule.rules) {
						this.parseRules(rule.rules);
					}

					this.emit('mediaEnd', rule.media, rule.rules);
					break;

				// {
				//  "type":"rule",
				//  "selectors":[".ui-header .ui-btn-up-a",".ui-header .ui-btn-hover-a"],
				//  "declarations":[{"type":"declaration","property":"border","value":"0"},{"type":"declaration","property":"background","value":"none"}]
				// }
				case 'rule':
					this.emit('rule', rule);

					if (!rule.selectors || !rule.declarations) {
						return;
					}

					// analyze each selector and declaration
					rule.selectors.forEach(function(selector) {
						var parsedSelector,
							expressions = [],
							i, len;

						// "#features > div:first-child" will become two expressions:
						//  {"combinator":" ","tag":"*","id":"features"}
						//  {"combinator":">","tag":"div","pseudos":[{"key":"first-child","value":null}]}
						parsedSelector = slickParse(selector)[0];

						if (typeof parsedSelector === 'undefined') {
							throw 'Unable to parse "' + selector + '" selector';
						}

						// convert object with keys to array with numeric index
						for (i=0, len=parsedSelector.length; i<len; i++) {
							expressions.push(parsedSelector[i]);
						}

						this.emit('selector', rule, selector, expressions);

						expressions.forEach(function(expression) {
							this.emit('expression', selector, expression);
						}, this);
					}, this);

					rule.declarations.forEach(function(declaration) {
						switch(declaration.type) {
							case 'declaration':
								this.emit('declaration', rule, declaration.property, declaration.value);
								break;

							case 'comment':
								this.emit('comment', declaration.comment);
								break;
						}
					}, this);
					break;

				// {"type":"comment","comment":" Cached as static-css-r518-9b0f5ab4632defb55d67a1d672aa31bd120f4414 "}
				case 'comment':
					this.emit('comment', rule.comment);
					break;

				// {"type":"import","import":"url('/css/styles.css')"}
				case 'import':
					this.emit('import', rule.import);
					break;
			}
		}, this);
	},

	run: function() {
		var rules = this.tree && this.tree.stylesheet.rules;
		this.parseRules(rules);
	},

	analyze: function(css) {
		var res,
			then = Date.now();

		this.metrics = {};
		this.offenders = {};

		// load and init all rules
		this.initRules();

		// parse CSS
		res = this.parseCss(css);

		if (res !== true) {
			return res;
		}

		this.emit('css', css);

		// now go through parsed CSS tree and emit events for rules
		this.run();

		this.emit('report');

		debug('Completed in %d ms', Date.now() - then);
		return true;
	}
};

module.exports = analyzer;
<|endoftext|>


<commit_before>'use strict'

var NODE_INDEX = '__set-dom-index__'
var HTML_ELEMENT = document.createElement('html')
var BODY_ELEMENT = document.createElement('body')

module.exports = setDOM

/**
 * Updates existing dom to match a new dom.
 *
 * @param {HTMLEntity} prev - The html entity to update.
 * @param {String|HTMLEntity} next - The updated html(entity).
 */
function setDOM (prev, next) {
  // Alias document element with document.
  if (prev === document) prev = document.documentElement

  // If a string was provided we will parse it as dom.
  if (typeof next === 'string') {
    if (prev === document.documentElement) {
      HTML_ELEMENT.innerHTML = next
      next = HTML_ELEMENT
    } else {
      BODY_ELEMENT.innerHTML = next
      next = BODY_ELEMENT.firstChild
    }
  }

  // Update the node.
  setNode(prev, next)
}

/**
 * Updates a specific htmlNode and does whatever it takes to convert it to another one.
 *
 * @param {Object} prev - The previous HTMLNode.
 * @param {Object} next - The updated HTMLNode.
 */
function setNode (prev, next) {
  // Handle text node update.
  if (next.nodeName === '#text') {
    if (prev.nodeName === '#text') {
      // If both are text nodes we can update directly.
      prev.nodeValue = next.nodeValue
    } else {
      // Otherwise we have to replace the node.
      prev.parentNode.replaceChild(next, prev)
    }
    return
  }

  // Update all children (and subchildren).
  setChildNodes(prev, prev.childNodes, next.childNodes)

  // Update the elements attributes / tagName.
  if (prev.nodeName === next.nodeName) {
    // If we have the same nodename then we can directly update the attributes.
    setAttributes(prev, prev.attributes, next.attributes)
  } else {
    // Otherwise clone the new node to use as the existing node.
    var newPrev = next.cloneNode()
    // Copy over all existing children from the original node.
    while (prev.firstChild) newPrev.appendChild(prev.firstChild)
    // Replace the original node with the new one with the right tag.
    prev.parentNode.replaceChild(newPrev, prev)
  }
}

/*
 * @private
 * @description
 * Utility that will update one list of attributes to match another.
 *
 * @param {Object} parent - The current parentNode being updated.
 * @param {Object} prev - The previous attributes.
 * @param {Object} next - The updated attributes.
 */
function setAttributes (parent, prev, next) {
  var i, a, b, attr

  // Set new attributes.
  for (i = next.length; i--;) {
    a = next.item(i)
    b = prev.getNamedItem(a.name)
    if (!b) {
      // Add a new attribute.
      attr = document.createAttribute(a.name)
      attr.value = a.value
      prev.setNamedItem(attr)
    } else if (b.value !== a.value) {
      // Update existing attribute.
      b.value = a.value
    }
  }

  // Remove old attributes.
  for (i = prev.length; i--;) {
    a = prev.item(i)
    b = next.getNamedItem(a.name)
    if (!b) prev.removeNamedItem(a.name)
  }
}

/*
 * @private
 * @description
 * Utility that will update one list of childNodes to match another.
 *
 * @param {Object} parent - The current parentNode being updated.
 * @param {Object} prevChildNodes - The previous children.
 * @param {Object} nextChildNodes - The updated children.
 */
function setChildNodes (parent, prevChildNodes, nextChildNodes) {
  var key, a, b

  // Convert nodelists into a usuable map.
  var prev = keyNodes(prevChildNodes)
  var next = keyNodes(nextChildNodes)

  // Set new nodes.
  for (key in next) {
    a = prev[key]
    b = next[key]

    if (a) {
      // Update an existing node.
      setNode(a, b)
      // Check if we should reposition the node.
      if (a[NODE_INDEX] === b[NODE_INDEX]) continue
      // Reposition node.
      parent.insertBefore(a, prevChildNodes[b[NODE_INDEX]])
    } else {
      // Append the new node.
      parent.appendChild(b)
    }
  }

  // Remove old nodes.
  for (key in prev) {
    if (next[key]) continue
    parent.removeChild(prev[key])
  }
}

/**
 * @private
 * @description
 * Converts a nodelist into a keyed map.
 * This is used for diffing while keeping elements with 'data-key' or 'id' if possible.
 *
 * @param {Object} childNodes - The childNodes to convert.
 * @return {Object}
 */
function keyNodes (childNodes) {
  var result = {}
  for (var i = childNodes.length, el; i--;) {
    el = childNodes[i]
    el[NODE_INDEX] = i
    result[getKey(el, i)] = el
  }
  return result
}

/**
 * Utility to try to pull a key out of an element.
 * (Uses 'id' if possible and falls back to 'data-key')
 *
 * @param {Object} node - The node to get the key for.
 * @param {*} alt - A fallback key for the node.
 * @return {String|undefined}
 */
function getKey (node, alt) {
  if (typeof node.getAttribute !== 'function') return alt
  return node.id || node.getAttribute('data-key') || alt
}
<commit_msg>Cleanup inline docs<commit_after>'use strict'

var NODE_INDEX = '__set-dom-index__'
var HTML_ELEMENT = document.createElement('html')
var BODY_ELEMENT = document.createElement('body')

module.exports = setDOM

/**
 * @description
 * Updates existing dom to match a new dom.
 *
 * @param {HTMLEntity} prev - The html entity to update.
 * @param {String|HTMLEntity} next - The updated html(entity).
 */
function setDOM (prev, next) {
  // Alias document element with document.
  if (prev === document) prev = document.documentElement

  // If a string was provided we will parse it as dom.
  if (typeof next === 'string') {
    if (prev === document.documentElement) {
      HTML_ELEMENT.innerHTML = next
      next = HTML_ELEMENT
    } else {
      BODY_ELEMENT.innerHTML = next
      next = BODY_ELEMENT.firstChild
    }
  }

  // Update the node.
  setNode(prev, next)
}

/**
 * @private
 * @description
 * Updates a specific htmlNode and does whatever it takes to convert it to another one.
 *
 * @param {HTMLEntity} prev - The previous HTMLNode.
 * @param {HTMLEntity} next - The updated HTMLNode.
 */
function setNode (prev, next) {
  // Handle text node update.
  if (next.nodeName === '#text') {
    if (prev.nodeName === '#text') {
      // If both are text nodes we can update directly.
      prev.nodeValue = next.nodeValue
    } else {
      // Otherwise we have to replace the node.
      prev.parentNode.replaceChild(next, prev)
    }
    return
  }

  // Update all children (and subchildren).
  setChildNodes(prev, prev.childNodes, next.childNodes)

  // Update the elements attributes / tagName.
  if (prev.nodeName === next.nodeName) {
    // If we have the same nodename then we can directly update the attributes.
    setAttributes(prev, prev.attributes, next.attributes)
  } else {
    // Otherwise clone the new node to use as the existing node.
    var newPrev = next.cloneNode()
    // Copy over all existing children from the original node.
    while (prev.firstChild) newPrev.appendChild(prev.firstChild)
    // Replace the original node with the new one with the right tag.
    prev.parentNode.replaceChild(newPrev, prev)
  }
}

/*
 * @private
 * @description
 * Utility that will update one list of attributes to match another.
 *
 * @param {HTMLEntity} parent - The current parentNode being updated.
 * @param {Attributes} prev - The previous attributes.
 * @param {Attributes} next - The updated attributes.
 */
function setAttributes (parent, prev, next) {
  var i, a, b, attr

  // Set new attributes.
  for (i = next.length; i--;) {
    a = next.item(i)
    b = prev.getNamedItem(a.name)
    if (!b) {
      // Add a new attribute.
      attr = document.createAttribute(a.name)
      attr.value = a.value
      prev.setNamedItem(attr)
    } else if (b.value !== a.value) {
      // Update existing attribute.
      b.value = a.value
    }
  }

  // Remove old attributes.
  for (i = prev.length; i--;) {
    a = prev.item(i)
    b = next.getNamedItem(a.name)
    if (!b) prev.removeNamedItem(a.name)
  }
}

/*
 * @private
 * @description
 * Utility that will update one list of childNodes to match another.
 *
 * @param {HTMLEntity} parent - The current parentNode being updated.
 * @param {NodeList} prevChildNodes - The previous children.
 * @param {NodeList} nextChildNodes - The updated children.
 */
function setChildNodes (parent, prevChildNodes, nextChildNodes) {
  var key, a, b

  // Convert nodelists into a usuable map.
  var prev = keyNodes(prevChildNodes)
  var next = keyNodes(nextChildNodes)

  // Set new nodes.
  for (key in next) {
    a = prev[key]
    b = next[key]

    if (a) {
      // Update an existing node.
      setNode(a, b)
      // Check if we should reposition the node.
      if (a[NODE_INDEX] === b[NODE_INDEX]) continue
      // Reposition node.
      parent.insertBefore(a, prevChildNodes[b[NODE_INDEX]])
    } else {
      // Append the new node.
      parent.appendChild(b)
    }
  }

  // Remove old nodes.
  for (key in prev) {
    if (next[key]) continue
    parent.removeChild(prev[key])
  }
}

/**
 * @private
 * @description
 * Converts a nodelist into a keyed map.
 * This is used for diffing while keeping elements with 'data-key' or 'id' if possible.
 *
 * @param {NodeList} childNodes - The childNodes to convert.
 * @return {Object}
 */
function keyNodes (childNodes) {
  var result = {}
  for (var i = childNodes.length, el; i--;) {
    el = childNodes[i]
    el[NODE_INDEX] = i
    result[getKey(el, i)] = el
  }
  return result
}

/**
 * @private
 * @description
 * Utility to try to pull a key out of an element.
 * (Uses 'id' if possible and falls back to 'data-key')
 *
 * @param {HTMLEntity} node - The node to get the key for.
 * @param {String} alt - A fallback key for the node.
 * @return {String}
 */
function getKey (node, alt) {
  if (typeof node.getAttribute !== 'function') return alt
  return node.id || node.getAttribute('data-key') || alt
}
<|endoftext|>


<commit_before>'use strict';

// Load modules

const Fs = require('fs');
const Os = require('os');
const Querystring = require('querystring');
const Stream = require('stream');
const Zlib = require('zlib');

const Boom = require('boom');
const Content = require('content');
const Hoek = require('hoek');
const Pez = require('pez');
const Wreck = require('wreck');

// Declare internals

const internals = {
    decoders: {
        gzip: (options) => Zlib.createGunzip(options),
        deflate: (options) => Zlib.createInflate(options)
    }
};

exports.parse = async function (req, tap, options) {

    Hoek.assert(options, 'Missing options');
    Hoek.assert(options.parse !== undefined, 'Missing parse option setting');
    Hoek.assert(options.output !== undefined, 'Missing output option setting');

    // Content size

    const contentLength = req.headers['content-length'];
    if (options.maxBytes !== undefined &&
        contentLength &&
        parseInt(contentLength, 10) > options.maxBytes) {

        throw Boom.entityTooLarge('Payload content length greater than maximum allowed: ' + options.maxBytes);
    }

    // Content type

    const contentType = Content.type(options.override || req.headers['content-type'] || options.defaultContentType || 'application/octet-stream');
    try {
        if (options.allow &&
            options.allow.indexOf(contentType.mime) === -1) {

            throw Boom.unsupportedMediaType();
        }

        const parsed = { mime: contentType.mime };

        // Parse: true

        if (options.parse === true) {
            parsed.payload = await internals.parse(req, tap, options, contentType);
            return parsed;
        }

        // Parse: false, 'gunzip'

        parsed.payload = await internals.raw(req, tap, options);
        return parsed;
    }
    catch (err) {
        err.mime = contentType.mime;
        throw err;
    }
};

internals.parse = async function (req, tap, options, contentType) {

    const output = options.output;        // Output: 'data', 'stream', 'file'
    let source = internals.decoder(req, options);

    // Tap request

    if (tap) {
        source = internals.pipe(source, tap);
    }

    // Multipart

    if (contentType.mime === 'multipart/form-data') {
        if (options.multipart === false) {                            // Defaults to true
            throw Boom.unsupportedMediaType();
        }

        return await internals.multipart(req, options, source, contentType);
    }

    // Output: 'stream'

    if (output === 'stream') {
        return source;
    }

    // Output: 'file'

    if (output === 'file') {
        return await internals.writeFile(req, options, source);
    }

    // Output: 'data'

    const payload = await Wreck.read(source, { timeout: options.timeout, maxBytes: options.maxBytes });
    return internals.object(options, payload, contentType.mime);
};

internals.decoder = function (source, options) {

    const contentEncoding = source.headers['content-encoding'];
    const decoder = (options.decoders || internals.decoders)[contentEncoding];
    if (!decoder) {
        return source;
    }

    const decoderOptions = (options.compression && options.compression[contentEncoding]) || null;
    const stream = decoder(decoderOptions);

    const orig = stream.emit;
    stream.emit = (event, ...args) => {

        if (event === 'error') {
            args = [Boom.badRequest('Invalid compressed payload', args[0])];
        }

        return orig.call(stream, event, ...args);
    };

    return internals.pipe(source, stream);
};

internals.raw = async function (req, tap, options) {

    const output = options.output;      // Output: 'data', 'stream', 'file'
    let source = req;

    // Content-encoding (optional)

    if (options.parse === 'gunzip') {
        source = internals.decoder(source, options);
    }

    // Setup source

    if (tap) {
        source = internals.pipe(source, tap);
    }

    // Output: 'stream'

    if (output === 'stream') {
        return source;
    }

    // Output: 'file'

    if (output === 'file') {
        return await internals.writeFile(req, options, source);
    }

    // Output: 'data'

    return await Wreck.read(source, { timeout: options.timeout, maxBytes: options.maxBytes });
};

internals.object = function (options, payload, mime) {

    // Binary

    if (mime === 'application/octet-stream') {
        return (payload.length ? payload : null);
    }

    // Text

    if (mime.match(/^text\/.+$/)) {
        return payload.toString('utf8');
    }

    // JSON

    if (/^application\/(?:.+\+)?json$/.test(mime)) {
        if (!payload.length) {
            return null;
        }

        try {
            return JSON.parse(payload.toString('utf8'));
        }
        catch (err) {
            const error = Boom.badRequest('Invalid request payload JSON format', err);
            error.raw = payload;
            throw error;
        }
    }

    // Form-encoded

    if (mime === 'application/x-www-form-urlencoded') {
        const parse = (options.querystring || Querystring.parse);
        return (payload.length ? parse(payload.toString('utf8')) : {});
    }

    const error = Boom.unsupportedMediaType();
    error.raw = payload;
    throw error;
};

internals.multipart = function (req, options, source, contentType) {

    return new Promise((resolve, reject) => {

        internals._multipart(req, options, source, contentType, (err, payload) => {

            if (err) {
                return reject(err);
            }

            return resolve(payload);
        });
    });
};

internals._multipart = function (req, options, source, contentType, next) {

    // Set stream timeout

    const clientTimeout = options.timeout;
    let clientTimeoutId = null;

    const dispenserOptions = Hoek.applyToDefaults(contentType, { maxBytes: options.maxBytes });
    const dispenser = new Pez.Dispenser(dispenserOptions);

    const onError = (err) => {

        return next(Boom.badRequest('Invalid multipart payload format', err));
    };

    dispenser.once('error', onError);

    const data = {};
    const finalize = () => {

        clearTimeout(clientTimeoutId);
        dispenser.removeListener('error', onError);
        dispenser.removeListener('part', onPart);
        dispenser.removeListener('field', onField);
        dispenser.removeListener('close', onClose);

        return next(null, data);
    };

    if (clientTimeout &&
        clientTimeout > 0) {

        clientTimeoutId = setTimeout(() => {

            return next(Boom.clientTimeout());
        }, clientTimeout);
    }

    const set = (name, value) => {

        if (!data.hasOwnProperty(name)) {
            data[name] = value;
        }
        else if (Array.isArray(data[name])) {
            data[name].push(value);
        }
        else {
            data[name] = [data[name], value];
        }
    };

    const pendingFiles = {};
    let nextId = 0;
    let closed = false;

    const output = (options.multipart ? options.multipart.output : options.output);

    const onPart = async (part) => {

        if (output === 'file') {                                                                // Output: 'file'
            const id = nextId++;
            pendingFiles[id] = true;
            try {
                const { path, bytes } = await internals.writeFile(req, options, part);

                const item = {
                    filename: part.filename,
                    path,
                    headers: part.headers,
                    bytes
                };

                set(part.name, item);
            }
            catch (err) {
                return next(err);
            }

            delete pendingFiles[id];

            if (closed &&
                !Object.keys(pendingFiles).length) {

                return finalize(data);
            }
        }
        else {                                                                                // Output: 'data'
            const payload = await Wreck.read(part); // Error handled by dispenser.once('error')

            if (output === 'stream') {                                                      // Output: 'stream'
                const item = Wreck.toReadableStream(payload);

                item.hapi = {
                    filename: part.filename,
                    headers: part.headers
                };

                return set(part.name, item);
            }

            const ct = part.headers['content-type'] || '';
            const mime = ct.split(';')[0].trim().toLowerCase();
            const annotate = (value) => set(part.name, output === 'annotated' ? { filename: part.filename, headers: part.headers, payload: value } : value);

            if (!mime) {
                return annotate(payload);
            }

            if (!payload.length) {
                return annotate({});
            }

            try {
                const object = internals.object(options, payload, mime);
                annotate(object);
            }
            catch (err) {
                annotate(payload);
            }
        }
    };

    dispenser.on('part', onPart);

    const onField = (name, value) => set(name, value);

    dispenser.on('field', onField);

    const onClose = () => {

        if (Object.keys(pendingFiles).length) {
            closed = true;
            return;
        }

        return finalize(data);
    };

    dispenser.once('close', onClose);

    source.pipe(dispenser);
};

internals.writeFile = function (req, options, stream) {

    return new Promise((resolve, reject) => {

        const path = Hoek.uniqueFilename(options.uploads || Os.tmpdir());
        const file = Fs.createWriteStream(path, { flags: 'wx' });
        const counter = new internals.Counter();

        const finalize = Hoek.once((err) => {

            req.removeListener('aborted', onAbort);
            file.removeListener('close', finalize);
            file.removeListener('error', finalize);

            if (!err) {
                return resolve({ path, bytes: counter.bytes });
            }

            file.destroy();
            Fs.unlink(path, (/* fsErr */) => reject(err));      // Ignore unlink errors
        });

        file.once('close', finalize);
        file.once('error', finalize);

        const onAbort = () => finalize(Boom.badRequest('Client connection aborted'));
        req.once('aborted', onAbort);

        stream.pipe(counter).pipe(file);
    });
};

internals.pipe = function (from, to) {

    from.once('error', (err) => to.emit('error', err));
    return from.pipe(to);
};

internals.Counter = class extends Stream.Transform {

    constructor() {

        super();
        this.bytes = 0;
    }

    _transform(chunk, encoding, next) {

        this.bytes = this.bytes + chunk.length;
        return next(null, chunk);
    }
};
<commit_msg>node 11. Closes #58<commit_after>'use strict';

// Load modules

const Fs = require('fs');
const Os = require('os');
const Querystring = require('querystring');
const Stream = require('stream');
const Zlib = require('zlib');

const Boom = require('boom');
const Content = require('content');
const Hoek = require('hoek');
const Pez = require('pez');
const Wreck = require('wreck');

// Declare internals

const internals = {
    decoders: {
        gzip: (options) => Zlib.createGunzip(options),
        deflate: (options) => Zlib.createInflate(options)
    }
};

exports.parse = async function (req, tap, options) {

    Hoek.assert(options, 'Missing options');
    Hoek.assert(options.parse !== undefined, 'Missing parse option setting');
    Hoek.assert(options.output !== undefined, 'Missing output option setting');

    // Content size

    const contentLength = req.headers['content-length'];
    if (options.maxBytes !== undefined &&
        contentLength &&
        parseInt(contentLength, 10) > options.maxBytes) {

        throw Boom.entityTooLarge('Payload content length greater than maximum allowed: ' + options.maxBytes);
    }

    // Content type

    const contentType = Content.type(options.override || req.headers['content-type'] || options.defaultContentType || 'application/octet-stream');
    try {
        if (options.allow &&
            options.allow.indexOf(contentType.mime) === -1) {

            throw Boom.unsupportedMediaType();
        }

        const parsed = { mime: contentType.mime };

        // Parse: true

        if (options.parse === true) {
            parsed.payload = await internals.parse(req, tap, options, contentType);
            return parsed;
        }

        // Parse: false, 'gunzip'

        parsed.payload = await internals.raw(req, tap, options);
        return parsed;
    }
    catch (err) {
        err.mime = contentType.mime;
        throw err;
    }
};

internals.parse = async function (req, tap, options, contentType) {

    const output = options.output;        // Output: 'data', 'stream', 'file'
    let source = internals.decoder(req, options);

    // Tap request

    if (tap) {
        source = internals.pipe(source, tap);
    }

    // Multipart

    if (contentType.mime === 'multipart/form-data') {
        if (options.multipart === false) {                            // Defaults to true
            throw Boom.unsupportedMediaType();
        }

        return await internals.multipart(req, options, source, contentType);
    }

    // Output: 'stream'

    if (output === 'stream') {
        return source;
    }

    // Output: 'file'

    if (output === 'file') {
        const file = await internals.writeFile(req, options, source);
        return file.item;
    }

    // Output: 'data'

    const payload = await Wreck.read(source, { timeout: options.timeout, maxBytes: options.maxBytes });
    return internals.object(options, payload, contentType.mime);
};

internals.decoder = function (source, options) {

    const contentEncoding = source.headers['content-encoding'];
    const decoder = (options.decoders || internals.decoders)[contentEncoding];
    if (!decoder) {
        return source;
    }

    const decoderOptions = (options.compression && options.compression[contentEncoding]) || null;
    const stream = decoder(decoderOptions);

    const orig = stream.emit;
    stream.emit = (event, ...args) => {

        if (event === 'error') {
            args = [Boom.badRequest('Invalid compressed payload', args[0])];
        }

        return orig.call(stream, event, ...args);
    };

    return internals.pipe(source, stream);
};

internals.raw = async function (req, tap, options) {

    const output = options.output;      // Output: 'data', 'stream', 'file'
    let source = req;

    // Content-encoding (optional)

    if (options.parse === 'gunzip') {
        source = internals.decoder(source, options);
    }

    // Setup source

    if (tap) {
        source = internals.pipe(source, tap);
    }

    // Output: 'stream'

    if (output === 'stream') {
        return source;
    }

    // Output: 'file'

    if (output === 'file') {
        const file = await internals.writeFile(req, options, source);
        return file.item;
    }

    // Output: 'data'

    return await Wreck.read(source, { timeout: options.timeout, maxBytes: options.maxBytes });
};

internals.object = function (options, payload, mime) {

    // Binary

    if (mime === 'application/octet-stream') {
        return (payload.length ? payload : null);
    }

    // Text

    if (mime.match(/^text\/.+$/)) {
        return payload.toString('utf8');
    }

    // JSON

    if (/^application\/(?:.+\+)?json$/.test(mime)) {
        if (!payload.length) {
            return null;
        }

        try {
            return JSON.parse(payload.toString('utf8'));
        }
        catch (err) {
            const error = Boom.badRequest('Invalid request payload JSON format', err);
            error.raw = payload;
            throw error;
        }
    }

    // Form-encoded

    if (mime === 'application/x-www-form-urlencoded') {
        const parse = (options.querystring || Querystring.parse);
        return (payload.length ? parse(payload.toString('utf8')) : {});
    }

    const error = Boom.unsupportedMediaType();
    error.raw = payload;
    throw error;
};

internals.multipart = function (req, options, source, contentType) {

    return new Promise((resolve, reject) => {

        // Set stream timeout

        const clientTimeout = options.timeout;
        const clientTimeoutId = (clientTimeout ? setTimeout(() => reject(Boom.clientTimeout()), clientTimeout) : null);

        // Create parser

        const dispenserOptions = Hoek.applyToDefaults(contentType, { maxBytes: options.maxBytes });
        const dispenser = new Pez.Dispenser(dispenserOptions);

        const onError = (err) => reject(Boom.badRequest('Invalid multipart payload format', err));
        dispenser.once('error', onError);

        const data = {};
        const pendingFiles = [];

        const set = (name, value) => {

            if (!data.hasOwnProperty(name)) {
                data[name] = value;
            }
            else if (Array.isArray(data[name])) {
                data[name].push(value);
            }
            else {
                data[name] = [data[name], value];
            }
        };

        const finalize = async () => {

            // Clean up

            clearTimeout(clientTimeoutId);
            dispenser.removeListener('error', onError);
            dispenser.removeListener('part', onPart);
            dispenser.removeListener('field', onField);
            dispenser.removeListener('close', onClose);

            // Wait for files

            try {
                const files = await Promise.all(pendingFiles);
                for (const { item, name } of files) {
                    set(name, item);
                }
            }
            catch (err) {
                reject(err);
                return;
            }

            resolve(data);
        };

        const output = (options.multipart ? options.multipart.output : options.output);

        const onPart = async (part) => {

            if (output === 'file') {                                                                // Output: 'file'
                pendingFiles.push(internals.writeFile(req, options, part));
            }
            else {                                                                                // Output: 'data'
                const payload = await Wreck.read(part);     // Error handled by dispenser.once('error')

                if (output === 'stream') {                                                          // Output: 'stream'
                    const item = Wreck.toReadableStream(payload);
                    item.hapi = {
                        filename: part.filename,
                        headers: part.headers
                    };

                    return set(part.name, item);
                }

                const ct = part.headers['content-type'] || '';
                const mime = ct.split(';')[0].trim().toLowerCase();
                const annotate = (value) => set(part.name, output === 'annotated' ? { filename: part.filename, headers: part.headers, payload: value } : value);

                if (!mime) {
                    return annotate(payload);
                }

                if (!payload.length) {
                    return annotate({});
                }

                try {
                    const object = internals.object(options, payload, mime);
                    annotate(object);
                }
                catch (err) {
                    annotate(payload);
                }
            }
        };

        dispenser.on('part', onPart);

        const onField = (name, value) => set(name, value);
        dispenser.on('field', onField);

        const onClose = () => finalize();
        dispenser.once('close', onClose);

        source.pipe(dispenser);
    });
};

internals.writeFile = function (req, options, stream) {

    return new Promise((resolve, reject) => {

        const path = Hoek.uniqueFilename(options.uploads || Os.tmpdir());
        const file = Fs.createWriteStream(path, { flags: 'wx' });
        const counter = new internals.Counter();

        const finalize = Hoek.once((err) => {

            req.removeListener('aborted', onAbort);
            file.removeListener('close', finalize);
            file.removeListener('error', finalize);

            if (err) {
                stream.unpipe(counter);
                counter.unpipe(file);

                file.destroy();
                Fs.unlink(path, (/* fsErr */) => reject(err));      // Ignore unlink errors
                return;
            }

            const result = {
                item: {
                    path,
                    bytes: counter.bytes
                }
            };

            if (stream.filename) {
                result.item.filename = stream.filename;
            }

            if (stream.headers) {
                result.item.headers = stream.headers;
            }

            if (stream.name) {
                result.name = stream.name;
            }

            resolve(result);
        });

        file.once('close', finalize);
        file.once('error', finalize);

        const onAbort = () => finalize(Boom.badRequest('Client connection aborted'));
        req.once('aborted', onAbort);

        internals.pipe(stream, counter);
        internals.pipe(counter, file);
    });
};

internals.pipe = function (from, to) {

    from.once('error', (err) => {

        from.unpipe(to);
        to.emit('error', err);
    });

    return from.pipe(to);
};

internals.Counter = class extends Stream.Transform {

    constructor() {

        super();
        this.bytes = 0;
    }

    _transform(chunk, encoding, next) {

        this.bytes = this.bytes + chunk.length;
        return next(null, chunk);
    }
};
<|endoftext|>


<commit_before>'use strict';

var gulp = require('gulp');
var gutil = require('gulp-util');
var connect = require('gulp-connect');
var runSequence = require('run-sequence');
var concat = require('gulp-concat');
var jshint = require('gulp-jshint');
var stylish = require('jshint-stylish');
var uglify = require('gulp-uglify');
var sourcemaps = require('gulp-sourcemaps');
var slim = require('gulp-slim');
var minifyHTML = require('gulp-minify-html');
var sass = require('gulp-sass');
var minifyCSS = require('gulp-minify-css');
var imagemin = require('gulp-imagemin');
var optipng = require('imagemin-optipng');
var karma = require('karma').server;
var _ = require('lodash');
var protractor = require('gulp-protractor').protractor;
/*jshint camelcase: false */
var webdriverStandalone = require('gulp-protractor').webdriver_standalone;
/*jshint camelcase: false */
var webdriverUpdate = require('gulp-protractor').webdriver_update;
var sh = require('shelljs');
var bower = require('bower');
var gopen = require('gulp-open');

var port = 9001;

gulp.task('default', ['run']);

gulp.task('run', function(callback) {
  return runSequence(['build', 'connect', 'watch'], ['open-index', 'tdd'], callback);
});

gulp.task('build', function(callback) {
  return runSequence('clean', 'bower', ['slim', 'scripts', 'ionic', 'img'], 'scss', callback);
});

gulp.task('watch', function() {
  return gulp.watch('app/**/*.*', ['build-and-reload']);
});

gulp.task('build-and-reload', function(callback) {
  return runSequence('build', 'reload-app', callback);
});

gulp.task('reload-app', function() {
  return gulp.src('app/**/*.*')
    .pipe(connect.reload());
});

gulp.task('run-android', ['build'], function() {
  sh.exec('phonegap local run android');
});

gulp.task('open-index', function(){
  var options = {
    url: 'http://localhost:' + port,
    app: 'google-chrome'
  };
  gulp.src('www/index.html')
  .pipe(gopen('', options));
});

var karmaCommonConf = {
  basepaths : '',
  browsers: ['Chrome'],
  frameworks: ['jasmine'],
  files : [
    'bower_components/ionic/js/ionic.bundle.js',
    'bower_components/angular-mocks/angular-mocks.js',
    'app/**/*.js',
    'test/unit/**/*.js'
  ],
  autoWatch : true,
  usePolling: true,
  plugins : [
    'karma-chrome-launcher',
    'karma-phantomjs-launcher',
    'karma-jasmine',
    'karma-coverage'
  ],
  reporters: ['progress', 'coverage'],
  preprocessors: {
    'app/**/*.js': ['coverage']
  },
  coverageReporter: {
    type : 'html',
    dir : 'coverage/'
  }
};

gulp.task('tdd', function (done) {
  return karma.start(karmaCommonConf, done);
});

gulp.task('test-single-run', function (done) {
  return karma.start(_.assign({}, karmaCommonConf, {singleRun: true}), done);
});

gulp.task('e2e', ['webdriver-update'], function(callback) {
  return gulp.src('e2e/*.js')
    .pipe(protractor({
        configFile: 'test/protractor-conf.js',
        args: ['--baseUrl', 'http://127.0.0.1:' + port]
    }))
    .on('error', function(e) { throw e; });
});

gulp.task('webdriver-standalone', webdriverStandalone);

gulp.task('webdriver-update', webdriverUpdate);

gulp.task('img', function(callback) {
  return runSequence(['png'], callback);
});

gulp.task('png', function() {
  return gulp.src('src/img/**/*.png')
    .pipe(imagemin({
        progressive: true,
        svgoPlugins: [{removeViewBox: false}],
        use: [optipng()]
    }))
    .pipe(gulp.dest('www/img'));
});

gulp.task('slim', ['slim-main', 'slim-views']);

gulp.task('slim-main', function() {
  return gulp.src('app/*.slim')
    .pipe(slim({ pretty: true }))
    .pipe(minifyHTML({ empty: true }))
    .pipe(gulp.dest('www/'));
});

gulp.task('slim-views', function() {
  return gulp.src('app/views/*.slim')
    .pipe(slim({ pretty: true }))
    .pipe(minifyHTML({ empty: true }))
    .pipe(gulp.dest('www/views/'));
});

gulp.task('scss', function() {
  return gulp.src('app/assets/scss/**/*.scss')
    .pipe(sass())
    .pipe(minifyCSS({
      keepSpecialComments: 0
    }))
    .pipe(concat('style.min.css'))
    .pipe(gulp.dest('www/css/'));
});

gulp.task('jshint', function() {
  return gulp.src(['app/**/*.js', 'gulpfile.js'])
    .pipe(jshint())
    .pipe(jshint.reporter(stylish));
});

gulp.task('scripts', ['jshint'], function() {
  return gulp.src(['app.module.js', 'app/**/*.js'])
    .pipe(sourcemaps.init())
    .pipe(uglify({mangle: true}))
    .pipe(concat('app-script.min.js'))
    .pipe(sourcemaps.write('maps'))
    .pipe(gulp.dest('www/js/'));
});

gulp.task('ionic', function(callback) {
  return runSequence(['ionic-fonts', 'ionic-js'], callback);
});

gulp.task('ionic-fonts', function(cb) {
  return gulp.src('bower_components/ionic/fonts/**/*.*')
    .pipe(gulp.dest('www/fonts'));
});

gulp.task('ionic-js', function(cb) {
  return gulp.src('bower_components/ionic/js/ionic.bundle.js')
    .pipe(gulp.dest('www/js/lib'));
});

gulp.task('git-check', function(done) {
  if (!sh.which('git')) {
    console.log(
      '  ' + gutil.colors.red('Git is not installed.'),
      '\n  Git, the version control system, is required to download Ionic.',
      '\n  Download git here:', gutil.colors.cyan('http://git-scm.com/downloads') + '.',
      '\n  Once git is installed, run \'' + gutil.colors.cyan('gulp install') + '\' again.'
    );
    process.exit(1);
  }
  done();
});

gulp.task('bower', ['git-check'], function() {
  return bower.commands.install()
    .on('log', function(data) {
      gutil.log('bower', gutil.colors.cyan(data.id), data.message);
    });
});

gulp.task('clean', function() {
  sh.rm('-rf', 'www');
});

gulp.task('connect', function() {
  return connect.server({
    root: 'www',
    port: port,
    livereload: true
  });
});
<commit_msg>Add build-for-device gulp taks<commit_after>'use strict';

var gulp = require('gulp');
var gutil = require('gulp-util');
var connect = require('gulp-connect');
var runSequence = require('run-sequence');
var concat = require('gulp-concat');
var jshint = require('gulp-jshint');
var stylish = require('jshint-stylish');
var uglify = require('gulp-uglify');
var sourcemaps = require('gulp-sourcemaps');
var slim = require('gulp-slim');
var minifyHTML = require('gulp-minify-html');
var sass = require('gulp-sass');
var minifyCSS = require('gulp-minify-css');
var imagemin = require('gulp-imagemin');
var optipng = require('imagemin-optipng');
var karma = require('karma').server;
var _ = require('lodash');
var protractor = require('gulp-protractor').protractor;
/*jshint camelcase: false */
var webdriverStandalone = require('gulp-protractor').webdriver_standalone;
/*jshint camelcase: false */
var webdriverUpdate = require('gulp-protractor').webdriver_update;
var sh = require('shelljs');
var bower = require('bower');
var gopen = require('gulp-open');

var port = 9001;

gulp.task('default', ['run']);

gulp.task('run', function(callback) {
  return runSequence(['build', 'connect', 'watch'], ['open-index', 'tdd'], callback);
});

gulp.task('build', function(callback) {
  return runSequence('clean', 'bower', ['test-single-run', 'slim', 'scripts', 'ionic', 'img'], 'scss', callback);
});

gulp.task('watch', function() {
  return gulp.watch('app/**/*.*', ['build-and-reload']);
});

gulp.task('build-and-reload', function(callback) {
  return runSequence('build', 'reload-app', callback);
});

gulp.task('reload-app', function() {
  return gulp.src('app/**/*.*')
    .pipe(connect.reload());
});

gulp.task('open-index', function(){
  var options = {
    url: 'http://localhost:' + port,
    app: 'google-chrome'
  };
  gulp.src('www/index.html')
  .pipe(gopen('', options));
});

var karmaCommonConf = {
  basepaths : '',
  browsers: ['Chrome'],
  frameworks: ['jasmine'],
  files : [
    'bower_components/ionic/js/ionic.bundle.js',
    'bower_components/angular-mocks/angular-mocks.js',
    'app/**/*.js',
    'test/unit/**/*.js'
  ],
  autoWatch : true,
  usePolling: true,
  plugins : [
    'karma-chrome-launcher',
    'karma-phantomjs-launcher',
    'karma-jasmine',
    'karma-coverage'
  ],
  reporters: ['progress', 'coverage'],
  preprocessors: {
    'app/**/*.js': ['coverage']
  },
  coverageReporter: {
    type : 'html',
    dir : 'coverage/'
  }
};

gulp.task('tdd', function (done) {
  return karma.start(karmaCommonConf, done);
});

gulp.task('test-single-run', function (done) {
  return karma.start(_.assign({}, karmaCommonConf, {singleRun: true}), done);
});

gulp.task('e2e', ['webdriver-update'], function(callback) {
  return gulp.src('e2e/*.js')
    .pipe(protractor({
        configFile: 'test/protractor-conf.js',
        args: ['--baseUrl', 'http://127.0.0.1:' + port]
    }))
    .on('error', function(e) { throw e; });
});

gulp.task('webdriver-standalone', webdriverStandalone);

gulp.task('webdriver-update', webdriverUpdate);

gulp.task('img', function(callback) {
  return runSequence(['png'], callback);
});

gulp.task('png', function() {
  return gulp.src('src/img/**/*.png')
    .pipe(imagemin({
        progressive: true,
        svgoPlugins: [{removeViewBox: false}],
        use: [optipng()]
    }))
    .pipe(gulp.dest('www/img'));
});

gulp.task('slim', ['slim-main', 'slim-views']);

gulp.task('slim-main', function() {
  return gulp.src('app/*.slim')
    .pipe(slim({ pretty: true }))
    .pipe(minifyHTML({ empty: true }))
    .pipe(gulp.dest('www/'));
});

gulp.task('slim-views', function() {
  return gulp.src('app/views/*.slim')
    .pipe(slim({ pretty: true }))
    .pipe(minifyHTML({ empty: true }))
    .pipe(gulp.dest('www/views/'));
});

gulp.task('scss', function() {
  return gulp.src('app/assets/scss/**/*.scss')
    .pipe(sass())
    .pipe(minifyCSS({
      keepSpecialComments: 0
    }))
    .pipe(concat('style.min.css'))
    .pipe(gulp.dest('www/css/'));
});

gulp.task('jshint', function() {
  return gulp.src(['app/**/*.js', 'gulpfile.js'])
    .pipe(jshint())
    .pipe(jshint.reporter(stylish));
});

gulp.task('scripts', ['jshint'], function() {
  return gulp.src(['app.module.js', 'app/**/*.js'])
    .pipe(sourcemaps.init())
    .pipe(uglify({mangle: true}))
    .pipe(concat('app-script.min.js'))
    .pipe(sourcemaps.write('maps'))
    .pipe(gulp.dest('www/js/'));
});

gulp.task('ionic', function(callback) {
  return runSequence(['ionic-fonts', 'ionic-js'], callback);
});

gulp.task('ionic-fonts', function(cb) {
  return gulp.src('bower_components/ionic/fonts/**/*.*')
    .pipe(gulp.dest('www/fonts'));
});

gulp.task('ionic-js', function(cb) {
  return gulp.src('bower_components/ionic/js/ionic.bundle.js')
    .pipe(gulp.dest('www/js/lib'));
});

gulp.task('git-check', function(done) {
  if (!sh.which('git')) {
    console.log(
      '  ' + gutil.colors.red('Git is not installed.'),
      '\n  Git, the version control system, is required to download Ionic.',
      '\n  Download git here:', gutil.colors.cyan('http://git-scm.com/downloads') + '.',
      '\n  Once git is installed, run \'' + gutil.colors.cyan('gulp install') + '\' again.'
    );
    process.exit(1);
  }
  done();
});

gulp.task('bower', ['git-check'], function() {
  return bower.commands.install()
    .on('log', function(data) {
      gutil.log('bower', gutil.colors.cyan(data.id), data.message);
    });
});

gulp.task('clean', function() {
  sh.rm('-rf', 'www');
});

var cordovaConfig = {
  platforms: [
    'android'
  ],
  plugins: [
    'org.apache.cordova.console'
  ]
};

gulp.task('build-for-device', function(callback) {
  return runSequence('build', 'build-cordova', callback);
});

gulp.task('build-cordova', function(callback) {
  return runSequence('clean-cordova', 'install-platforms', 'install-plugins', callback);
});

gulp.task('install-platforms', function() {
  var platforms = cordovaConfig.platforms;
  for (var i = 0; i < platforms.length; i++) {
    var platform = platforms[i];
    var command = 'cordova platform add ' + platform;
    sh.echo('run comand: ' + command);
    sh.exec(command);
  }
});

gulp.task('install-plugins', function() {
  var plugins = cordovaConfig.plugins;
  for (var i = 0; i < plugins.length; i++) {
    var plugin = plugins[i];
    var command = 'cordova plugin add ' + plugin;
    sh.echo('run comand: ' + command);
    sh.exec(command);
  }
});

gulp.task('clean-cordova', function() {
  var cordovaPaths = [
    'platforms/',
    'plugins/'
  ];
  sh.rm('-rf', cordovaPaths);
});

gulp.task('emulate-to-android', ['build-for-device'], function() {
  sh.exec('cordova emulate android');
});

gulp.task('connect', function() {
  return connect.server({
    root: 'www',
    port: port,
    livereload: true
  });
});
<|endoftext|>


<commit_before>var crypto = require('crypto');
var fs = require('fs');
var path = require('path');
var zlib = require('zlib');

var mime = require('mime');
var watch = require('watch');

var gzipContentTypes = require('./gzipContentTypes.js');

exports.static = function static(directory, options) {
    directory = path.resolve(directory || 'public');

    if (options) {
        if (options.hostname) {
            if (typeof options.hostname !== 'string') {
                throw Error('Hostname must be a string');
            } else if (options.hostname.slice(-1) === '/') {
                options.hostname = options.hostname.slice(0, options.hostname.length - 1);
            }
        }
    }

    var files = {};

    function cacheFile(filePath, stat) {
        var data = fs.readFileSync(filePath);

        // Ensure the path we use is relative to the public dir
        var relativeUrl = toRelativeUrl(filePath);

        files[relativeUrl] = {
            content: data,
            contentLength: typeof data === 'string' ? Buffer.byteLength(data) : data.length,
            contentType: mime.lookup(relativeUrl),
            hash: crypto.createHash('md5').update(data).digest('hex'),
            modified: stat.mtime
        };
    }

    function cacheIfNotDir(f, stat) {
        if(!stat.isDirectory()) {
            cacheFile(f, stat);
        }
    }

    function dehashifyPath(filePath) {
        var hashRegex = /-[0-9a-f]+(\.[^\.]*$)/;
        var hashMatch = filePath.match(hashRegex);

        return {
            path: filePath.replace(hashRegex, '$1'),
            hash: hashMatch ? hashMatch[0].slice(1).replace(/\.([^\.]*$)/, '') : null
        };
    }

    function gzip(file, callback) {
        if (file.gzippedContent && file.gzippedContentLength) {
            return callback(null, file);
        }

        zlib.gzip(file.content, function(err, gzippedContent) {
            if (err) {
                return callback(err);
            }

            file.gzippedContent = gzippedContent;
            file.gzippedContentLength = gzippedContent.length;

            callback(null, file);
        });
    }

    function hashifyPath(filePath, hash) {
        if (filePath.indexOf('.') != -1) {
            return filePath.replace(/\.([^\.]*$)/, '-' + hash + '.$1');
        }

        return filePath + '-' + hash;
    }

    function prefixSlash(path) {
        return path[0] === '/' ? path : '/' + path;
    }

    function toRelativeUrl(filePath) {
        var relativeUrl = path.relative(directory, path.resolve(filePath));

        // Make URI-friendly and prepend a /
        return prefixSlash(relativeUrl).replace(/\\/g, '/');
    }

    function urlBuilder(filePath) {
        var file = files[prefixSlash(filePath)];

        if (file) {
            var uri = hashifyPath(prefixSlash(filePath), file.hash);

            if (options && options.hostname) {
                uri = '//' + options.hostname + uri;
            }

            return uri;
        } else {
            return filePath;
        }
    }

    // Load all files synchronously
    (function loadFiles(workingDir) {
        var contents;

        try {
            contents = fs.readdirSync(workingDir);
        }
        catch (e) {
            switch (e.code) {
                case 'ENOENT':
                    console.error('Directory does not exist');
                break;

                case 'ENOTDIR':
                    console.error('Not a directory');
                break;

                case 'EACCES':
                    console.error('Insufficient permissions to access directory');
                break;
            }

            throw e;
        }

        contents.forEach(function loadDirectory(file) {
            var filePath = path.join(workingDir, file);
            var stat = fs.statSync(filePath);

            if (stat.isDirectory()) {
                return loadFiles(filePath);
            }

            cacheFile(filePath, stat);
        });
    })(directory);

    // Watch directory
    watch.createMonitor(directory, function monitorFiles(monitor) {
        monitor.on('changed', cacheIfNotDir);
        monitor.on('created', cacheIfNotDir);

        monitor.on('removed', function fileRemoved(f, stat) {
            files[toRelativeUrl(f)] = null;
        });
    });

    return function staticMiddleware(req, res, next) {
        // Ignore anything that's not a GET or HEAD request
        if (req.method !== 'GET' && req.method !== 'HEAD') {
            return next();
        }

        // Register view helper if we haven't already
        if (!req.app.locals.electricity) {
            req.app.locals.electricity = {
                url: urlBuilder
            };
        }

        var reqInfo = dehashifyPath(req.path);
        var file = files[reqInfo.path];

        if (!file) {
            return next();
        }

        // Verify file matches the requested hash, otherwise 302
        if (reqInfo.hash !== file.hash) {
            return res.redirect(hashifyPath(reqInfo.path, file.hash));
        }

        var expires = new Date();
        expires.setYear(expires.getFullYear() + 1);

        res.set({
            'Cache-Control': 'public, max-age=31536000', // One year
            'Content-Type': file.contentType,
            'ETag': file.hash,
            'Expires': expires.toUTCString(),
            'Last-Modified': file.modified.toUTCString()
        });

        if (req.get('If-None-Match') === file.hash) {
            res.send(304);
        } else if (new Date(req.get('If-Modified-Since')) >= file.modified) {
            res.send(304);
        } else {
            res.set({
                'Content-Length': file.contentLength
            });

            res.status(200);

            if (req.method === 'HEAD') {
                return res.end();
            }

            var acceptEncoding = req.get('Accept-Encoding');

            if (acceptEncoding && acceptEncoding.split(/,\s?/).indexOf('gzip') !== -1 && gzipContentTypes.indexOf(file.contentType) !== -1) {
                gzip(file, function(err, gzippedFile) {
                    if (err) {
                        return next(err);
                    }

                    res.set({
                        'Content-Encoding': 'gzip',
                        'Content-Length': gzippedFile.gzippedContentLength
                    });

                    res.send(file.gzippedContent);
                });
            }
            else {
                res.send(file.content);
            }
        }
    };
};
<commit_msg>cleanup<commit_after>var crypto = require('crypto');
var fs = require('fs');
var path = require('path');
var zlib = require('zlib');

var mime = require('mime');
var watch = require('watch');

var gzipContentTypes = require('./gzipContentTypes.js');

exports.static = function static(directory, options) {
    directory = path.resolve(directory || 'public');

    if (options) {
        if (options.hostname) {
            if (typeof options.hostname !== 'string') {
                throw Error('Hostname must be a string');
            } else if (options.hostname.slice(-1) === '/') {
                options.hostname = options.hostname.slice(0, options.hostname.length - 1);
            }
        }
    }

    var files = {};

    function cacheFile(filePath, stat) {
        if (!stat.isDirectory()) {
            var data = fs.readFileSync(filePath);

            // Ensure the path we use is relative to the public dir
            var relativeUrl = toRelativeUrl(filePath);

            files[relativeUrl] = {
                content: data,
                contentLength: typeof data === 'string' ? Buffer.byteLength(data) : data.length,
                contentType: mime.lookup(relativeUrl),
                hash: crypto.createHash('md5').update(data).digest('hex'),
                modified: stat.mtime
            };
        }
    }

    function dehashifyPath(filePath) {
        var hashRegex = /-[0-9a-f]+(\.[^\.]*$)/;
        var hashMatch = filePath.match(hashRegex);

        return {
            path: filePath.replace(hashRegex, '$1'),
            hash: hashMatch ? hashMatch[0].slice(1).replace(/\.([^\.]*$)/, '') : null
        };
    }

    function gzip(file, callback) {
        if (file.gzippedContent && file.gzippedContentLength) {
            return callback(null, file);
        }

        zlib.gzip(file.content, function(err, gzippedContent) {
            if (err) {
                return callback(err);
            }

            file.gzippedContent = gzippedContent;
            file.gzippedContentLength = gzippedContent.length;

            callback(null, file);
        });
    }

    function hashifyPath(filePath, hash) {
        if (filePath.indexOf('.') != -1) {
            return filePath.replace(/\.([^\.]*$)/, '-' + hash + '.$1');
        }

        return filePath + '-' + hash;
    }

    function prefixSlash(path) {
        return path[0] === '/' ? path : '/' + path;
    }

    function toRelativeUrl(filePath) {
        var relativeUrl = path.relative(directory, path.resolve(filePath));

        // Make URI-friendly and prepend a /
        return prefixSlash(relativeUrl).replace(/\\/g, '/');
    }

    function urlBuilder(filePath) {
        var file = files[prefixSlash(filePath)];

        if (file) {
            var uri = hashifyPath(prefixSlash(filePath), file.hash);

            if (options && options.hostname) {
                uri = '//' + options.hostname + uri;
            }

            return uri;
        } else {
            return filePath;
        }
    }

    // Load all files synchronously
    (function loadFiles(workingDir) {
        var contents;

        try {
            contents = fs.readdirSync(workingDir);
        }
        catch (e) {
            switch (e.code) {
                case 'ENOENT':
                    console.error('Directory does not exist');
                break;

                case 'ENOTDIR':
                    console.error('Not a directory');
                break;

                case 'EACCES':
                    console.error('Insufficient permissions to access directory');
                break;
            }

            throw e;
        }

        contents.forEach(function loadDirectory(file) {
            var filePath = path.join(workingDir, file);
            var stat = fs.statSync(filePath);

            if (stat.isDirectory()) {
                return loadFiles(filePath);
            }

            cacheFile(filePath, stat);
        });
    })(directory);

    // Watch directory
    watch.createMonitor(directory, function monitorFiles(monitor) {
        monitor.on('changed', cacheFile);
        monitor.on('created', cacheFile);

        monitor.on('removed', function removeFile(f, stat) {
            files[toRelativeUrl(f)] = null;
        });
    });

    return function staticMiddleware(req, res, next) {
        // Ignore anything that's not a GET or HEAD request
        if (req.method !== 'GET' && req.method !== 'HEAD') {
            return next();
        }

        // Register view helper if we haven't already
        if (!req.app.locals.electricity) {
            req.app.locals.electricity = {
                url: urlBuilder
            };
        }

        var reqInfo = dehashifyPath(req.path);
        var file = files[reqInfo.path];

        if (!file) {
            return next();
        }

        // Verify file matches the requested hash, otherwise 302
        if (reqInfo.hash !== file.hash) {
            return res.redirect(hashifyPath(reqInfo.path, file.hash));
        }

        var expires = new Date();
        expires.setYear(expires.getFullYear() + 1);

        res.set({
            'Cache-Control': 'public, max-age=31536000', // One year
            'Content-Type': file.contentType,
            'ETag': file.hash,
            'Expires': expires.toUTCString(),
            'Last-Modified': file.modified.toUTCString()
        });

        if (req.get('If-None-Match') === file.hash) {
            res.send(304);
        } else if (new Date(req.get('If-Modified-Since')) >= file.modified) {
            res.send(304);
        } else {
            res.set({
                'Content-Length': file.contentLength
            });

            res.status(200);

            if (req.method === 'HEAD') {
                return res.end();
            }

            var acceptEncoding = req.get('Accept-Encoding');

            if (acceptEncoding && acceptEncoding.split(/,\s?/).indexOf('gzip') !== -1 && gzipContentTypes.indexOf(file.contentType) !== -1) {
                gzip(file, function(err, gzippedFile) {
                    if (err) {
                        return next(err);
                    }

                    res.set({
                        'Content-Encoding': 'gzip',
                        'Content-Length': gzippedFile.gzippedContentLength
                    });

                    res.send(file.gzippedContent);
                });
            }
            else {
                res.send(file.content);
            }
        }
    };
};
<|endoftext|>


<commit_before>const store = {
  ALLOWLIST_URL: '/shavar-prod-lists/disconnect-entitylist.json',
  db: null,

  async init() {
    if (!this.db) {
      this.makeNewDatabase();
    }
    browser.runtime.onMessage.addListener((m) => store.messageHandler(m));
    await this.getAllowList();
  },

  async isFirstRun() {
    let isFirstRun = await browser.storage.local.get('isFirstRun');
    if (! ('isFirstRun' in isFirstRun)) {
      isFirstRun = true;
    } else if (isFirstRun) {
      isFirstRun = false;
    }
    await browser.storage.local.set({ isFirstRun });
    return isFirstRun;
  },

  indexes: [
    'hostname', // Primary key
    'firstRequestTime',
    'lastRequestTime',
    'isVisible',
    'firstParty'
  ],

  makeNewDatabase() {
    this.db = new Dexie('websites_database');
    const websites = this.indexes.join(', ');
    this.db.version(1).stores({websites});
    this.db.open();
  },

  // get Disconnect Entity List from shavar-prod-lists submodule
  async getAllowList() {
    let allowList;
    try {
      allowList = await fetch(this.ALLOWLIST_URL);
      allowList = await allowList.json();
    } catch (error) {
      allowList = {};
      const explanation = 'See README.md for how to import submodule file';
      // eslint-disable-next-line no-console
      console.error(`${error.message} ${explanation} ${this.ALLOWLIST_URL}`);
    }
    const { firstPartyAllowList, thirdPartyAllowList }
      = this.reformatList(allowList);
    this.firstPartyAllowList = firstPartyAllowList;
    this.thirdPartyAllowList = thirdPartyAllowList;
  },

  /*
  disconnect-entitylist.json is expected to match this format, where:
    - 'properties' keys are first parties
    - 'resources' keys are third parties

  {
    "Facebook" : {
      "properties": [
        "facebook.com",
        "facebook.de",
        ...
        "messenger.com"
      ],
      "resources": [
        "facebook.com",
        "facebook.de",
        ...
        "akamaihd.net"
      ]
    }

    "Google" : {
      ...
    }
  }

  this.firstPartyAllowList is expected to match this format:
  {
    "google.com": 1,
    "abc.xyz": 1
    ....
    "facebook.com": 2,
    ...
  }

  this.thirdPartyAllowList is expected to match this format:
  {
    1: [
      "google.com",
      "googleanalytics.com",
      "weloveevilstuff.com"
    ]
  }
*/

  reformatList(allowList) {
    const firstPartyAllowList = {};
    const thirdPartyAllowList = {};
    let counter = 0;
    for (const siteOwner in allowList) {
      const firstParties = allowList[siteOwner].properties;
      for (let i = 0; i < firstParties.length; i++) {
        firstPartyAllowList[firstParties[i]] = counter;
      }
      const thirdParties = allowList[siteOwner].resources;
      thirdPartyAllowList[counter] = [];
      for (let i = 0; i < thirdParties.length; i++) {
        thirdPartyAllowList[counter].push(thirdParties[i]);
      }
      counter++;
    }

    return {
      firstPartyAllowList,
      thirdPartyAllowList
    };
  },

  // send message to storeChild when data in store is changed
  updateChild(...args) {
    return browser.runtime.sendMessage({
      type: 'storeChildCall',
      args
    });
  },

  messageHandler(m) {
    if (m.type !== 'storeCall') {
      return;
    }

    const publicMethods = [
      'getAll',
      'reset',
      'getFirstRequestTime',
      'getNumFirstParties',
      'getNumThirdParties',
      'isFirstRun'
    ];

    if (publicMethods.includes(m['method'])) {
      const args = m.args;
      return this[m['method']](...args);
    } else {
      return new Error(`Unsupported method ${m.method}`);
    }
  },

  async _write(website) {
    return await this.db.websites.put(website);
  },

  outputWebsite(hostname, website) {
    const output = {
      hostname: hostname,
      favicon: website.faviconUrl || '',
      firstPartyHostnames: website.firstPartyHostnames || false,
      firstParty: !!website.firstParty,
      thirdParties: []
    };
    if ('thirdPartyHostnames' in website) {
      output.thirdParties = website.thirdPartyHostnames;
    }
    return output;
  },

  async getAll() {
    const websites = await this.db.websites.filter((website) => {
      return website.isVisible || website.firstParty;
    }).toArray();
    const output = {};
    for (const website of websites) {
      output[website.hostname]
        = this.outputWebsite(website.hostname, website);
    }
    return output;
  },

  /*
  if not {}, getWebsite returns an object:
  - in this format for first parties:
    {
      thirdPartyHostnames: [
        "www.thirdpartydomain.com",
        ...
      ]
    }
  and in this format for third parties:
    {
      firstPartyHostnames: [
        "www.firstpartydomain.com",
        ...
      ],
      isVisible: false,
    }
  */
  async getWebsite(hostname) {
    if (!hostname) {
      throw new Error('getWebsite requires a valid hostname argument');
    }

    const website = await this.db.websites.get(hostname);
    if (website) {
      const websiteOutput = {};
      Object.keys(website).forEach((key) => {
        websiteOutput[key] = this.mungeDataOutbound(key, website[key]);
      });
      return websiteOutput;
    } else {
      return {};
    }
  },

  mungeDataInbound(key, value) {
    if (this.indexes.includes(key)) {
      // IndexedDB does not accept boolean values for indexes; using 0/1 instead
      if (value === true) {
        value = 1;
      }
      if (value === false) {
        value = 0;
      }
    }
    return value;
  },

  mungeDataOutbound(key, value) {
    if (this.indexes.includes(key)) {
      // IndexedDB does not accept boolean values for indexes; using 0/1 instead
      if (value === 1) {
        value = true;
      }
      if (value === 0) {
        value = false;
      }
    }
    return value;
  },

  async setWebsite(hostname, data) {
    const website = await this.getWebsite(hostname);

    if (!('hostname' in website)) {
      website['hostname'] = hostname;
    }

    for (const key in data) {
      const value = this.mungeDataInbound(key, data[key]);
      switch (key) {
        case 'requestTime':
          // store first and last request times for clearing data every X days
          if (!('firstRequestTime' in website)) {
            website.firstRequestTime = value;
          } else {
            website.lastRequestTime = value;
          }
          break;
        case 'isVisible':
          if ('isVisible' in website
              && website.isVisible === true) {
            break;
          }
          website.isVisible = value;
          break;
        case 'firstParty':
          if ('firstParty' in website
              && website.firstParty === true) {
            break;
          }
          website.firstParty = value;
          if (value) {
            website.isVisible = true;
          }
          break;
        default:
          website[key] = value;
          break;
      }
    }

    await this._write(website);
    return website;
  },

  async isNewWebsite(hostname) {
    if (!(await this.db.websites.get(hostname))) {
      return true;
    }
    return false;
  },

  getHostnameVariants(hostname) {
    const hostnameVariants = [hostname];
    const hostnameArr = hostname.split('.');
    const numDots = hostnameArr.length - 1;
    for (let i = 0; i < numDots - 1; i++) {
      hostnameArr.shift();
      hostname = hostnameArr.join('.');
      hostnameVariants.push(hostname);
    }
    return hostnameVariants;
  },

  // check if third party is on the allowlist (owned by the first party)
  // returns true if it is and false otherwise
  onAllowList(firstPartyFromRequest, thirdPartyFromRequest) {
    if (thirdPartyFromRequest && this.firstPartyAllowList) {
      const hostnameVariantsFirstParty
        = this.getHostnameVariants(firstPartyFromRequest);
      for (let i = 0; i < hostnameVariantsFirstParty.length; i++) {
        if (this.firstPartyAllowList
          .hasOwnProperty(hostnameVariantsFirstParty[i])) {
          // first party is in the allowlist
          const index = this.firstPartyAllowList[hostnameVariantsFirstParty[i]];
          const hostnameVariantsThirdParty
            = this.getHostnameVariants(thirdPartyFromRequest);
          for (let j = 0; j < hostnameVariantsThirdParty.length; j++) {
            if (this.thirdPartyAllowList[index]
              .includes(hostnameVariantsThirdParty[j])) {
              return true;
            }
          }
          return false;
        }
      }
    }
    return false;
  },

  async setFirstParty(hostname, data) {
    if (!hostname) {
      throw new Error('setFirstParty requires a valid hostname argument');
    }

    const isNewWebsite = await this.isNewWebsite(hostname);

    const responseData = await this.setWebsite(hostname, data);

    if (isNewWebsite) {
      this.updateChild(this.outputWebsite(hostname, responseData));
    }
  },

  async setThirdParty(origin, target, data) {
    if (!origin) {
      throw new Error('setThirdParty requires a valid origin argument');
    }

    let isNewThirdParty = false;
    let shouldUpdate = false;

    const firstParty = await this.getWebsite(origin);
    const thirdParty = await this.getWebsite(target);

    // add link in third party if it doesn't exist yet
    if (!('firstPartyHostnames' in thirdParty)) {
      thirdParty['firstPartyHostnames'] = [];
    }
    if (!thirdParty['firstPartyHostnames'].includes(origin)) {
      thirdParty['firstPartyHostnames'].push(origin);
    }

    // add link in first party if it doesn't exist yet
    // and the third party is visible (i.e. not allowlisted)
    if (!this.isFirstPartyLinkedToThirdParty(firstParty, target)) {
      if (!this.isVisibleThirdParty(thirdParty)) {
        if (this.onAllowList(origin, target)) {
          // hide third party
          thirdParty['isVisible'] = false;
        } else {
          // show third party; it either became visible or is brand new
          thirdParty['isVisible'] = true;
          isNewThirdParty = true;
          for (let i = 0; i < thirdParty['firstPartyHostnames'].length; i++) {
            const firstPartyHostname = thirdParty['firstPartyHostnames'][i];
            await this.addFirstPartyLink(firstPartyHostname, target);
          }
          shouldUpdate = true;
        }
      }
      if (this.isVisibleThirdParty(thirdParty) && !isNewThirdParty) {
        // an existing visible third party links to a new first party
        await this.addFirstPartyLink(origin, target);
        shouldUpdate = true;
      }
    }

    // merge data with thirdParty
    for (const key in data) {
      thirdParty[key] = data[key];
    }

    const responseData = await this.setWebsite(target, thirdParty);

    if (shouldUpdate) {
      this.updateChild(this.outputWebsite(target, responseData));
    }
  },

  isFirstPartyLinkedToThirdParty(firstParty, thirdPartyHostname) {
    if (!('thirdPartyHostnames' in firstParty)
      || !firstParty['thirdPartyHostnames'].includes(thirdPartyHostname)) {
      return false;
    }
    return true;
  },

  isVisibleThirdParty(thirdParty) {
    if (!('isVisible' in thirdParty) || !thirdParty['isVisible']) {
      return false;
    }
    return true;
  },

  async addFirstPartyLink(firstPartyHostname, thirdPartyHostname) {
    const firstParty = await this.getWebsite(firstPartyHostname);
    // We are likely storing the tp first, lets account for that
    if (!('firstParty' in firstParty)) {
      firstParty.firstParty = true;
    }
    if (!('thirdPartyHostnames' in firstParty)) {
      firstParty['thirdPartyHostnames'] = [];
    }
    if (!this.isFirstPartyLinkedToThirdParty(firstParty, thirdPartyHostname)) {
      firstParty['thirdPartyHostnames'].push(thirdPartyHostname);
      await this.setFirstParty(firstPartyHostname, firstParty);
    }
  },

  async reset() {
    // empty out request processing queue
    capture.queue = [];
    return await this.db.websites.clear();
  },

  async getFirstRequestTime() {
    const oldestSite
      = await this.db.websites.orderBy('firstRequestTime').first();
    if (!oldestSite) {
      return false;
    }
    return oldestSite.firstRequestTime;
  },

  async getNumFirstParties() {
    return await this.db.websites.where('firstParty').equals(1).count();
  },

  async getNumThirdParties() {
    return await this.db.websites
      .where('firstParty').equals(0).and((website) => {
        return website.isVisible;
      }).count();
  }
};

store.init();
<commit_msg>Properly munge inbound data in all cases. Fixes #187. (#188)<commit_after>const store = {
  ALLOWLIST_URL: '/shavar-prod-lists/disconnect-entitylist.json',
  db: null,

  async init() {
    if (!this.db) {
      this.makeNewDatabase();
    }
    browser.runtime.onMessage.addListener((m) => store.messageHandler(m));
    await this.getAllowList();
  },

  async isFirstRun() {
    let isFirstRun = await browser.storage.local.get('isFirstRun');
    if (! ('isFirstRun' in isFirstRun)) {
      isFirstRun = true;
    } else if (isFirstRun) {
      isFirstRun = false;
    }
    await browser.storage.local.set({ isFirstRun });
    return isFirstRun;
  },

  indexes: [
    'hostname', // Primary key
    'firstRequestTime',
    'lastRequestTime',
    'isVisible',
    'firstParty'
  ],

  makeNewDatabase() {
    this.db = new Dexie('websites_database');
    const websites = this.indexes.join(', ');
    this.db.version(1).stores({websites});
    this.db.open();
  },

  // get Disconnect Entity List from shavar-prod-lists submodule
  async getAllowList() {
    let allowList;
    try {
      allowList = await fetch(this.ALLOWLIST_URL);
      allowList = await allowList.json();
    } catch (error) {
      allowList = {};
      const explanation = 'See README.md for how to import submodule file';
      // eslint-disable-next-line no-console
      console.error(`${error.message} ${explanation} ${this.ALLOWLIST_URL}`);
    }
    const { firstPartyAllowList, thirdPartyAllowList }
      = this.reformatList(allowList);
    this.firstPartyAllowList = firstPartyAllowList;
    this.thirdPartyAllowList = thirdPartyAllowList;
  },

  /*
  disconnect-entitylist.json is expected to match this format, where:
    - 'properties' keys are first parties
    - 'resources' keys are third parties

  {
    "Facebook" : {
      "properties": [
        "facebook.com",
        "facebook.de",
        ...
        "messenger.com"
      ],
      "resources": [
        "facebook.com",
        "facebook.de",
        ...
        "akamaihd.net"
      ]
    }

    "Google" : {
      ...
    }
  }

  this.firstPartyAllowList is expected to match this format:
  {
    "google.com": 1,
    "abc.xyz": 1
    ....
    "facebook.com": 2,
    ...
  }

  this.thirdPartyAllowList is expected to match this format:
  {
    1: [
      "google.com",
      "googleanalytics.com",
      "weloveevilstuff.com"
    ]
  }
*/

  reformatList(allowList) {
    const firstPartyAllowList = {};
    const thirdPartyAllowList = {};
    let counter = 0;
    for (const siteOwner in allowList) {
      const firstParties = allowList[siteOwner].properties;
      for (let i = 0; i < firstParties.length; i++) {
        firstPartyAllowList[firstParties[i]] = counter;
      }
      const thirdParties = allowList[siteOwner].resources;
      thirdPartyAllowList[counter] = [];
      for (let i = 0; i < thirdParties.length; i++) {
        thirdPartyAllowList[counter].push(thirdParties[i]);
      }
      counter++;
    }

    return {
      firstPartyAllowList,
      thirdPartyAllowList
    };
  },

  // send message to storeChild when data in store is changed
  updateChild(...args) {
    return browser.runtime.sendMessage({
      type: 'storeChildCall',
      args
    });
  },

  messageHandler(m) {
    if (m.type !== 'storeCall') {
      return;
    }

    const publicMethods = [
      'getAll',
      'reset',
      'getFirstRequestTime',
      'getNumFirstParties',
      'getNumThirdParties',
      'isFirstRun'
    ];

    if (publicMethods.includes(m['method'])) {
      const args = m.args;
      return this[m['method']](...args);
    } else {
      return new Error(`Unsupported method ${m.method}`);
    }
  },

  async _write(website) {
    for (const key in website) {
      website[key] = this.mungeDataInbound(key, website[key]);
    }
    return await this.db.websites.put(website);
  },

  outputWebsite(hostname, website) {
    const output = {
      hostname: hostname,
      favicon: website.faviconUrl || '',
      firstPartyHostnames: website.firstPartyHostnames || false,
      firstParty: !!website.firstParty,
      thirdParties: []
    };
    if ('thirdPartyHostnames' in website) {
      output.thirdParties = website.thirdPartyHostnames;
    }
    return output;
  },

  async getAll() {
    const websites = await this.db.websites.filter((website) => {
      return website.isVisible || website.firstParty;
    }).toArray();
    const output = {};
    for (const website of websites) {
      output[website.hostname]
        = this.outputWebsite(website.hostname, website);
    }
    return output;
  },

  /*
  if not {}, getWebsite returns an object:
  - in this format for first parties:
    {
      thirdPartyHostnames: [
        "www.thirdpartydomain.com",
        ...
      ]
    }
  and in this format for third parties:
    {
      firstPartyHostnames: [
        "www.firstpartydomain.com",
        ...
      ],
      isVisible: false,
    }
  */
  async getWebsite(hostname) {
    if (!hostname) {
      throw new Error('getWebsite requires a valid hostname argument');
    }

    const website = await this.db.websites.get(hostname);
    if (website) {
      const websiteOutput = {};
      Object.keys(website).forEach((key) => {
        websiteOutput[key] = this.mungeDataOutbound(key, website[key]);
      });
      return websiteOutput;
    } else {
      return {};
    }
  },

  mungeDataInbound(key, value) {
    if (this.indexes.includes(key)) {
      // IndexedDB does not accept boolean values for indexes; using 0/1 instead
      if (value === true) {
        value = 1;
      }
      if (value === false) {
        value = 0;
      }
    }
    return value;
  },

  mungeDataOutbound(key, value) {
    if (this.indexes.includes(key)) {
      // IndexedDB does not accept boolean values for indexes; using 0/1 instead
      if (value === 1) {
        value = true;
      }
      if (value === 0) {
        value = false;
      }
    }
    return value;
  },

  async setWebsite(hostname, data) {
    const website = await this.getWebsite(hostname);

    if (!('hostname' in website)) {
      website['hostname'] = hostname;
    }

    for (const key in data) {
      const value = data[key];
      switch (key) {
        case 'requestTime':
          // store first and last request times for clearing data every X days
          if (!('firstRequestTime' in website)) {
            website.firstRequestTime = value;
          } else {
            website.lastRequestTime = value;
          }
          break;
        case 'isVisible':
          if ('isVisible' in website
              && website.isVisible === true) {
            // once a website is visible, it will always be visible
            break;
          }
          website.isVisible = value;
          break;
        case 'firstParty':
          if ('firstParty' in website
              && website.firstParty === true) {
            // once a website is a first party, it will always be drawn as one
            break;
          }
          website.firstParty = value;
          if (value) {
            website.isVisible = value;
          }
          break;
        default:
          website[key] = value;
          break;
      }
    }

    await this._write(website);

    return website;
  },

  async isNewWebsite(hostname) {
    if (!(await this.db.websites.get(hostname))) {
      return true;
    }
    return false;
  },

  getHostnameVariants(hostname) {
    const hostnameVariants = [hostname];
    const hostnameArr = hostname.split('.');
    const numDots = hostnameArr.length - 1;
    for (let i = 0; i < numDots - 1; i++) {
      hostnameArr.shift();
      hostname = hostnameArr.join('.');
      hostnameVariants.push(hostname);
    }
    return hostnameVariants;
  },

  // check if third party is on the allowlist (owned by the first party)
  // returns true if it is and false otherwise
  onAllowList(firstPartyFromRequest, thirdPartyFromRequest) {
    if (thirdPartyFromRequest && this.firstPartyAllowList) {
      const hostnameVariantsFirstParty
        = this.getHostnameVariants(firstPartyFromRequest);
      for (let i = 0; i < hostnameVariantsFirstParty.length; i++) {
        if (this.firstPartyAllowList
          .hasOwnProperty(hostnameVariantsFirstParty[i])) {
          // first party is in the allowlist
          const index = this.firstPartyAllowList[hostnameVariantsFirstParty[i]];
          const hostnameVariantsThirdParty
            = this.getHostnameVariants(thirdPartyFromRequest);
          for (let j = 0; j < hostnameVariantsThirdParty.length; j++) {
            if (this.thirdPartyAllowList[index]
              .includes(hostnameVariantsThirdParty[j])) {
              return true;
            }
          }
          return false;
        }
      }
    }
    return false;
  },

  async setFirstParty(hostname, data) {
    if (!hostname) {
      throw new Error('setFirstParty requires a valid hostname argument');
    }

    const isNewWebsite = await this.isNewWebsite(hostname);

    const responseData = await this.setWebsite(hostname, data);

    if (isNewWebsite) {
      this.updateChild(this.outputWebsite(hostname, responseData));
    }
  },

  async setThirdParty(origin, target, data) {
    if (!origin) {
      throw new Error('setThirdParty requires a valid origin argument');
    }

    let isNewThirdParty = false;
    let shouldUpdate = false;

    const firstParty = await this.getWebsite(origin);
    const thirdParty = await this.getWebsite(target);

    // add link in third party if it doesn't exist yet
    if (!('firstPartyHostnames' in thirdParty)) {
      thirdParty['firstPartyHostnames'] = [];
    }
    if (!thirdParty['firstPartyHostnames'].includes(origin)) {
      thirdParty['firstPartyHostnames'].push(origin);
    }

    // add link in first party if it doesn't exist yet
    // and the third party is visible (i.e. not allowlisted)
    if (!this.isFirstPartyLinkedToThirdParty(firstParty, target)) {
      if (!this.isVisibleThirdParty(thirdParty)) {
        if (this.onAllowList(origin, target)) {
          // hide third party
          thirdParty['isVisible'] = false;
        } else {
          // show third party; it either became visible or is brand new
          thirdParty['isVisible'] = true;
          isNewThirdParty = true;
          for (let i = 0; i < thirdParty['firstPartyHostnames'].length; i++) {
            const firstPartyHostname = thirdParty['firstPartyHostnames'][i];
            await this.addFirstPartyLink(firstPartyHostname, target);
          }
          shouldUpdate = true;
        }
      }
      if (this.isVisibleThirdParty(thirdParty) && !isNewThirdParty) {
        // an existing visible third party links to a new first party
        await this.addFirstPartyLink(origin, target);
        shouldUpdate = true;
      }
    }

    // merge data with thirdParty
    for (const key in data) {
      thirdParty[key] = data[key];
    }

    const responseData = await this.setWebsite(target, thirdParty);

    if (shouldUpdate) {
      this.updateChild(this.outputWebsite(target, responseData));
    }
  },

  isFirstPartyLinkedToThirdParty(firstParty, thirdPartyHostname) {
    if (!('thirdPartyHostnames' in firstParty)
      || !firstParty['thirdPartyHostnames'].includes(thirdPartyHostname)) {
      return false;
    }
    return true;
  },

  isVisibleThirdParty(thirdParty) {
    if (!('isVisible' in thirdParty) || !thirdParty['isVisible']) {
      return false;
    }
    return true;
  },

  async addFirstPartyLink(firstPartyHostname, thirdPartyHostname) {
    const firstParty = await this.getWebsite(firstPartyHostname);
    // We are likely storing the tp first, lets account for that
    if (!('firstParty' in firstParty)) {
      firstParty.firstParty = true;
    }
    if (!('thirdPartyHostnames' in firstParty)) {
      firstParty['thirdPartyHostnames'] = [];
    }
    if (!this.isFirstPartyLinkedToThirdParty(firstParty, thirdPartyHostname)) {
      firstParty['thirdPartyHostnames'].push(thirdPartyHostname);
      await this.setFirstParty(firstPartyHostname, firstParty);
    }
  },

  async reset() {
    // empty out request processing queue
    capture.queue = [];
    return await this.db.websites.clear();
  },

  async getFirstRequestTime() {
    const oldestSite
      = await this.db.websites.orderBy('firstRequestTime').first();
    if (!oldestSite) {
      return false;
    }
    return oldestSite.firstRequestTime;
  },

  async getNumFirstParties() {
    return await this.db.websites.where('firstParty').equals(1).count();
  },

  async getNumThirdParties() {
    return await this.db.websites
      .where('firstParty').equals(0).and((website) => {
        return website.isVisible;
      }).count();
  }
};

store.init();
<|endoftext|>