<commit_before>'use strict';

var gulp = require('gulp');
var yargs = require('yargs');
var sass = require('gulp-sass');
var concat = require('gulp-concat');
var uglify = require('gulp-uglify');
var cheerio = require('gulp-cheerio');
var cleanhtml = require('gulp-cleanhtml');
var footer = require('gulp-footer');
var header = require('gulp-header');
var uncss = require('gulp-uncss');
var glob = require('glob');
var minifycss = require('gulp-minify-css');
var autoprefixer = require('gulp-autoprefixer');
var htmlmin = require('gulp-htmlmin');
var imageresize = require('gulp-image-resize');
var tap = require('gulp-tap');
var changed = require('gulp-changed');
var Q = require('q');
var node_static = require('node-static');
var chalk = require('chalk');
var moment = require('moment');

var default_task_deps = ['base', 'scss', 'images', 'js', 'geomicons', 'site'];

if(!yargs.argv.dev) {

    default_task_deps.push('uncss', 'htmlmin');
}

gulp.task('default', default_task_deps);

gulp.task('base', function(){

    var stream = gulp.src('base/**').pipe(gulp.dest('site'));

    return stream;
});

gulp.task('scss', function () {

    var outputStyle = yargs.argv.dev ? 'nested' : 'compressed';

    var stream = gulp.src('assets/scss/site.scss')
        .pipe(sass({outputStyle: outputStyle}))
        .pipe(autoprefixer('> 1%', 'last 2 versions'))
        .pipe(gulp.dest('site/assets'));

    return stream;
});

gulp.task('images', function () {

    var stream = gulp.src('content/uploads/*.jpg')
        .pipe(changed('site/uploads'))
        .pipe(gulp.dest('site/uploads'))
        .pipe(imageresize({
            width : 688,
            height : 0,
            imageMagick: true
        }))
        .pipe(gulp.dest('site/uploads/thumbnails'));

    return stream;
});

gulp.task('js', function () {

    var stream = gulp.src([
            'assets/js/prism.js',
            'assets/js/turbolinks.min.js',
            'assets/js/geomicons.min.js',
            'assets/js/ender.min.js',
            'assets/js/site.js'
        ])
        .pipe(concat("site.js"));

    if(!yargs.argv.dev) {

        stream.pipe(uglify({preserveComments: 'some'}))
    }

    stream.pipe(gulp.dest('site/assets'));

    return stream;
});

gulp.task('geomicons', ['geomicons-sprite', 'geomicons-defs']);

gulp.task('geomicons-sprite', function() {

    var stream = gulp.src('assets/geomicons/enabled/*.svg')
        .pipe(concat('geomicons.svg'))
        .pipe(header('<svg xmlns="http://www.w3.org/2000/svg">'))
        .pipe(footer('</svg>'))
        .pipe(cleanhtml())
        .pipe(gulp.dest('site/assets'));

    return stream;
});

gulp.task('geomicons-defs', function() {

    var stream = gulp.src('assets/geomicons/enabled/*.svg')
        .pipe(cheerio({
            run: function($) {
                var $path = $('svg').children('path'), id = $('svg').attr('id');
                $path.attr('id', id);
                $('svg').replaceWith($path[0]);
            }
        }))
        .pipe(concat('geomicons-defs.svg'))
        .pipe(header('<svg xmlns="http://www.w3.org/2000/svg" width="0" height="0"><defs>'))
        .pipe(footer('</defs></svg>'))
        .pipe(cleanhtml())
        .pipe(gulp.dest('site/assets'));

    return stream;
});

gulp.task('site', function(cb){

    var site = require('./lib/site.js');
    var defaults = require('./lib/plugins/defaults.js');
    var content = require('./lib/plugins/content.js');
    var pager = require('./lib/plugins/pager.js');
    var marked_converter = require('./lib/converters/marked.js');
    var nunjucks = require('nunjucks');
    var _ = require('lodash');

    pager = pager();

    content.configure('./content/', {
        converters: { md: marked_converter() },
        date_formats: [ "YYYY-MM-DD" ]
    });

    nunjucks.configure('./templates/', { autoescape: true })

    site = site('./site/');

    site.engine(nunjucks.render);

    site.route('/')
        .alias('home')
        .use(content('posts/*'))
        .use(pager)
        .use(function(pages, next) {

            next([_.last(pages)]);
        })
        .render('post.html');

    site.route('/posts/{slug}/')
        .alias('post')
        .use(content('posts/*'))
        .use(pager)
        .render('post.html');

    site.route('/posts/')
        .use(content('posts.md'))
        .use(function(pages, next) {

            var posts = content('posts/*');

            posts([], function(posts){

                _.map(posts, function(v, k){

                    var page = v.page;

                    page.year_month = page.date.format('MMMM YYYY');

                    posts[k] = page;
                });

                pages[0].page.posts = posts.reverse();

                next(pages);
            });
        })
        .render('posts.html');

    site.route('/404.html').use(content('404.md')).render('404.html');

    site.after(defaults(site, './content/defaults.yml'));

    return site.build();
});

gulp.task('htmlmin', ['site'], function () {

    var stream = gulp.src('site/**/**.html')
        .pipe(htmlmin({collapseWhitespace: true}))
        .pipe(gulp.dest('site'));

    return stream;
});

gulp.task('uncss', ['htmlmin', 'scss'], function (cb) {

    var ignore = [
        /\.token.*/,
        /\.style.*/,
        /\.namespace.*/
    ];

    glob('site/**/**.html', function(err, files){

        gulp.src('site/assets/site.css')
            .pipe(uncss({html: files, ignore: ignore}))
            .pipe(minifycss())
            .pipe(gulp.dest('site/assets'))
            .pipe(tap(function(){ cb(); }))
        ;
    });

});

gulp.task('watch', ['default'], function () {

    gulp.watch('base/**', ['base']);
    gulp.watch('content/uploads/*.jpg', ['images']);
    gulp.watch('assets/js/**.js', ['js']);
    gulp.watch('assets/geomicons/**/**.svg', ['geomicons']);

    if(!yargs.argv.dev) {

        gulp.watch('assets/scss/**.scss', ['scss', 'uncss']);
        gulp.watch('templates/**/**.html', ['site', 'htmlmin', 'uncss']);
        gulp.watch('content/**', ['site', 'htmlmin', 'uncss']);

    }
    else {

        gulp.watch('assets/scss/**.scss', ['scss']);
        gulp.watch('templates/**/**.html', ['site']);
        gulp.watch('content/**', ['site']);
    }
});

gulp.task('serve', ['default'], function () {

    var server = new node_static.Server('./site/');

    require('http').createServer(function (request, response) {
        request.addListener('end', function () {
            server.serve(request, response, function (err, result) {

                var message;
                var end = true

                if (err) {

                    if(err.status === 404) {

                        end = false;

                        server.serveFile('404.html', 404, {}, request, response);
                    }
                    else {
                        response.writeHead(err.status, err.headers);
                    }

                    message = '[' + chalk.red(err.status) + ']';
                }
                else {

                    message = '[' + chalk.green(response.statusCode) + ']';
                }

                console.log('[' + chalk.gray(moment().format('HH:mm:ss')) + '] ' + message + ' serving ' + request.url);

                if(end) {

                    response.end();
                }
            });
        }).resume();

    }).listen(8080);
});
<commit_msg>Only one geomicon task is needed<commit_after>'use strict';

var gulp = require('gulp');
var yargs = require('yargs');
var sass = require('gulp-sass');
var concat = require('gulp-concat');
var uglify = require('gulp-uglify');
var cheerio = require('gulp-cheerio');
var cleanhtml = require('gulp-cleanhtml');
var footer = require('gulp-footer');
var header = require('gulp-header');
var uncss = require('gulp-uncss');
var glob = require('glob');
var minifycss = require('gulp-minify-css');
var autoprefixer = require('gulp-autoprefixer');
var htmlmin = require('gulp-htmlmin');
var imageresize = require('gulp-image-resize');
var tap = require('gulp-tap');
var changed = require('gulp-changed');
var Q = require('q');
var node_static = require('node-static');
var chalk = require('chalk');
var moment = require('moment');

var default_task_deps = ['base', 'scss', 'images', 'js', 'geomicons', 'site'];

if(!yargs.argv.dev) {

    default_task_deps.push('uncss', 'htmlmin');
}

gulp.task('default', default_task_deps);

gulp.task('base', function(){

    var stream = gulp.src('base/**').pipe(gulp.dest('site'));

    return stream;
});

gulp.task('scss', function () {

    var outputStyle = yargs.argv.dev ? 'nested' : 'compressed';

    var stream = gulp.src('assets/scss/site.scss')
        .pipe(sass({outputStyle: outputStyle}))
        .pipe(autoprefixer('> 1%', 'last 2 versions'))
        .pipe(gulp.dest('site/assets'));

    return stream;
});

gulp.task('images', function () {

    var stream = gulp.src('content/uploads/*.jpg')
        .pipe(changed('site/uploads'))
        .pipe(gulp.dest('site/uploads'))
        .pipe(imageresize({
            width : 688,
            height : 0,
            imageMagick: true
        }))
        .pipe(gulp.dest('site/uploads/thumbnails'));

    return stream;
});

gulp.task('js', function () {

    var stream = gulp.src([
            'assets/js/prism.js',
            'assets/js/turbolinks.min.js',
            'assets/js/geomicons.min.js',
            'assets/js/ender.min.js',
            'assets/js/site.js'
        ])
        .pipe(concat("site.js"));

    if(!yargs.argv.dev) {

        stream.pipe(uglify({preserveComments: 'some'}))
    }

    stream.pipe(gulp.dest('site/assets'));

    return stream;
});

gulp.task('geomicons', function() {

    var stream = gulp.src('assets/geomicons/enabled/*.svg')
        .pipe(concat('geomicons.svg'))
        .pipe(header('<svg xmlns="http://www.w3.org/2000/svg">'))
        .pipe(footer('</svg>'))
        .pipe(cleanhtml())
        .pipe(gulp.dest('site/assets'));

    return stream;
});

// gulp.task('geomicons-defs', function() {
//
//     var stream = gulp.src('assets/geomicons/enabled/*.svg')
//         .pipe(cheerio({
//             run: function($) {
//                 var $path = $('svg').children('path'), id = $('svg').attr('id');
//                 $path.attr('id', id);
//                 $('svg').replaceWith($path[0]);
//             }
//         }))
//         .pipe(concat('geomicons-defs.svg'))
//         .pipe(header('<svg xmlns="http://www.w3.org/2000/svg" width="0" height="0"><defs>'))
//         .pipe(footer('</defs></svg>'))
//         .pipe(cleanhtml())
//         .pipe(gulp.dest('site/assets'));
//
//     return stream;
// });

gulp.task('site', function(cb){

    var site = require('./lib/site.js');
    var defaults = require('./lib/plugins/defaults.js');
    var content = require('./lib/plugins/content.js');
    var pager = require('./lib/plugins/pager.js');
    var marked_converter = require('./lib/converters/marked.js');
    var nunjucks = require('nunjucks');
    var _ = require('lodash');

    pager = pager();

    content.configure('./content/', {
        converters: { md: marked_converter() },
        date_formats: [ "YYYY-MM-DD" ]
    });

    nunjucks.configure('./templates/', { autoescape: true })

    site = site('./site/');

    site.engine(nunjucks.render);

    site.route('/')
        .alias('home')
        .use(content('posts/*'))
        .use(pager)
        .use(function(pages, next) {

            next([_.last(pages)]);
        })
        .render('post.html');

    site.route('/posts/{slug}/')
        .alias('post')
        .use(content('posts/*'))
        .use(pager)
        .render('post.html');

    site.route('/posts/')
        .use(content('posts.md'))
        .use(function(pages, next) {

            var posts = content('posts/*');

            posts([], function(posts){

                _.map(posts, function(v, k){

                    var page = v.page;

                    page.year_month = page.date.format('MMMM YYYY');

                    posts[k] = page;
                });

                pages[0].page.posts = posts.reverse();

                next(pages);
            });
        })
        .render('posts.html');

    site.route('/404.html').use(content('404.md')).render('404.html');

    site.after(defaults(site, './content/defaults.yml'));

    return site.build();
});

gulp.task('htmlmin', ['site'], function () {

    var stream = gulp.src('site/**/**.html')
        .pipe(htmlmin({collapseWhitespace: true}))
        .pipe(gulp.dest('site'));

    return stream;
});

gulp.task('uncss', ['htmlmin', 'scss'], function (cb) {

    var ignore = [
        /\.token.*/,
        /\.style.*/,
        /\.namespace.*/
    ];

    glob('site/**/**.html', function(err, files){

        gulp.src('site/assets/site.css')
            .pipe(uncss({html: files, ignore: ignore}))
            .pipe(minifycss())
            .pipe(gulp.dest('site/assets'))
            .pipe(tap(function(){ cb(); }))
        ;
    });

});

gulp.task('watch', ['default'], function () {

    gulp.watch('base/**', ['base']);
    gulp.watch('content/uploads/*.jpg', ['images']);
    gulp.watch('assets/js/**.js', ['js']);
    gulp.watch('assets/geomicons/**/**.svg', ['geomicons']);

    if(!yargs.argv.dev) {

        gulp.watch('assets/scss/**.scss', ['scss', 'uncss']);
        gulp.watch('templates/**/**.html', ['site', 'htmlmin', 'uncss']);
        gulp.watch('content/**', ['site', 'htmlmin', 'uncss']);

    }
    else {

        gulp.watch('assets/scss/**.scss', ['scss']);
        gulp.watch('templates/**/**.html', ['site']);
        gulp.watch('content/**', ['site']);
    }
});

gulp.task('serve', ['default'], function () {

    var server = new node_static.Server('./site/');

    require('http').createServer(function (request, response) {
        request.addListener('end', function () {
            server.serve(request, response, function (err, result) {

                var message;
                var end = true

                if (err) {

                    if(err.status === 404) {

                        end = false;

                        server.serveFile('404.html', 404, {}, request, response);
                    }
                    else {
                        response.writeHead(err.status, err.headers);
                    }

                    message = '[' + chalk.red(err.status) + ']';
                }
                else {

                    message = '[' + chalk.green(response.statusCode) + ']';
                }

                console.log('[' + chalk.gray(moment().format('HH:mm:ss')) + '] ' + message + ' serving ' + request.url);

                if(end) {

                    response.end();
                }
            });
        }).resume();

    }).listen(8080);
});
<|endoftext|>


<commit_before>'use strict';

const path = require('path');
const browserify = require('browserify');
const gulp = require('gulp');
const source = require('vinyl-source-stream');
const buffer = require('vinyl-buffer');
const gutil = require('gulp-util');
const reactify = require('reactify');
const envify = require('envify/custom');
const del = require('del');

const minifier = require('gulp-uglify/minifier');
const uglifyJS = require('uglify-js');

const packager = require('electron-packager');

gulp.task('clean', function (cb) {
    del(['dist', 'release'], cb);
});

gulp.task('bundle', function () {
    const b = browserify({
        entries: './src/index.js',
        debug: false,
        node: true,
        bundleExternal: false,
        transform: [reactify, envify({
            NODE_ENV: 'production'
        })]
    });

    const options = {
        preserveComments: 'license',
        // FIXME: if mangle is set to true, then I get "SyntaxError: Identifier 'n' has already been declared" ?!
        // it's not needed, but it would be nice if it worked :)
        mangle: false
    };

    return b.bundle()
        .pipe(source('index.js'))
        .pipe(buffer())
        .pipe(minifier(options, uglifyJS))
        .pipe(gulp.dest('./dist/'));
});

gulp.task('html', function () {
    return gulp.src('./src/index.html')
        .pipe(gulp.dest('dist'));
});

gulp.task('assets', function () {
    return gulp.src('./src/assets/**/*', {"base": "./src"})
        .pipe(gulp.dest('dist'));
});

gulp.task('default', ['html', 'assets', 'bundle']);

gulp.task('hello', function () {
    console.log(path.dirname(__dirname + '/src'));
});

// PACKAGING

function defaultPackOptions() {
    return {
        dir: '.',
        asar: false,
        version: '1.4.5',
        //platform: '',
        //arch: '',
        out: 'release',
        overwrite: true,
        prune: true,
        ignore: [
            '^/.idea',
            '^/release',
            '^/src',
            '^/test',
            '^/.gitignore',
            '^/gulpfile.js',
            '^/README.md'
        ]
    };
}

gulp.task('pack-mac', function () {
    const options = Object.assign(defaultPackOptions(), {
        platform: 'darwin',
        arch: 'x64',
    });

    return packager(options, (err, appPaths) => {
        if (err) {
            console.log(err);
        } else {
            console.log('Done', appPaths);
        }
    });
});<commit_msg>Update gulp<commit_after>'use strict';

const path = require('path');
const browserify = require('browserify');
const gulp = require('gulp');
const source = require('vinyl-source-stream');
const buffer = require('vinyl-buffer');
const gutil = require('gulp-util');
const reactify = require('reactify');
const envify = require('envify/custom');
const del = require('del');

const minifier = require('gulp-uglify/minifier');
const uglifyJS = require('uglify-js');

const packager = require('electron-packager');

gulp.task('clean', function (cb) {
    del(['dist', 'release'], cb);
});

gulp.task('bundle', function () {
    const b = browserify({
        entries: './src/index.js',
        debug: false,
        node: true,
        bundleExternal: false,
        transform: [reactify, envify({
            NODE_ENV: 'production'
        })]
    });

    const options = {
        preserveComments: 'license',
        // FIXME: if mangle is set to true, then I get "SyntaxError: Identifier 'n' has already been declared" ?!
        // it's not needed, but it would be nice if it worked :)
        mangle: false
    };

    return b.bundle()
        .pipe(source('index.js'))
        .pipe(buffer())
        .pipe(minifier(options, uglifyJS))
        .pipe(gulp.dest('./dist/'));
});

gulp.task('html', function () {
    return gulp.src('./src/index.html')
        .pipe(gulp.dest('dist'));
});

gulp.task('assets', function () {
    return gulp.src('./src/assets/**/*', {"base": "./src"})
        .pipe(gulp.dest('dist'));
});

gulp.task('default', ['html', 'assets', 'bundle']);

gulp.task('hello', function () {
    console.log(path.dirname(__dirname + '/src'));
});

// PACKAGING

function defaultPackOptions() {
    return {
        dir: '.',
        asar: false,
        version: '1.4.5',
        //platform: '',
        //arch: '',
        out: 'release',
        overwrite: true,
        prune: true,
        ignore: [
            '^/.idea',
            '^/release',
            '^/src',
            '^/test',
            '^/.gitignore',
            '^/gulpfile.js',
            '^/readme-res',
            '^/README.md'
        ]
    };
}

gulp.task('pack-mac', function () {
    const options = Object.assign(defaultPackOptions(), {
        platform: 'darwin',
        arch: 'x64',
    });

    return packager(options, (err, appPaths) => {
        if (err) {
            console.log(err);
        } else {
            console.log('Done', appPaths);
        }
    });
});<|endoftext|>


<commit_before>"use strict";

const req = require("cheerio-req")
    , typpy = require("typpy")
    , iterateObj = require("iterate-object")
    , Err = require("err")
    , objDef = require("obj-def")
    , emptyObj = require("is-empty-obj")
    , assured = require("assured")
    , cheerio = require("cheerio")
    ;

/**
 * scrapeIt
 * A scraping module for humans.
 *
 * @name scrapeIt
 * @function
 * @param {String|Object} url The page url or request options.
 * @param {Object} opts The options passed to `scrapeHTML` method.
 * @param {Function} cb The callback function.
 * @return {Promise} A promise object.
 */
function scrapeIt (url, opts, cb) {
    cb = assured(cb);
    req(url, (err, $, res, body) => {
        if (err) { return cb(err); }
        cb(null, scrapeIt.scrapeHTML($, opts), $, res, body);
    });
    return cb._;
}

/**
 * scrapeIt.scrapeHTML
 * Scrapes the data in the provided element.
 *
 * @name scrapeIt.scrapeHTML
 * @function
 * @param {Cheerio} $ The input element.
 * @param {Object} opts An object containing the scraping information.
 *
 *   If you want to scrape a list, you have to use the `listItem` selector:
 *
 *    - `listItem` (String): The list item selector.
 *    - `data` (Object): The fields to include in the list objects:
 *       - `<fieldName>` (Object|String): The selector or an object containing:
 *          - `selector` (String): The selector.
 *          - `convert` (Function): An optional function to change the value.
 *          - `how` (Function|String): A function or function name to access the
 *            value.
 *          - `attr` (String): If provided, the value will be taken based on
 *            the attribute name.
 *          - `trim` (Boolean): If `false`, the value will *not* be trimmed
 *            (default: `true`).
 *          - `closest` (String): If provided, returns the first ancestor of
 *            the given element.
 *          - `eq` (Number): If provided, it will select the *nth* element.
 *          - `listItem` (Object): An object, keeping the recursive schema of
 *            the `listItem` object. This can be used to create nested lists.
 *
 *   **Example**:
 *   ```js
 *   {
 *      articles: {
 *          listItem: ".article"
 *        , data: {
 *              createdAt: {
 *                  selector: ".date"
 *                , convert: x => new Date(x)
 *              }
 *            , title: "a.article-title"
 *            , tags: {
 *                  listItem: ".tags > span"
 *              }
 *            , content: {
 *                  selector: ".article-content"
 *                , how: "html"
 *              }
 *            , traverseOtherNode: {
 *                  selector: ".upperNode"
 *                , closest: "div"
 *                , convert: x => x.length
 *              }
 *          }
 *      }
 *   }
 *   ```
 *
 *   If you want to collect specific data from the page, just use the same
 *   schema used for the `data` field.
 *
 *   **Example**:
 *   ```js
 *   {
 *        title: ".header h1"
 *      , desc: ".header h2"
 *      , avatar: {
 *            selector: ".header img"
 *          , attr: "src"
 *        }
 *   }
 *   ```
 *
 * @returns {Object} The scraped data.
 */
scrapeIt.scrapeHTML = ($, opts) => {

    if (typeof $ === "string") {
        $ = cheerio.load($);
    }

    let normalizeOpt = v => {
            if (typpy(v, String)) {
                v = { selector: v };
            }
            objDef(v, "data", {});
            objDef(v, "how", "text", true);
            if (v.attr) {
                v.how = $elm => $elm.attr(v.attr);
            }
            objDef(v, "trimValue", true);
            objDef(v, "closest", "");
            return v;
        }
      , handleDataObj = (data, $context) => {
            let pageData = {};
            iterateObj(data, (cOpt, optName) => {
                cOpt = normalizeOpt(cOpt);
                cOpt.name = optName;

                let $cContext = $context === $ ? undefined : $context;
                if (!$cContext && !cOpt.selector && !cOpt.listItem) {
                    throw new Err("There is no element selected for the '<option.name>' field. Please provide a selector, list item or use nested object structure.", {
                        option: cOpt
                      , code: "NO_ELEMENT_SELECTED"
                    });
                }

                let $elm = cOpt.selector ? $(cOpt.selector, $cContext) : $cContext;

                // Handle lists
                if (cOpt.listItem) {
                    let docs = pageData[cOpt.name] = []
                      , $items = $(cOpt.listItem, $cContext)
                      , isEmpty = emptyObj(cOpt.data)
                      ;

                    if (isEmpty) {
                        cOpt.data.___raw = {};
                    }

                    for (let i = 0; i < $items.length; ++i) {
                        let cDoc = handleDataObj(cOpt.data, $items.eq(i));
                        docs.push(cDoc.___raw || cDoc);
                    }
                } else {

                    if (typpy(cOpt.eq, Number)) {
                        $elm = $elm.eq(cOpt.eq);
                    }

                    if (!emptyObj(cOpt.data)) {
                        pageData[cOpt.name] = handleDataObj(cOpt.data, $elm);
                        return pageData;
                    }

                    let value = typpy(cOpt.how, Function) ? cOpt.how($elm) : $elm[cOpt.how]();

                    if (cOpt.trimValue && typpy(value, String)) {
                        value = value.trim();
                    }

                    if (cOpt.convert) {
                        if (cOpt.closest && $cContext) {
                            let $node = $cContext.closest(cOpt.closest);
                            value = cOpt.convert($node.html(), $node);
                        } else {
                            value = cOpt.convert(value, $elm);
                        }
                    }

                    pageData[cOpt.name] = value;
                }
            });
            return pageData;
        }
      ;

    return handleDataObj(opts);
};

module.exports = scrapeIt;
<commit_msg>Minor<commit_after>"use strict";

const req = require("cheerio-req")
    , typpy = require("typpy")
    , iterateObj = require("iterate-object")
    , Err = require("err")
    , objDef = require("obj-def")
    , emptyObj = require("is-empty-obj")
    , assured = require("assured")
    , cheerio = require("cheerio")
    ;

/**
 * scrapeIt
 * A scraping module for humans.
 *
 * @name scrapeIt
 * @function
 * @param {String|Object} url The page url or request options.
 * @param {Object} opts The options passed to `scrapeHTML` method.
 * @param {Function} cb The callback function.
 * @return {Promise} A promise object.
 */
function scrapeIt (url, opts, cb) {
    cb = assured(cb);
    req(url, (err, $, res, body) => {
        if (err) { return cb(err); }
        cb(null, scrapeIt.scrapeHTML($, opts), $, res, body);
    });
    return cb._;
}

/**
 * scrapeIt.scrapeHTML
 * Scrapes the data in the provided element.
 *
 * @name scrapeIt.scrapeHTML
 * @function
 * @param {Cheerio} $ The input element.
 * @param {Object} opts An object containing the scraping information.
 *
 *   If you want to scrape a list, you have to use the `listItem` selector:
 *
 *    - `listItem` (String): The list item selector.
 *    - `data` (Object): The fields to include in the list objects:
 *       - `<fieldName>` (Object|String): The selector or an object containing:
 *          - `selector` (String): The selector.
 *          - `convert` (Function): An optional function to change the value.
 *          - `how` (Function|String): A function or function name to access the
 *            value.
 *          - `attr` (String): If provided, the value will be taken based on
 *            the attribute name.
 *          - `trim` (Boolean): If `false`, the value will *not* be trimmed
 *            (default: `true`).
 *          - `closest` (String): If provided, returns the first ancestor of
 *            the given element.
 *          - `eq` (Number): If provided, it will select the *nth* element.
 *          - `listItem` (Object): An object, keeping the recursive schema of
 *            the `listItem` object. This can be used to create nested lists.
 *
 *   **Example**:
 *   ```js
 *   {
 *      articles: {
 *          listItem: ".article"
 *        , data: {
 *              createdAt: {
 *                  selector: ".date"
 *                , convert: x => new Date(x)
 *              }
 *            , title: "a.article-title"
 *            , tags: {
 *                  listItem: ".tags > span"
 *              }
 *            , content: {
 *                  selector: ".article-content"
 *                , how: "html"
 *              }
 *            , traverseOtherNode: {
 *                  selector: ".upperNode"
 *                , closest: "div"
 *                , convert: x => x.length
 *              }
 *          }
 *      }
 *   }
 *   ```
 *
 *   If you want to collect specific data from the page, just use the same
 *   schema used for the `data` field.
 *
 *   **Example**:
 *   ```js
 *   {
 *        title: ".header h1"
 *      , desc: ".header h2"
 *      , avatar: {
 *            selector: ".header img"
 *          , attr: "src"
 *        }
 *   }
 *   ```
 *
 * @returns {Object} The scraped data.
 */
scrapeIt.scrapeHTML = ($, opts) => {

    if (typeof $ === "string") {
        $ = cheerio.load($);
    }

    let normalizeOpt = v => {
            if (typpy(v, String)) {
                v = { selector: v };
            }
            objDef(v, "data", {});
            objDef(v, "how", "text", true);
            if (v.attr) {
                v.how = $elm => $elm.attr(v.attr);
            }
            objDef(v, "trimValue", true);
            objDef(v, "closest", "");
            return v;
        }
      , handleDataObj = (data, $context) => {
            let pageData = {};
            iterateObj(data, (cOpt, optName) => {
                cOpt = normalizeOpt(cOpt);
                cOpt.name = optName;

                let $cContext = $context === $ ? undefined : $context;
                if (!$cContext && !cOpt.selector && !cOpt.listItem) {
                    throw new Err("There is no element selected for the '<option.name>' field. Please provide a selector, list item or use nested object structure.", {
                        option: cOpt
                      , code: "NO_ELEMENT_SELECTED"
                    });
                }

                let $elm = cOpt.selector ? $(cOpt.selector, $cContext) : $cContext;

                // Handle lists
                if (cOpt.listItem) {
                    let docs = pageData[cOpt.name] = []
                      , $items = $(cOpt.listItem, $cContext)
                      , isEmpty = emptyObj(cOpt.data)
                      ;

                    if (isEmpty) {
                        cOpt.data.___raw = {};
                    }

                    for (let i = 0; i < $items.length; ++i) {
                        let cDoc = handleDataObj(cOpt.data, $items.eq(i));
                        docs.push(cDoc.___raw || cDoc);
                    }
                } else {

                    if (typpy(cOpt.eq, Number)) {
                        $elm = $elm.eq(cOpt.eq);
                    }

                    if (!emptyObj(cOpt.data)) {
                        pageData[cOpt.name] = handleDataObj(cOpt.data, $elm);
                        return pageData;
                    }

                    let value = typpy(cOpt.how, Function) ? cOpt.how($elm) : $elm[cOpt.how]();

                    if (cOpt.trimValue && typpy(value, String)) {
                        value = value.trim();
                    }

                    if (cOpt.convert) {
                        if (cOpt.closest && $cContext) {
                            let $node = $cContext.closest(cOpt.closest);
                            value = cOpt.convert($node.html(), $node);
                        } else {
                            value = cOpt.convert(value, $elm);
                        }
                    }

                    pageData[cOpt.name] = value;
                }
            });
            return pageData;
        }
      ;

    return handleDataObj(opts);
};

module.exports = scrapeIt;
<|endoftext|>


<commit_before>var gulp = require('gulp'),
    eventStream = require('event-stream'),
    gulpLoadPlugins = require('gulp-load-plugins'),
    fs = require('fs'),
    del  = require('del'),
    path = require('path'),
    s = require('underscore.string'),
    argv = require('yargs').argv,
    logger = require('js-logger'),
    hawtio = require('@hawtio/node-backend'),
    Server = require('karma').Server;

var plugins = gulpLoadPlugins({});

var config = {
  proxyPort: argv.port || 8181,
  targetPath: argv.path || '/jolokia',
  logLevel: argv.debug ? logger.DEBUG : logger.INFO,
  ts: ['plugins/**/*.ts'],
  less: ['plugins/**/*.less'],
  templates: ['plugins/**/*.html', 'plugins/**/*.md'],
  templateModule: 'hawtio-jmx-templates',
  dist: argv.out || './dist/',
  js: 'hawtio-jmx.js',
  dts: 'hawtio-jmx.d.ts',
  css: 'hawtio-jmx.css',
  tsProject: plugins.typescript.createProject('tsconfig.json'),
  sourceMap: argv.sourcemap,
  vendor: './vendor/'
};

gulp.task('clean-defs', function() {
  return del(config.dist + '*.d.ts');
});

gulp.task('tsc', ['clean-defs'], function() {
  var tsResult = config.tsProject.src()
    .pipe(plugins.if(config.sourceMap, plugins.sourcemaps.init()))
    .pipe(config.tsProject());

  return eventStream.merge(
    tsResult.js
      .pipe(plugins.ngAnnotate())
      .pipe(plugins.if(config.sourceMap, plugins.sourcemaps.write()))
      .pipe(gulp.dest('.')),
    tsResult.dts
      .pipe(plugins.rename(config.dts))
      .pipe(gulp.dest(config.dist)));
});

gulp.task('less', function () {
  return gulp.src(config.less)
    .pipe(plugins.less({
      paths: [
        path.join(__dirname, 'plugins'),
        path.join(__dirname, 'node_modules')
      ]
    }))
    .pipe(plugins.concat(config.css))
    .pipe(gulp.dest(config.dist));
});

gulp.task('template', ['tsc'], function() {
  return gulp.src(config.templates)
    .pipe(plugins.angularTemplatecache({
      filename: 'templates.js',
      root: 'plugins/',
      standalone: true,
      module: config.templateModule,
      templateFooter: '}]); hawtioPluginLoader.addModule("' + config.templateModule + '");'
    }))
    .pipe(gulp.dest('.'));
});

gulp.task('concat', ['template'], function() {
  return gulp.src([
    config.vendor + 'dangle.js',
    config.vendor + 'jolokia.js',
    config.vendor + 'jolokia-simple.js',
    config.vendor + 'cubism.v1.js',
    config.vendor + 'jolokia-cubism.js',
    'compiled.js',
    'templates.js'
  ])
  .pipe(plugins.concat(config.js))
  .pipe(gulp.dest(config.dist));
});

gulp.task('clean', ['concat'], function() {
  return del(['templates.js', 'compiled.js']);
});

gulp.task('watch-less', function() {
  gulp.watch(config.less, ['less']);
});

gulp.task('watch', ['build', 'watch-less'], function() {
  gulp.watch(['index.html', config.dist + '/*'], ['reload']);
  gulp.watch([config.ts, config.templates], ['tsc', 'template', 'concat', 'clean']);
});

gulp.task('connect', ['watch'], function() {
  hawtio.setConfig({
    logLevel: config.logLevel,
    port: 2772,
    proxy: '/hawtio/proxy',
    staticProxies: [{
      port: config.proxyPort,
      path: '/jolokia',
      targetPath: config.targetPath
    }],
    staticAssets: [{
      path: '/hawtio/',
      dir: '.'

    }],
    fallback: 'index.html',
    liveReload: {
      enabled: true
    }
  });
  hawtio.use('/', function(req, res, next) {
          var path = req.originalUrl;
          if (path === '/') {
            res.redirect('/hawtio');
          } else if (s.startsWith(path, '/plugins/') && s.endsWith(path, 'html')) {
            // avoid returning these files, they should get pulled from js
            console.log("returning 404 for: ", path);
            res.statusCode = 404;
            res.end();
          } else {
            // console.log("allowing: ", path);
            next();
          }
        });
  hawtio.listen(function(server) {
    var host = server.address().address;
    var port = server.address().port;
    console.log("started from gulp file at ", host, ":", port);
  });
});

gulp.task('reload', function() {
  gulp.src('.')
    .pipe(hawtio.reload());
});

gulp.task('test', ['build'], function (done) {
  new Server({
    configFile: __dirname + '/karma.conf.js'
  }, done).start();
});

gulp.task('build', ['tsc', 'less', 'template', 'concat', 'clean']);

gulp.task('default', ['connect']);
<commit_msg>Fully rely on tsconfig.json for TS files<commit_after>var gulp = require('gulp'),
    eventStream = require('event-stream'),
    gulpLoadPlugins = require('gulp-load-plugins'),
    fs = require('fs'),
    del  = require('del'),
    path = require('path'),
    s = require('underscore.string'),
    argv = require('yargs').argv,
    logger = require('js-logger'),
    hawtio = require('@hawtio/node-backend'),
    Server = require('karma').Server;

var plugins = gulpLoadPlugins({});

const config = {
  proxyPort: argv.port || 8181,
  targetPath: argv.path || '/jolokia',
  logLevel: argv.debug ? logger.DEBUG : logger.INFO,
  ts: ['plugins/**/*.ts'],
  less: ['plugins/**/*.less'],
  templates: ['plugins/**/*.html', 'plugins/**/*.md'],
  templateModule: 'hawtio-jmx-templates',
  dist: argv.out || './dist/',
  js: 'hawtio-jmx.js',
  dts: 'hawtio-jmx.d.ts',
  css: 'hawtio-jmx.css',
  sourceMap: argv.sourcemap,
  vendor: './vendor/',
};

const tsProject = plugins.typescript.createProject('tsconfig.json');

gulp.task('clean-defs', function() {
  return del(config.dist + '*.d.ts');
});

gulp.task('tsc', ['clean-defs'], function() {
  var tsResult = tsProject.src()
    .pipe(plugins.if(config.sourceMap, plugins.sourcemaps.init()))
    .pipe(tsProject());

  return eventStream.merge(
    tsResult.js
      .pipe(plugins.ngAnnotate())
      .pipe(plugins.if(config.sourceMap, plugins.sourcemaps.write()))
      .pipe(gulp.dest('.')),
    tsResult.dts
      .pipe(plugins.rename(config.dts))
      .pipe(gulp.dest(config.dist)));
});

gulp.task('less', function () {
  return gulp.src(config.less)
    .pipe(plugins.less({
      paths: [
        path.join(__dirname, 'plugins'),
        path.join(__dirname, 'node_modules')
      ]
    }))
    .pipe(plugins.concat(config.css))
    .pipe(gulp.dest(config.dist));
});

gulp.task('template', ['tsc'], function() {
  return gulp.src(config.templates)
    .pipe(plugins.angularTemplatecache({
      filename: 'templates.js',
      root: 'plugins/',
      standalone: true,
      module: config.templateModule,
      templateFooter: '}]); hawtioPluginLoader.addModule("' + config.templateModule + '");'
    }))
    .pipe(gulp.dest('.'));
});

gulp.task('concat', ['template'], function() {
  return gulp.src([
    config.vendor + 'dangle.js',
    config.vendor + 'jolokia.js',
    config.vendor + 'jolokia-simple.js',
    config.vendor + 'cubism.v1.js',
    config.vendor + 'jolokia-cubism.js',
    'compiled.js',
    'templates.js'
  ])
  .pipe(plugins.concat(config.js))
  .pipe(gulp.dest(config.dist));
});

gulp.task('clean', ['concat'], function() {
  return del(['templates.js', 'compiled.js']);
});

gulp.task('watch-less', function() {
  gulp.watch(config.less, ['less']);
});

gulp.task('watch', ['build', 'watch-less'], function() {
  gulp.watch(['index.html', config.dist + '/*'], ['reload']);

  const tsconfig = require('./tsconfig.json');
  gulp.watch([...tsconfig.include, ...(tsconfig.exclude || []).map(e => `!${e}`),config.templates],
    ['tsc', 'template', 'concat', 'clean']);
});

gulp.task('connect', ['watch'], function() {
  hawtio.setConfig({
    logLevel: config.logLevel,
    port: 2772,
    proxy: '/hawtio/proxy',
    staticProxies: [{
      port: config.proxyPort,
      path: '/jolokia',
      targetPath: config.targetPath
    }],
    staticAssets: [{
      path: '/hawtio/',
      dir: '.'

    }],
    fallback: 'index.html',
    liveReload: {
      enabled: true
    }
  });
  hawtio.use('/', function(req, res, next) {
          var path = req.originalUrl;
          if (path === '/') {
            res.redirect('/hawtio');
          } else if (s.startsWith(path, '/plugins/') && s.endsWith(path, 'html')) {
            // avoid returning these files, they should get pulled from js
            console.log("returning 404 for: ", path);
            res.statusCode = 404;
            res.end();
          } else {
            // console.log("allowing: ", path);
            next();
          }
        });
  hawtio.listen(function(server) {
    var host = server.address().address;
    var port = server.address().port;
    console.log("started from gulp file at ", host, ":", port);
  });
});

gulp.task('reload', function() {
  gulp.src('.')
    .pipe(hawtio.reload());
});

gulp.task('test', ['build'], function (done) {
  new Server({
    configFile: __dirname + '/karma.conf.js'
  }, done).start();
});

gulp.task('build', ['tsc', 'less', 'template', 'concat', 'clean']);

gulp.task('default', ['connect']);
<|endoftext|>


<commit_before>const fs = require('fs')

const gitRevSync = require('git-rev-sync')
const gulp = require('gulp')
const ghPages = require('gulp-gh-pages')
const htmlhint = require('gulp-htmlhint')
const htmlmin = require('gulp-htmlmin')
const plumber = require('gulp-plumber')
const sassLint = require('gulp-sass-lint')
const standard = require('gulp-standard')
const watch = require('gulp-watch')

const pkg = require('./package.json')

gulp.task('default', ['lint', 'watch'])
gulp.task('lint', ['standard', 'sass-lint'])

gulp.task('htmlhint', () => {
  return gulp.src('public/**/*.html')
    .pipe(htmlhint())
    .pipe(htmlhint.failReporter())
})

gulp.task('standard', () => {
  return gulp.src('app/**/*.js')
    .pipe(standard())
    .pipe(standard.reporter('default', {
        breakOnError: true
    }));
})

gulp.task('sass-lint', () => {
  return gulp.src('app/**/*.scss')
    .pipe(sassLint())
    .pipe(sassLint.format())
    .pipe(sassLint.failOnError());
})

gulp.task('watch', () => {
  gulp.src('public/**/*.html')
    .pipe(watch('public/**/*.html'))
    .pipe(plumber())
    .pipe(htmlhint())
    .pipe(htmlhint.reporter())

  gulp.src('app/**/*.js')
    .pipe(watch('app/**/*.js'))
    .pipe(plumber())
    .pipe(standard())
    .pipe(standard.reporter('default'))

  return gulp.src('app/*/**/.scss')
    .pipe(watch('app/**/*.scss'))
    .pipe(plumber())
    .pipe(sassLint())
    .pipe(sassLint.format());
})

gulp.task('minify', () => {
  return gulp.src('public/**/*.html')
    .pipe(htmlmin({
      collapseWhitespace: true,
      preserveLineBreaks: true,
      minifyJS: true
    }))
    .pipe(gulp.dest('public'))
})

gulp.task('deploy', () => {
  fs.openSync('public/.nojekyll', 'w')

  return gulp.src(['public/.nojekyll', 'public/**/*'])
    .pipe(ghPages({
      remoteUrl: `git@github.com:${pkg.repository}.git`,
      message: `Deploy ${gitRevSync.short()} from v${pkg.version}`
    }));
})
<commit_msg>Format gulpfile with standard<commit_after>const fs = require('fs')

const gitRevSync = require('git-rev-sync')
const gulp = require('gulp')
const ghPages = require('gulp-gh-pages')
const htmlhint = require('gulp-htmlhint')
const htmlmin = require('gulp-htmlmin')
const plumber = require('gulp-plumber')
const sassLint = require('gulp-sass-lint')
const standard = require('gulp-standard')
const watch = require('gulp-watch')

const pkg = require('./package.json')

gulp.task('default', ['lint', 'watch'])
gulp.task('lint', ['standard', 'sass-lint'])

gulp.task('htmlhint', () => {
  return gulp.src('public/**/*.html')
    .pipe(htmlhint())
    .pipe(htmlhint.failReporter())
})

gulp.task('standard', () => {
  return gulp.src('app/**/*.js')
    .pipe(standard())
    .pipe(standard.reporter('default', {
      breakOnError: true
    }))
})

gulp.task('sass-lint', () => {
  return gulp.src('app/**/*.scss')
    .pipe(sassLint())
    .pipe(sassLint.format())
    .pipe(sassLint.failOnError())
})

gulp.task('watch', () => {
  gulp.src('public/**/*.html')
    .pipe(watch('public/**/*.html'))
    .pipe(plumber())
    .pipe(htmlhint())
    .pipe(htmlhint.reporter())

  gulp.src('app/**/*.js')
    .pipe(watch('app/**/*.js'))
    .pipe(plumber())
    .pipe(standard())
    .pipe(standard.reporter('default'))

  return gulp.src('app/*/**/.scss')
    .pipe(watch('app/**/*.scss'))
    .pipe(plumber())
    .pipe(sassLint())
    .pipe(sassLint.format())
})

gulp.task('minify', () => {
  return gulp.src('public/**/*.html')
    .pipe(htmlmin({
      collapseWhitespace: true,
      preserveLineBreaks: true,
      minifyJS: true
    }))
    .pipe(gulp.dest('public'))
})

gulp.task('deploy', () => {
  fs.openSync('public/.nojekyll', 'w')

  return gulp.src(['public/.nojekyll', 'public/**/*'])
    .pipe(ghPages({
      remoteUrl: `git@github.com:${pkg.repository}.git`,
      message: `Deploy ${gitRevSync.short()} from v${pkg.version}`
    }))
})
<|endoftext|>


<commit_before>var vectorator = require('./vectorator.js');
var wordList = require('./wordlist.js');
var objectMutator = require('./objectMutator.js');
var mjs = require('mathjs');
var surku = require('./surkuWrapper.js')

var lcg = (function() {
  // Set to values from http://en.wikipedia.org/wiki/Numerical_Recipes
  // m is basically chosen to be large (as it is the max period)
  // and for its relationships to a and c
  var m = 4294967296,
    // a - 1 should be divisible by m's prime factors
    a = 1664525,
    // c and m should be co-prime
    c = 1013904223,
    seed, z;
  return {
    setSeed: function(val) {
      z = seed = val || Math.round(Math.random() * m);
    },
    getSeed: function() {
      return seed;
    },
    rand: function() {
      // define the recurrence relationship
      z = (a * z + c) % m;
      // return a float in [0, 1)
      // if z = m then z / m = 0 therefore (z % m) / m < 1 always
      return z / m;
    }
  };
}());
lcg.setSeed(new Date().getTime());

var sorrowSync = {};
var sorrow = {};

function createRandomObj(fieldCount, allowNested, nestedCount) {
  fieldCount = fieldCount || mjs.randomInt(10);
  allowNested = allowNested || true;
  nestedCount = nestedCount || 0;
  if (nestedCount > 1) {
    allowNested = false;
  } else {
    allowNested = allowNested || true;
  }
  var generatedObj = {};

  for (var i = 0; i < fieldCount; i++) {
    var generatedObjField;

    switch (mjs.randomInt(allowNested ? 6 : 5)) {

      case 0:
        generatedObjField = lcg.rand();
        break;

      case 1:
        generatedObjField = lcg.rand() + Math.random();
        break;

      case 2:
        generatedObjField = sorrowSync.boolean;
        break;

      case 3:
        generatedObjField = sorrowSync.string;
        break;

      case 4:
        generatedObjField = null;
        break;

      case 5:
        generatedObjField = createRandomObj(fieldCount, allowNested, nestedCount++);
        break;
    }
    generatedObj[wordList.getRandomWord()] = generatedObjField;
  }
  return generatedObj;
}

var mutate = {
    string: (function() {
      return function(seed){
        return (seed !== null && seed !== undefined)? surku['string'](seed) : vectorize.rand()
      }
    })(),
    number: (function() {
      return function(seed){
        return (seed !== null && seed !== undefined)? surku['number'](String(seed)) : surku['number'](String(lcg.rand()))
      }
    })(),
    date: (function() {
        return function(seed) {
            if (seed !== null && seed !== undefined) {
              return surku['date'](seed)
            } else {
              var seedVal = lcg.rand();
              var currEpoch = new Date().getTime();
              while (seedVal > currEpoch * 1.2) {
                seedVal = seedVal * Math.random();
              }
              return surku['date'](
                new Date(
                  Math.random() * (
                    Math.max(currEpoch - seedVal, 0)
                  )
                ).toString()
              );

            }
          }
        })(),
      binary: (function() {
      return function(seed){
          return (seed !== null && seed !== undefined)? surku['binary'](seed) : surku['binary'](
            new Buffer(Math.floor(Math.random() * (500 - 10 + 1)) + 10)
            ).toString()
        }
    })(),
      object: (function() {
      return function(seed){
          return (seed !== null && seed !== undefined)? om(seed) :  createRandomObj();
        }
    })(),
      any: (function() {
      return function(seed){
          return sorrowSync[mjs.pickRandom(['string', 'number', 'date', 'binary', 'object', 'boolean', 'array'])];
      }
    })(),
      array: (function() {
      return function(seed){
        if(seed !== null && seed !== undefined){
          var retVal = [];
          var len = seed.length;
          for (var i = 0; i < len; i++) {
            type = typeof seed[i]
            type = (type === 'object'|| type === 'array')? 'string': type;
            retVal.push(sorrowAsync[type](seed[i]));
          }
          return retVal;
        } else {
          var retVal = [];
          var len = mjs.randomInt(20);
          for (var i = 0; i < len; i++) {
            retVal.push(sorrowSync[mjs.pickRandom(['string', 'number', 'date', 'binary', 'object', 'boolean'])]);
          }
          return retVal;
        }
      }
    })(),
      boolean: (function() {

      return function(seed){
          return (seed !== null && seed !== undefined)? surku['boolean'](seed) : mjs.pickRandom(['true', 'false', 1, 0]);
        }
    })(),
    };

    Object.defineProperties(sorrowSync, {
      "string": {
        enumerable: true,
        get: function() {
          var b = vectorator.rand();
          while (b === undefined) {
            b = vectorator.rand();
          }
          return b;
        },
      },
      "number": {
        enumerable: true,
        get: mutate.number,
      },
      "date": {
        enumerable: true,
        get: mutate.date,
      },
      "binary": {
        enumerable: true,
        get: mutate.binary,
      },
      "object": {
        enumerable: true,
        get: mutate.object,
      },
      "boolean": {
        enumerable: true,
        get: mutate.boolean,
      },
      "any": {
        enumerable: true,
        get: mutate.any,
      },
      "array": {
        enumerable: true,
        get: mutate.array,
      }
    });

    function asyncify(type) {
      return function(seed, cb) {
        var payload;
        payload = mutate[type]((seed !== null && seed !== undefined)? String(seed): undefined);
        if (!cb || typeof cb !== 'function') {
          return payload;
        } else {
          return cb(payload);
        }
      }
    }

    var sorrowAsync = {
      "string": asyncify('string'),
      "number": asyncify('number'),
      "date": asyncify('date'),
      "binary": asyncify('binary'),
      "object": asyncify('object'),
      "boolean": asyncify('boolean'),
      "any": asyncify('any'),
      "array": asyncify('array'),
      // etc. etc.
    };
    sorrowSync._internals = vectorator
    sorrowSync.async = sorrowAsync;

    module.exports = sorrowSync;<commit_msg>push update to index from previous commit<commit_after>var vectorator = require('./vectorator.js');
var wordList = require('./wordlist.js');
var mjs = require('mathjs');
var surku = require('./surkuWrapper.js')

var objectMutator = function(o, immutableProperties){

    var build, key, destKey, ix, value;

    build = {};
    for (key in o) {
        // Get the destination key
        if(!immutableProperties || immutableProperties.indexOf(key) !== -1){
        // Get the value

        // If this is an object, recurse
        if (typeof value === "object") {
            value = objectMutator(value);
        } else {

        var types = ['string','date','number', 'boolean', 'binary']
        var type = types.indexOf(typeof o[key]);
        if (type === -1) {
            type = 0;
        };
        type = types[type];
        // Set it on the result using the destination key
        build[key] = sorrowAsync[type](o[key]);
    }
    }
}
    return build;

}
var lcg = (function() {
  // Set to values from http://en.wikipedia.org/wiki/Numerical_Recipes
  // m is basically chosen to be large (as it is the max period)
  // and for its relationships to a and c
  var m = 4294967296,
    // a - 1 should be divisible by m's prime factors
    a = 1664525,
    // c and m should be co-prime
    c = 1013904223,
    seed, z;
  return {
    setSeed: function(val) {
      z = seed = val || Math.round(Math.random() * m);
    },
    getSeed: function() {
      return seed;
    },
    rand: function() {
      // define the recurrence relationship
      z = (a * z + c) % m;
      // return a float in [0, 1)
      // if z = m then z / m = 0 therefore (z % m) / m < 1 always
      return z / m;
    }
  };
}());
lcg.setSeed(new Date().getTime());

var sorrowSync = {};
var sorrow = {};

function createRandomObj(fieldCount, allowNested, nestedCount) {
  fieldCount = fieldCount || mjs.randomInt(10);
  allowNested = allowNested || true;
  nestedCount = nestedCount || 0;
  if (nestedCount > 1) {
    allowNested = false;
  } else {
    allowNested = allowNested || true;
  }
  var generatedObj = {};

  for (var i = 0; i < fieldCount; i++) {
    var generatedObjField;

    switch (mjs.randomInt(allowNested ? 6 : 5)) {

      case 0:
        generatedObjField = lcg.rand();
        break;

      case 1:
        generatedObjField = lcg.rand() + Math.random();
        break;

      case 2:
        generatedObjField = sorrowSync.boolean;
        break;

      case 3:
        generatedObjField = sorrowSync.string;
        break;

      case 4:
        generatedObjField = null;
        break;

      case 5:
        generatedObjField = createRandomObj(fieldCount, allowNested, nestedCount++);
        break;
    }
    generatedObj[wordList.getRandomWord()] = generatedObjField;
  }
  return generatedObj;
}

var mutate = {
    string: (function() {
      return function(seed){
        return (seed !== null && seed !== undefined)? surku['string'](seed) : vectorize.rand()
      }
    })(),
    number: (function() {
      return function(seed){
        return (seed !== null && seed !== undefined)? surku['number'](String(seed)) : surku['number'](String(lcg.rand()))
      }
    })(),
    date: (function() {
        return function(seed) {
            if (seed !== null && seed !== undefined) {
              return surku['date'](seed)
            } else {
              var seedVal = lcg.rand();
              var currEpoch = new Date().getTime();
              while (seedVal > currEpoch * 1.2) {
                seedVal = seedVal * Math.random();
              }
              return surku['date'](
                new Date(
                  Math.random() * (
                    Math.max(currEpoch - seedVal, 0)
                  )
                ).toString()
              );

            }
          }
        })(),
      binary: (function() {
      return function(seed){
          return (seed !== null && seed !== undefined)? surku['binary'](seed) : surku['binary'](
            new Buffer(Math.floor(Math.random() * (500 - 10 + 1)) + 10)
            ).toString()
        }
    })(),
      object: (function() {
      return function(seed){
          return (seed !== null && seed !== undefined)? objectMutator(seed) :  createRandomObj();
        }
    })(),
      any: (function() {
      return function(seed){
          return sorrowSync[mjs.pickRandom(['string', 'number', 'date', 'binary', 'object', 'boolean', 'array'])];
      }
    })(),
      array: (function() {
      return function(seed){
        if(seed !== null && seed !== undefined){
          var retVal = [];
          var len = seed.length;
          for (var i = 0; i < len; i++) {
            type = typeof seed[i]
            type = (type === 'object'|| type === 'array')? 'string': type;
            retVal.push(sorrowAsync[type](seed[i]));
          }
          return retVal;
        } else {
          var retVal = [];
          var len = mjs.randomInt(20);
          for (var i = 0; i < len; i++) {
            retVal.push(sorrowSync[mjs.pickRandom(['string', 'number', 'date', 'binary', 'object', 'boolean'])]);
          }
          return retVal;
        }
      }
    })(),
      boolean: (function() {

      return function(seed){
          return (seed !== null && seed !== undefined)? surku['boolean'](seed) : mjs.pickRandom(['true', 'false', 1, 0]);
        }
    })(),
    };

    Object.defineProperties(sorrowSync, {
      "string": {
        enumerable: true,
        get: function() {
          var b = vectorator.rand();
          while (b === undefined) {
            b = vectorator.rand();
          }
          return b;
        },
      },
      "number": {
        enumerable: true,
        get: mutate.number,
      },
      "date": {
        enumerable: true,
        get: mutate.date,
      },
      "binary": {
        enumerable: true,
        get: mutate.binary,
      },
      "object": {
        enumerable: true,
        get: mutate.object,
      },
      "boolean": {
        enumerable: true,
        get: mutate.boolean,
      },
      "any": {
        enumerable: true,
        get: mutate.any,
      },
      "array": {
        enumerable: true,
        get: mutate.array,
      }
    });

    function asyncify(type) {
      return function(seed, cb) {
        var payload;
        payload = mutate[type]((seed !== null && seed !== undefined)? seed: undefined);
        if (!cb || typeof cb !== 'function') {
          return payload;
        } else {
          return cb(payload);
        }
      }
    }

    var sorrowAsync = {
      "string": asyncify('string'),
      "number": asyncify('number'),
      "date": asyncify('date'),
      "binary": asyncify('binary'),
      "object": asyncify('object'),
      "boolean": asyncify('boolean'),
      "any": asyncify('any'),
      "array": asyncify('array'),
      // etc. etc.
    };
    sorrowSync._internals = vectorator
    sorrowSync.async = sorrowAsync;

    module.exports = sorrowSync;<|endoftext|>


<commit_before>/**
 * This file contains gulp (https://gulpjs.com/) tasks to run at build time.
 * See also npm tasks in package.json.
 */

const fs = require('fs');

const argv = require("yargs").argv;
const path = require("path");
const runSequence = require('run-sequence');

const gulp = require("gulp");
const textSimple = require("gulp-text-simple");
const rename = require("gulp-rename");
const exec = require("gulp-exec");
const git = require("gulp-git");
var jest = require('gulp-jest').default;
const jsonEditor = require("gulp-json-editor");

const semver = require("semver");
const mjml = require("mjml");

const TYPESCRIPT_SOURCE_DIR = "lib";

const TEMPLATES_SOURCE_DIR = "templates/mjml";
const TEMPLATES_SOURCE = `${TEMPLATES_SOURCE_DIR}/*.mjml`;
const TEMPLATES_OUTPUT_DIR = `${TYPESCRIPT_SOURCE_DIR}/templates/html`;

const GIT_RELEASE_BRANCH = "master";
const GIT_ORIGIN = "origin";

// resolve the root directory of this project
const rootDir = path.resolve(argv.rootDir || './') + '/';

// path to the root `package.json`
const packageJsonPath = path.join(rootDir, "package.json");

// the content of the root `package.json`
const currentPackageJson = JSON.parse(fs.readFileSync(packageJsonPath));

// the following values are calculated for every task but are only used during
// the release process, not super efficient but it's the easiest way to share
// them across tasks
const currentVersion = semver.parse(currentPackageJson.version);
const releaseVersionValue = `${currentVersion.major}.${currentVersion.minor}.${currentVersion.patch}`;
const nextVersionValue = `${semver.inc(releaseVersionValue, "minor")}-SNAPSHOT`;
const funcpackBranchPrefix = "funcpack-release";
const releaseVersionFuncpackBranchName = `${funcpackBranchPrefix}-v${releaseVersionValue}`;

/**
 * Transform a mjml template to a Typescript function outputting HTML.
 *
 * @param content the content of mjml template as string
 * @param options options object (see gulp-text-simple plugin https://www.npmjs.com/package/gulp-text-simple)
 */
const toMjml = (content, options) => {
  const name = path.basename(options.sourcePath);
  return [
    `// DO NOT EDIT THIS FILE`,
    `// this file was auto generated from '${name}'`,
    `export default function(`,
    `  title: string,`,
    `  headlineText: string,`,
    `  senderOrganizationName: string,`,
    `  senderServiceName: string,`,
    `  titleText: string,`,
    `  contentHtml: string,`,
    `  footerHtml: string`,
    `): string {`,
    `  return \``,
    `${mjml.mjml2html(content, { minify: true }).html}\`;`,
    `}`,
    ""
  ].join("\n");
};

/**
 * Generate Typescript template files from mjml (https://mjml.io/).
 */
gulp.task("generate:templates", () => {
  return gulp
    .src(TEMPLATES_SOURCE)
    .pipe(textSimple(toMjml)())
    .pipe(rename((filepath) => (filepath.extname = ".ts")))
    .pipe(gulp.dest(TEMPLATES_OUTPUT_DIR));
});

/**
 * Run the build task
 */
gulp.task("yarn:build", () => {
  return gulp.src(TYPESCRIPT_SOURCE_DIR)
    .pipe(exec(`yarn build`))
    .pipe(exec.reporter());
});

/**
 * Run the lint task
 */
gulp.task("yarn:lint", () => {
  return gulp.src(TYPESCRIPT_SOURCE_DIR)
    .pipe(exec(`yarn run lint`))
    .pipe(exec.reporter());
});

/**
 * Run unit tests
 */
gulp.task("unit:test", () => {
  return gulp.src(TYPESCRIPT_SOURCE_DIR)
    .pipe(jest({
      "coverage": true
    }))
    .pipe(exec.reporter());
});

/**
 * Run the test task
 */
gulp.task("test", ["yarn:lint"], () =>
  runSequence(["unit:test"],
    (err) => {
      if (err) {
        console.log(err.message);
      }
      cb(err);
  })
);

/**
 * Package Azure Functions code and dependencines in a single file
 */
gulp.task("yarn:funcpack", () => {
  return gulp.src(TYPESCRIPT_SOURCE_DIR)
    .pipe(exec("yarn run funcpack pack ./"))
    .pipe(exec.reporter());
});

/**
 * Checks out the release branch
 */
gulp.task("git:checkout:release", (cb) => {
  return git.checkout(GIT_RELEASE_BRANCH, {}, cb);
});

/**
 * Fails if repository has untracked files
 */
gulp.task("git:check:untracked", (cb) => {
  return git.exec({
    args: "ls-files --other --exclude-standard"
  }, (err, stdout) => {
    if (err) {
      throw(err);
    };
    if (stdout.trim().length > 0) {
      return cb("Repository must not have untracked files");
    }
    cb();
  });
});

/**
 * Fails if repository has modified files
 */
gulp.task("git:check:modified", (cb) => {
  return git.exec({
    args: "ls-files --modified --exclude-standard"
  }, (err, stdout) => {
    if (err) {
      throw(err);
    };
    if (stdout.trim().length > 0) {
      return cb("Repository must not have modified files");
    }
    cb();
  });
});

/**
 * Fails if repository is not clean
 */
gulp.task("git:check:clean", ["git:check:untracked", "git:check:modified"]);

/**
 * Fails if current branch is not GIT_RELEASE_BRANCH
 */
gulp.task("git:check:branch", (cb) => {
  return git.exec({
    args: "symbolic-ref HEAD"
  }, (err, stdout) => {
    if (err) {
      throw(err);
    };
    if (stdout.trim() !== `refs/heads/${GIT_RELEASE_BRANCH}`) {
      return cb(`You must be on the ${GIT_RELEASE_BRANCH} branch`);
    }
    cb();
  });
});

/**
 * Push master to origin
 */
gulp.task("git:push:origin", (cb) => {
  return git.push(GIT_ORIGIN, GIT_RELEASE_BRANCH, {}, cb);
});

/**
 * Push the tags to origin
 */
gulp.task("git:push:tags", (cb) => {
  return git.push(GIT_ORIGIN, GIT_RELEASE_BRANCH, { args: "--tags" }, cb);
});

/**
 * Bump the version to the release version
 */
gulp.task("release:bump:release", () => {
  return gulp.src(packageJsonPath)
    .pipe(jsonEditor({
      "version": releaseVersionValue
    }))
    .pipe(gulp.dest(rootDir));
});

/**
 * Commit package.json with updated release version
 */
gulp.task("release:git:commit:release", () => {
  return gulp.src(packageJsonPath)
    .pipe(git.add())
    .pipe(git.commit(`Bumped release ${releaseVersionValue}`));
})

/**
 * Tag last commit with the release version
 */
gulp.task("release:git:tag:release", (cb) => {
  const tag = `v${releaseVersionValue}`;
  return git.tag(tag, `Created tag for version ${tag}`, cb);
})

/**
 * Creates a new branch for storing funcpack assets
 */
gulp.task("release:git:checkout:funcpack", (cb) => {
  return git.checkout(releaseVersionFuncpackBranchName, { args: "-b" }, cb);
});

/**
 * Adds funcpack assets
 */
gulp.task("release:git:commit:funcpack", (cb) => {
  return gulp.src(["*/function.json", ".funcpack/index.js"])
    .pipe(git.add({ args: "-f" })) // force because .gitignore contains *.js
    .pipe(git.commit(`Adds funcpack assets for release ${releaseVersionValue}`));
});

/**
 * Pushes funcpack branch to origin
 */
gulp.task("release:git:push:funcpack", (cb) => {
  return git.push(GIT_ORIGIN, releaseVersionFuncpackBranchName, { args: "-u" }, cb);
});

/**
 * Updates the remote "latest" funcpack branch to the current funcpack branch
 */
gulp.task("release:git:push:funcpack:latest", (cb) => {
  return git.push(GIT_ORIGIN, `${releaseVersionFuncpackBranchName}:${funcpackBranchPrefix}-latest`, { args: "-f" }, cb);
});

/**
 * Bump the version to the snapshot version
 */
gulp.task("release:bump:next", () => {
  return gulp.src(packageJsonPath)
    .pipe(jsonEditor({
      "version": nextVersionValue
    }))
    .pipe(gulp.dest(rootDir));
});

/**
 * Commit package.json with updated snapshot version
 */
gulp.task("release:git:commit:next", () => {
  return gulp.src(packageJsonPath)
    .pipe(git.add())
    .pipe(git.commit(`Bumped release ${nextVersionValue}`));
})

gulp.task("release", function (cb) {
  runSequence(
    // check that the release is running on the GIT_RELEASE_BRANCH branch
    "git:check:branch",
    // check that the working directory is a git repository and
    // the repository has no outstanding changes.
    "git:check:clean",
    // run tests
    "test",
    // bumps the version to the next release version:
    // current version without the qualifier (eg. 1.2-SNAPSHOT -> 1.2)
    "release:bump:release",
    // commit the changes in package.json
    "release:git:commit:release",
    // tag the previous commit with v$version (eg. v1.2, v1.2.3).
    "release:git:tag:release",
    // checkout a new branch funcpack-release-vx.x.x
    "release:git:checkout:funcpack",
    // build and run funcpack
    "yarn:build",
    "yarn:funcpack",
    // commits and pushes funcpack branch
    "release:git:commit:funcpack",
    "release:git:push:funcpack",
    "release:git:push:funcpack:latest",
    // check out the release branch (master)
    "git:checkout:release",
    // bumps the version to the next snapshot version:
    // increase the minor version segment of the current version and set the
    // qualifier to '-SNAPSHOT' (eg. 1.2.1-SNAPSHOT -> 1.3.0-SNAPSHOT)
    "release:bump:next",
    // commit the changes in package.json
    "release:git:commit:next",
    // push changes to origin
    "git:push:origin",
    "git:push:tags",
    (err) => {
      if (err) {
        console.log(err.message);
      } else {
        console.log('RELEASE FINISHED SUCCESSFULLY');
      }
      cb(err);
    });
});

gulp.task("default", ["generate:templates"]);
<commit_msg>Update gulpfile.js (#89)<commit_after>/**
 * This file contains gulp (https://gulpjs.com/) tasks to run at build time.
 * See also npm tasks in package.json.
 */

const fs = require('fs');

const argv = require("yargs").argv;
const path = require("path");
const runSequence = require('run-sequence');

const gulp = require("gulp");
const textSimple = require("gulp-text-simple");
const rename = require("gulp-rename");
const exec = require("gulp-exec");
const git = require("gulp-git");
var jest = require('gulp-jest').default;
const jsonEditor = require("gulp-json-editor");

const semver = require("semver");
const mjml = require("mjml");

const TYPESCRIPT_SOURCE_DIR = "lib";

const TEMPLATES_SOURCE_DIR = "templates/mjml";
const TEMPLATES_SOURCE = `${TEMPLATES_SOURCE_DIR}/*.mjml`;
const TEMPLATES_OUTPUT_DIR = `${TYPESCRIPT_SOURCE_DIR}/templates/html`;

const GIT_RELEASE_BRANCH = "master";
const GIT_ORIGIN = "origin";

// resolve the root directory of this project
const rootDir = path.resolve(argv.rootDir || './') + '/';

// path to the root `package.json`
const packageJsonPath = path.join(rootDir, "package.json");

// the content of the root `package.json`
const currentPackageJson = JSON.parse(fs.readFileSync(packageJsonPath));

// the following values are calculated for every task but are only used during
// the release process, not super efficient but it's the easiest way to share
// them across tasks
const currentVersion = semver.parse(currentPackageJson.version);
const releaseVersionValue = `${currentVersion.major}.${currentVersion.minor}.${currentVersion.patch}`;
const nextVersionValue = `${semver.inc(releaseVersionValue, "minor")}-SNAPSHOT`;
const funcpackBranchPrefix = "funcpack-release";
const releaseVersionFuncpackBranchName = `${funcpackBranchPrefix}-v${releaseVersionValue}`;

/**
 * Transform a mjml template to a Typescript function outputting HTML.
 *
 * @param content the content of mjml template as string
 * @param options options object (see gulp-text-simple plugin https://www.npmjs.com/package/gulp-text-simple)
 */
const toMjml = (content, options) => {
  const name = path.basename(options.sourcePath);
  return [
    `// DO NOT EDIT THIS FILE`,
    `// this file was auto generated from '${name}'`,
    `export default function(`,
    `  title: string,`,
    `  headlineText: string,`,
    `  senderOrganizationName: string,`,
    `  senderServiceName: string,`,
    `  titleText: string,`,
    `  contentHtml: string,`,
    `  footerHtml: string`,
    `): string {`,
    `  return \``,
    `${mjml.mjml2html(content, { minify: true }).html}\`;`,
    `}`,
    ""
  ].join("\n");
};

/**
 * Generate Typescript template files from mjml (https://mjml.io/).
 */
gulp.task("generate:templates", () => {
  return gulp
    .src(TEMPLATES_SOURCE)
    .pipe(textSimple(toMjml)())
    .pipe(rename((filepath) => (filepath.extname = ".ts")))
    .pipe(gulp.dest(TEMPLATES_OUTPUT_DIR));
});

/**
 * Run the build task
 */
gulp.task("yarn:build", () => {
  return gulp.src(TYPESCRIPT_SOURCE_DIR)
    .pipe(exec(`yarn build`))
    .pipe(exec.reporter());
});

/**
 * Run the lint task
 */
gulp.task("yarn:lint", () => {
  return gulp.src(TYPESCRIPT_SOURCE_DIR)
    .pipe(exec(`yarn run lint`))
    .pipe(exec.reporter());
});

/**
 * Run unit tests
 */
gulp.task("unit:test", () => {
  return gulp.src(TYPESCRIPT_SOURCE_DIR)
    .pipe(jest({
      "coverage": true
    }))
    .pipe(exec.reporter());
});

/**
 * Run the test task
 */
gulp.task("test", ["yarn:lint"], (cb) =>
  runSequence(["unit:test"],
    (err) => {
      if (err) {
        console.log(err.message);
      }
      cb(err);
  })
);

/**
 * Package Azure Functions code and dependencines in a single file
 */
gulp.task("yarn:funcpack", () => {
  return gulp.src(TYPESCRIPT_SOURCE_DIR)
    .pipe(exec("yarn run funcpack pack ./"))
    .pipe(exec.reporter());
});

/**
 * Checks out the release branch
 */
gulp.task("git:checkout:release", (cb) => {
  return git.checkout(GIT_RELEASE_BRANCH, {}, cb);
});

/**
 * Fails if repository has untracked files
 */
gulp.task("git:check:untracked", (cb) => {
  return git.exec({
    args: "ls-files --other --exclude-standard"
  }, (err, stdout) => {
    if (err) {
      throw(err);
    };
    if (stdout.trim().length > 0) {
      return cb("Repository must not have untracked files");
    }
    cb();
  });
});

/**
 * Fails if repository has modified files
 */
gulp.task("git:check:modified", (cb) => {
  return git.exec({
    args: "ls-files --modified --exclude-standard"
  }, (err, stdout) => {
    if (err) {
      throw(err);
    };
    if (stdout.trim().length > 0) {
      return cb("Repository must not have modified files");
    }
    cb();
  });
});

/**
 * Fails if repository is not clean
 */
gulp.task("git:check:clean", ["git:check:untracked", "git:check:modified"]);

/**
 * Fails if current branch is not GIT_RELEASE_BRANCH
 */
gulp.task("git:check:branch", (cb) => {
  return git.exec({
    args: "symbolic-ref HEAD"
  }, (err, stdout) => {
    if (err) {
      throw(err);
    };
    if (stdout.trim() !== `refs/heads/${GIT_RELEASE_BRANCH}`) {
      return cb(`You must be on the ${GIT_RELEASE_BRANCH} branch`);
    }
    cb();
  });
});

/**
 * Push master to origin
 */
gulp.task("git:push:origin", (cb) => {
  return git.push(GIT_ORIGIN, GIT_RELEASE_BRANCH, {}, cb);
});

/**
 * Push the tags to origin
 */
gulp.task("git:push:tags", (cb) => {
  return git.push(GIT_ORIGIN, GIT_RELEASE_BRANCH, { args: "--tags" }, cb);
});

/**
 * Bump the version to the release version
 */
gulp.task("release:bump:release", () => {
  return gulp.src(packageJsonPath)
    .pipe(jsonEditor({
      "version": releaseVersionValue
    }))
    .pipe(gulp.dest(rootDir));
});

/**
 * Commit package.json with updated release version
 */
gulp.task("release:git:commit:release", () => {
  return gulp.src(packageJsonPath)
    .pipe(git.add())
    .pipe(git.commit(`Bumped release ${releaseVersionValue}`));
})

/**
 * Tag last commit with the release version
 */
gulp.task("release:git:tag:release", (cb) => {
  const tag = `v${releaseVersionValue}`;
  return git.tag(tag, `Created tag for version ${tag}`, cb);
})

/**
 * Creates a new branch for storing funcpack assets
 */
gulp.task("release:git:checkout:funcpack", (cb) => {
  return git.checkout(releaseVersionFuncpackBranchName, { args: "-b" }, cb);
});

/**
 * Adds funcpack assets
 */
gulp.task("release:git:commit:funcpack", (cb) => {
  return gulp.src(["*/function.json", ".funcpack/index.js"])
    .pipe(git.add({ args: "-f" })) // force because .gitignore contains *.js
    .pipe(git.commit(`Adds funcpack assets for release ${releaseVersionValue}`));
});

/**
 * Pushes funcpack branch to origin
 */
gulp.task("release:git:push:funcpack", (cb) => {
  return git.push(GIT_ORIGIN, releaseVersionFuncpackBranchName, { args: "-u" }, cb);
});

/**
 * Updates the remote "latest" funcpack branch to the current funcpack branch
 */
gulp.task("release:git:push:funcpack:latest", (cb) => {
  return git.push(GIT_ORIGIN, `${releaseVersionFuncpackBranchName}:${funcpackBranchPrefix}-latest`, { args: "-f" }, cb);
});

/**
 * Bump the version to the snapshot version
 */
gulp.task("release:bump:next", () => {
  return gulp.src(packageJsonPath)
    .pipe(jsonEditor({
      "version": nextVersionValue
    }))
    .pipe(gulp.dest(rootDir));
});

/**
 * Commit package.json with updated snapshot version
 */
gulp.task("release:git:commit:next", () => {
  return gulp.src(packageJsonPath)
    .pipe(git.add())
    .pipe(git.commit(`Bumped release ${nextVersionValue}`));
})

gulp.task("release", function (cb) {
  runSequence(
    // check that the release is running on the GIT_RELEASE_BRANCH branch
    "git:check:branch",
    // check that the working directory is a git repository and
    // the repository has no outstanding changes.
    "git:check:clean",
    // run tests
    "test",
    // bumps the version to the next release version:
    // current version without the qualifier (eg. 1.2-SNAPSHOT -> 1.2)
    "release:bump:release",
    // commit the changes in package.json
    "release:git:commit:release",
    // tag the previous commit with v$version (eg. v1.2, v1.2.3).
    "release:git:tag:release",
    // checkout a new branch funcpack-release-vx.x.x
    "release:git:checkout:funcpack",
    // build and run funcpack
    "yarn:build",
    "yarn:funcpack",
    // commits and pushes funcpack branch
    "release:git:commit:funcpack",
    "release:git:push:funcpack",
    "release:git:push:funcpack:latest",
    // check out the release branch (master)
    "git:checkout:release",
    // bumps the version to the next snapshot version:
    // increase the minor version segment of the current version and set the
    // qualifier to '-SNAPSHOT' (eg. 1.2.1-SNAPSHOT -> 1.3.0-SNAPSHOT)
    "release:bump:next",
    // commit the changes in package.json
    "release:git:commit:next",
    // push changes to origin
    "git:push:origin",
    "git:push:tags",
    (err) => {
      if (err) {
        console.log(err.message);
      } else {
        console.log('RELEASE FINISHED SUCCESSFULLY');
      }
      cb(err);
    });
});

gulp.task("default", ["generate:templates"]);
<|endoftext|>


<commit_before>var mongoose = require('mongoose'),
    async    = require('async'),
    _        = require('underscore'),
    schemas  = require('./schemas'),
    utils    = require('./utils'),
    config   = require('config');

var connection_cache = {
    // empty to start with
};

var settings_cache = {
    // empty to start with
};

var PopIt = (function() {

    function PopIt () {
        this._instance_name = null;
    }

    PopIt.prototype.set_instance = function ( name ) {
        if (!name) throw new Error("Need a instance name");
        this._instance_name = name;
        return name;
    };

    PopIt.prototype.instance_name = function () {
        return this._instance_name;
    };

    PopIt.prototype.set_as_master = function () {
        return this.set_instance( config.MongoDB.master_name );
    };

    PopIt.prototype.is_master = function () {
        return this._instance_name === config.MongoDB.master_name;
    };

    PopIt.prototype._get_connection = function (name) {
        if (!name) throw new Error("Need a connection_name");

        if ( !connection_cache[name] ) {
            var uri = utils.mongodb_connection_string(name);
            connection_cache[name] = mongoose.createConnection( uri );
        }

        return connection_cache[name];
    };

    PopIt.prototype.master_db = function () {
        return this._get_connection( config.MongoDB.master_name );
    };

    PopIt.prototype.instance_db = function () {
        var name = this._instance_name;
        if (!name) throw new Error("Must set_instance before calling instance_db");
        return this._get_connection( this._instance_name );
    };

    PopIt.prototype.close_db_connections = function () {
        _.each(
            _.keys( connection_cache ),
            function ( key ) {
                var connection = connection_cache[key];
                delete connection_cache[key];
                connection.close();
            }
        );
    };

    PopIt.prototype.schema = function ( name ) {
        return schemas[name];
    };

    PopIt.prototype.model = function (name) {
        var db = this.instance_db();
        var model = null;

        try {
            // perhaps we've already got one set up?
            model = db.model( name );
        } catch (e) {

            // is this the expected error about the schema not being registered yet?
            if ( e.message.match( /^Schema hasn't been registered/ ) ) {

                var schema = this.schema(name);
                if ( !schema )
                    throw new Error("Could not find a schema for " + name );

                model = db.model( name, schema );
            } else {
                // otherwise re-throw the error
                throw e;
            }
        }

        return model;
    };

    // settings

    PopIt.prototype._get_cached_settings = function clear_stored_settings () {
        var settings = settings_cache[ this._instance_name ];

        if ( ! settings )
            throw new Error("Settings not loaded - have you called load_settings?");

        return settings;
    };

    PopIt.prototype.clear_cached_settings = function clear_stored_settings () {
        settings_cache = {};
        return true;
    };

    PopIt.prototype.load_settings = function load_settings ( cb ) {
        var instance_name = this._instance_name;

        // If settings already loaded then return
        if ( settings_cache[ instance_name ] ) {
            cb(null);
            return;
        }

        // load settings from db, store in hash
        var Setting = this.model('Setting');
        Setting.find( {}, function (err, docs) {
            if ( err ) throw cb(err);

            var loaded_settings = {};

            _.each( docs, function (doc) {
               loaded_settings[doc.key] = doc.value;
            });

            settings_cache[ instance_name ] = loaded_settings;
            cb(null);
        });
    };

    PopIt.prototype.set_setting = function set_setting (key, value, cb) {
        var settings = this._get_cached_settings();
        var Setting = this.model('Setting');

        Setting.update(
            { key: key },
            { value: value},
            { upsert: true },
            function (err) {
                if ( ! err ) settings[key] = value;
                cb(err);
            }
        );
    };

    PopIt.prototype.setting = function setting (key) {
        var settings = this._get_cached_settings();
        return settings[key] || config.default_settings[key];
    };

    return PopIt;

})();

module.exports = PopIt;<commit_msg>Adding note about clustering<commit_after>var mongoose = require('mongoose'),
    async    = require('async'),
    _        = require('underscore'),
    schemas  = require('./schemas'),
    utils    = require('./utils'),
    config   = require('config');

var connection_cache = {
    // empty to start with
};

// FIXME - will not be shared when running under something like cluster
var settings_cache = {
    // empty to start with
};

var PopIt = (function() {

    function PopIt () {
        this._instance_name = null;
    }

    PopIt.prototype.set_instance = function ( name ) {
        if (!name) throw new Error("Need a instance name");
        this._instance_name = name;
        return name;
    };

    PopIt.prototype.instance_name = function () {
        return this._instance_name;
    };

    PopIt.prototype.set_as_master = function () {
        return this.set_instance( config.MongoDB.master_name );
    };

    PopIt.prototype.is_master = function () {
        return this._instance_name === config.MongoDB.master_name;
    };

    PopIt.prototype._get_connection = function (name) {
        if (!name) throw new Error("Need a connection_name");

        if ( !connection_cache[name] ) {
            var uri = utils.mongodb_connection_string(name);
            connection_cache[name] = mongoose.createConnection( uri );
        }

        return connection_cache[name];
    };

    PopIt.prototype.master_db = function () {
        return this._get_connection( config.MongoDB.master_name );
    };

    PopIt.prototype.instance_db = function () {
        var name = this._instance_name;
        if (!name) throw new Error("Must set_instance before calling instance_db");
        return this._get_connection( this._instance_name );
    };

    PopIt.prototype.close_db_connections = function () {
        _.each(
            _.keys( connection_cache ),
            function ( key ) {
                var connection = connection_cache[key];
                delete connection_cache[key];
                connection.close();
            }
        );
    };

    PopIt.prototype.schema = function ( name ) {
        return schemas[name];
    };

    PopIt.prototype.model = function (name) {
        var db = this.instance_db();
        var model = null;

        try {
            // perhaps we've already got one set up?
            model = db.model( name );
        } catch (e) {

            // is this the expected error about the schema not being registered yet?
            if ( e.message.match( /^Schema hasn't been registered/ ) ) {

                var schema = this.schema(name);
                if ( !schema )
                    throw new Error("Could not find a schema for " + name );

                model = db.model( name, schema );
            } else {
                // otherwise re-throw the error
                throw e;
            }
        }

        return model;
    };

    // settings

    PopIt.prototype._get_cached_settings = function clear_stored_settings () {
        var settings = settings_cache[ this._instance_name ];

        if ( ! settings )
            throw new Error("Settings not loaded - have you called load_settings?");

        return settings;
    };

    PopIt.prototype.clear_cached_settings = function clear_stored_settings () {
        settings_cache = {};
        return true;
    };

    PopIt.prototype.load_settings = function load_settings ( cb ) {
        var instance_name = this._instance_name;

        // If settings already loaded then return
        if ( settings_cache[ instance_name ] ) {
            cb(null);
            return;
        }

        // load settings from db, store in hash
        var Setting = this.model('Setting');
        Setting.find( {}, function (err, docs) {
            if ( err ) throw cb(err);

            var loaded_settings = {};

            _.each( docs, function (doc) {
               loaded_settings[doc.key] = doc.value;
            });

            settings_cache[ instance_name ] = loaded_settings;
            cb(null);
        });
    };

    PopIt.prototype.set_setting = function set_setting (key, value, cb) {
        var settings = this._get_cached_settings();
        var Setting = this.model('Setting');

        Setting.update(
            { key: key },
            { value: value},
            { upsert: true },
            function (err) {
                if ( ! err ) settings[key] = value;
                cb(err);
            }
        );
    };

    PopIt.prototype.setting = function setting (key) {
        var settings = this._get_cached_settings();
        return settings[key] || config.default_settings[key];
    };

    return PopIt;

})();

module.exports = PopIt;<|endoftext|>


<commit_before>'use strict';

var webshot = require('webshot');
var ProgressBar = require('progress');

var path = require('path');
var exec = require('child_process').exec;
var spawn = require('child_process').spawn;
var spawnSync = require('child_process').spawnSync;
var request = require('request');

var logger = require('../lib/logger');

var GitWalker = require('../lib/GitWalker');
var Slideshow = require('../lib/Slideshow');

class GitLapse {
  constructor(config) {
    this.config = Object.assign({
      startRevision: '',
      endRevision: '',
      name: 'Gitlapse Project',
      repo: __dirname,
      steps: 1,
      scripts: {
        setup: 'scripts/setup.sh',
        server: 'scripts/server.sh'
      },
      uri: 'http://localhost:8080',
      timeout: 10000,
      attemptLimit: 30,
      delayBetweenAttemps: 6000,
      output: 'dist',
      window: {
        width: 1280,
        height: 800
      },
      debug: false
    }, config);

    logger.debug = this.config.debug;

    this.gitWalker = new GitWalker(config);
    this.slides = [];
    this.serverAttempts = 0;
  }

  go() {
    this.gitWalker.init((err) => {
      if (err) {
        console.log('There was an error finding those commits.\nHave you set the working directory?\nAre those revisions correct?');
        console.log(err);
        return false;
      }

      // Git repo is prepared, let's check out the commits and build screenshots
      logger.timer('gitlapse');

      this.progressBar = new ProgressBar('  generating [:bar] :percent :current/:total', {
        complete: '=',
        incomplete: ' ',
        width: 20,
        total: this.gitWalker.commits.length
      });

      this._processCommits();
    });

    return this;
  }

  onComplete() {
    var slideshow = new Slideshow(this.slides, {
      output: path.resolve(this.config.output),
      name: this.config.name,
      width: this.config.window.width,
      height: this.config.window.height
    });

    logger.log('Generating slideshow', 'bold');
    logger.timer('slideshow');
    slideshow.save((err) => {
      if (err) throw new Error(err);

      logger.timer('slideshow', true);
      console.log(`All done! Screenshots are in ${path.resolve(this.config.output)}`, 'bgGreen');
      return;
    });

    logger.timer('gitlapse', true);
  }

  checkServerAlive(done) {
    logger.log(`Attempting to connect to server (${this.serverAttempts}/${this.config.attemptLimit})`, 'blue');

    request({ url: this.config.uri, timeout: this.config.timeout }, (err, res, body) => {
      if (!err && res.statusCode == 200) {
        logger.log(`Got OK response from server`, 'green');
        return done(true)
      }
      else {
        logger.log(`Server connection error: ${err}`, 'yellow');
        return done(false);
      }
    });
  }

  pollServer(done) {
    this.serverAttempts++;

    this.checkServerAlive((alive) => {
      if (alive) {
        return done();
      }

      if (this.serverAttempts >= this.config.attemptLimit) {
        return done(`Failed to connect to server ${this.serverAttempts} times, aborting.`);
      }

      // Wait a second and then try again
      setTimeout(() => {
        this.pollServer(done);
      }, this.config.delayBetweenAttemps);
    });
  }

  takeScreenshot(commit, done) {
    let filename = `${path.normalize(this.config.output)}/${commit.hash}.png`;
    webshot(this.config.uri,
      filename,
      {
        screenSize: {
          width: this.config.window.width,
          height: this.config.window.height
        }
      },
      (err) => {
        this.slides.push({
          src: `${commit.hash}.png`,
          caption: commit.hash
        });

        done(filename);
    });
  }

  _processCommits() {
    this.gitWalker.checkoutNextCommit((err, out, commit) => {
      if (out === 'FINISHED') return this.onComplete();

      this.progressBar.tick(1);

      logger.log(`\n\nRunning setup script for ${commit.hash} (${commit.message})`, 'blue');
      logger.timer(commit.hash);

      let setup = spawn('sh', [path.resolve(this.config.scripts.setup)], {
        cwd: this.config.repo,
        stdio: 'pipe',
        encoding: 'utf-8',
        shell: true
      });

      setup.stdout.on('data', (chunk) => {
        logger.log(`SETUP STDOUT: ${chunk}`, 'dim');
      });

      setup.stderr.on('data', (chunk) => {
        logger.log(`SETUP STDERR: ${chunk}`, 'red');
      });

      setup.on('close', (code) => {
        logger.log(`\n\nSetup done. Starting server for ${commit.hash} (${commit.message})`, 'blue');

        let server = spawn('sh', [path.resolve(this.config.scripts.server)], {
          cwd: this.config.repo,
          stdio: 'pipe',
          encoding: 'utf-8',
          detached: true
        });

        server.stdout.on('data', (chunk) => {
          logger.log(`SERVER STDOUT: ${chunk}`, 'dim');
        });

        server.stderr.on('data', (chunk) => {
          logger.log(`SERVER STDOUT: ${chunk}`, 'red');
        });

        // Wait for the web server to come online within specified timeout
        this.pollServer((err) => {
          if (err) throw new Error(err);

          this.takeScreenshot(commit, (filename) => {
            logger.timer(commit.hash, true);
            logger.log(`Screenshot saved as ${filename}`, 'yellow');
            logger.log(`Shutting down server.\n`, 'yellow');

            if (process.platform != "win32") {
              spawnSync('sh', ['-c', 'kill -INT -'+server.pid]);
            }
            else {
              server.kill("SIGINT");
            }

            this._processCommits();
          });
        });
      });
    });
  }
}

module.exports = GitLapse;
<commit_msg>Reset server attempt count after each commit checkout<commit_after>'use strict';

var webshot = require('webshot');
var ProgressBar = require('progress');

var path = require('path');
var exec = require('child_process').exec;
var spawn = require('child_process').spawn;
var spawnSync = require('child_process').spawnSync;
var request = require('request');

var logger = require('../lib/logger');

var GitWalker = require('../lib/GitWalker');
var Slideshow = require('../lib/Slideshow');

class GitLapse {
  constructor(config) {
    this.config = Object.assign({
      startRevision: '',
      endRevision: '',
      name: 'Gitlapse Project',
      repo: __dirname,
      steps: 1,
      scripts: {
        setup: 'scripts/setup.sh',
        server: 'scripts/server.sh'
      },
      uri: 'http://localhost:8080',
      timeout: 10000,
      attemptLimit: 30,
      delayBetweenAttemps: 6000,
      output: 'dist',
      window: {
        width: 1280,
        height: 800
      },
      debug: false
    }, config);

    logger.debug = this.config.debug;

    this.gitWalker = new GitWalker(config);
    this.slides = [];
    this.serverAttempts = 0;
  }

  go() {
    this.gitWalker.init((err) => {
      if (err) {
        console.log('There was an error finding those commits.\nHave you set the working directory?\nAre those revisions correct?');
        console.log(err);
        return false;
      }

      // Git repo is prepared, let's check out the commits and build screenshots
      logger.timer('gitlapse');

      this.progressBar = new ProgressBar('  generating [:bar] :percent :current/:total', {
        complete: '=',
        incomplete: ' ',
        width: 20,
        total: this.gitWalker.commits.length
      });

      this._processCommits();
    });

    return this;
  }

  onComplete() {
    var slideshow = new Slideshow(this.slides, {
      output: path.resolve(this.config.output),
      name: this.config.name,
      width: this.config.window.width,
      height: this.config.window.height
    });

    logger.log('Generating slideshow', 'bold');
    logger.timer('slideshow');
    slideshow.save((err) => {
      if (err) throw new Error(err);

      logger.timer('slideshow', true);
      console.log(`All done! Screenshots are in ${path.resolve(this.config.output)}`, 'bgGreen');
      return;
    });

    logger.timer('gitlapse', true);
  }

  checkServerAlive(done) {
    logger.log(`Attempting to connect to server (${this.serverAttempts}/${this.config.attemptLimit})`, 'blue');

    request({ url: this.config.uri, timeout: this.config.timeout }, (err, res, body) => {
      if (!err && res.statusCode == 200) {
        logger.log(`Got OK response from server`, 'green');
        return done(true)
      }
      else {
        logger.log(`Server connection error: ${err}`, 'yellow');
        return done(false);
      }
    });
  }

  pollServer(done) {
    this.serverAttempts++;

    this.checkServerAlive((alive) => {
      if (alive) {
        return done();
      }

      if (this.serverAttempts >= this.config.attemptLimit) {
        return done(`Failed to connect to server ${this.serverAttempts} times, aborting.`);
      }

      // Wait a second and then try again
      setTimeout(() => {
        this.pollServer(done);
      }, this.config.delayBetweenAttemps);
    });
  }

  takeScreenshot(commit, done) {
    let filename = `${path.normalize(this.config.output)}/${commit.hash}.png`;
    webshot(this.config.uri,
      filename,
      {
        screenSize: {
          width: this.config.window.width,
          height: this.config.window.height
        }
      },
      (err) => {
        this.slides.push({
          src: `${commit.hash}.png`,
          caption: commit.hash
        });

        done(filename);
    });
  }

  _processCommits() {
    this.gitWalker.checkoutNextCommit((err, out, commit) => {
      if (out === 'FINISHED') return this.onComplete();

      this.progressBar.tick(1);

      logger.log(`\n\nRunning setup script for ${commit.hash} (${commit.message})`, 'blue');
      logger.timer(commit.hash);

      let setup = spawn('sh', [path.resolve(this.config.scripts.setup)], {
        cwd: this.config.repo,
        stdio: 'pipe',
        encoding: 'utf-8',
        shell: true
      });

      setup.stdout.on('data', (chunk) => {
        logger.log(`SETUP STDOUT: ${chunk}`, 'dim');
      });

      setup.stderr.on('data', (chunk) => {
        logger.log(`SETUP STDERR: ${chunk}`, 'red');
      });

      setup.on('close', (code) => {
        logger.log(`\n\nSetup done. Starting server for ${commit.hash} (${commit.message})`, 'blue');

        let server = spawn('sh', [path.resolve(this.config.scripts.server)], {
          cwd: this.config.repo,
          stdio: 'pipe',
          encoding: 'utf-8',
          detached: true
        });

        server.stdout.on('data', (chunk) => {
          logger.log(`SERVER STDOUT: ${chunk}`, 'dim');
        });

        server.stderr.on('data', (chunk) => {
          logger.log(`SERVER STDOUT: ${chunk}`, 'red');
        });

        // Wait for the web server to come online within specified timeout
        this.serverAttempts = 0;
        this.pollServer((err) => {
          if (err) throw new Error(err);

          this.takeScreenshot(commit, (filename) => {
            logger.timer(commit.hash, true);
            logger.log(`Screenshot saved as ${filename}`, 'yellow');
            logger.log(`Shutting down server.\n`, 'yellow');

            if (process.platform != "win32") {
              spawnSync('sh', ['-c', 'kill -INT -'+server.pid]);
            }
            else {
              server.kill("SIGINT");
            }

            this._processCommits();
          });
        });
      });
    });
  }
}

module.exports = GitLapse;
<|endoftext|>


<commit_before>var gulp   = require('gulp');
var jshint = require('gulp-jshint');
var gulp   = require('gulp');
var jscs = require('gulp-jscs');
var exec = require('child_process').exec;

var scripts = [
    './**/*.js',
    '!./node_modules/**/*.js',
    '!./coverage/**/*.js'
];

gulp.task('lint', function() {
    return gulp.src(scripts)
        .pipe(jshint('./.jshintrc'))
        .pipe(jshint.reporter('jshint-stylish'))
        .pipe(jshint.reporter('fail'))
        .pipe(jscs());
});

gulp.task('test', function(cb) {
    exec('npm test', function(err, stdout, stderr) {
        console.log(stdout);
        console.log(stderr);
        cb(err);
    });
});

gulp.task('testcoverage', function(cb) {
    exec('rm -rf coverage && node_modules/.bin/istanbul cover --report lcov --report html node_modules/mocha/bin/_mocha -- --reporter dot', function(err, stdout, stderr) {
        console.log(stdout);
        console.log(stderr);
        cb(err);
    });
});

gulp.task('default', [ 'lint' ]);
gulp.task('travis', [ 'lint', 'test' ]);
<commit_msg>Upgrade gulpfile to gulp 4<commit_after>var gulp   = require('gulp');
var jshint = require('gulp-jshint');
var gulp   = require('gulp');
var jscs = require('gulp-jscs');
var exec = require('child_process').exec;

var scripts = [
    './**/*.js',
    '!./node_modules/**/*.js',
    '!./coverage/**/*.js'
];

gulp.task('lint', function() {
    return gulp.src(scripts)
        .pipe(jshint('./.jshintrc'))
        .pipe(jshint.reporter('jshint-stylish'))
        .pipe(jshint.reporter('fail'))
        .pipe(jscs());
});

gulp.task('test', function(cb) {
    exec('npm test', function(err, stdout, stderr) {
        console.log(stdout);
        console.log(stderr);
        cb(err);
    });
});

gulp.task('testcoverage', function(cb) {
    exec('rm -rf coverage && node_modules/.bin/nyc cover --report lcov --report html node_modules/mocha/bin/_mocha -- --reporter dot', function(err, stdout, stderr) {
        console.log(stdout);
        console.log(stderr);
        cb(err);
    });
});

gulp.task('default', gulp.series('lint'));
gulp.task('travis', gulp.series('lint', 'test'));
<|endoftext|>