<commit_before>/**
   MiracleGrue - Model Generator for toolpathing. <http://www.grue.makerbot.com>
   Copyright (C) 2011 Far McKon <Far@makerbot.com>, Hugo Boyer (hugo@makerbot.com)

   This program is free software: you can redistribute it and/or modify
   it under the terms of the GNU Affero General Public License as
   published by the Free Software Foundation, either version 3 of the
   License, or (at your option) any later version.

 */

#include <list>
#include <vector>

#include "pather.h"
#include "limits.h"
#include "pather_optimizer_graph.h"
#include "pather_optimizer_fastgraph.h"
#include "dump_restore.h"

namespace mgl {
using namespace std;

Pather::Pather(const PatherConfig& pCfg, ProgressBar* progress)
		: Progressive(progress), patherCfg(pCfg) {}
Pather::Pather(const GrueConfig& grueConf, ProgressBar* progress)
        : Progressive(progress) {
    patherCfg.doGraphOptimization = grueConf.get_doGraphOptimization();
    patherCfg.coarseness = grueConf.get_coarseness();
    patherCfg.directionWeight = grueConf.get_directionWeight();
}

void Pather::generatePaths(const GrueConfig& grueCfg,
		const RegionList &skeleton,
		const LayerMeasure &layerMeasure,
		const Grid &grid,
		LayerPaths &layerpaths,
		int sfirstSliceIdx, // =-1
		int slastSliceIdx) //
{
	size_t firstSliceIdx = 0;
	size_t lastSliceIdx = INT_MAX;

	if (sfirstSliceIdx > 0) {
		firstSliceIdx = (size_t) sfirstSliceIdx;
	}

	if (slastSliceIdx > 0) {
		lastSliceIdx = (size_t) slastSliceIdx;
	}

	bool direction = false;
	unsigned int currentSlice = 0;

	initProgress("Path generation", skeleton.size());

    abstract_optimizer* optimizer = NULL;
    if(grueCfg.get_doGraphOptimization()) {
        optimizer = new pather_optimizer_fastgraph(grueCfg);
    } else {
        optimizer = new pather_optimizer();
    }

	for (RegionList::const_iterator layerRegions = skeleton.begin();
			layerRegions != skeleton.end(); ++layerRegions) {
		tick();
        try {
		if (currentSlice < firstSliceIdx) continue;
		if (currentSlice > lastSliceIdx) break;
        if(grueCfg.get_doRaft() && currentSlice > 1 &&
                currentSlice < grueCfg.get_raftLayers() &&
                grueCfg.get_raftAligned()) {
            //don't flip direction
        } else {
            direction = !direction;
        }
		const layer_measure_index_t layerMeasureId =
				layerRegions->layerMeasureId;

		//adding these should be handled in gcoder
		const Scalar z = layerMeasure.getLayerPosition(layerMeasureId);
		const Scalar h = layerMeasure.getLayerThickness(layerMeasureId);
		const Scalar w = layerMeasure.getLayerWidth(layerMeasureId);

		layerpaths.push_back(LayerPaths::Layer(z, h, w, layerMeasureId));

		LayerPaths::Layer& lp_layer = layerpaths.back();

		//TODO: this only handles the case where the user specifies the extruder
		// it does not handle a dualstrusion print
		lp_layer.extruders.push_back(
				LayerPaths::Layer::ExtruderLayer(grueCfg.get_defaultExtruder()));
		LayerPaths::Layer::ExtruderLayer& extruderlayer =
				lp_layer.extruders.back();

//        Json::Value spurLoops;
//        for(std::list<LoopList>::const_iterator depthIter =
//                layerRegions->spurLoops.begin();
//                depthIter != layerRegions->spurLoops.end();
//                ++depthIter) {
//            dumpLoopList(*depthIter, spurLoops);
//        }
//        std::cerr << Json::FastWriter().write(spurLoops);

		optimizer->clearBoundaries();
        optimizer->clearPaths();

		const std::list<LoopList>& insetLoops = layerRegions->insetLoops;
		const std::list<OpenPathList>& spurPaths = layerRegions->spurs;

        if(grueCfg.get_doOutlines()) {
            for(LoopList::const_iterator iter = layerRegions->outlines.begin();
                    iter != layerRegions->outlines.end();
                    ++iter) {
                const LoopPath outlinePath(*iter, iter->clockwise(),
                        iter->counterClockwise());
                extruderlayer.paths.push_back(PathLabel(PathLabel::TYP_OUTLINE,
                        PathLabel::OWN_MODEL));
                OpenPath& path = extruderlayer.paths.back().myPath;
                for(LoopPath::const_iterator pointIter = outlinePath.fromStart();
                        pointIter != outlinePath.end();
                        ++pointIter) {
                    path.appendPoint(*pointIter);
                }
            }
            for(LoopList::const_iterator iter = layerRegions->supportLoops.begin();
                    iter != layerRegions->supportLoops.end();
                    ++iter) {
                const LoopPath outlinePath(*iter, iter->clockwise(),
                        iter->counterClockwise());
                extruderlayer.paths.push_back(PathLabel(PathLabel::TYP_OUTLINE,
                        PathLabel::OWN_SUPPORT));
                OpenPath& path = extruderlayer.paths.back().myPath;
                for(LoopPath::const_iterator pointIter = outlinePath.fromStart();
                        pointIter != outlinePath.end();
                        ++pointIter) {
                    path.appendPoint(*pointIter);
                }
            }
        }

		optimizer->addBoundaries(layerRegions->outlines);

        bool hasInfill = grueCfg.get_doInfills() &&
                grueCfg.get_infillDensity() > 0;
        bool hasSolidLayers = grueCfg.get_roofLayerCount() > 0 ||
                grueCfg.get_floorLayerCount() > 0;

        if(!hasInfill && !hasSolidLayers) {
            optimizer->addBoundaries(layerRegions->interiorLoops);
        }

        const GridRanges& infillRanges = layerRegions->infill;

		const std::vector<Scalar>& values =
				!direction ? grid.getXValues() : grid.getYValues();
		axis_e axis = direction ? X_AXIS : Y_AXIS;

        if(grueCfg.get_doRaft() || grueCfg.get_doSupport()) {
            LoopList outsetSupportLoops;
            loopsOffset(outsetSupportLoops, layerRegions->supportLoops,
                    0.01);
            optimizer->addBoundaries(outsetSupportLoops);

            const GridRanges& supportRanges = layerRegions->support;
            OpenPathList supportPaths;
            grid.gridRangesToOpenPaths(
                    direction ? supportRanges.xRays : supportRanges.yRays,
                    values,
                    axis,
                    supportPaths);
            optimizer->addPaths(supportPaths, PathLabel(PathLabel::TYP_INFILL,
                    PathLabel::OWN_SUPPORT, 0));
        }
		if(grueCfg.get_doInsets()) {
            int currentShell = LayerPaths::Layer::ExtruderLayer::INSET_LABEL_VALUE;
            for(std::list<LoopList>::const_iterator listIter = insetLoops.begin();
                    listIter != insetLoops.end();
                    ++listIter) {
                int shellVal = currentShell;
                optimizer->addPaths(*listIter,
                        PathLabel(PathLabel::TYP_INSET,
                        PathLabel::OWN_MODEL, shellVal));
                ++currentShell;
            }

            currentShell = LayerPaths::Layer::ExtruderLayer::INSET_LABEL_VALUE;
            for(std::list<OpenPathList>::const_iterator spurIter = spurPaths.begin();
                spurIter != spurPaths.end();
                    ++spurIter) {
                int shellVal = currentShell;
                optimizer->addPaths(*spurIter,
                        PathLabel(PathLabel::TYP_INSET,
                        PathLabel::OWN_MODEL, shellVal));
                ++currentShell;
            }
        }

		OpenPathList infillPaths;
		grid.gridRangesToOpenPaths(
				direction ? infillRanges.xRays : infillRanges.yRays,
				values,
				axis,
				infillPaths);

		LabeledOpenPaths preoptimized;

        if(grueCfg.get_doInfills()) {
            optimizer->addPaths(infillPaths, PathLabel(PathLabel::TYP_INFILL,
                    PathLabel::OWN_MODEL,
                    LayerPaths::Layer::ExtruderLayer::INFILL_LABEL_VALUE));
        }
        optimizer->optimize(preoptimized);
        smoothCollection(preoptimized, grueCfg.get_coarseness(),
                grueCfg.get_directionWeight());
        cleanPaths(preoptimized);
        smoothCollection(preoptimized, grueCfg.get_coarseness(),
                grueCfg.get_directionWeight());

        extruderlayer.paths.insert(extruderlayer.paths.end(),
                preoptimized.begin(), preoptimized.end());
        } catch (const std::exception& our) {
            std::cout << "Error " << our.what() << " on layer " <<
                    currentSlice << std::endl;
        }
		++currentSlice;
	}
    delete optimizer;
}

void Pather::cleanPaths(LabeledOpenPaths& result) {
    std::vector<LabeledOpenPaths::iterator> eraseMe;
    typedef LabeledOpenPaths::iterator iterator;
    if(result.empty())
        return;
    iterator current = result.begin();
    iterator next = current;
    for(++next;
            next != result.end();
            ++current, ++next) {
        const Scalar dropThreshold = patherCfg.coarseness * 0.5;
        while(current != result.end() && current->myPath.distance() <
                dropThreshold) {
            current = result.erase(current);
            next = current;
            ++next;
        }
        while(next != result.end() && next->myPath.distance() <
                dropThreshold) {
            next = result.erase(next);
        }
        if(current == result.end() || next == result.end())
            break;
        Point2Type currentStart = *(current->myPath.fromStart());
        Point2Type currentEnd = *(current->myPath.fromEnd());
        Point2Type nextStart = *(next->myPath.fromStart());
        Point2Type nextEnd = *(next->myPath.fromEnd());
        if((currentEnd - nextStart).squaredMagnitude() >
                (patherCfg.coarseness * patherCfg.coarseness)) { //separate paths
            continue;
        }
        if((current->myLabel.isConnection() ||
                current->myLabel.isInset()) &&
                (next->myLabel.isConnection() ||
                next->myLabel.isInset())) {
            //we have adjacent paths of the correct types
            if((currentStart == currentEnd && current->myPath.size() > 2) ||
                    (nextStart == nextEnd && next->myPath.size() > 2)) //one is an inset, don't join
                continue;
            OpenPath::iterator nextPoint = next->myPath.fromStart();
            ++nextPoint;
            current->myPath.appendPoints(nextPoint, next->myPath.end());
            if(current->myLabel.isInset()) {
                next->myLabel = current->myLabel;
            }
            next->myPath = current->myPath;
            eraseMe.push_back(current);
        }
    }
    while(!eraseMe.empty()) {
        result.erase(eraseMe.back());
        eraseMe.pop_back();
    }
}

}
<commit_msg>Reduce level of filtering to fix mangled paths<commit_after>/**
   MiracleGrue - Model Generator for toolpathing. <http://www.grue.makerbot.com>
   Copyright (C) 2011 Far McKon <Far@makerbot.com>, Hugo Boyer (hugo@makerbot.com)

   This program is free software: you can redistribute it and/or modify
   it under the terms of the GNU Affero General Public License as
   published by the Free Software Foundation, either version 3 of the
   License, or (at your option) any later version.

    MiracleGrue - Toolpath generator for 3D printing.
    Copyright (C) 2012 Joseph Sadusk <jsadusk@makerbot.com>, Filipp Gelman <filipp@makerbot.com>

    This program is free software: you can redistribute it and/or modify
    it under the terms of the GNU Affero General Public License as
    published by the Free Software Foundation, either version 3 of the
    License, or (at your option) any later version.

    This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU Affero General Public License for more details.

    You should have received a copy of the GNU Affero General Public License
    along with this program.  If not, see <http://www.gnu.org/licenses/>.

 */

#include <list>
#include <vector>

#include "pather.h"
#include "limits.h"
#include "pather_optimizer_graph.h"
#include "pather_optimizer_fastgraph.h"
#include "dump_restore.h"

namespace mgl {
using namespace std;

Pather::Pather(const PatherConfig& pCfg, ProgressBar* progress)
		: Progressive(progress), patherCfg(pCfg) {}
Pather::Pather(const GrueConfig& grueConf, ProgressBar* progress)
        : Progressive(progress) {
    patherCfg.doGraphOptimization = grueConf.get_doGraphOptimization();
    patherCfg.coarseness = grueConf.get_coarseness();
    patherCfg.directionWeight = grueConf.get_directionWeight();
}

void Pather::generatePaths(const GrueConfig& grueCfg,
		const RegionList &skeleton,
		const LayerMeasure &layerMeasure,
		const Grid &grid,
		LayerPaths &layerpaths,
		int sfirstSliceIdx, // =-1
		int slastSliceIdx) //
{
	size_t firstSliceIdx = 0;
	size_t lastSliceIdx = INT_MAX;

	if (sfirstSliceIdx > 0) {
		firstSliceIdx = (size_t) sfirstSliceIdx;
	}

	if (slastSliceIdx > 0) {
		lastSliceIdx = (size_t) slastSliceIdx;
	}

	bool direction = false;
	unsigned int currentSlice = 0;

	initProgress("Path generation", skeleton.size());

    abstract_optimizer* optimizer = NULL;
    if(grueCfg.get_doGraphOptimization()) {
        optimizer = new pather_optimizer_fastgraph(grueCfg);
    } else {
        optimizer = new pather_optimizer();
    }

	for (RegionList::const_iterator layerRegions = skeleton.begin();
			layerRegions != skeleton.end(); ++layerRegions) {
		tick();
        try {
		if (currentSlice < firstSliceIdx) continue;
		if (currentSlice > lastSliceIdx) break;
        if(grueCfg.get_doRaft() && currentSlice > 1 &&
                currentSlice < grueCfg.get_raftLayers() &&
                grueCfg.get_raftAligned()) {
            //don't flip direction
        } else {
            direction = !direction;
        }
		const layer_measure_index_t layerMeasureId =
				layerRegions->layerMeasureId;

		//adding these should be handled in gcoder
		const Scalar z = layerMeasure.getLayerPosition(layerMeasureId);
		const Scalar h = layerMeasure.getLayerThickness(layerMeasureId);
		const Scalar w = layerMeasure.getLayerWidth(layerMeasureId);

		layerpaths.push_back(LayerPaths::Layer(z, h, w, layerMeasureId));

		LayerPaths::Layer& lp_layer = layerpaths.back();

		//TODO: this only handles the case where the user specifies the extruder
		// it does not handle a dualstrusion print
		lp_layer.extruders.push_back(
				LayerPaths::Layer::ExtruderLayer(grueCfg.get_defaultExtruder()));
		LayerPaths::Layer::ExtruderLayer& extruderlayer =
				lp_layer.extruders.back();

//        Json::Value spurLoops;
//        for(std::list<LoopList>::const_iterator depthIter =
//                layerRegions->spurLoops.begin();
//                depthIter != layerRegions->spurLoops.end();
//                ++depthIter) {
//            dumpLoopList(*depthIter, spurLoops);
//        }
//        std::cerr << Json::FastWriter().write(spurLoops);

		optimizer->clearBoundaries();
        optimizer->clearPaths();

		const std::list<LoopList>& insetLoops = layerRegions->insetLoops;
		const std::list<OpenPathList>& spurPaths = layerRegions->spurs;

        if(grueCfg.get_doOutlines()) {
            for(LoopList::const_iterator iter = layerRegions->outlines.begin();
                    iter != layerRegions->outlines.end();
                    ++iter) {
                const LoopPath outlinePath(*iter, iter->clockwise(),
                        iter->counterClockwise());
                extruderlayer.paths.push_back(PathLabel(PathLabel::TYP_OUTLINE,
                        PathLabel::OWN_MODEL));
                OpenPath& path = extruderlayer.paths.back().myPath;
                for(LoopPath::const_iterator pointIter = outlinePath.fromStart();
                        pointIter != outlinePath.end();
                        ++pointIter) {
                    path.appendPoint(*pointIter);
                }
            }
            for(LoopList::const_iterator iter = layerRegions->supportLoops.begin();
                    iter != layerRegions->supportLoops.end();
                    ++iter) {
                const LoopPath outlinePath(*iter, iter->clockwise(),
                        iter->counterClockwise());
                extruderlayer.paths.push_back(PathLabel(PathLabel::TYP_OUTLINE,
                        PathLabel::OWN_SUPPORT));
                OpenPath& path = extruderlayer.paths.back().myPath;
                for(LoopPath::const_iterator pointIter = outlinePath.fromStart();
                        pointIter != outlinePath.end();
                        ++pointIter) {
                    path.appendPoint(*pointIter);
                }
            }
        }

		optimizer->addBoundaries(layerRegions->outlines);

        bool hasInfill = grueCfg.get_doInfills() &&
                grueCfg.get_infillDensity() > 0;
        bool hasSolidLayers = grueCfg.get_roofLayerCount() > 0 ||
                grueCfg.get_floorLayerCount() > 0;

        if(!hasInfill && !hasSolidLayers) {
            optimizer->addBoundaries(layerRegions->interiorLoops);
        }

        const GridRanges& infillRanges = layerRegions->infill;

		const std::vector<Scalar>& values =
				!direction ? grid.getXValues() : grid.getYValues();
		axis_e axis = direction ? X_AXIS : Y_AXIS;

        if(grueCfg.get_doRaft() || grueCfg.get_doSupport()) {
            LoopList outsetSupportLoops;
            loopsOffset(outsetSupportLoops, layerRegions->supportLoops,
                    0.01);
            optimizer->addBoundaries(outsetSupportLoops);

            const GridRanges& supportRanges = layerRegions->support;
            OpenPathList supportPaths;
            grid.gridRangesToOpenPaths(
                    direction ? supportRanges.xRays : supportRanges.yRays,
                    values,
                    axis,
                    supportPaths);
            optimizer->addPaths(supportPaths, PathLabel(PathLabel::TYP_INFILL,
                    PathLabel::OWN_SUPPORT, 0));
        }
		if(grueCfg.get_doInsets()) {
            int currentShell = LayerPaths::Layer::ExtruderLayer::INSET_LABEL_VALUE;
            for(std::list<LoopList>::const_iterator listIter = insetLoops.begin();
                    listIter != insetLoops.end();
                    ++listIter) {
                int shellVal = currentShell;
                optimizer->addPaths(*listIter,
                        PathLabel(PathLabel::TYP_INSET,
                        PathLabel::OWN_MODEL, shellVal));
                ++currentShell;
            }

            currentShell = LayerPaths::Layer::ExtruderLayer::INSET_LABEL_VALUE;
            for(std::list<OpenPathList>::const_iterator spurIter = spurPaths.begin();
                spurIter != spurPaths.end();
                    ++spurIter) {
                int shellVal = currentShell;
                optimizer->addPaths(*spurIter,
                        PathLabel(PathLabel::TYP_INSET,
                        PathLabel::OWN_MODEL, shellVal));
                ++currentShell;
            }
        }

		OpenPathList infillPaths;
		grid.gridRangesToOpenPaths(
				direction ? infillRanges.xRays : infillRanges.yRays,
				values,
				axis,
				infillPaths);

		LabeledOpenPaths preoptimized;

        if(grueCfg.get_doInfills()) {
            optimizer->addPaths(infillPaths, PathLabel(PathLabel::TYP_INFILL,
                    PathLabel::OWN_MODEL,
                    LayerPaths::Layer::ExtruderLayer::INFILL_LABEL_VALUE));
        }
        optimizer->optimize(preoptimized);
//        smoothCollection(preoptimized, grueCfg.get_coarseness(),
//                grueCfg.get_directionWeight());
        cleanPaths(preoptimized);
        smoothCollection(preoptimized, grueCfg.get_coarseness(),
                grueCfg.get_directionWeight());

        extruderlayer.paths.insert(extruderlayer.paths.end(),
                preoptimized.begin(), preoptimized.end());
        } catch (const std::exception& our) {
            std::cout << "Error " << our.what() << " on layer " <<
                    currentSlice << std::endl;
        }
		++currentSlice;
	}
    delete optimizer;
}

void Pather::cleanPaths(LabeledOpenPaths& result) {
    std::vector<LabeledOpenPaths::iterator> eraseMe;
    typedef LabeledOpenPaths::iterator iterator;
    if(result.empty())
        return;
    iterator current = result.begin();
    iterator next = current;
    for(++next;
            next != result.end();
            ++current, ++next) {
        if(false) {
            //paths below dropThreshold are too short to be valid.
            //even connections must be longer, so we drop them
            const Scalar dropThreshold = patherCfg.coarseness * 0.5;
            //keep dropping current until length above threshold
            while(current != result.end() && current->myPath.distance() <
                    dropThreshold) {
                current = result.erase(current);
                next = current;
                ++next;
            }
            //keep dropping next until length above threshold
            while(next != result.end() && next->myPath.distance() <
                    dropThreshold) {
                next = result.erase(next);
            }
            if(current == result.end() || next == result.end())
                break;
        }
        Point2Type currentStart = *(current->myPath.fromStart());
        Point2Type currentEnd = *(current->myPath.fromEnd());
        Point2Type nextStart = *(next->myPath.fromStart());
        Point2Type nextEnd = *(next->myPath.fromEnd());
        if((currentEnd - nextStart).squaredMagnitude() >
                (patherCfg.coarseness * patherCfg.coarseness)) { //separate paths
            continue;
        }
        //here we only join spurs and connections
        if((current->myLabel.isConnection() ||
                current->myLabel.isInset()) &&
                (next->myLabel.isConnection() ||
                next->myLabel.isInset())) {
            //we have adjacent paths of the correct types
            if((currentStart == currentEnd && current->myPath.size() > 2) ||
                    (nextStart == nextEnd && next->myPath.size() > 2)) //one is an inset, don't join
                continue;
            OpenPath::iterator nextPoint = next->myPath.fromStart();
            ++nextPoint;
            current->myPath.appendPoints(nextPoint, next->myPath.end());
            if(current->myLabel.isInset()) {
                next->myLabel = current->myLabel;
            }
            next->myPath = current->myPath;
            eraseMe.push_back(current);
        }
    }
    while(!eraseMe.empty()) {
        result.erase(eraseMe.back());
        eraseMe.pop_back();
    }
}

}
<|endoftext|>


<commit_before><commit_msg>Minor comment changes.<commit_after><|endoftext|>


<commit_before>/**
 * This file is part of the CernVM File System.
 */

#include "config.h"
#include "auto_umount.h"

#include <unistd.h>
#include <errno.h>
#include <sys/types.h>
#include <dirent.h>

#include <string>
#include <vector>
#include <set>
#include <map>

#include "platform.h"
#include "logging.h"
#include "util.h"

using namespace std;

namespace auto_umount {

string *mountpoint_ = NULL;

void SetMountpoint(const string &mountpoint) {
  if (mountpoint == "") {
    delete mountpoint_;
    mountpoint_ = NULL;
  } else {
    mountpoint_ = new string(mountpoint);
  }
}

void UmountOnCrash() {
  if (!mountpoint_) {
    LogCvmfs(kLogCvmfs, kLogSyslogErr, "crash cleanup handler: no mountpoint");
    return;
  }

  std::vector<std::string> all_mountpoints = platform_mountlist();
  if (all_mountpoints.empty()) {
    LogCvmfs(kLogCvmfs, kLogSyslogErr, "crash cleanup handler: "
             "failed to read mount point list");
    return;
  }

  // Mitigate auto-mount - crash - umount - auto-mount loops
  SafeSleepMs(2000);

  // Check if *mountpoint_ is still mounted
  // (we don't want to trigger a mount by immediately doing stat *mountpoint_)
  bool still_mounted = false;
  for (unsigned i = 0; i < all_mountpoints.size(); ++i) {
    if (*mountpoint_ == all_mountpoints[i]) {
      still_mounted = true;
      break;
    }
  }
  if (!still_mounted) {
    LogCvmfs(kLogCvmfs, kLogDebug, "crash cleanup handler: %s not mounted",
             mountpoint_->c_str());
    return;
  }

  // stat() might be served from caches.  Opendir ensures fuse module is called.
  DIR *dirp = opendir(mountpoint_->c_str());
  if (dirp || (errno != ENOTCONN)) {
    if (dirp) closedir(dirp);
    LogCvmfs(kLogCvmfs, kLogSyslog, "crash cleanup handler: "
             "%s seems not to be stalled (%d)", mountpoint_->c_str(), errno);
    return;
  }

  // sudo umount -l *mountpoint_
  if (!SwitchCredentials(0, getegid(), true)) {
    LogCvmfs(kLogCvmfs, kLogSyslogErr, "crash cleanup handler: "
             "failed to re-gain root privileges");
    return;
  }
  const bool lazy = true;
  bool retval = platform_umount(mountpoint_->c_str(), lazy);
  if (!retval) {
    LogCvmfs(kLogCvmfs, kLogSyslogErr, "crash cleanup handler: "
             "failed to unmount %s", mountpoint_->c_str());
    return;
  }

  LogCvmfs(kLogCvmfs, kLogSyslog, "crash cleanup handler: unmounted %s",
           mountpoint_->c_str());
}

}  // namespace auto_umount
<commit_msg>cosmetics<commit_after>/**
 * This file is part of the CernVM File System.
 */

#include "config.h"
#include "auto_umount.h"

#include <unistd.h>
#include <errno.h>
#include <sys/types.h>
#include <dirent.h>

#include <string>
#include <vector>
#include <set>
#include <map>

#include "platform.h"
#include "logging.h"
#include "util.h"

using namespace std;

namespace auto_umount {

string *mountpoint_ = NULL;

void SetMountpoint(const string &mountpoint) {
  if (mountpoint == "") {
    delete mountpoint_;
    mountpoint_ = NULL;
  } else {
    mountpoint_ = new string(mountpoint);
  }
}

void UmountOnCrash() {
  if (!mountpoint_) {
    LogCvmfs(kLogCvmfs, kLogSyslogErr, "crash cleanup handler: no mountpoint");
    return;
  }

  std::vector<std::string> all_mountpoints = platform_mountlist();
  if (all_mountpoints.empty()) {
    LogCvmfs(kLogCvmfs, kLogSyslogErr, "crash cleanup handler: "
             "failed to read mount point list");
    return;
  }

  // Mitigate auto-mount - crash - umount - auto-mount loops
  SafeSleepMs(2000);

  // Check if *mountpoint_ is still mounted
  // (we don't want to trigger a mount by immediately doing stat *mountpoint_)
  bool still_mounted = false;
  for (unsigned i = 0; i < all_mountpoints.size(); ++i) {
    if (*mountpoint_ == all_mountpoints[i]) {
      still_mounted = true;
      break;
    }
  }
  if (!still_mounted) {
    LogCvmfs(kLogCvmfs, kLogDebug, "crash cleanup handler: %s not mounted",
             mountpoint_->c_str());
    return;
  }

  // stat() might be served from caches.  Opendir ensures fuse module is called.
  DIR *dirp = opendir(mountpoint_->c_str());
  if (dirp || (errno != ENOTCONN)) {
    if (dirp) closedir(dirp);
    LogCvmfs(kLogCvmfs, kLogSyslog, "crash cleanup handler: "
             "%s seems not to be stalled (%d)", mountpoint_->c_str(), errno);
    return;
  }

  // sudo umount -l *mountpoint_
  if (!SwitchCredentials(0, getegid(), true)) {
    LogCvmfs(kLogCvmfs, kLogSyslogErr, "crash cleanup handler: "
             "failed to re-gain root privileges");
    return;
  }
  const bool lazy = true;
  bool retval = platform_umount(mountpoint_->c_str(), lazy);
  if (!retval) {
    LogCvmfs(kLogCvmfs, kLogSyslogErr, "crash cleanup handler: "
             "failed to unmount %s", mountpoint_->c_str());
    return;
  }

  LogCvmfs(kLogCvmfs, kLogSyslog, "crash cleanup handler unmounted stalled %s",
           mountpoint_->c_str());
}

}  // namespace auto_umount
<|endoftext|>


<commit_before>#include <vector>

#include "mrmc.h"
#include "typedbytes.h"

void MatrixHandler::read_full_row(std::vector<double>& row) {
  row.clear();
  TypedBytesType code = in_.next_type();
  typedbytes_length len;
  TypedBytesType nexttype;
  switch (code) {
  case TypedBytesVector:
    len = in_.read_typedbytes_sequence_length();
    row.reserve((size_t)len);
    for (size_t i = 0; i < (size_t) len; ++i) {
      nexttype = in_.next_type();
      if (in_.can_be_double(nexttype)) {
	row.push_back(in_.convert_double());
      } else {
	hadoop_error("row %zi, col %zi has a non-double-convertable type\n",
		     num_total_rows_, row.size());
      }
    }
    break;
  case TypedBytesByteSequence:
    len = in_.read_byte_sequence_length();
    row.resize((size_t) len / sizeof(double));
    in_.read_byte_sequence((unsigned char *) &row[0], len);
    break;
  case TypedBytesList:
    hadoop_message("TypedBytesList!\n");
    nexttype = in_.next_type();
    while (nexttype != TypedBytesListEnd) {
      if (in_.can_be_double(nexttype)) {
	row.push_back(in_.convert_double());
      } else {
	hadoop_message("row has a non-double-convertable type!!\n");
	hadoop_error("row %zi, col %zi has a non-double-convertable type\n",
		     num_total_rows_, row.size());
      }
      nexttype = in_.next_type();
    }
    break;
  case TypedBytesString:
    len = in_.read_string_length();
    row.resize(len / 8);
    in_.read_string_data((unsigned char *) &row[0], (size_t) len);
    break;
  default:
    hadoop_error("row %zi is an unknown type (code is: %d)\n",
		 num_total_rows_, code);
  }
}

bool MatrixHandler::read_key_val_pair(typedbytes_opaque& key,
				      std::vector<double>& value) {
  if (!in_.read_opaque(key)) {
    return false;
  }
  read_full_row(value);
  return true;
}

void MatrixHandler::mapper() {
  std::vector<double> row;
  first_row();
  while (!feof(in_.get_stream())) {
    typedbytes_opaque key;
    if (!read_key_val_pair(key, row)) {
      if (feof(in_.get_stream())) {
	break;
      } else {
	hadoop_error("invalid key: row %i\n", num_total_rows_);
      }
    }
    collect(key, row);
  }
  hadoop_status("final output");
  output();
}

// Allocate the local matrix and set to zero
void MatrixHandler::alloc(size_t num_rows, size_t num_cols) {
  local_matrix_.resize(num_rows * num_cols);
  for (size_t i = 0; i < num_rows * num_cols; ++i) {
    local_matrix_[i] = 0.;
  }
  num_rows_ = num_rows;
  num_cols_ = num_cols;
  num_local_rows_ = 0;
}

// Handle the first input row.  We use the first row to gather data
// about the matrix.
void MatrixHandler::first_row() {
  typedbytes_opaque key;
  std::vector<double> row;
  read_key_val_pair(key, row);
  // TODO(arbenson) check for error here
  num_cols_ = row.size();
  hadoop_message("matrix size: %zi num_cols_, up to %i localrows\n",
		 num_cols_, blocksize_ * num_cols_);
  alloc(blocksize_ * num_cols_, num_cols_);
  add_row(row);
}

// read in a row and add it to the local matrix
void MatrixHandler::add_row(const std::vector<double>& row) {
  assert(row.size() == num_cols_);
  assert(num_local_rows_ < num_rows_);
  // store by column
  for (size_t k = 0; k < rows_per_record_; ++k) {
    size_t i = 0;
    for (size_t j = 0; j < num_cols_; ++j) {
      local_matrix_[num_local_rows_ + j * num_rows_] = row[i];
      ++i;
    }
    // increment the number of local rows
    ++num_local_rows_;
    ++num_total_rows_;
  }
}

<commit_msg>handle the case where no data is passed to the task<commit_after>#include <vector>

#include "mrmc.h"
#include "typedbytes.h"

void MatrixHandler::read_full_row(std::vector<double>& row) {
  row.clear();
  TypedBytesType code = in_.next_type();
  typedbytes_length len;
  TypedBytesType nexttype;
  switch (code) {
  case TypedBytesVector:
    len = in_.read_typedbytes_sequence_length();
    row.reserve((size_t)len);
    for (size_t i = 0; i < (size_t) len; ++i) {
      nexttype = in_.next_type();
      if (in_.can_be_double(nexttype)) {
	row.push_back(in_.convert_double());
      } else {
	hadoop_error("row %zi, col %zi has a non-double-convertable type\n",
		     num_total_rows_, row.size());
      }
    }
    break;
  case TypedBytesByteSequence:
    len = in_.read_byte_sequence_length();
    row.resize((size_t) len / sizeof(double));
    in_.read_byte_sequence((unsigned char *) &row[0], len);
    break;
  case TypedBytesList:
    hadoop_message("TypedBytesList!\n");
    nexttype = in_.next_type();
    while (nexttype != TypedBytesListEnd) {
      if (in_.can_be_double(nexttype)) {
	row.push_back(in_.convert_double());
      } else {
	hadoop_message("row has a non-double-convertable type!!\n");
	hadoop_error("row %zi, col %zi has a non-double-convertable type\n",
		     num_total_rows_, row.size());
      }
      nexttype = in_.next_type();
    }
    break;
  case TypedBytesString:
    len = in_.read_string_length();
    row.resize(len / 8);
    in_.read_string_data((unsigned char *) &row[0], (size_t) len);
    break;
  default:
    hadoop_error("row %zi is an unknown type (code is: %d)\n",
		 num_total_rows_, code);
  }
}

bool MatrixHandler::read_key_val_pair(typedbytes_opaque& key,
				      std::vector<double>& value) {
  if (!in_.read_opaque(key)) {
    return false;
  }
  read_full_row(value);
  return true;
}

void MatrixHandler::mapper() {
  std::vector<double> row;
  first_row();
  while (!feof(in_.get_stream())) {
    typedbytes_opaque key;
    if (!read_key_val_pair(key, row)) {
      if (feof(in_.get_stream())) {
	break;
      } else {
	hadoop_error("invalid key: row %i\n", num_total_rows_);
      }
    }
    collect(key, row);
  }
  hadoop_status("final output");
  output();
}

// Allocate the local matrix and set to zero
void MatrixHandler::alloc(size_t num_rows, size_t num_cols) {
  local_matrix_.resize(num_rows * num_cols);
  for (size_t i = 0; i < num_rows * num_cols; ++i) {
    local_matrix_[i] = 0.;
  }
  num_rows_ = num_rows;
  num_cols_ = num_cols;
  num_local_rows_ = 0;
}

// Handle the first input row.  We use the first row to gather data
// about the matrix.
void MatrixHandler::first_row() {
  typedbytes_opaque key;
  std::vector<double> row;
  read_key_val_pair(key, row);
  // TODO(arbenson) check for error here
  num_cols_ = row.size();
  hadoop_message("matrix size: %zi columns, up to %i localrows\n",
		 num_cols_, blocksize_ * num_cols_);
  if (num_cols_ == 0) {
    hadoop_message("no data received on this task\n");
    return;
  }
  alloc(blocksize_ * num_cols_, num_cols_);
  add_row(row);
}

// read in a row and add it to the local matrix
void MatrixHandler::add_row(const std::vector<double>& row) {
  assert(row.size() == num_cols_);
  assert(num_local_rows_ < num_rows_);
  // store by column
  for (size_t k = 0; k < rows_per_record_; ++k) {
    size_t i = 0;
    for (size_t j = 0; j < num_cols_; ++j) {
      local_matrix_[num_local_rows_ + j * num_rows_] = row[i];
      ++i;
    }
    // increment the number of local rows
    ++num_local_rows_;
    ++num_total_rows_;
  }
}
<|endoftext|>


<commit_before>/**
Copyright (c) 2016, Ubiquity Robotics
All rights reserved.

Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are met:

* Redistributions of source code must retain the above copyright notice, this
  list of conditions and the following disclaimer.

* Redistributions in binary form must reproduce the above copyright notice,
  this list of conditions and the following disclaimer in the documentation
  and/or other materials provided with the distribution.

* Neither the name of ubiquity_motor nor the names of its
  contributors may be used to endorse or promote products derived from
  this software without specific prior written permission.

THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
**/

#include <dynamic_reconfigure/server.h>
#include <ros/ros.h>
#include <time.h>
#include <ubiquity_motor/PIDConfig.h>
#include <ubiquity_motor/motor_hardware.h>
#include <ubiquity_motor/motor_message.h>
#include <ubiquity_motor/motor_parmeters.h>
#include <boost/thread.hpp>
#include <string>
#include "controller_manager/controller_manager.h"

static const double BILLION = 1000000000.0;

static FirmwareParams firmware_params;
static CommsParams serial_params;
static NodeParams node_params;

void PID_update_callback(const ubiquity_motor::PIDConfig& config,
                         uint32_t level) {
    if (level == 1) {
        firmware_params.pid_proportional = config.PID_P;
    } else if (level == 2) {
        firmware_params.pid_integral = config.PID_I;
    } else if (level == 4) {
        firmware_params.pid_derivative = config.PID_D;
    } else if (level == 8) {
        firmware_params.pid_denominator = config.PID_C;
    } else if (level == 16) {
        firmware_params.pid_moving_buffer_size = config.PID_W;
    } else {
        ROS_ERROR("Unsupported dynamic_reconfigure level %u", level);
    }
}

int main(int argc, char* argv[]) {
    ros::init(argc, argv, "motor_node");
    ros::NodeHandle nh;

    firmware_params = FirmwareParams(nh);
    serial_params = CommsParams(nh);
    node_params = NodeParams(nh);

    MotorHardware robot(nh, serial_params, firmware_params);
    controller_manager::ControllerManager cm(&robot, nh);

    ros::AsyncSpinner spinner(1);
    spinner.start();

    dynamic_reconfigure::Server<ubiquity_motor::PIDConfig> server;
    dynamic_reconfigure::Server<ubiquity_motor::PIDConfig>::CallbackType f;

    f = boost::bind(&PID_update_callback, _1, _2);
    server.setCallback(f);

    robot.setParams(firmware_params);

    ros::Rate r(node_params.controller_loop_rate);
    robot.requestVersion();

    int times = 0;
    while (robot.firmware_version == 0) {
        if (times >= 10)
            throw std::runtime_error("Firmware not reporting its version");
        robot.readInputs();
        r.sleep();
        times++;
    }

    ros::Time last_time;
    ros::Time current_time;
    ros::Duration elapsed;
    last_time = ros::Time::now();

    for (int i = 0; i < 5; i++) {
        r.sleep();
        robot.sendParams();
    }

    while (ros::ok()) {
        current_time = ros::Time::now();
        elapsed = last_time - current_time;
        last_time = current_time;
        robot.readInputs();
        cm.update(ros::Time::now(), elapsed);
        robot.setParams(firmware_params);
        robot.sendParams();
        robot.writeSpeeds();

        r.sleep();
    }

    return 0;
}
<commit_msg>Fix computaion of elapsed time so that it is +ve<commit_after>/**
Copyright (c) 2016, Ubiquity Robotics
All rights reserved.

Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are met:

* Redistributions of source code must retain the above copyright notice, this
  list of conditions and the following disclaimer.

* Redistributions in binary form must reproduce the above copyright notice,
  this list of conditions and the following disclaimer in the documentation
  and/or other materials provided with the distribution.

* Neither the name of ubiquity_motor nor the names of its
  contributors may be used to endorse or promote products derived from
  this software without specific prior written permission.

THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
**/

#include <dynamic_reconfigure/server.h>
#include <ros/ros.h>
#include <time.h>
#include <ubiquity_motor/PIDConfig.h>
#include <ubiquity_motor/motor_hardware.h>
#include <ubiquity_motor/motor_message.h>
#include <ubiquity_motor/motor_parmeters.h>
#include <boost/thread.hpp>
#include <string>
#include "controller_manager/controller_manager.h"

static const double BILLION = 1000000000.0;

static FirmwareParams firmware_params;
static CommsParams serial_params;
static NodeParams node_params;

void PID_update_callback(const ubiquity_motor::PIDConfig& config,
                         uint32_t level) {
    if (level == 1) {
        firmware_params.pid_proportional = config.PID_P;
    } else if (level == 2) {
        firmware_params.pid_integral = config.PID_I;
    } else if (level == 4) {
        firmware_params.pid_derivative = config.PID_D;
    } else if (level == 8) {
        firmware_params.pid_denominator = config.PID_C;
    } else if (level == 16) {
        firmware_params.pid_moving_buffer_size = config.PID_W;
    } else {
        ROS_ERROR("Unsupported dynamic_reconfigure level %u", level);
    }
}

int main(int argc, char* argv[]) {
    ros::init(argc, argv, "motor_node");
    ros::NodeHandle nh;

    firmware_params = FirmwareParams(nh);
    serial_params = CommsParams(nh);
    node_params = NodeParams(nh);

    MotorHardware robot(nh, serial_params, firmware_params);
    controller_manager::ControllerManager cm(&robot, nh);

    ros::AsyncSpinner spinner(1);
    spinner.start();

    dynamic_reconfigure::Server<ubiquity_motor::PIDConfig> server;
    dynamic_reconfigure::Server<ubiquity_motor::PIDConfig>::CallbackType f;

    f = boost::bind(&PID_update_callback, _1, _2);
    server.setCallback(f);

    robot.setParams(firmware_params);

    ros::Rate r(node_params.controller_loop_rate);
    robot.requestVersion();

    int times = 0;
    while (robot.firmware_version == 0) {
        if (times >= 10)
            throw std::runtime_error("Firmware not reporting its version");
        robot.readInputs();
        r.sleep();
        times++;
    }

    ros::Time last_time;
    ros::Time current_time;
    ros::Duration elapsed;
    last_time = ros::Time::now();

    for (int i = 0; i < 5; i++) {
        r.sleep();
        robot.sendParams();
    }

    while (ros::ok()) {
        current_time = ros::Time::now();
        elapsed = current_time - last_time;
        last_time = current_time;
        robot.readInputs();
        cm.update(ros::Time::now(), elapsed);
        robot.setParams(firmware_params);
        robot.sendParams();
        robot.writeSpeeds();

        r.sleep();
    }

    return 0;
}
<|endoftext|>


<commit_before>#include "process.hpp"

//////////////
// Includes //
#include "image.hpp"

//////////
// Code //

// Verifying the validity of a set of images.
bool validateImages(int count, Image** imgs) {
    for (int i = 1; i < count; i++) {
        if (imgs[i - 1]->width    != imgs[i]->width  ||
            imgs[i - 1]->height   != imgs[i]->height ||
            imgs[i - 1]->maxValue != imgs[i]->maxValue)
            return false;
    }
    return true;
}

// Getting a single pixel at a given index of every image.
Pixel* getPixelsAt(int count, Image** imgs, int index) {
    Pixel* pixels = new Pixel[count];
    for (int i = 0; i < count; i++)
        pixels[i] = imgs[i]->pixels[index];
    return pixels;
}

Pixel choosePixel(int count, Pixel* pixels) {
    // TODO: Develop a consensus.
    Pixel p;
    return p;
}

// Processing a set of images.
Image* processImages(int count, Image** imgs) {
    if (!validateImages(count, imgs))
        return nullptr;

    int length = imgs[0]->width * imgs[0]->height;
    Pixel* finalPixels = new Pixel[length];
    Pixel* pixels;

    for (int i = 0; i < length; i++) {
        pixels = getPixelsAt(count, imgs, i);
        finalPixels[i] = choosePixel(count, pixels);
        delete[] pixels;
    }

    return new Image(finalPixels,
                     imgs[0]->width,
                     imgs[0]->height,
                     imgs[0]->maxValue);
}
<commit_msg>made the thing work.<commit_after>#include "process.hpp"

//////////////
// Includes //
#include "image.hpp"

//////////
// Code //

// Verifying the validity of a set of images.
bool validateImages(int count, Image** imgs) {
    for (int i = 1; i < count; i++) {
        if (imgs[i - 1]->width    != imgs[i]->width  ||
            imgs[i - 1]->height   != imgs[i]->height ||
            imgs[i - 1]->maxValue != imgs[i]->maxValue)
            return false;
    }
    return true;
}

// Getting a single pixel at a given index of every image.
Pixel* getPixelsAt(int count, Image** imgs, int index) {
    Pixel* pixels = new Pixel[count];
    for (int i = 0; i < count; i++)
        pixels[i] = imgs[i]->pixels[index];
    return pixels;
}

// Choosing a pixel from a vector of pixels.
Pixel choosePixel(int count, Pixel* pixels) {
    int n = 0;

    Pixel ips[] = {
        Pixel(0, 0, 0),
        Pixel(0, 0, 0)
    };

    for (int i = 0; i < count; i++) {
        if (n == 0)
            ips[1] = pixels[i];

        if (pixels[i] == ips[1])
            n++;
        else
            n--;
    }

    n = 0;
    for (int i = 0; i < count; i++)
        if (pixels[i] == ips[1])
            n++;

    if (n > count / 2)
        return ips[1];
    return ips[0];
}

// Processing a set of images.
Image* processImages(int count, Image** imgs) {
    if (!validateImages(count, imgs))
        return nullptr;

    int length = imgs[0]->width * imgs[0]->height;
    Pixel* finalPixels = new Pixel[length];
    Pixel* pixels;

    for (int i = 0; i < length; i++) {
        pixels = getPixelsAt(count, imgs, i);
        finalPixels[i] = choosePixel(count, pixels);
        delete[] pixels;
    }

    return new Image(finalPixels,
                     imgs[0]->width,
                     imgs[0]->height,
                     imgs[0]->maxValue);
}
<|endoftext|>


<commit_before>#include <string>
#include <vector>
#include <list>
#include <map>
#include <set>
#include <deque>
#include <stack>
#include <bitset>
#include <algorithm>
#include <functional>
#include <numeric>
#include <utility>
#include <sstream>
#include <iostream>
#include <iomanip>
#include <cstdio>
#include <cmath>
#include <cstdlib>
#include <ctime>
#include <iterator>
#include <tuple>
#include <regex>
#include <array>
#include <valarray>
#define all(v)begin(v),end(v)
#define dump(v)copy(all(v),ostream_iterator<decltype(*v.begin())>(cout,"\n"))
#define rg(i,a,b)for(int i=a,i##e=b;i<i##e;++i)
#define fr(i,n)for(int i=0,i##e=n;i<i##e;++i)
#define rf(i,n)for(int i=n-1;i>=0;--i)
#define ei(a,m)for(auto&a:m)if(int a##i=&a-&*begin(m)+1)if(--a##i,1)
#define sz(v)int(v.size())
#define sr(v)sort(all(v))
#define rs(v)sort(all(v),greater<int>())
#define rev(v)reverse(all(v))
#define eb emplace_back
#define stst stringstream
#define big numeric_limits<int>::max()
#define g(t,i)get<i>(t)
#define cb(v,w)copy(all(v),back_inserter(w))
#define uni(v)sort(all(v));v.erase(unique(all(v)),end(v))
#define vt(...)vector<tuple<__VA_ARGS__>>
#define smx(a,b)a=max(a,b)
#define smn(a,b)a=min(a,b)
#define words(w,q)vector<string>w;[&w](string&&s){stringstream u(s);string r;while(u>>r)w.eb(r);}(q);
#define digits(d,n,s)vector<int>d;[&d](int m){while(m)d.eb(m%s),m/=s;}(n);
typedef long long ll;
using namespace std;

struct MountainWalk {
	int cellsVisited(vector <string> m, int k) {
		int x = 0, y = 0;
		int w = sz(m), h = sz(m[0]);
		int dx[] = {1, 0, -1, 0}, dy[] = {0, -1, 0, 1};
		int c = 1;
		vector<string> t = m;
		for (;;) {
			bool f = true;
			fr(i, 4) {
				int nx = x + dx[i];
				int ny = y + dy[i];
				if (0 <= nx && nx < w && 0 <= ny && ny < h && t[nx][ny] != '-' && abs(m[x][y] - m[nx][ny]) <= k) {
					cout << nx << ' ' << ny << endl;
					x = nx;
					y = ny;
					t[x][y] = '-';
					++c;
					f = false;
					break;
				}
			}
			if (f) break;
		}
		return c;
	}
};

// BEGIN KAWIGIEDIT TESTING
// Generated by KawigiEdit-pf 2.3.0
#include <iostream>
#include <string>
#include <vector>
#include <ctime>
#include <cmath>
using namespace std;
bool KawigiEdit_RunTest(int testNum, vector <string> p0, int p1, bool hasAnswer, int p2) {
	cout << "Test " << testNum << ": [" << "{";
	for (int i = 0; int(p0.size()) > i; ++i) {
		if (i > 0) {
			cout << ",";
		}
		cout << "\"" << p0[i] << "\"";
	}
	cout << "}" << "," << p1;
	cout << "]" << endl;
	MountainWalk *obj;
	int answer;
	obj = new MountainWalk();
	clock_t startTime = clock();
	answer = obj->cellsVisited(p0, p1);
	clock_t endTime = clock();
	delete obj;
	bool res;
	res = true;
	cout << "Time: " << double(endTime - startTime) / CLOCKS_PER_SEC << " seconds" << endl;
	if (hasAnswer) {
		cout << "Desired answer:" << endl;
		cout << "\t" << p2 << endl;
	}
	cout << "Your answer:" << endl;
	cout << "\t" << answer << endl;
	if (hasAnswer) {
		res = answer == p2;
	}
	if (!res) {
		cout << "DOESN'T MATCH!!!!" << endl;
	} else if (double(endTime - startTime) / CLOCKS_PER_SEC >= 2) {
		cout << "FAIL the timeout" << endl;
		res = false;
	} else if (hasAnswer) {
		cout << "Match :-)" << endl;
	} else {
		cout << "OK, but is it right?" << endl;
	}
	cout << "" << endl;
	return res;
}
int main() {
	bool all_right;
	bool disabled;
	bool tests_disabled;
	all_right = true;
	tests_disabled = false;

	vector <string> p0;
	int p1;
	int p2;

	// ----- test 0 -----
	disabled = false;
	p0 = {"056","135","234"};
	p1 = 1;
	p2 = 5;
	all_right = (disabled || KawigiEdit_RunTest(0, p0, p1, true, p2) ) && all_right;
	tests_disabled = tests_disabled || disabled;
	// ------------------

	// ----- test 1 -----
	disabled = false;
	p0 = {"056","195","234"};
	p1 = 1;
	p2 = 8;
	all_right = (disabled || KawigiEdit_RunTest(1, p0, p1, true, p2) ) && all_right;
	tests_disabled = tests_disabled || disabled;
	// ------------------

	// ----- test 2 -----
	disabled = false;
	p0 = {"865","123","111"};
	p1 = 3;
	p2 = 9;
	all_right = (disabled || KawigiEdit_RunTest(2, p0, p1, true, p2) ) && all_right;
	tests_disabled = tests_disabled || disabled;
	// ------------------

	// ----- test 3 -----
	disabled = false;
	p0 = {"00009876543210","00009876543210","00009876543210","00009876543210"};
	p1 = 8;
	p2 = 16;
	all_right = (disabled || KawigiEdit_RunTest(3, p0, p1, true, p2) ) && all_right;
	tests_disabled = tests_disabled || disabled;
	// ------------------

	// ----- test 4 -----
	disabled = false;
	p0 = {"0000","0000","0000","0000","9999","8888","7777","6666","5555","4444","3333","2222","1111","0000"};
	p1 = 3;
	p2 = 16;
	all_right = (disabled || KawigiEdit_RunTest(4, p0, p1, true, p2) ) && all_right;
	tests_disabled = tests_disabled || disabled;
	// ------------------

	// ----- test 5 -----
	disabled = false;
	p0 = {"173642855131893831828253420","126290035950506994475683704","381277675415026563959463393","019782700912864681764582260","496448425114634806770407597","049628433145840178727435051","117194708226266248973780562","398138380998246682323622510","408178777661559971959512111"};
	p1 = 8;
	p2 = 135;
	all_right = (disabled || KawigiEdit_RunTest(5, p0, p1, true, p2) ) && all_right;
	tests_disabled = tests_disabled || disabled;
	// ------------------

	// ----- test 6 -----
	disabled = false;
	p0 = {"9"};
	p1 = 0;
	p2 = 1;
	all_right = (disabled || KawigiEdit_RunTest(6, p0, p1, true, p2) ) && all_right;
	tests_disabled = tests_disabled || disabled;
	// ------------------

	if (all_right) {
		if (tests_disabled) {
			cout << "You're a stud (but some test cases were disabled)!" << endl;
		} else {
			cout << "You're a stud (at least on given cases)!" << endl;
		}
	} else {
		cout << "Some of the test cases had errors." << endl;
	}
	return 0;
}
// END KAWIGIEDIT TESTING
<commit_msg>MountainWalk<commit_after>#include <string>
#include <vector>
#include <list>
#include <map>
#include <set>
#include <deque>
#include <stack>
#include <bitset>
#include <algorithm>
#include <functional>
#include <numeric>
#include <utility>
#include <sstream>
#include <iostream>
#include <iomanip>
#include <cstdio>
#include <cmath>
#include <cstdlib>
#include <ctime>
#include <iterator>
#include <tuple>
#include <regex>
#include <array>
#include <valarray>
#define all(v)begin(v),end(v)
#define dump(v)copy(all(v),ostream_iterator<decltype(*v.begin())>(cout,"\n"))
#define rg(i,a,b)for(int i=a,i##e=b;i<i##e;++i)
#define fr(i,n)for(int i=0,i##e=n;i<i##e;++i)
#define rf(i,n)for(int i=n-1;i>=0;--i)
#define ei(a,m)for(auto&a:m)if(int a##i=&a-&*begin(m)+1)if(--a##i,1)
#define sz(v)int(v.size())
#define sr(v)sort(all(v))
#define rs(v)sort(all(v),greater<int>())
#define rev(v)reverse(all(v))
#define eb emplace_back
#define stst stringstream
#define big numeric_limits<int>::max()
#define g(t,i)get<i>(t)
#define cb(v,w)copy(all(v),back_inserter(w))
#define uni(v)sort(all(v));v.erase(unique(all(v)),end(v))
#define vt(...)vector<tuple<__VA_ARGS__>>
#define smx(a,b)a=max(a,b)
#define smn(a,b)a=min(a,b)
#define words(w,q)vector<string>w;[&w](string&&s){stringstream u(s);string r;while(u>>r)w.eb(r);}(q);
#define digits(d,n,s)vector<int>d;[&d](int m){while(m)d.eb(m%s),m/=s;}(n);
typedef long long ll;
using namespace std;

struct MountainWalk {
	int cellsVisited(vector <string> m, int k) {
		int x = 0, y = 0;
		int w = sz(m), h = sz(m[0]);
		int dx[] = {1, 0, -1, 0}, dy[] = {0, -1, 0, 1};
		int c = 1;
		vector<string> t = m;
		t[0][0] = '-';
		for (;;) {
			bool f = true;
			fr(i, 4) {
				int nx = x + dx[i];
				int ny = y + dy[i];
				if (0 <= nx && nx < w && 0 <= ny && ny < h && t[nx][ny] != '-' && abs(m[x][y] - m[nx][ny]) <= k) {
					cout << nx << ' ' << ny << endl;
					x = nx;
					y = ny;
					t[x][y] = '-';
					++c;
					f = false;
					break;
				}
			}
			if (f) break;
		}
		return c;
	}
};

// BEGIN KAWIGIEDIT TESTING
// Generated by KawigiEdit-pf 2.3.0
#include <iostream>
#include <string>
#include <vector>
#include <ctime>
#include <cmath>
using namespace std;
bool KawigiEdit_RunTest(int testNum, vector <string> p0, int p1, bool hasAnswer, int p2) {
	cout << "Test " << testNum << ": [" << "{";
	for (int i = 0; int(p0.size()) > i; ++i) {
		if (i > 0) {
			cout << ",";
		}
		cout << "\"" << p0[i] << "\"";
	}
	cout << "}" << "," << p1;
	cout << "]" << endl;
	MountainWalk *obj;
	int answer;
	obj = new MountainWalk();
	clock_t startTime = clock();
	answer = obj->cellsVisited(p0, p1);
	clock_t endTime = clock();
	delete obj;
	bool res;
	res = true;
	cout << "Time: " << double(endTime - startTime) / CLOCKS_PER_SEC << " seconds" << endl;
	if (hasAnswer) {
		cout << "Desired answer:" << endl;
		cout << "\t" << p2 << endl;
	}
	cout << "Your answer:" << endl;
	cout << "\t" << answer << endl;
	if (hasAnswer) {
		res = answer == p2;
	}
	if (!res) {
		cout << "DOESN'T MATCH!!!!" << endl;
	} else if (double(endTime - startTime) / CLOCKS_PER_SEC >= 2) {
		cout << "FAIL the timeout" << endl;
		res = false;
	} else if (hasAnswer) {
		cout << "Match :-)" << endl;
	} else {
		cout << "OK, but is it right?" << endl;
	}
	cout << "" << endl;
	return res;
}
int main() {
	bool all_right;
	bool disabled;
	bool tests_disabled;
	all_right = true;
	tests_disabled = false;

	vector <string> p0;
	int p1;
	int p2;

	// ----- test 0 -----
	disabled = false;
	p0 = {"056","135","234"};
	p1 = 1;
	p2 = 5;
	all_right = (disabled || KawigiEdit_RunTest(0, p0, p1, true, p2) ) && all_right;
	tests_disabled = tests_disabled || disabled;
	// ------------------

	// ----- test 1 -----
	disabled = false;
	p0 = {"056","195","234"};
	p1 = 1;
	p2 = 8;
	all_right = (disabled || KawigiEdit_RunTest(1, p0, p1, true, p2) ) && all_right;
	tests_disabled = tests_disabled || disabled;
	// ------------------

	// ----- test 2 -----
	disabled = false;
	p0 = {"865","123","111"};
	p1 = 3;
	p2 = 9;
	all_right = (disabled || KawigiEdit_RunTest(2, p0, p1, true, p2) ) && all_right;
	tests_disabled = tests_disabled || disabled;
	// ------------------

	// ----- test 3 -----
	disabled = false;
	p0 = {"00009876543210","00009876543210","00009876543210","00009876543210"};
	p1 = 8;
	p2 = 16;
	all_right = (disabled || KawigiEdit_RunTest(3, p0, p1, true, p2) ) && all_right;
	tests_disabled = tests_disabled || disabled;
	// ------------------

	// ----- test 4 -----
	disabled = false;
	p0 = {"0000","0000","0000","0000","9999","8888","7777","6666","5555","4444","3333","2222","1111","0000"};
	p1 = 3;
	p2 = 16;
	all_right = (disabled || KawigiEdit_RunTest(4, p0, p1, true, p2) ) && all_right;
	tests_disabled = tests_disabled || disabled;
	// ------------------

	// ----- test 5 -----
	disabled = false;
	p0 = {"173642855131893831828253420","126290035950506994475683704","381277675415026563959463393","019782700912864681764582260","496448425114634806770407597","049628433145840178727435051","117194708226266248973780562","398138380998246682323622510","408178777661559971959512111"};
	p1 = 8;
	p2 = 135;
	all_right = (disabled || KawigiEdit_RunTest(5, p0, p1, true, p2) ) && all_right;
	tests_disabled = tests_disabled || disabled;
	// ------------------

	// ----- test 6 -----
	disabled = false;
	p0 = {"9"};
	p1 = 0;
	p2 = 1;
	all_right = (disabled || KawigiEdit_RunTest(6, p0, p1, true, p2) ) && all_right;
	tests_disabled = tests_disabled || disabled;
	// ------------------

	if (all_right) {
		if (tests_disabled) {
			cout << "You're a stud (but some test cases were disabled)!" << endl;
		} else {
			cout << "You're a stud (at least on given cases)!" << endl;
		}
	} else {
		cout << "Some of the test cases had errors." << endl;
	}
	return 0;
}
// END KAWIGIEDIT TESTING
<|endoftext|>


<commit_before>/******************************************************************************
 *                    _   _____   __________                                  *
 *                   | | / / _ | / __/_  __/     Visibility                   *
 *                   | |/ / __ |_\ \  / /          Across                     *
 *                   |___/_/ |_/___/ /_/       Space and Time                 *
 *                                                                            *
 * This file is part of VAST. It is subject to the license terms in the       *
 * LICENSE file found in the top-level directory of this distribution and at  *
 * http://vast.io/license. No part of VAST, including this file, may be       *
 * copied, modified, propagated, or distributed except according to the terms *
 * contained in the LICENSE file.                                             *
 ******************************************************************************/

#include "vast/system/importer.hpp"

#include <fstream>

#include <caf/config_value.hpp>
#include <caf/dictionary.hpp>

#include "vast/concept/printable/to_string.hpp"
#include "vast/concept/printable/vast/error.hpp"
#include "vast/concept/printable/vast/filesystem.hpp"
#include "vast/defaults.hpp"
#include "vast/detail/fill_status_map.hpp"
#include "vast/detail/notifying_stream_manager.hpp"
#include "vast/logger.hpp"
#include "vast/system/atoms.hpp"
#include "vast/table_slice.hpp"

using namespace std::chrono;
using namespace std::chrono_literals;
using namespace caf;

namespace vast::system {

importer_state::importer_state(event_based_actor* self_ptr) : self(self_ptr) {
  // nop
}

importer_state::~importer_state() {
  write_state();
}

caf::error importer_state::read_state() {
  VAST_TRACE("");
  id_generators.clear();
  auto file = dir / "available_ids";
  if (exists(file)) {
    VAST_DEBUG(self, "reads persistent state from", to_string(file));
    std::ifstream available{to_string(file)};
    std::string line;
    while (std::getline(available, line)) {
      id i;
      id last;
      std::istringstream in{line};
      if (in >> i >> last) {
        VAST_DEBUG(self, "found ID range:", i, "to", last);
        id_generators.emplace_back(i, last);
      } else {
        VAST_ERROR(self, "got an invalidly formatted persistence file:",
                   to_string(file));
        return ec::parse_error;
      }
    }
  }
  return caf::none;
}

caf::error importer_state::write_state() {
  VAST_TRACE("");
  if (id_generators.empty() || available_ids() == 0)
    return caf::none;
  if (!exists(dir)) {
    auto result = mkdir(dir);
    if (!result)
      return std::move(result.error());
  }
  std::ofstream available{to_string(dir / "available_ids")};
  auto i = id_generators.begin();
  available << i->i << ' ' << i->last;
  for (++i; i != id_generators.end(); ++i) {
    available << '\n' << i->i << ' ' << i->last;
  }
  VAST_DEBUG(self, "saved", available_ids(), "available IDs");
  return caf::none;
}

int32_t importer_state::available_ids() const noexcept {
  auto f = [](int32_t x, const id_generator& y) {
    return x + y.remaining();
  };
  return std::accumulate(id_generators.begin(), id_generators.end(),
                         int32_t{0}, f);
}

id importer_state::next_id_block() {
  VAST_ASSERT(!id_generators.empty());
  auto& g = id_generators.front();
  VAST_ASSERT(!g.at_end());
  auto result = g.next(max_table_slice_size);
  if (g.at_end())
    id_generators.erase(id_generators.begin());
  return result;
}

caf::dictionary<caf::config_value> importer_state::status() const {
  caf::dictionary<caf::config_value> result;
  // Misc parameters.
  result.emplace("in-flight-slices", in_flight_slices);
  result.emplace("max-table-slice-size", max_table_slice_size);
  result.emplace("blocks-per-replenish", blocks_per_replenish);
  result.emplace("last-replenish", caf::deep_to_string(last_replenish));
  result.emplace("awaiting-ids", awaiting_ids);
  result.emplace("available-ids", available_ids());
  if (!id_generators.empty())
    result.emplace("next-id", id_generators.front().i);
  // General state such as open streams.
  detail::fill_status_map(result, self);
  return result;
}

void importer_state::send_report() {
  auto now = stopwatch::now();
  if (measurement_.events > 0) {
    using namespace std::string_literals;
    auto elapsed = std::chrono::duration_cast<measurement::timespan>(
      now - last_report);
    auto node_throughput = measurement{elapsed, measurement_.events};
    auto r = performance_report{
      {{"importer"s, measurement_}, {"node_throughput"s, node_throughput}}};
    measurement_ = measurement{};
    self->send(accountant, std::move(r));
  }
  last_report = now;
}

void importer_state::notify_flush_listeners() {
  VAST_DEBUG(self, "forwards 'flush' subscribers to", index_actors.size(),
             "INDEX actors");
  for (auto& listener : flush_listeners)
    for (auto& next : index_actors)
      self->send(next, subscribe_atom::value, flush_atom::value, listener);
  flush_listeners.clear();
}

namespace {

// Asks the consensus module for more IDs.
void replenish(stateful_actor<importer_state>* self) {
  VAST_TRACE("");
  auto& st = self->state;
  // Do nothing if we're already waiting for a response from the consensus
  // module.
  if (st.awaiting_ids)
    return;
  // Check whether we obtain new IDs too frequently.
  auto now = steady_clock::now();
  if ((now - st.last_replenish) < 10s) {
    VAST_DEBUG(self, "had to replenish twice within 10 secs");
    VAST_DEBUG(self, "increase blocks_per_replenish:", st.blocks_per_replenish,
                    "->", st.blocks_per_replenish + 100);
    st.blocks_per_replenish += 100;
  }
  st.last_replenish = now;
  VAST_DEBUG(self, "replenishes", st.blocks_per_replenish, "ID blocks");
  // If we get an EXIT message while expecting a response from the consensus
  // module, we'll give it a bit of time to come back.
  self->set_default_handler(skip);
  // Trigger consensus module and wait for response.
  auto n = st.max_table_slice_size * st.blocks_per_replenish;
  self->send(st.consensus, add_atom::value, "id", data{n});
  st.awaiting_ids = true;
  self->become(
    keep_behavior,
    [=](const data& old) {
      auto x = caf::holds_alternative<caf::none_t>(old) ? count{0}
                                                        : caf::get<count>(old);
      VAST_DEBUG(self, "got", n, "new IDs starting at", x);
      auto& st = self->state;
      // Add a new ID generator for the available range.
      VAST_ASSERT(st.awaiting_ids);
      st.id_generators.emplace_back(x, x + n);
      // Save state.
      auto err = self->state.write_state();
      if (err) {
        VAST_ERROR(self, "failed to save state:", self->system().render(err));
        self->quit(std::move(err));
        return;
      }
      // Try to emit more credit with out new IDs.
      st.stg->advance();
      // Return to previous behavior.
      st.awaiting_ids = false;
      self->set_default_handler(print_and_drop);
      self->unbecome();
    }
  );
}

class driver : public importer_state::driver_base {
public:
  using super = importer_state::driver_base;

  using pointer = importer_actor*;

  driver(importer_state::downstream_manager& out, pointer self)
    : super(out),
      self_(self) {
    // nop
  }

  void process(caf::downstream<output_type>& out,
               std::vector<input_type>& xs) override {
    VAST_TRACE(VAST_ARG(xs));
    auto& st = self_->state;
    auto t = timer::start(st.measurement_);
    VAST_DEBUG(self_, "has", st.available_ids(), "IDs available");
    VAST_DEBUG(self_, "got", xs.size(), "slices with", st.in_flight_slices,
               "in-flight slices");
    VAST_ASSERT(xs.size() <= static_cast<size_t>(st.available_ids()));
    VAST_ASSERT(xs.size() <= static_cast<size_t>(st.in_flight_slices));
    st.in_flight_slices -= static_cast<int32_t>(xs.size());
    uint64_t events = 0;
    for (auto& x : xs) {
      events += x->rows();
      x.unshared().offset(st.next_id_block());
      out.push(std::move(x));
    }
    t.stop(events);
  }

  int32_t acquire_credit(inbound_path* path, int32_t desired) override {
    VAST_TRACE(VAST_ARG(path) << VAST_ARG(desired));
    CAF_IGNORE_UNUSED(path);
    // This function makes sure that we never hand out more credit than we have
    // IDs available.
    if (desired == 0) {
      // Easy decision if the path acquires no new credit.
      return 0;
    }
    // Calculate how much more in-flight events we can allow.
    auto& st = self_->state;
    CAF_ASSERT(st.available_ids() % st.max_table_slice_size == 0);
    auto max_credit = (st.available_ids() / st.max_table_slice_size)
                      - st.in_flight_slices;
    VAST_ASSERT(max_credit >= 0);
    if (max_credit <= desired) {
      // Get more IDs if we're running out.
      VAST_DEBUG(self_, "had to limit acquired credit to", max_credit);
      replenish(self_);
      st.in_flight_slices += max_credit;
      return max_credit;
    }
    st.in_flight_slices += desired;
    return desired;
  }

  pointer self() const {
    return self_;
  }

private:
  pointer self_;
};

auto make_importer_stage(importer_actor* self) {
  using impl = detail::notifying_stream_manager<driver>;
  auto result = caf::make_counted<impl>(self);
  result->continuous(true);
  return result;
}

} // namespace <anonymous>

behavior importer(stateful_actor<importer_state>* self, path dir,
                  size_t max_table_slice_size) {
  VAST_TRACE(VAST_ARG(dir), VAST_ARG(max_table_slice_size));
  self->state.dir = dir;
  self->state.last_replenish = steady_clock::time_point::min();
  self->state.max_table_slice_size = static_cast<int32_t>(max_table_slice_size);
  auto err = self->state.read_state();
  if (err) {
    VAST_ERROR(self, "failed to load state:", self->system().render(err));
    self->quit(std::move(err));
    return {};
  }
  namespace defs = defaults::system;
  if (auto a = self->system().registry().get(accountant_atom::value)) {
    self->state.accountant = actor_cast<accountant_type>(a);
    self->send(self->state.accountant, announce_atom::value, self->name());
    self->delayed_send(self, defs::telemetry_rate, telemetry_atom::value);
  }
  self->set_exit_handler(
    [=](const exit_msg& msg) {
      self->state.send_report();
      self->quit(msg.reason);
    });
  self->state.stg = make_importer_stage(self);
  return {
    [=](const consensus_type& c) {
      VAST_DEBUG(self, "registers consensus module");
      VAST_ASSERT(c != self->state.consensus);
      self->monitor(c);
      self->state.consensus = c;
    },
    [=](const archive_type& archive) {
      VAST_DEBUG(self, "registers archive", archive);
      return self->state.stg->add_outbound_path(archive);
    },
    [=](index_atom, const actor& index) {
      VAST_DEBUG(self, "registers index", index);
      self->state.index_actors.emplace_back(index);
      // TODO: currently, the subscriber expects only a single 'flush' message.
      //       Adding multiple INDEX actors will cause the subscriber to
      //       receive more than one 'flush'  message, but the subscriber only
      //       expects one and will stop waiting after the first one. Once we
      //       support multiple INDEX actors at the IMPORTER, we also need to
      //       revise the signaling of these 'flush' messages.
      if (self->state.index_actors.size() > 1)
        VAST_WARNING(self, "registered more than one INDEX actor",
                     "(currently unsupported!)");
      return self->state.stg->add_outbound_path(index);
    },
    [=](exporter_atom, const actor& exporter) {
      VAST_DEBUG(self, "registers exporter", exporter);
      return self->state.stg->add_outbound_path(exporter);
    },
    [=](stream<importer_state::input_type>& in) {
      auto& st = self->state;
      if (!st.consensus) {
        VAST_ERROR(self, "has no consensus module configured");
        return;
      }
      VAST_INFO(self, "adds a new source");
      st.stg->add_inbound_path(in);
    },
    [=](add_atom, const actor& subscriber) {
      auto& st = self->state;
      VAST_INFO(self, "adds a new sink");
      st.stg->add_outbound_path(subscriber);
    },
    [=](subscribe_atom, flush_atom, actor& listener) {
      auto& st = self->state;
      VAST_ASSERT(st.stg != nullptr);
      st.flush_listeners.emplace_back(std::move(listener));
      detail::notify_listeners_if_clean(st, *st.stg);
    },
    [=](status_atom) {
      return self->state.status();
    },
    [=](telemetry_atom) {
      self->state.send_report();
      self->delayed_send(self, defs::telemetry_rate, telemetry_atom::value);
    }
  };
}

} // namespace vast::system
<commit_msg>Include last_replenish for status only after use<commit_after>/******************************************************************************
 *                    _   _____   __________                                  *
 *                   | | / / _ | / __/_  __/     Visibility                   *
 *                   | |/ / __ |_\ \  / /          Across                     *
 *                   |___/_/ |_/___/ /_/       Space and Time                 *
 *                                                                            *
 * This file is part of VAST. It is subject to the license terms in the       *
 * LICENSE file found in the top-level directory of this distribution and at  *
 * http://vast.io/license. No part of VAST, including this file, may be       *
 * copied, modified, propagated, or distributed except according to the terms *
 * contained in the LICENSE file.                                             *
 ******************************************************************************/

#include "vast/system/importer.hpp"

#include <fstream>

#include <caf/config_value.hpp>
#include <caf/dictionary.hpp>

#include "vast/concept/printable/to_string.hpp"
#include "vast/concept/printable/vast/error.hpp"
#include "vast/concept/printable/vast/filesystem.hpp"
#include "vast/defaults.hpp"
#include "vast/detail/fill_status_map.hpp"
#include "vast/detail/notifying_stream_manager.hpp"
#include "vast/logger.hpp"
#include "vast/system/atoms.hpp"
#include "vast/table_slice.hpp"

using namespace std::chrono;
using namespace std::chrono_literals;
using namespace caf;

namespace vast::system {

importer_state::importer_state(event_based_actor* self_ptr) : self(self_ptr) {
  // nop
}

importer_state::~importer_state() {
  write_state();
}

caf::error importer_state::read_state() {
  VAST_TRACE("");
  id_generators.clear();
  auto file = dir / "available_ids";
  if (exists(file)) {
    VAST_DEBUG(self, "reads persistent state from", to_string(file));
    std::ifstream available{to_string(file)};
    std::string line;
    while (std::getline(available, line)) {
      id i;
      id last;
      std::istringstream in{line};
      if (in >> i >> last) {
        VAST_DEBUG(self, "found ID range:", i, "to", last);
        id_generators.emplace_back(i, last);
      } else {
        VAST_ERROR(self, "got an invalidly formatted persistence file:",
                   to_string(file));
        return ec::parse_error;
      }
    }
  }
  return caf::none;
}

caf::error importer_state::write_state() {
  VAST_TRACE("");
  if (id_generators.empty() || available_ids() == 0)
    return caf::none;
  if (!exists(dir)) {
    auto result = mkdir(dir);
    if (!result)
      return std::move(result.error());
  }
  std::ofstream available{to_string(dir / "available_ids")};
  auto i = id_generators.begin();
  available << i->i << ' ' << i->last;
  for (++i; i != id_generators.end(); ++i) {
    available << '\n' << i->i << ' ' << i->last;
  }
  VAST_DEBUG(self, "saved", available_ids(), "available IDs");
  return caf::none;
}

int32_t importer_state::available_ids() const noexcept {
  auto f = [](int32_t x, const id_generator& y) {
    return x + y.remaining();
  };
  return std::accumulate(id_generators.begin(), id_generators.end(),
                         int32_t{0}, f);
}

id importer_state::next_id_block() {
  VAST_ASSERT(!id_generators.empty());
  auto& g = id_generators.front();
  VAST_ASSERT(!g.at_end());
  auto result = g.next(max_table_slice_size);
  if (g.at_end())
    id_generators.erase(id_generators.begin());
  return result;
}

caf::dictionary<caf::config_value> importer_state::status() const {
  caf::dictionary<caf::config_value> result;
  // Misc parameters.
  result.emplace("in-flight-slices", in_flight_slices);
  result.emplace("max-table-slice-size", max_table_slice_size);
  result.emplace("blocks-per-replenish", blocks_per_replenish);
  if (last_replenish > steady_clock::time_point::min())
    result.emplace("last-replenish", caf::deep_to_string(last_replenish));
  result.emplace("awaiting-ids", awaiting_ids);
  result.emplace("available-ids", available_ids());
  if (!id_generators.empty())
    result.emplace("next-id", id_generators.front().i);
  // General state such as open streams.
  detail::fill_status_map(result, self);
  return result;
}

void importer_state::send_report() {
  auto now = stopwatch::now();
  if (measurement_.events > 0) {
    using namespace std::string_literals;
    auto elapsed = std::chrono::duration_cast<measurement::timespan>(
      now - last_report);
    auto node_throughput = measurement{elapsed, measurement_.events};
    auto r = performance_report{
      {{"importer"s, measurement_}, {"node_throughput"s, node_throughput}}};
    measurement_ = measurement{};
    self->send(accountant, std::move(r));
  }
  last_report = now;
}

void importer_state::notify_flush_listeners() {
  VAST_DEBUG(self, "forwards 'flush' subscribers to", index_actors.size(),
             "INDEX actors");
  for (auto& listener : flush_listeners)
    for (auto& next : index_actors)
      self->send(next, subscribe_atom::value, flush_atom::value, listener);
  flush_listeners.clear();
}

namespace {

// Asks the consensus module for more IDs.
void replenish(stateful_actor<importer_state>* self) {
  VAST_TRACE("");
  auto& st = self->state;
  // Do nothing if we're already waiting for a response from the consensus
  // module.
  if (st.awaiting_ids)
    return;
  // Check whether we obtain new IDs too frequently.
  auto now = steady_clock::now();
  if ((now - st.last_replenish) < 10s) {
    VAST_DEBUG(self, "had to replenish twice within 10 secs");
    VAST_DEBUG(self, "increase blocks_per_replenish:", st.blocks_per_replenish,
                    "->", st.blocks_per_replenish + 100);
    st.blocks_per_replenish += 100;
  }
  st.last_replenish = now;
  VAST_DEBUG(self, "replenishes", st.blocks_per_replenish, "ID blocks");
  // If we get an EXIT message while expecting a response from the consensus
  // module, we'll give it a bit of time to come back.
  self->set_default_handler(skip);
  // Trigger consensus module and wait for response.
  auto n = st.max_table_slice_size * st.blocks_per_replenish;
  self->send(st.consensus, add_atom::value, "id", data{n});
  st.awaiting_ids = true;
  self->become(
    keep_behavior,
    [=](const data& old) {
      auto x = caf::holds_alternative<caf::none_t>(old) ? count{0}
                                                        : caf::get<count>(old);
      VAST_DEBUG(self, "got", n, "new IDs starting at", x);
      auto& st = self->state;
      // Add a new ID generator for the available range.
      VAST_ASSERT(st.awaiting_ids);
      st.id_generators.emplace_back(x, x + n);
      // Save state.
      auto err = self->state.write_state();
      if (err) {
        VAST_ERROR(self, "failed to save state:", self->system().render(err));
        self->quit(std::move(err));
        return;
      }
      // Try to emit more credit with out new IDs.
      st.stg->advance();
      // Return to previous behavior.
      st.awaiting_ids = false;
      self->set_default_handler(print_and_drop);
      self->unbecome();
    }
  );
}

class driver : public importer_state::driver_base {
public:
  using super = importer_state::driver_base;

  using pointer = importer_actor*;

  driver(importer_state::downstream_manager& out, pointer self)
    : super(out),
      self_(self) {
    // nop
  }

  void process(caf::downstream<output_type>& out,
               std::vector<input_type>& xs) override {
    VAST_TRACE(VAST_ARG(xs));
    auto& st = self_->state;
    auto t = timer::start(st.measurement_);
    VAST_DEBUG(self_, "has", st.available_ids(), "IDs available");
    VAST_DEBUG(self_, "got", xs.size(), "slices with", st.in_flight_slices,
               "in-flight slices");
    VAST_ASSERT(xs.size() <= static_cast<size_t>(st.available_ids()));
    VAST_ASSERT(xs.size() <= static_cast<size_t>(st.in_flight_slices));
    st.in_flight_slices -= static_cast<int32_t>(xs.size());
    uint64_t events = 0;
    for (auto& x : xs) {
      events += x->rows();
      x.unshared().offset(st.next_id_block());
      out.push(std::move(x));
    }
    t.stop(events);
  }

  int32_t acquire_credit(inbound_path* path, int32_t desired) override {
    VAST_TRACE(VAST_ARG(path) << VAST_ARG(desired));
    CAF_IGNORE_UNUSED(path);
    // This function makes sure that we never hand out more credit than we have
    // IDs available.
    if (desired == 0) {
      // Easy decision if the path acquires no new credit.
      return 0;
    }
    // Calculate how much more in-flight events we can allow.
    auto& st = self_->state;
    CAF_ASSERT(st.available_ids() % st.max_table_slice_size == 0);
    auto max_credit = (st.available_ids() / st.max_table_slice_size)
                      - st.in_flight_slices;
    VAST_ASSERT(max_credit >= 0);
    if (max_credit <= desired) {
      // Get more IDs if we're running out.
      VAST_DEBUG(self_, "had to limit acquired credit to", max_credit);
      replenish(self_);
      st.in_flight_slices += max_credit;
      return max_credit;
    }
    st.in_flight_slices += desired;
    return desired;
  }

  pointer self() const {
    return self_;
  }

private:
  pointer self_;
};

auto make_importer_stage(importer_actor* self) {
  using impl = detail::notifying_stream_manager<driver>;
  auto result = caf::make_counted<impl>(self);
  result->continuous(true);
  return result;
}

} // namespace <anonymous>

behavior importer(stateful_actor<importer_state>* self, path dir,
                  size_t max_table_slice_size) {
  VAST_TRACE(VAST_ARG(dir), VAST_ARG(max_table_slice_size));
  self->state.dir = dir;
  self->state.last_replenish = steady_clock::time_point::min();
  self->state.max_table_slice_size = static_cast<int32_t>(max_table_slice_size);
  auto err = self->state.read_state();
  if (err) {
    VAST_ERROR(self, "failed to load state:", self->system().render(err));
    self->quit(std::move(err));
    return {};
  }
  namespace defs = defaults::system;
  if (auto a = self->system().registry().get(accountant_atom::value)) {
    self->state.accountant = actor_cast<accountant_type>(a);
    self->send(self->state.accountant, announce_atom::value, self->name());
    self->delayed_send(self, defs::telemetry_rate, telemetry_atom::value);
  }
  self->set_exit_handler(
    [=](const exit_msg& msg) {
      self->state.send_report();
      self->quit(msg.reason);
    });
  self->state.stg = make_importer_stage(self);
  return {
    [=](const consensus_type& c) {
      VAST_DEBUG(self, "registers consensus module");
      VAST_ASSERT(c != self->state.consensus);
      self->monitor(c);
      self->state.consensus = c;
    },
    [=](const archive_type& archive) {
      VAST_DEBUG(self, "registers archive", archive);
      return self->state.stg->add_outbound_path(archive);
    },
    [=](index_atom, const actor& index) {
      VAST_DEBUG(self, "registers index", index);
      self->state.index_actors.emplace_back(index);
      // TODO: currently, the subscriber expects only a single 'flush' message.
      //       Adding multiple INDEX actors will cause the subscriber to
      //       receive more than one 'flush'  message, but the subscriber only
      //       expects one and will stop waiting after the first one. Once we
      //       support multiple INDEX actors at the IMPORTER, we also need to
      //       revise the signaling of these 'flush' messages.
      if (self->state.index_actors.size() > 1)
        VAST_WARNING(self, "registered more than one INDEX actor",
                     "(currently unsupported!)");
      return self->state.stg->add_outbound_path(index);
    },
    [=](exporter_atom, const actor& exporter) {
      VAST_DEBUG(self, "registers exporter", exporter);
      return self->state.stg->add_outbound_path(exporter);
    },
    [=](stream<importer_state::input_type>& in) {
      auto& st = self->state;
      if (!st.consensus) {
        VAST_ERROR(self, "has no consensus module configured");
        return;
      }
      VAST_INFO(self, "adds a new source");
      st.stg->add_inbound_path(in);
    },
    [=](add_atom, const actor& subscriber) {
      auto& st = self->state;
      VAST_INFO(self, "adds a new sink");
      st.stg->add_outbound_path(subscriber);
    },
    [=](subscribe_atom, flush_atom, actor& listener) {
      auto& st = self->state;
      VAST_ASSERT(st.stg != nullptr);
      st.flush_listeners.emplace_back(std::move(listener));
      detail::notify_listeners_if_clean(st, *st.stg);
    },
    [=](status_atom) {
      return self->state.status();
    },
    [=](telemetry_atom) {
      self->state.send_report();
      self->delayed_send(self, defs::telemetry_rate, telemetry_atom::value);
    }
  };
}

} // namespace vast::system
<|endoftext|>