<commit_before>// Copyright Hugh Perkins 2014 hughperkins at gmail
//
// This Source Code Form is subject to the terms of the Mozilla Public License,
// v. 2.0. If a copy of the MPL was not distributed with this file, You can
// obtain one at http://mozilla.org/MPL/2.0/.

#include "util/StatefulTimer.h"

#include "layer/LayerMaker.h"
#include "loss/SoftMaxLayer.h"

using namespace std;

#undef VIRTUAL
#define VIRTUAL
#undef STATIC
#define STATIC

SoftMaxLayer::SoftMaxLayer(Layer *previousLayer, SoftMaxMaker *maker) :
    LossLayer(previousLayer, maker),
        perPlane(maker->_perPlane),
        imageSize(previousLayer->getOutputSize()),
        numPlanes(previousLayer->getOutputPlanes()),
        imageSizeSquared(previousLayer->getOutputSize() * previousLayer->getOutputSize()),
        output(0),
        gradInput(0),
        allocatedSize(0),
        batchSize(0)
         {
}
VIRTUAL SoftMaxLayer::~SoftMaxLayer() {
    if(gradInput != 0) {
        delete[] gradInput;
    }
    if(output != 0) {
        delete[] output;
    }
}
VIRTUAL std::string SoftMaxLayer::getClassName() const {
    return "SoftMaxLayer";
}
VIRTUAL float *SoftMaxLayer::getOutput() {
    return output;
}
VIRTUAL float *SoftMaxLayer::getGradInput() {
    return gradInput;
}
VIRTUAL void SoftMaxLayer::setBatchSize(int batchSize) {
    this->batchSize = batchSize;
    if(batchSize <= this->allocatedSize) {
        return;
    }
    if(output != 0) {
        delete[] output;
    }
    if(gradInput != 0) {
        delete[] gradInput;
    }
    output = new float[ getOutputNumElements() ];
    gradInput = new float[ previousLayer-> getOutputNumElements() ];
    allocatedSize = batchSize;
}
VIRTUAL int SoftMaxLayer::getBatchSize() {
    return this->batchSize;
}
// need to calculate multinomial logistic /cross-entropy loss
VIRTUAL float SoftMaxLayer::calcLossFromLabels(int const *labels) {
//    cout << "softmaxlayer::calcloss" << endl;
    StatefulTimer::timeCheck("start SoftMaxLayer calcLossfromlabels");
    float loss = 0;
    if(perPlane) {
        for(int n = 0; n < batchSize; n++) {
            for(int plane = 0; plane < numPlanes; plane++) {
                int label = labels[n * numPlanes + plane];
                int imageOffset = (n * numPlanes + plane) * imageSizeSquared;
                loss += - log(std::max(output[ imageOffset + label ], FLT_MIN));
            }
        }
    } else {
        // force imagesize of 1 for now
        if(imageSize != 1) {
            throw std::runtime_error("perColumn only supported for imagesize 1 for now.  Sit tight :-)  (But please raise an issue to highlight your need)");
        }
        for(int n = 0; n < batchSize; n++) {
            int imageOffset = n * numPlanes * imageSizeSquared;
            int label = labels[n];
            loss += - log(std::max(output[imageOffset + label], FLT_MIN));
        }
    }
    StatefulTimer::timeCheck("end SoftMaxLayer calcLossfromlabels");
    return loss;
}
// need to calculate multinomial logistic /cross-entropy loss
VIRTUAL float SoftMaxLayer::calcLoss(float const *expectedValues) {
    StatefulTimer::timeCheck("start SoftMaxLayer calcLoss");
    float loss = 0;
    if(perPlane) {
        for(int n = 0; n < batchSize; n++) {
            for(int plane = 0; plane < numPlanes; plane++) {
                int imageOffset = (n * numPlanes + plane) * imageSizeSquared;
                for(int i = 0; i < imageSizeSquared; i++) {
                    if(expectedValues[ imageOffset + i ] != 0) {
                        float thisloss = - expectedValues[ imageOffset + i ] * log(std::max(output[ imageOffset + i ], FLT_MIN));
                        loss += thisloss;
                    }
                }
            }
        }
    } else {
        // force imagesize of 1 for now
        if(imageSize != 1) {
            throw std::runtime_error("perColumn only supported for imagesize 1 for now.  Sit tight :-)  (But please raise an issue to highlight your need)");
        }
        for(int n = 0; n < batchSize; n++) {
            int imageOffset = n * numPlanes * imageSizeSquared;
            for(int plane = 0; plane < numPlanes; plane++) {
                float thisloss = - expectedValues[imageOffset + plane] * log(std::max(output[imageOffset + plane], FLT_MIN));
                loss += thisloss;
            }
        }
    }
    StatefulTimer::timeCheck("end SoftMaxLayer calcLoss");
    return loss;
}
// calculate partial deriv loss wrt our inputs, in other words, product of
// (multinomial cross-entropy) loss derivative wrt our output, and
// derivative of softmax wrt our inputs
VIRTUAL void SoftMaxLayer::calcGradInputFromLabels(int const *labels) {
//    cout << "softmaxlayer::calcerrors" << endl;
    StatefulTimer::timeCheck("start SoftMaxLayer calcGradInputfromlabels");
    if(perPlane) {
        for(int n = 0; n < batchSize; n++) {
            for(int plane = 0; plane < numPlanes; plane++) {
                int imageOffset = (n * numPlanes + plane) * imageSizeSquared;
                int label = labels[n * numPlanes + plane];
                for(int i = 0; i < imageSizeSquared; i++) {
                    float value = output[imageOffset + i];
                    if (std::isfinite(value) == false)
                        throw runtime_error("Output is a non-finite number, this usually means the learning rate is too high");
                    gradInput[imageOffset + i] = value;
                }
                gradInput[imageOffset + label] -= 1;
            }
        }
    } else {
        // force imagesize of 1 for now
        if(imageSize != 1) {
            throw std::runtime_error("perColumn only supported for imagesize 1 for now.  Sit tight :-)  (But please raise an issue to highlight your need)");
        }
        for(int n = 0; n < batchSize; n++) {
            int imageOffset = n * numPlanes * imageSizeSquared;
            int label = labels[n];
            for(int plane = 0; plane < numPlanes; plane++) {
                float value = output[imageOffset + plane];
                if (std::isfinite(value) == false)
                    throw runtime_error("Output is a non-finite number, this usually means the learning rate is too high");
                gradInput[imageOffset + plane] = value;
            }
            if(label >= numPlanes) {
                throw runtime_error("Label " + toString(label) + " exceeds number of softmax planes " + toString(numPlanes) );
            } else if(label < 0) {
                throw runtime_error("Label " + toString(label) + " negative");
            }
            gradInput[imageOffset + label] -= 1;
        }
    }
    StatefulTimer::timeCheck("end SoftMaxLayer calcGradInputfromlabels");
}
// calculate partial deriv loss wrt our inputs, in other words, product of
// (multinomial cross-entropy) loss derivative wrt our output, and
// derivative of softmax wrt our inputs
VIRTUAL void SoftMaxLayer::calcGradInput(float const *expectedValues) {
//    cout << "softmaxlayer::calcerrors" << endl;
    StatefulTimer::timeCheck("start SoftMaxLayer calcGradInput");
    if(perPlane) {
        for(int n = 0; n < batchSize; n++) {
            for(int plane = 0; plane < numPlanes; plane++) {
                int imageOffset = (n * numPlanes + plane) * imageSizeSquared;
                for(int i = 0; i < imageSizeSquared; i++) {
                    int resultIndex = imageOffset + i;
                    float value = output[resultIndex];
                    if (std::isfinite(value) == false)
                        throw runtime_error("Output is a non-finite number, this usually means the learning rate is too high");
                    gradInput[resultIndex] = value - expectedValues[resultIndex];
                }
            }
        }
    } else {
        // force imagesize of 1 for now
        if(imageSize != 1) {
            throw std::runtime_error("perColumn only supported for imagesize 1 for now.  Sit tight :-)  (But please raise an issue to highlight your need)");
        }
        for(int n = 0; n < batchSize; n++) {
            int imageOffset = n * numPlanes * imageSizeSquared;
            for(int plane = 0; plane < numPlanes; plane++) {
                int resultIndex = imageOffset + plane;
                float value = output[resultIndex];
                if (std::isfinite(value) == false)
                    throw runtime_error("Output is a non-finite number, this usually means the learning rate is too high");
                gradInput[resultIndex] = value - expectedValues[resultIndex];
            }
        }
    }
    StatefulTimer::timeCheck("end SoftMaxLayer calcGradInput");
}
VIRTUAL int SoftMaxLayer::getNumLabelsPerExample() {
    if(perPlane) {
        return numPlanes;
    } else {
        return imageSizeSquared;
    }
}
VIRTUAL int SoftMaxLayer::getPersistSize(int version) const {
    return 0;
}
VIRTUAL int SoftMaxLayer::calcNumRightFromLabels(int const*labels) {
    StatefulTimer::timeCheck("start SoftMaxLayer calcNumRight");
//    float *input = previousLayer->getOutput(); // just retrieve as host-side array for now
    int numRight = 0;
    if(perPlane) {
        for(int n = 0; n < batchSize; n++) {
            for(int plane = 0; plane < numPlanes; plane++) {
                int imageOffset = (n * numPlanes + plane) * imageSizeSquared;
                int label = labels[n * numPlanes + plane];
                float thisMax = output[imageOffset + 0];
                int iMax = 0;
                for(int i = 1; i < imageSizeSquared; i++) {
                    if(output[imageOffset + i] > thisMax) {
                        thisMax = output[imageOffset + i];
                        iMax = i;
                    }
                }
                if(label == iMax) {
//                    cout << "n " << n << " plane " << plane << " label " << label << endl;
                    numRight++;
                }
            }
        }
    } else {
        // force imagesize of 1 for now
        if(imageSize != 1) {
            throw std::runtime_error("perColumn only supported for imagesize 1 for now.  Sit tight :-)  (But please raise an issue to highlight your need)");
        }
        for(int n = 0; n < batchSize; n++) {
            int imageOffset = n * numPlanes * imageSizeSquared;
            int label = labels[n];
            float thisMax = output[imageOffset + 0];
            int iMax = 0;
            for(int i = 1; i < numPlanes; i++) {
                if(output[imageOffset + i] > thisMax) {
                    thisMax = output[imageOffset + i];
                    iMax = i;
                }
            }
            if(label == iMax) {
                numRight++;
            }
        }
    }

    StatefulTimer::timeCheck("start SoftMaxLayer calcNumRight");
    return numRight;
}
// for forward, we just need to apply the softmax activation. "just" :-P
VIRTUAL void SoftMaxLayer::forward() {
//    cout << "softmaxlayer::forward" << endl;
    StatefulTimer::timeCheck("start SoftMaxLayer forward");
    float *input = previousLayer->getOutput(); // just retrieve as host-side array for now
    if(perPlane) {
        for(int n = 0; n < batchSize; n++) {
            for(int plane = 0; plane < numPlanes; plane++) {
                int imageOffset = (n * numPlanes + plane) * imageSizeSquared;
                float maxValue = input[imageOffset + 0];
                for(int i = 1; i < imageSizeSquared; i++) {
                    maxValue = std::max(maxValue, input[imageOffset + i]);
                }
                float denominator = 0;
                for(int i = 0; i < imageSizeSquared; i++) {
                    denominator += exp(input[imageOffset + i] - maxValue);
                }
                for(int i = 0; i < imageSizeSquared; i++) {
                    output[imageOffset + i] = exp(input[imageOffset + i] - maxValue) / denominator;
                }
            }
        }
    } else {
        // force imagesize of 1 for now
        if(imageSize != 1) {
            throw std::runtime_error("perColumn only supported for imagesize 1 for now.  Sit tight :-)  (But please raise an issue to highlight your need)");
        }
        for(int n = 0; n < batchSize; n++) {
            int imageOffset = n * numPlanes * imageSizeSquared;
            // first get the max
            float maxValue = input[imageOffset + 0]; // since we assume imagesize 1, this is correct
            for(int plane = 1; plane < numPlanes; plane++) {
                maxValue = std::max(maxValue, input[imageOffset + plane]);
            }
            // calculate sum, under this max
            float denominator = 0;
            for(int plane = 0; plane < numPlanes; plane++) {
                denominator += exp(input[imageOffset + plane] - maxValue);
            }
            // now calc the softmaxes:
            for(int plane = 0; plane < numPlanes; plane++) {
                output[imageOffset + plane] = exp(input[imageOffset + plane] - maxValue) / denominator;
            }
        }
    }
    StatefulTimer::timeCheck("end SoftMaxLayer forward");
}
VIRTUAL void SoftMaxLayer::getLabels(int *labels) { // need to allocate labels array first, and have called 'forward' first
    if(perPlane) {
        throw std::runtime_error("getLabels doesnt work with 'perPlane' option currently, though it wouldnt be hard to add, so ask if you need");
    }
    if(imageSize != 1) {
        throw std::runtime_error("perColumn only supported for imagesize 1 for now.  Sit tight :-)  (But please raise an issue to highlight your need)");
    }
    for(int n = 0; n < batchSize; n++) {
        float *outputStack = output + n * numPlanes;
        float highestProb = outputStack[0];
        int bestPlane = 0;
        for(int plane = 1; plane < numPlanes; plane++) {
            if(outputStack[plane] > highestProb) {
                bestPlane = plane;
                highestProb = outputStack[plane];
            }
        }
        labels[n] = bestPlane;
    }
}
// this seems to be handled by calcGradInput? So, just to a nop?
// (cos this layer kind of combines loss layer and a 'normal' propagation layer)
// certainly, we dont have any weights to update, and we already handled error
// propagation in 'calcGradInput' method above
//VIRTUAL void SoftMaxLayer::backward(float learningRate) {
//    cout << "softmaxlayer::backproperrors" << endl;
    // nop, do nothing :-)
//}
VIRTUAL std::string SoftMaxLayer::asString() const {
    return "SoftMaxLayer{ perPlane=" + toString(perPlane) + " numPlanes=" + toString(numPlanes)
        + " imageSize=" + toString(imageSize) + " }";
}

<commit_msg>SoftMaxLayer: Added missing include<commit_after>// Copyright Hugh Perkins 2014 hughperkins at gmail
//
// This Source Code Form is subject to the terms of the Mozilla Public License,
// v. 2.0. If a copy of the MPL was not distributed with this file, You can
// obtain one at http://mozilla.org/MPL/2.0/.

#include "util/StatefulTimer.h"

#include "layer/LayerMaker.h"
#include "loss/SoftMaxLayer.h"
#include <cfloat>

using namespace std;

#undef VIRTUAL
#define VIRTUAL
#undef STATIC
#define STATIC

SoftMaxLayer::SoftMaxLayer(Layer *previousLayer, SoftMaxMaker *maker) :
    LossLayer(previousLayer, maker),
        perPlane(maker->_perPlane),
        imageSize(previousLayer->getOutputSize()),
        numPlanes(previousLayer->getOutputPlanes()),
        imageSizeSquared(previousLayer->getOutputSize() * previousLayer->getOutputSize()),
        output(0),
        gradInput(0),
        allocatedSize(0),
        batchSize(0)
         {
}
VIRTUAL SoftMaxLayer::~SoftMaxLayer() {
    if(gradInput != 0) {
        delete[] gradInput;
    }
    if(output != 0) {
        delete[] output;
    }
}
VIRTUAL std::string SoftMaxLayer::getClassName() const {
    return "SoftMaxLayer";
}
VIRTUAL float *SoftMaxLayer::getOutput() {
    return output;
}
VIRTUAL float *SoftMaxLayer::getGradInput() {
    return gradInput;
}
VIRTUAL void SoftMaxLayer::setBatchSize(int batchSize) {
    this->batchSize = batchSize;
    if(batchSize <= this->allocatedSize) {
        return;
    }
    if(output != 0) {
        delete[] output;
    }
    if(gradInput != 0) {
        delete[] gradInput;
    }
    output = new float[ getOutputNumElements() ];
    gradInput = new float[ previousLayer-> getOutputNumElements() ];
    allocatedSize = batchSize;
}
VIRTUAL int SoftMaxLayer::getBatchSize() {
    return this->batchSize;
}
// need to calculate multinomial logistic /cross-entropy loss
VIRTUAL float SoftMaxLayer::calcLossFromLabels(int const *labels) {
//    cout << "softmaxlayer::calcloss" << endl;
    StatefulTimer::timeCheck("start SoftMaxLayer calcLossfromlabels");
    float loss = 0;
    if(perPlane) {
        for(int n = 0; n < batchSize; n++) {
            for(int plane = 0; plane < numPlanes; plane++) {
                int label = labels[n * numPlanes + plane];
                int imageOffset = (n * numPlanes + plane) * imageSizeSquared;
                loss += - log(std::max(output[ imageOffset + label ], FLT_MIN));
            }
        }
    } else {
        // force imagesize of 1 for now
        if(imageSize != 1) {
            throw std::runtime_error("perColumn only supported for imagesize 1 for now.  Sit tight :-)  (But please raise an issue to highlight your need)");
        }
        for(int n = 0; n < batchSize; n++) {
            int imageOffset = n * numPlanes * imageSizeSquared;
            int label = labels[n];
            loss += - log(std::max(output[imageOffset + label], FLT_MIN));
        }
    }
    StatefulTimer::timeCheck("end SoftMaxLayer calcLossfromlabels");
    return loss;
}
// need to calculate multinomial logistic /cross-entropy loss
VIRTUAL float SoftMaxLayer::calcLoss(float const *expectedValues) {
    StatefulTimer::timeCheck("start SoftMaxLayer calcLoss");
    float loss = 0;
    if(perPlane) {
        for(int n = 0; n < batchSize; n++) {
            for(int plane = 0; plane < numPlanes; plane++) {
                int imageOffset = (n * numPlanes + plane) * imageSizeSquared;
                for(int i = 0; i < imageSizeSquared; i++) {
                    if(expectedValues[ imageOffset + i ] != 0) {
                        float thisloss = - expectedValues[ imageOffset + i ] * log(std::max(output[ imageOffset + i ], FLT_MIN));
                        loss += thisloss;
                    }
                }
            }
        }
    } else {
        // force imagesize of 1 for now
        if(imageSize != 1) {
            throw std::runtime_error("perColumn only supported for imagesize 1 for now.  Sit tight :-)  (But please raise an issue to highlight your need)");
        }
        for(int n = 0; n < batchSize; n++) {
            int imageOffset = n * numPlanes * imageSizeSquared;
            for(int plane = 0; plane < numPlanes; plane++) {
                float thisloss = - expectedValues[imageOffset + plane] * log(std::max(output[imageOffset + plane], FLT_MIN));
                loss += thisloss;
            }
        }
    }
    StatefulTimer::timeCheck("end SoftMaxLayer calcLoss");
    return loss;
}
// calculate partial deriv loss wrt our inputs, in other words, product of
// (multinomial cross-entropy) loss derivative wrt our output, and
// derivative of softmax wrt our inputs
VIRTUAL void SoftMaxLayer::calcGradInputFromLabels(int const *labels) {
//    cout << "softmaxlayer::calcerrors" << endl;
    StatefulTimer::timeCheck("start SoftMaxLayer calcGradInputfromlabels");
    if(perPlane) {
        for(int n = 0; n < batchSize; n++) {
            for(int plane = 0; plane < numPlanes; plane++) {
                int imageOffset = (n * numPlanes + plane) * imageSizeSquared;
                int label = labels[n * numPlanes + plane];
                for(int i = 0; i < imageSizeSquared; i++) {
                    float value = output[imageOffset + i];
                    if (std::isfinite(value) == false)
                        throw runtime_error("Output is a non-finite number, this usually means the learning rate is too high");
                    gradInput[imageOffset + i] = value;
                }
                gradInput[imageOffset + label] -= 1;
            }
        }
    } else {
        // force imagesize of 1 for now
        if(imageSize != 1) {
            throw std::runtime_error("perColumn only supported for imagesize 1 for now.  Sit tight :-)  (But please raise an issue to highlight your need)");
        }
        for(int n = 0; n < batchSize; n++) {
            int imageOffset = n * numPlanes * imageSizeSquared;
            int label = labels[n];
            for(int plane = 0; plane < numPlanes; plane++) {
                float value = output[imageOffset + plane];
                if (std::isfinite(value) == false)
                    throw runtime_error("Output is a non-finite number, this usually means the learning rate is too high");
                gradInput[imageOffset + plane] = value;
            }
            if(label >= numPlanes) {
                throw runtime_error("Label " + toString(label) + " exceeds number of softmax planes " + toString(numPlanes) );
            } else if(label < 0) {
                throw runtime_error("Label " + toString(label) + " negative");
            }
            gradInput[imageOffset + label] -= 1;
        }
    }
    StatefulTimer::timeCheck("end SoftMaxLayer calcGradInputfromlabels");
}
// calculate partial deriv loss wrt our inputs, in other words, product of
// (multinomial cross-entropy) loss derivative wrt our output, and
// derivative of softmax wrt our inputs
VIRTUAL void SoftMaxLayer::calcGradInput(float const *expectedValues) {
//    cout << "softmaxlayer::calcerrors" << endl;
    StatefulTimer::timeCheck("start SoftMaxLayer calcGradInput");
    if(perPlane) {
        for(int n = 0; n < batchSize; n++) {
            for(int plane = 0; plane < numPlanes; plane++) {
                int imageOffset = (n * numPlanes + plane) * imageSizeSquared;
                for(int i = 0; i < imageSizeSquared; i++) {
                    int resultIndex = imageOffset + i;
                    float value = output[resultIndex];
                    if (std::isfinite(value) == false)
                        throw runtime_error("Output is a non-finite number, this usually means the learning rate is too high");
                    gradInput[resultIndex] = value - expectedValues[resultIndex];
                }
            }
        }
    } else {
        // force imagesize of 1 for now
        if(imageSize != 1) {
            throw std::runtime_error("perColumn only supported for imagesize 1 for now.  Sit tight :-)  (But please raise an issue to highlight your need)");
        }
        for(int n = 0; n < batchSize; n++) {
            int imageOffset = n * numPlanes * imageSizeSquared;
            for(int plane = 0; plane < numPlanes; plane++) {
                int resultIndex = imageOffset + plane;
                float value = output[resultIndex];
                if (std::isfinite(value) == false)
                    throw runtime_error("Output is a non-finite number, this usually means the learning rate is too high");
                gradInput[resultIndex] = value - expectedValues[resultIndex];
            }
        }
    }
    StatefulTimer::timeCheck("end SoftMaxLayer calcGradInput");
}
VIRTUAL int SoftMaxLayer::getNumLabelsPerExample() {
    if(perPlane) {
        return numPlanes;
    } else {
        return imageSizeSquared;
    }
}
VIRTUAL int SoftMaxLayer::getPersistSize(int version) const {
    return 0;
}
VIRTUAL int SoftMaxLayer::calcNumRightFromLabels(int const*labels) {
    StatefulTimer::timeCheck("start SoftMaxLayer calcNumRight");
//    float *input = previousLayer->getOutput(); // just retrieve as host-side array for now
    int numRight = 0;
    if(perPlane) {
        for(int n = 0; n < batchSize; n++) {
            for(int plane = 0; plane < numPlanes; plane++) {
                int imageOffset = (n * numPlanes + plane) * imageSizeSquared;
                int label = labels[n * numPlanes + plane];
                float thisMax = output[imageOffset + 0];
                int iMax = 0;
                for(int i = 1; i < imageSizeSquared; i++) {
                    if(output[imageOffset + i] > thisMax) {
                        thisMax = output[imageOffset + i];
                        iMax = i;
                    }
                }
                if(label == iMax) {
//                    cout << "n " << n << " plane " << plane << " label " << label << endl;
                    numRight++;
                }
            }
        }
    } else {
        // force imagesize of 1 for now
        if(imageSize != 1) {
            throw std::runtime_error("perColumn only supported for imagesize 1 for now.  Sit tight :-)  (But please raise an issue to highlight your need)");
        }
        for(int n = 0; n < batchSize; n++) {
            int imageOffset = n * numPlanes * imageSizeSquared;
            int label = labels[n];
            float thisMax = output[imageOffset + 0];
            int iMax = 0;
            for(int i = 1; i < numPlanes; i++) {
                if(output[imageOffset + i] > thisMax) {
                    thisMax = output[imageOffset + i];
                    iMax = i;
                }
            }
            if(label == iMax) {
                numRight++;
            }
        }
    }

    StatefulTimer::timeCheck("start SoftMaxLayer calcNumRight");
    return numRight;
}
// for forward, we just need to apply the softmax activation. "just" :-P
VIRTUAL void SoftMaxLayer::forward() {
//    cout << "softmaxlayer::forward" << endl;
    StatefulTimer::timeCheck("start SoftMaxLayer forward");
    float *input = previousLayer->getOutput(); // just retrieve as host-side array for now
    if(perPlane) {
        for(int n = 0; n < batchSize; n++) {
            for(int plane = 0; plane < numPlanes; plane++) {
                int imageOffset = (n * numPlanes + plane) * imageSizeSquared;
                float maxValue = input[imageOffset + 0];
                for(int i = 1; i < imageSizeSquared; i++) {
                    maxValue = std::max(maxValue, input[imageOffset + i]);
                }
                float denominator = 0;
                for(int i = 0; i < imageSizeSquared; i++) {
                    denominator += exp(input[imageOffset + i] - maxValue);
                }
                for(int i = 0; i < imageSizeSquared; i++) {
                    output[imageOffset + i] = exp(input[imageOffset + i] - maxValue) / denominator;
                }
            }
        }
    } else {
        // force imagesize of 1 for now
        if(imageSize != 1) {
            throw std::runtime_error("perColumn only supported for imagesize 1 for now.  Sit tight :-)  (But please raise an issue to highlight your need)");
        }
        for(int n = 0; n < batchSize; n++) {
            int imageOffset = n * numPlanes * imageSizeSquared;
            // first get the max
            float maxValue = input[imageOffset + 0]; // since we assume imagesize 1, this is correct
            for(int plane = 1; plane < numPlanes; plane++) {
                maxValue = std::max(maxValue, input[imageOffset + plane]);
            }
            // calculate sum, under this max
            float denominator = 0;
            for(int plane = 0; plane < numPlanes; plane++) {
                denominator += exp(input[imageOffset + plane] - maxValue);
            }
            // now calc the softmaxes:
            for(int plane = 0; plane < numPlanes; plane++) {
                output[imageOffset + plane] = exp(input[imageOffset + plane] - maxValue) / denominator;
            }
        }
    }
    StatefulTimer::timeCheck("end SoftMaxLayer forward");
}
VIRTUAL void SoftMaxLayer::getLabels(int *labels) { // need to allocate labels array first, and have called 'forward' first
    if(perPlane) {
        throw std::runtime_error("getLabels doesnt work with 'perPlane' option currently, though it wouldnt be hard to add, so ask if you need");
    }
    if(imageSize != 1) {
        throw std::runtime_error("perColumn only supported for imagesize 1 for now.  Sit tight :-)  (But please raise an issue to highlight your need)");
    }
    for(int n = 0; n < batchSize; n++) {
        float *outputStack = output + n * numPlanes;
        float highestProb = outputStack[0];
        int bestPlane = 0;
        for(int plane = 1; plane < numPlanes; plane++) {
            if(outputStack[plane] > highestProb) {
                bestPlane = plane;
                highestProb = outputStack[plane];
            }
        }
        labels[n] = bestPlane;
    }
}
// this seems to be handled by calcGradInput? So, just to a nop?
// (cos this layer kind of combines loss layer and a 'normal' propagation layer)
// certainly, we dont have any weights to update, and we already handled error
// propagation in 'calcGradInput' method above
//VIRTUAL void SoftMaxLayer::backward(float learningRate) {
//    cout << "softmaxlayer::backproperrors" << endl;
    // nop, do nothing :-)
//}
VIRTUAL std::string SoftMaxLayer::asString() const {
    return "SoftMaxLayer{ perPlane=" + toString(perPlane) + " numPlanes=" + toString(numPlanes)
        + " imageSize=" + toString(imageSize) + " }";
}

<|endoftext|>


<commit_before>/*****************************************************************************
 *
 * This file is part of Mapnik (c++ mapping toolkit)
 *
 * Copyright (C) 2006 Artem Pavlenko
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2.1 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with this library; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
 *
 *****************************************************************************/

//$Id: postgisfs.cc 34 2005-04-04 13:27:23Z pavlenko $

#include <boost/lexical_cast.hpp>
#include <boost/algorithm/string.hpp>
#include <mapnik/global.hpp>
#include <mapnik/wkb.hpp>
#include <mapnik/unicode.hpp>
#include <mapnik/sql_utils.hpp>
#include "postgis.hpp"
#include <sstream>

using boost::lexical_cast;
using boost::bad_lexical_cast;
using boost::trim_copy;
using mapnik::Feature;
using mapnik::geometry_type;
using mapnik::byte;
using mapnik::geometry_utils;

postgis_featureset::postgis_featureset(boost::shared_ptr<IResultSet> const& rs,
                                       std::string const& encoding,
                                       bool multiple_geometries,
                                       unsigned num_attrs=0)
    : rs_(rs),
      multiple_geometries_(multiple_geometries),
      num_attrs_(num_attrs),
      tr_(new transcoder(encoding)),
      totalGeomSize_(0),
      feature_id_(1)  {}

feature_ptr postgis_featureset::next()
{
    if (rs_->next())
    {
        feature_ptr feature(new Feature(feature_id_));
        ++feature_id_;

        int size = rs_->getFieldLength(0);
        const char *data = rs_->getValue(0);
        geometry_utils::from_wkb(*feature,data,size,multiple_geometries_);
        totalGeomSize_+=size;

        for (unsigned pos=1;pos<num_attrs_+1;++pos)
        {
           std::string name = rs_->getFieldName(pos);

           if (!rs_->isNull(pos))
           {
              const char* buf=rs_->getValue(pos);
              int oid = rs_->getTypeOID(pos);

              if (oid==16) //bool
              {
                 boost::put(*feature,name,buf[0] != 0);
              }
              else if (oid==23) //int4
              {
                 int val = int4net(buf);
                 boost::put(*feature,name,val);
              }
              else if (oid==21) //int2
              {
                 int val = int2net(buf);
                 boost::put(*feature,name,val);
              }
              else if (oid==20) //int8/BigInt
              {
                 int val = int8net(buf);
                 boost::put(*feature,name,val);
              }
              else if (oid == 700) // float4
              {
                 float val;
                 float4net(val,buf);
                 boost::put(*feature,name,val);
              }
              else if (oid == 701) // float8
              {
                 double val;
                 float8net(val,buf);
                 boost::put(*feature,name,val);
              }
              else if (oid==25 || oid==1043) // text or varchar
              {
                 UnicodeString ustr = tr_->transcode(buf);
                 boost::put(*feature,name,ustr);
              }
              else if (oid==1042)
              {
                 UnicodeString ustr = tr_->transcode(trim_copy(string(buf)).c_str()); // bpchar
                 boost::put(*feature,name,ustr);
              }
              else if (oid == 1700) // numeric
              {
                 std::string str = mapnik::numeric2string(buf);
                 try
                 {
                    double val = boost::lexical_cast<double>(str);
                    boost::put(*feature,name,val);
                 }
                 catch (boost::bad_lexical_cast & ex)
                 {
                    std::clog << ex.what() << "\n";
                 }
              }
              else
              {
#ifdef MAPNIK_DEBUG
                 std::clog << "Postgis Plugin: uknown OID = " << oid << " FIXME " << std::endl;
#endif
              }
           }
        }
        return feature;
    }
    else
    {
        rs_->close();
        return feature_ptr();
    }
}

postgis_featureset::~postgis_featureset()
{
    rs_->close();
}
<commit_msg>+ use feature_factory to create features<commit_after>/*****************************************************************************
 *
 * This file is part of Mapnik (c++ mapping toolkit)
 *
 * Copyright (C) 2006 Artem Pavlenko
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2.1 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with this library; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
 *
 *****************************************************************************/

//$Id: postgisfs.cc 34 2005-04-04 13:27:23Z pavlenko $

#include <boost/lexical_cast.hpp>
#include <boost/algorithm/string.hpp>
#include <mapnik/global.hpp>
#include <mapnik/wkb.hpp>
#include <mapnik/unicode.hpp>
#include <mapnik/sql_utils.hpp>
#include <mapnik/feature_factory.hpp>

#include "postgis.hpp"
#include <sstream>

using boost::lexical_cast;
using boost::bad_lexical_cast;
using boost::trim_copy;
using mapnik::geometry_type;
using mapnik::byte;
using mapnik::geometry_utils;
using mapnik::feature_factory;

postgis_featureset::postgis_featureset(boost::shared_ptr<IResultSet> const& rs,
                                       std::string const& encoding,
                                       bool multiple_geometries,
                                       unsigned num_attrs=0)
    : rs_(rs),
      multiple_geometries_(multiple_geometries),
      num_attrs_(num_attrs),
      tr_(new transcoder(encoding)),
      totalGeomSize_(0),
      feature_id_(1)  {}

feature_ptr postgis_featureset::next()
{
    if (rs_->next())
    {
        feature_ptr feature(feature_factory::create(feature_id_));
        ++feature_id_;

        int size = rs_->getFieldLength(0);
        const char *data = rs_->getValue(0);
        geometry_utils::from_wkb(*feature,data,size,multiple_geometries_);
        totalGeomSize_+=size;

        for (unsigned pos=1;pos<num_attrs_+1;++pos)
        {
           std::string name = rs_->getFieldName(pos);

           if (!rs_->isNull(pos))
           {
              const char* buf=rs_->getValue(pos);
              int oid = rs_->getTypeOID(pos);

              if (oid==16) //bool
              {
                 boost::put(*feature,name,buf[0] != 0);
              }
              else if (oid==23) //int4
              {
                 int val = int4net(buf);
                 boost::put(*feature,name,val);
              }
              else if (oid==21) //int2
              {
                 int val = int2net(buf);
                 boost::put(*feature,name,val);
              }
              else if (oid==20) //int8/BigInt
              {
                 int val = int8net(buf);
                 boost::put(*feature,name,val);
              }
              else if (oid == 700) // float4
              {
                 float val;
                 float4net(val,buf);
                 boost::put(*feature,name,val);
              }
              else if (oid == 701) // float8
              {
                 double val;
                 float8net(val,buf);
                 boost::put(*feature,name,val);
              }
              else if (oid==25 || oid==1043) // text or varchar
              {
                 UnicodeString ustr = tr_->transcode(buf);
                 boost::put(*feature,name,ustr);
              }
              else if (oid==1042)
              {
                 UnicodeString ustr = tr_->transcode(trim_copy(string(buf)).c_str()); // bpchar
                 boost::put(*feature,name,ustr);
              }
              else if (oid == 1700) // numeric
              {
                 std::string str = mapnik::numeric2string(buf);
                 try
                 {
                    double val = boost::lexical_cast<double>(str);
                    boost::put(*feature,name,val);
                 }
                 catch (boost::bad_lexical_cast & ex)
                 {
                    std::clog << ex.what() << "\n";
                 }
              }
              else
              {
#ifdef MAPNIK_DEBUG
                 std::clog << "Postgis Plugin: uknown OID = " << oid << " FIXME " << std::endl;
#endif
              }
           }
        }
        return feature;
    }
    else
    {
        rs_->close();
        return feature_ptr();
    }
}

postgis_featureset::~postgis_featureset()
{
    rs_->close();
}
<|endoftext|>


<commit_before>// implementation of the dijkstra's algorithm.
#include <cmath>
#include <cstddef>
#include <cstdint>
#include <iostream>
#include <limits>
#include <queue>
#include <vector>

class Point {
  const double x;
  const double y;

public:
  Point(const double &x, const double &y) : x(x), y(y) {}

  double dist(const Point &other) const {
    double dx = x - other.x;
    double dy = y - other.y;
    return pow(pow(dx, 2) + pow(dy, 2), 0.5);
  }
};

template <typename P> class PointCluster {
  const std::vector<P> points;

public:
  PointCluster(const std::vector<P> &points) : points(points) {}
  double dist(const PointCluster &other) const {
    double smallest = std::numeric_limits<double>::max();
    for (auto i = points.begin(); i < points.end(); ++i) {
      for (auto j = other.points.begin(); j < other.points.end(); ++j) {
        smallest = std::min(smallest, i->dist(*j));
      }
    }
    return smallest;
  }
};

template <typename P>
double dijkstra(const std::vector<P> &points, size_t begin, size_t end) {
  auto n = points.size();
  std::vector<bool> visited = std::vector<bool>(n, false);
  std::vector<double> distance =
      std::vector<double>(n, std::numeric_limits<double>::max());
  visited[begin] = true;
  distance[begin] = 0.0;
  for (auto i = 0; i < n; i++) {
    distance[i] = points[i].dist(points[begin]);
  }

  // put a bound on iterations.
  for (auto iteration = 0; iteration < n; iteration++) {
    size_t index;
    bool found = false;
    double lowest = 0;
    for (auto i = 0; i < n; i++) {
      // TODO - use a priority queue here.
      if (visited[i]) {
        continue;
      } else if (!found) {
        index = i;
        found = true;
        lowest = distance[i];
      } else if (distance[i] < lowest) {
        lowest = distance[i];
        index = i;
      }
    }
    if (index == end) {
      return distance[index];
    }
    for (auto i = 0; i < n; i++) {
      distance[i] = std::min(distance[i],
                             distance[index] + points[i].dist(points[index]));
    }
    visited[index] = true;
  }
  return std::numeric_limits<double>::max();
}

int main() {
  std::cout << "Case 1" << std::endl;
  std::vector<Point> points1;
  points1.push_back(Point(0, 0));
  points1.push_back(Point(10, 10));
  std::cout << dijkstra(points1, 0, 1) << std::endl;

  std::cout << "Case 2" << std::endl;
  std::vector<PointCluster<Point>> points2;
  points2.push_back(PointCluster<Point>(std::vector<Point> { Point(0, 0) }));
  points2.push_back(PointCluster<Point>(std::vector<Point> { Point(10, 10) }));
  points2.push_back(PointCluster<Point>(std::vector<Point> { Point(0, 1), Point(100, 100) }));
  points2.push_back(PointCluster<Point>(std::vector<Point> { Point(100, 101), Point(10, 9) }));
  points2.push_back(PointCluster<Point>(std::vector<Point> { Point(0, 2), Point(10, 8) }));
  std::cout << dijkstra(points2, 0, 1) << std::endl;
}
<commit_msg>Update<commit_after>// implementation of the dijkstra's algorithm.
#include <cmath>
#include <cstddef>
#include <cstdint>
#include <iostream>
#include <limits>
#include <queue>
#include <vector>

class Point {
  const double x;
  const double y;

public:
  Point(const double &x, const double &y) : x(x), y(y) {}

  double dist(const Point &other) const {
    double dx = x - other.x;
    double dy = y - other.y;
    return pow(pow(dx, 2) + pow(dy, 2), 0.5);
  }
};

template <typename P> class PointCluster {
  const std::vector<P> points;

public:
  PointCluster(const std::vector<P> &points) : points(points) {}
  double dist(const PointCluster &other) const {
    double smallest = std::numeric_limits<double>::max();
    for (auto i = points.begin(); i < points.end(); ++i) {
      for (auto j = other.points.begin(); j < other.points.end(); ++j) {
        smallest = std::min(smallest, i->dist(*j));
      }
    }
    return smallest;
  }
};

template <typename P>
double dijkstra(const std::vector<P> &points, size_t begin, size_t end) {
  auto n = points.size();
  std::vector<bool> visited = std::vector<bool>(n, false);
  std::vector<double> distance =
      std::vector<double>(n, std::numeric_limits<double>::max());
  visited[begin] = true;
  distance[begin] = 0.0;
  for (auto i = 0; i < n; i++) {
    distance[i] = points[i].dist(points[begin]);
  }

  // put a bound on iterations.
  for (auto iteration = 0; iteration < n; iteration++) {
    size_t index;
    bool found = false;
    double lowest = 0;
    for (auto i = 0; i < n; i++) {
      // TODO - use a priority queue here.
      if (visited[i]) {
        continue;
      } else if (!found) {
        index = i;
        found = true;
        lowest = distance[i];
      } else if (distance[i] < lowest) {
        lowest = distance[i];
        index = i;
      }
    }
    if (index == end) {
      return distance[index];
    }
    for (auto i = 0; i < n; i++) {
      distance[i] = std::min(distance[i],
                             distance[index] + points[i].dist(points[index]));
    }
    visited[index] = true;
  }
  return std::numeric_limits<double>::max();
}

int main() {
  std::cout << "Case 1" << std::endl;
  std::vector<Point> points1;
  points1.push_back(Point(0, 0));
  points1.push_back(Point(10, 10));
  std::cout << dijkstra(points1, 0, 1) << std::endl;

  std::cout << "Case 2" << std::endl;
  std::vector<PointCluster<Point>> points2;
  points2.push_back(PointCluster<Point>(std::vector<Point>{Point(0, 0)}));
  points2.push_back(PointCluster<Point>(std::vector<Point>{Point(10, 10)}));
  points2.push_back(
      PointCluster<Point>(std::vector<Point>{Point(0, 1), Point(100, 100)}));
  points2.push_back(
      PointCluster<Point>(std::vector<Point>{Point(100, 101), Point(10, 9)}));
  points2.push_back(
      PointCluster<Point>(std::vector<Point>{Point(0, 2), Point(10, 8)}));
  std::cout << dijkstra(points2, 0, 1) << std::endl;
}
<|endoftext|>


<commit_before>#include <cassert>
#include <iostream>
#include <memory>
#include <string>
#include <vector>

using namespace std;

// fwd decl
template <typename T> class object;
template <typename T> int count_object(const object<T>&);
template <typename T> T* get_object_ptr(const object<T>&);

template <typename T> class object {
public:
  object() = delete;
  ~object() noexcept = default;

  // Not required (because destructor declaration automatically removes move
  // constructor and move assignment)
  // object(object<T>&&) = delete;
  // object(const object<T>&&) = delete;
  // object<T>& operator=(object<T>&&) = delete;

  explicit object(T* p) : px{ p } { assert(px != nullptr); }

  operator T&() const noexcept { return *px; }

  T& operator*() const noexcept { return *px; }

  T* operator->() const noexcept { return px.get(); }

  // not possible
  // T& operator.() const noexcept { return *px; }

private:
  shared_ptr<T> px;

  friend int count_object<T>(const object<T>&);
  friend T* get_object_ptr<T>(const object<T>&);
};

// auxiliary functions
template <typename T, typename... Args> object<T> make_object(Args&&... args) {
  return object<T>{ new T{ std::forward<Args>(args)... } };
}

template <typename T> int count_object(const object<T>& obj) {
  return obj.px.use_count();
}

template <typename T> T* get_object_ptr(const object<T>& obj) {
  return obj.px.get();
}

struct base {
  virtual void f() { cout << "base" << endl; }
};

struct extended : public base {
  virtual void f() override { cout << "extended" << endl; }
};

int main() {
  // runtime errors:
  // object<int> bang {nullptr};
  // object<int> bad;

  object<int> o0{ new int{ 5 } };
  cout << *o0 << endl;

  auto mo1 = make_object<int>(10);
  // will actually make a copy instead of 'move' (because a destructor was
  // declared)
  auto mo2 = std::move(mo1);
  cout << get_object_ptr(mo1) << ",count: " << count_object(mo1) << endl;
  cout << *mo1 << endl;

  auto o1 = object<string>{ new string{ "abc" } };
  cout << *o1 << endl;

  vector<object<double> > v{ make_object<double>(1.0),
                             make_object<double>(2.0) };
  cout << "{ ";
  for (const auto& o : v) {
    cout << *o << " ";
  }
  cout << "}" << endl;

  extended ext;
  base& base_ref = ext;
  base_ref.f();

  auto o2 = make_object<extended>();
  o2->f();
  extended& ref2 = o2;
  ref2.f();
  (static_cast<extended&>(o2)).f();
  // operator. can not be overriden in c++ (and operator T&() won't do it)
  // o2.f();

  // why? how to do it?
  //  object<base> o3 = make_object<extended>();
  //  o3->f();
}
<commit_msg>fixup! fixup! some shared_ptr wrapper<commit_after>#include <cassert>
#include <iostream>
#include <memory>
#include <string>
#include <vector>

using namespace std;

template <typename T> class object;
template <typename T, typename... Args> object<T> make_object(Args&&...);
template <typename T> int count_object(const object<T>&);
template <typename T> T* get_object_ptr(const object<T>&);

template <typename T> class object {
private:
  explicit object(T* p) : px{ p } { assert(px != nullptr); }

public:
  object() = delete;
  ~object() noexcept = default;
  //destructor declaration automatically removes the implicity move constructor

  operator T&() const noexcept { return *px; }

  T& operator*() const noexcept { return *px; }

  T* operator->() const noexcept { return px.get(); }

  // not possible
  // T& operator.() const noexcept { return *px; }

private:
  shared_ptr<T> px;

  friend int count_object<T>(const object<T>&);
  friend T* get_object_ptr<T>(const object<T>&);
  template<typename T1, typename... A> friend object<T1> make_object(A&&...);
};

// auxiliary functions
template <typename T, typename... Args> object<T> make_object(Args&&... args) {
  return object<T>{ new T{ std::forward<Args>(args)... } };
}

template <typename T> int count_object(const object<T>& obj) {
  return obj.px.use_count();
}

template <typename T> T* get_object_ptr(const object<T>& obj) {
  return obj.px.get();
}

struct base {
  virtual void f() { cout << "base" << endl; }
};

struct extended : public base {
  virtual void f() override { cout << "extended" << endl; }
};

int main() {
  // runtime errors:
  // object<int> bang {nullptr};
  // object<int> bad;

  //object<int> o0{ new int{ 5 } };
  //cout << *o0 << endl;

  auto mo1 = make_object<int>(10);
  // will actually make a copy instead of 'move' (because a destructor was
  // declared)

  auto mo2 = std::move(mo1);
  cout << get_object_ptr(mo1) << ",count: " << count_object(mo1) << endl;
  cout << *mo1 << endl;

  auto o1 = object<string>{ make_object<string>("abc") };
  cout << *o1 << endl;

  vector<object<double> > v{ make_object<double>(1.0),
                             make_object<double>(2.0) };
  cout << "{ ";
  for (const auto& o : v) {
    cout << *o << " ";
  }
  cout << "}" << endl;

  extended ext;
  base& base_ref = ext;
  base_ref.f();

  auto o2 = make_object<extended>();
  o2->f();
  extended& ref2 = o2;
  ref2.f();
  (static_cast<extended&>(o2)).f();

  // operator. can not be overriden in c++ (and operator T&() won't do it)
  // o2.f();

  // why? how to do it?
  //  object<base> o3 = make_object<extended>();
  //  o3->f();
//  */
}

<|endoftext|>


<commit_before>//
//  Copper.cpp
//  Clock Signal
//
//  Created by Thomas Harte on 16/09/2021.
//  Copyright © 2021 Thomas Harte. All rights reserved.
//

#ifndef NDEBUG
#define NDEBUG
#endif

#define LOG_PREFIX "[Copper] "
#include "../../Outputs/Log.hpp"

#include "Chipset.hpp"
#include "Copper.hpp"

using namespace Amiga;

namespace {

bool satisfies_raster(uint16_t position, uint16_t blitter_status, uint16_t *instruction) {
	const uint16_t mask = 0x8000 | (instruction[1] & 0x7ffe);
	return
		(position & mask & 0xff00) >= (instruction[0] & mask & 0xff00) &&
		(position & mask & 0x00ff) >= (instruction[0] & mask & 0x00ff) &&
		(!(blitter_status & 0x4000) || (instruction[1] & 0x8000));
}

}

//
// Quick notes on the Copper:
//
// There are three instructions: move, wait and skip. All are two words in length.
//
// Move writes a value to one of the Chipset registers; it is encoded as:
//
//		First word:
//			b0:		0
//			b1–b8:	register address
//			b9+:	unused ("should be set to 0")
//
//		Second word:
//			b0–b15:	value to move.
//
//
// Wait waits until the raster gets to at least a certain position, and
// optionally until the Blitter has finished. It is encoded as:
//
//		First word:
//			b0:		1
//			b1–b7:	horizontal beam position
//			b8+:	vertical beam position
//
//		Second word:
//			b0:		0
//			b1–b7:	horizontal beam comparison mask
//			b8–b14:	vertical beam comparison mask
//			b15:	1 => don't also wait for the Blitter to be finished; 0 => wait.
//
//
// Skip skips the next instruction if the raster has already reached a certain
// position, and optionally only if the Blitter has finished, and only if the
// next instruction is a move.
//
//		First word:
//			b0:		1
//			b1–b7:	horizontal beam position
//			b8+:	vertical beam position
//
//		Second word:
//			b0:		1
//			b1–b7:	horizontal beam comparison mask
//			b8–b14:	vertical beam comparison mask
//			b15:	1 => don't also test whether the Blitter is finished; 0 => test.
//
bool Copper::advance_dma(uint16_t position, uint16_t blitter_status) {
	switch(state_) {
		default: return false;

		case State::Waiting:
			if(!satisfies_raster(position, blitter_status, instruction_)) {
				return false;
			}
			LOG("Unblocked waiting for " << PADHEX(4) << instruction_[0] << " at " << PADHEX(4) << position << " with mask " << PADHEX(4) << (instruction_[1] & 0x7ffe));
			state_ = State::FetchFirstWord;
			[[fallthrough]];

		case State::FetchFirstWord:
			instruction_[0] = ram_[address_ & ram_mask_];
			++address_;
			state_ = State::FetchSecondWord;
			LOG("First word fetch at " << PADHEX(4) << position);
		break;

		case State::FetchSecondWord: {
			// Get and reset the should-skip-next flag.
			const bool should_skip_move = skip_next_;
			skip_next_ = false;

			// Read in the second instruction word.
			instruction_[1] = ram_[address_ & ram_mask_];
			++address_;
			LOG("Second word fetch at " << PADHEX(4) << position);

			// Check for a MOVE.
			if(!(instruction_[0] & 1)) {
				if(!should_skip_move) {
					// Stop if this move would be a privilege violation.
					instruction_[0] &= 0x1fe;
					if((instruction_[0] < 0x10) || (instruction_[0] < 0x20 && !(control_&1))) {
						LOG("Invalid MOVE to " << PADHEX(4) << instruction_[0] << "; stopping");
						state_ = State::Stopped;
						break;
					}

					// Construct a 68000-esque Microcycle in order to be able to perform the access.
					CPU::MC68000::Microcycle cycle;
					cycle.operation = CPU::MC68000::Microcycle::SelectWord;
					uint32_t full_address = instruction_[0];
					CPU::RegisterPair16 data = instruction_[1];
					cycle.address = &full_address;
					cycle.value = &data;
					chipset_.perform(cycle);
				}

				// Roll onto the next command.
				state_ = State::FetchFirstWord;
				break;
			}

			// Got to here => this is a WAIT or a SKIP.

			if(!(instruction_[1] & 1)) {
				// A WAIT. Empirically, I don't think this waits at all if the test is
				// already satisfied.
				state_ = State::Waiting;
				LOG("Will wait from " << PADHEX(4) << position);
				break;
			}

			// Neither a WAIT nor a MOVE => a SKIP.

			skip_next_ = satisfies_raster(position, blitter_status, instruction_);
			state_ = State::FetchFirstWord;
		} break;
	}

	return true;
}
<commit_msg>Withdraw separate x-and-y guess, make MOVE lose a cycle if a sleep/wake occurs.<commit_after>//
//  Copper.cpp
//  Clock Signal
//
//  Created by Thomas Harte on 16/09/2021.
//  Copyright © 2021 Thomas Harte. All rights reserved.
//

#ifndef NDEBUG
#define NDEBUG
#endif

#define LOG_PREFIX "[Copper] "
#include "../../Outputs/Log.hpp"

#include "Chipset.hpp"
#include "Copper.hpp"

using namespace Amiga;

namespace {

bool satisfies_raster(uint16_t position, uint16_t blitter_status, uint16_t *instruction) {
	const uint16_t mask = 0x8000 | (instruction[1] & 0x7ffe);
	return
		(position & mask) >= (instruction[0] & mask) &&
		(!(blitter_status & 0x4000) || (instruction[1] & 0x8000));
}

}

//
// Quick notes on the Copper:
//
// There are three instructions: move, wait and skip. All are two words in length.
//
// Move writes a value to one of the Chipset registers; it is encoded as:
//
//		First word:
//			b0:		0
//			b1–b8:	register address
//			b9+:	unused ("should be set to 0")
//
//		Second word:
//			b0–b15:	value to move.
//
//
// Wait waits until the raster gets to at least a certain position, and
// optionally until the Blitter has finished. It is encoded as:
//
//		First word:
//			b0:		1
//			b1–b7:	horizontal beam position
//			b8+:	vertical beam position
//
//		Second word:
//			b0:		0
//			b1–b7:	horizontal beam comparison mask
//			b8–b14:	vertical beam comparison mask
//			b15:	1 => don't also wait for the Blitter to be finished; 0 => wait.
//
//
// Skip skips the next instruction if the raster has already reached a certain
// position, and optionally only if the Blitter has finished, and only if the
// next instruction is a move.
//
//		First word:
//			b0:		1
//			b1–b7:	horizontal beam position
//			b8+:	vertical beam position
//
//		Second word:
//			b0:		1
//			b1–b7:	horizontal beam comparison mask
//			b8–b14:	vertical beam comparison mask
//			b15:	1 => don't also test whether the Blitter is finished; 0 => test.
//
bool Copper::advance_dma(uint16_t position, uint16_t blitter_status) {
	switch(state_) {
		default: return false;

		case State::Waiting:
			if(satisfies_raster(position, blitter_status, instruction_)) {
				LOG("Unblocked waiting for " << PADHEX(4) << instruction_[0] << " at " << PADHEX(4) << position << " with mask " << PADHEX(4) << (instruction_[1] & 0x7ffe));
				state_ = State::FetchFirstWord;
			}
		return false;

		case State::FetchFirstWord:
			instruction_[0] = ram_[address_ & ram_mask_];
			++address_;
			state_ = State::FetchSecondWord;
			LOG("First word fetch at " << PADHEX(4) << position);
		break;

		case State::FetchSecondWord: {
			// Get and reset the should-skip-next flag.
			const bool should_skip_move = skip_next_;
			skip_next_ = false;

			// Read in the second instruction word.
			instruction_[1] = ram_[address_ & ram_mask_];
			++address_;
			LOG("Second word fetch at " << PADHEX(4) << position);

			// Check for a MOVE.
			if(!(instruction_[0] & 1)) {
				if(!should_skip_move) {
					// Stop if this move would be a privilege violation.
					instruction_[0] &= 0x1fe;
					if((instruction_[0] < 0x10) || (instruction_[0] < 0x20 && !(control_&1))) {
						LOG("Invalid MOVE to " << PADHEX(4) << instruction_[0] << "; stopping");
						state_ = State::Stopped;
						break;
					}

					// Construct a 68000-esque Microcycle in order to be able to perform the access.
					CPU::MC68000::Microcycle cycle;
					cycle.operation = CPU::MC68000::Microcycle::SelectWord;
					uint32_t full_address = instruction_[0];
					CPU::RegisterPair16 data = instruction_[1];
					cycle.address = &full_address;
					cycle.value = &data;
					chipset_.perform(cycle);
				}

				// Roll onto the next command.
				state_ = State::FetchFirstWord;
				break;
			}

			// Got to here => this is a WAIT or a SKIP.

			const bool raster_is_satisfied = satisfies_raster(position, blitter_status, instruction_);

			if(!(instruction_[1] & 1)) {
				// A WAIT. Empirically, I don't think this waits at all if the test is
				// already satisfied.
				state_ = raster_is_satisfied ? State::FetchFirstWord : State::Waiting;
				if(raster_is_satisfied) LOG("Will wait from " << PADHEX(4) << position);
				break;
			}

			// Neither a WAIT nor a MOVE => a SKIP.

			skip_next_ = satisfies_raster(position, blitter_status, instruction_);
			state_ = State::FetchFirstWord;
		} break;
	}

	return true;
}
<|endoftext|>


<commit_before>/*----------------------------------------------------------------------------*/
/* Copyright (c) Kauai Labs. All Rights Reserved.							  */
/*                                                                            */
/* Created in support of Team 2465 (Kauaibots).  Go Thunderchicken!           */
/*                                                                            */
/* Based upon the Open Source WPI Library released by FIRST robotics.         */
/* Open Source Software - may be modified and shared by FRC teams. The code   */
/* must be accompanied by the FIRST BSD license file in $(WIND_BASE)/WPILib.  */
/*----------------------------------------------------------------------------*/

#include "NetworkCommunication/UsageReporting.h"
#include "Timer.h"
#include "WPIErrors.h"
#include "LiveWindow/LiveWindow.h"
#include <time.h>
#include "IMU.h"
#include "IMUProtocol.h"
#include "Synchronized.h"

static SEM_ID cIMUStateSemaphore = semBCreate (SEM_Q_PRIORITY, SEM_FULL);
static int update_count = 0;
static int byte_count = 0;

static bool stop = false;

/*** Internal task.
 *
 * Task which retrieves yaw/pitch/roll updates from the IMU, via the
 * SerialPort.
 **/

static char protocol_buffer[1024];

static void imuTask(IMU *imu)
{
	stop = false;
	SerialPort *pport = imu->GetSerialPort();
	pport->SetReadBufferSize(512);
	pport->SetTimeout(1.0);
	pport->EnableTermination('\n');
	pport->Flush();
	pport->Reset();

	float yaw = 0.0;
	float pitch = 0.0;
	float roll = 0.0;
	float compass_heading = 0.0;

    // Give the nav6 circuit a few seconds to initialize, then send the stream configuration command.
    Wait(2.0);
    int cmd_packet_length = IMUProtocol::encodeStreamCommand( protocol_buffer, STREAM_CMD_STREAM_TYPE_YPR, imu->update_rate_hz );

    pport->Write( protocol_buffer, cmd_packet_length );
    pport->Flush();
    pport->Reset();

	while (!stop)
	{
//		INT32 bytes_received = pport->GetBytesReceived();
//		if ( bytes_received > 0 )
		{
			UINT32 bytes_read = pport->Read( protocol_buffer, sizeof(protocol_buffer) );
			if ( bytes_read > 0 )
			{
				int packets_received = 0;
				byte_count += bytes_read;
				UINT32 i = 0;
				// Scan the buffer looking for valid packets
				while ( i < bytes_read )
				{
					int bytes_remaining = bytes_read - i;
					int packet_length = IMUProtocol::decodeYPRUpdate( &protocol_buffer[i], bytes_remaining, yaw, pitch, roll, compass_heading );
					if ( packet_length > 0 )
					{
						packets_received++;
						update_count++;
						imu->SetYawPitchRoll(yaw,pitch,roll,compass_heading);
						i += packet_length;
					}
					else // current index is not the start of a valid packet; increment
					{
						i++;
					}
				}
                if ( ( packets_received == 0 ) && ( bytes_read == 256 ) ) {
                    // No packets received and 256 bytes received; this
                    // condition occurs in the SerialPort.  In this case,
                    // reset the serial port.
                	pport->Reset();
                }

			}
			else {
				double start_wait_timer = Timer::GetFPGATimestamp();
				// Timeout
				int bytes_received = pport->GetBytesReceived();
				while ( !stop && ( bytes_received == 0 ) ) {
					Wait(1.0/imu->update_rate_hz);
					bytes_received = pport->GetBytesReceived();
				}
                if ( !stop && (bytes_received > 0 ) ) {
                    if ( (Timer::GetFPGATimestamp() - start_wait_timer ) > 1.0 ) {
                    	Wait(2.0);
                        int cmd_packet_length = IMUProtocol::encodeStreamCommand( protocol_buffer, STREAM_CMD_STREAM_TYPE_YPR, imu->update_rate_hz );

                        pport->Write( protocol_buffer, cmd_packet_length );
                        pport->Flush();
                        pport->Reset();
                    }
                }
			}
		}
	}
}

IMU::IMU( SerialPort *pport, bool internal, uint8_t update_rate_hz )
{
	this->update_rate_hz = update_rate_hz;
	yaw = 0.0;
	pitch = 0.0;
	roll = 0.0;
	pserial_port = pport;
	pserial_port->Reset();
	InitIMU();
}

IMU::IMU( SerialPort *pport, uint8_t update_rate_hz )
{
	this->update_rate_hz = update_rate_hz;
	m_task = new Task("IMU", (FUNCPTR)imuTask,Task::kDefaultPriority+1);
	yaw = 0.0;
	pitch = 0.0;
	roll = 0.0;
	pserial_port = pport;
	pserial_port->Reset();
	InitIMU();
	m_task->Start((UINT32)this);
}

/**
 * Initialize the IMU.
 */
void IMU::InitIMU()
{
	// The IMU serial port configuration is 8 data bits, no parity, one stop bit.
	// No flow control is used.
	// Conveniently, these are the defaults used by the WPILib's SerialPort class.
	//
	// In addition, the WPILib's SerialPort class also defaults to:
	//
	// Timeout period of 5 seconds
	// Termination ('\n' character)
	// Transmit immediately
	InitializeYawHistory();
	yaw_offset = 0;

	// set the nav6 into "YPR" update mode

	int packet_length = IMUProtocol::encodeStreamCommand( protocol_buffer, STREAM_CMD_STREAM_TYPE_YPR, update_rate_hz );
	pserial_port->Write( protocol_buffer, packet_length );
}

/**
 * Delete the IMU.
 */
IMU::~IMU()
{
	m_task->Stop();
	delete m_task;
	m_task = 0;
}

void IMU::Restart()
{
	stop = true;
	pserial_port->Reset();
	m_task->Stop();

	pserial_port->Reset();
	InitializeYawHistory();
	update_count = 0;
	byte_count = 0;
	m_task->Restart();
}

bool IMU::IsConnected()
{
	double time_since_last_update = Timer::GetPPCTimestamp() - this->last_update_time;
	return time_since_last_update <= 1.0;
}

double IMU::GetByteCount()
{
	return byte_count;
}
double IMU::GetUpdateCount()
{
	return update_count;
}

void IMU::ZeroYaw()
{
	yaw_offset = GetAverageFromYawHistory();
}

/**
 * Return the yaw angle in degrees.
 *
 * This angle increases as the robot spins to the right.
 *
 * This angle ranges from -180 to 180 degrees.
 */
float IMU::GetYaw( void )
{
	Synchronized sync(cIMUStateSemaphore);
	double yaw = this->yaw;
	yaw -= yaw_offset;
	if ( yaw < -180 ) yaw += 360;
	if ( yaw > 180 ) yaw -= 360;
	return yaw;
}

float IMU::GetPitch( void )
{
	Synchronized sync(cIMUStateSemaphore);
	return this->pitch;
}

float IMU::GetRoll( void )
{
	Synchronized sync(cIMUStateSemaphore);
	return this->roll;
}

float IMU::GetCompassHeading( void )
{
	Synchronized sync(cIMUStateSemaphore);
	return this->compass_heading;
}

/**
 * Get the angle in degrees for the PIDSource base object.
 *
 * @return The angle in degrees.
 */
double IMU::PIDGet()
{
	return GetYaw();
}

void IMU::UpdateTable() {
	if (m_table != NULL) {
		m_table->PutNumber("Value", GetYaw());
	}
}

void IMU::StartLiveWindowMode() {

}

void IMU::StopLiveWindowMode() {

}

std::string IMU::GetSmartDashboardType() {
	return "Gyro";
}

void IMU::InitTable(ITable *subTable) {
	m_table = subTable;
	UpdateTable();
}

ITable * IMU::GetTable() {
	return m_table;
}

void IMU::SetYawPitchRoll(float yaw, float pitch, float roll, float compass_heading)
{
	{
		Synchronized sync(cIMUStateSemaphore);

		this->yaw = yaw;
		this->pitch = pitch;
		this->roll = roll;
		this->compass_heading = compass_heading;
	}
	UpdateYawHistory(this->yaw);
}

void IMU::InitializeYawHistory()
{
	for ( int i = 0; i < YAW_HISTORY_LENGTH; i++ )
	{
		yaw_history[i] = 0;
	}
	next_yaw_history_index = 0;
	last_update_time = 0.0;
}

void IMU::UpdateYawHistory(float curr_yaw )
{
	if ( next_yaw_history_index >= YAW_HISTORY_LENGTH )
	{
		next_yaw_history_index = 0;
	}
	yaw_history[next_yaw_history_index] = curr_yaw;
	last_update_time = Timer::GetPPCTimestamp();
	next_yaw_history_index++;
}

double IMU::GetAverageFromYawHistory()
{
	double yaw_history_sum = 0.0;
	for ( int i = 0; i < YAW_HISTORY_LENGTH; i++ )
	{
		yaw_history_sum += yaw_history[i];
	}
	double yaw_history_avg = yaw_history_sum / YAW_HISTORY_LENGTH;
	return yaw_history_avg;
}
<commit_msg>Moving files to nav6SimpleRobotExample subdirectory<commit_after><|endoftext|>


<commit_before><commit_msg>am cb262a21: am 10d817b3: mm-camera: change non-HDR frame rate to variable<commit_after><|endoftext|>


<commit_before><commit_msg>am 4c1072ae: camera: Reduce log verbosity.<commit_after><|endoftext|>


<commit_before><commit_msg>*** empty log message ***<commit_after><|endoftext|>


<commit_before><commit_msg>fixed part of rereplicate<commit_after><|endoftext|>


<commit_before>/*
 * (C) Copyright 2015 ETH Zurich Systems Group (http://www.systems.ethz.ch/) and others.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 * Contributors:
 *     Markus Pilman <mpilman@inf.ethz.ch>
 *     Simon Loesing <sloesing@inf.ethz.ch>
 *     Thomas Etter <etterth@gmail.com>
 *     Kevin Bocksrocker <kevin.bocksrocker@gmail.com>
 *     Lucas Braun <braunl@inf.ethz.ch>
 */
#pragma once
#include <type_traits>
#include <memory>

#include <crossbow/singleton.hpp>
#include <tellstore/ClientConfig.hpp>
#include <tellstore/ClientManager.hpp>
#include <tellstore/TransactionRunner.hpp>

#include "Transaction.hpp"

namespace tell {
namespace db {

template<class Context>
class TransactionFiber;
template<class Context>
class ClientManager;

class CounterImpl;

namespace impl {

class ClientTable {
    template<class T> friend class ::tell::db::ClientManager;
    ClientTable() {}
    void init(store::ClientHandle& handle);
    void destroy(store::ClientHandle& handle);
    uint64_t mClientId = 0;
    std::unique_ptr<store::Table> mClientsTable = nullptr;
    std::unique_ptr<store::Table> mTransactionsTable = nullptr;
public:
    /**
     * @brief Table where clients register themselves
     */
    const store::Table& clientsTable() const {
        return *mClientsTable;
    }

    const store::Table& txTable() const {
        return *mTransactionsTable;
    }
};

class Indexes;
Indexes* createIndexes(store::ClientHandle& handle);
struct TellDBContext {
    TellDBContext(ClientTable* table);
    ~TellDBContext();
    void setIndexes(Indexes* idxs);
    std::unordered_map<table_t, tell::store::Table*> tables;
    std::unordered_map<crossbow::string, table_t> tableNames;
    std::unique_ptr<Indexes> indexes;
    ClientTable* clientTable;
};

template<class Context>
struct FiberContext {
    typename std::conditional<std::is_void<Context>::value, char[0], Context>::type mUserContext;
    TellDBContext mContext;

    template<class Fun, class C = Context>
    typename std::enable_if<!std::is_void<C>::value, void>::type executeHandler(Fun& fun, Transaction& transaction) {
        fun(transaction, mUserContext);
    }

    template<class Fun, class C = Context>
    typename std::enable_if<std::is_void<C>::value, void>::type executeHandler(Fun& fun, Transaction& transaction) {
        fun(transaction);
    }

    template<class... Args>
    FiberContext(ClientTable* table, Args&&... args)
        : mUserContext(std::forward<Args>(args)...)
        , mContext(table)
    {}
};

class IteratorImpl;

} // namespace impl

/**
 * @brief Wrapper class around a transaction context.
 *
 * This class is used to run transactions. It will run in a fiber and can be used
 * to share information between the fiber and the main thread. The fiber can use
 * Transaction<Context>::block to block its execution and the main thread can use
 * Transaction<Context>::wait and Transaction<Context>::unblock to wait for the
 * fiber and unblock its execution
 */
template<class Context>
class TransactionFiber {
private: // types
    using telldb_context = typename impl::FiberContext<Context>;
    friend class ClientManager<Context>;
private: // members
    std::unique_ptr<tell::store::SingleTransactionRunner<telldb_context>> mTxRunner;
    tell::store::TransactionType mTxType;
private: // construction
    TransactionFiber(tell::store::ClientManager<telldb_context>& clientManager, tell::store::TransactionType txType)
        : mTxRunner(new tell::store::SingleTransactionRunner<telldb_context>(clientManager))
        , mTxType(txType)
    {}
private: // private access
    template<class Fun>
    void exec(Fun fun) {
        mTxRunner->execute([this, fun](tell::store::ClientHandle& handle, telldb_context& context) {
            if (context.mContext.indexes == nullptr) {
                context.mContext.setIndexes(impl::createIndexes(handle));
            }
            try {
                auto snapshot = handle.startTransaction(mTxType);
                Transaction transaction(handle, context.mContext, std::move(snapshot), mTxType);
                context.executeHandler(fun, transaction);
            } catch (std::exception& e) {
                std::cerr << "Exception: " << e.what() << std::endl;
            } catch (...) {
                // This should never happen
                std::cerr << "Got an unknown error" << std::endl;
            }
        });
    }
public: // construction
    TransactionFiber(const TransactionFiber&) = delete;
    TransactionFiber(TransactionFiber&& other)
        : mTxRunner(std::move(other.mTxRunner))
    {}
    TransactionFiber& operator=(const TransactionFiber&) = delete;
    TransactionFiber& operator=(TransactionFiber&& other) {
        mTxRunner = std::move(other.mTxRunner);
    }
public:
    /**
     * @brief Wait until the transaction completes or blocks
     *
     * Must only be called from outside the transaction thread.
     *
     * @return True if the transaction completed, false if it blocked
     */
    bool wait() {
        return mTxRunner->wait();
    }
    /**
     * @brief Blocks the transaction (and notifies the issuing thread)
     *
     * Must only be called from inside the transaction.
     */
    void block() {
        mTxRunner->block();
    }
    /**
     * @brief Unblocks the blocked transaction
     *
     * Must only be called from outside the transaction.
     *
     * @return Whether the transaction was unblocked
     */
    bool unblock() {
        return mTxRunner->unblock();
    }
};

/**
 * @brief ClientManager is the main class. It should be instantiated only once.
 *
 * This class is used to initialize the network, set up the cashes, start the
 * OS threads, opens connections to TellStore and the commit manager etc. It
 * prepares everything so that the user can run transactions. This class takes
 * a context class as template argument (which can be void if it is not needed).
 * This Context type will be created for each OS thread and passed to all fibers.
 * It will act like a thread local variable. This context will be passed to all
 * callback functions.
 */
template<class Context>
class ClientManager {
private:
    tell::store::ClientManager<impl::FiberContext<Context>> mClientManager;
    impl::ClientTable mClientTable;
public:
    /**
     * @brief Constructor
     *
     * This will construct a new ClientManager.
     *
     * @param[in] clientConfig
     * @parblock
     * The Configuration of the client. It contains information TellDB needs in
     * order to connect to TellStore and the commit manager.
     * @endparblock
     * @param[in] args
     * @parblock
     * For each processor (hardware level thread), the client manager will create a
     * new context object. To do so, it will pass the arguments to the Context class.
     * @endparblock
     */
    template<class... Args>
    ClientManager(tell::store::ClientConfig& clientConfig, Args... args)
        : mClientManager(clientConfig, &mClientTable, args...)
    {
        store::TransactionRunner::executeBlocking(mClientManager,
                [this](store::ClientHandle &handle, impl::FiberContext<Context>&){
            mClientTable.init(handle);
        });
    }

    ~ClientManager() {
        store::TransactionRunner::executeBlocking(mClientManager,
                [this](store::ClientHandle &handle, impl::FiberContext<Context>&){
            mClientTable.destroy(handle);
        });
    }

    /**
     * @brief starts a new transaction and executes fun within its context
     *
     * This function will create a new fiber which runs in the context of a new Transaction.
     * The function will get a context (if the context is not void) and a transaction object.
     *
     * @param[in] type The type of the transaction
     * @param[in] fun
     * @parblock
     * The function to call. This function will run in the new fiber. Therefore
     * it must not use any posix mutexes or other kinds of blocks. Using blocking within a
     * fiber might result in a dead lock, since blocking on a lock will not result in a
     * fiber rescheduling (since the underlying thread will block).
     * Furthermore it should not:
     *   - Call into system calls (yield).
     *   - Call any blocking operations.
     * In general it should not do anything that will schedule the underlying OS thread.
     * @endparblock
     */
    template<class Fun>
    TransactionFiber<Context> startTransaction(
            Fun& fun,
            tell::store::TransactionType type = tell::store::TransactionType::READ_WRITE)
    {
        TransactionFiber<Context> fiber(mClientManager, type);
        fiber.exec(fun);
        return fiber;
    }

    /**
     * @brief Shutdown everything
     *
     * This functions has to get called before the client application quits. After
     * calling shutdown, no other functionality within TellDB can be used - otherwise
     * the results will be non-deterministic (and it might crash).
     */
    void shutdown() {
        mClientManager->shutdown();
    }

    std::unique_ptr<store::ScanMemoryManager> allocateScanMemory(size_t chunkCount, size_t chunkLength) {
        return mClientManager.allocateScanMemory(chunkCount, chunkLength);
    }
};

} // namespace db
} // namespace tell

<commit_msg>exposed scan memory allocator reasonably<commit_after>/*
 * (C) Copyright 2015 ETH Zurich Systems Group (http://www.systems.ethz.ch/) and others.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 * Contributors:
 *     Markus Pilman <mpilman@inf.ethz.ch>
 *     Simon Loesing <sloesing@inf.ethz.ch>
 *     Thomas Etter <etterth@gmail.com>
 *     Kevin Bocksrocker <kevin.bocksrocker@gmail.com>
 *     Lucas Braun <braunl@inf.ethz.ch>
 */
#pragma once
#include <type_traits>
#include <memory>

#include <crossbow/singleton.hpp>
#include <tellstore/ClientConfig.hpp>
#include <tellstore/ClientManager.hpp>
#include <tellstore/TransactionRunner.hpp>

#include "Transaction.hpp"

namespace tell {
namespace db {

template<class Context>
class TransactionFiber;
template<class Context>
class ClientManager;

class CounterImpl;

namespace impl {

class ClientTable {
    template<class T> friend class ::tell::db::ClientManager;
    ClientTable() {}
    void init(store::ClientHandle& handle);
    void destroy(store::ClientHandle& handle);
    uint64_t mClientId = 0;
    std::unique_ptr<store::Table> mClientsTable = nullptr;
    std::unique_ptr<store::Table> mTransactionsTable = nullptr;
public:
    /**
     * @brief Table where clients register themselves
     */
    const store::Table& clientsTable() const {
        return *mClientsTable;
    }

    const store::Table& txTable() const {
        return *mTransactionsTable;
    }
};

class Indexes;
Indexes* createIndexes(store::ClientHandle& handle);
struct TellDBContext {
    TellDBContext(ClientTable* table);
    ~TellDBContext();
    void setIndexes(Indexes* idxs);
    std::unordered_map<table_t, tell::store::Table*> tables;
    std::unordered_map<crossbow::string, table_t> tableNames;
    std::unique_ptr<Indexes> indexes;
    ClientTable* clientTable;
};

template<class Context>
struct FiberContext {
    typename std::conditional<std::is_void<Context>::value, char[0], Context>::type mUserContext;
    TellDBContext mContext;

    template<class Fun, class C = Context>
    typename std::enable_if<!std::is_void<C>::value, void>::type executeHandler(Fun& fun, Transaction& transaction) {
        fun(transaction, mUserContext);
    }

    template<class Fun, class C = Context>
    typename std::enable_if<std::is_void<C>::value, void>::type executeHandler(Fun& fun, Transaction& transaction) {
        fun(transaction);
    }

    template<class... Args>
    FiberContext(ClientTable* table, Args&&... args)
        : mUserContext(std::forward<Args>(args)...)
        , mContext(table)
    {}
};

class IteratorImpl;

} // namespace impl

/**
 * @brief Wrapper class around a transaction context.
 *
 * This class is used to run transactions. It will run in a fiber and can be used
 * to share information between the fiber and the main thread. The fiber can use
 * Transaction<Context>::block to block its execution and the main thread can use
 * Transaction<Context>::wait and Transaction<Context>::unblock to wait for the
 * fiber and unblock its execution
 */
template<class Context>
class TransactionFiber {
private: // types
    using telldb_context = typename impl::FiberContext<Context>;
    friend class ClientManager<Context>;
private: // members
    std::unique_ptr<tell::store::SingleTransactionRunner<telldb_context>> mTxRunner;
    tell::store::TransactionType mTxType;
private: // construction
    TransactionFiber(tell::store::ClientManager<telldb_context>& clientManager, tell::store::TransactionType txType)
        : mTxRunner(new tell::store::SingleTransactionRunner<telldb_context>(clientManager))
        , mTxType(txType)
    {}
private: // private access
    template<class Fun>
    void exec(Fun fun) {
        mTxRunner->execute([this, fun](tell::store::ClientHandle& handle, telldb_context& context) {
            if (context.mContext.indexes == nullptr) {
                context.mContext.setIndexes(impl::createIndexes(handle));
            }
            try {
                auto snapshot = handle.startTransaction(mTxType);
                Transaction transaction(handle, context.mContext, std::move(snapshot), mTxType);
                context.executeHandler(fun, transaction);
            } catch (std::exception& e) {
                std::cerr << "Exception: " << e.what() << std::endl;
            } catch (...) {
                // This should never happen
                std::cerr << "Got an unknown error" << std::endl;
            }
        });
    }
public: // construction
    TransactionFiber(const TransactionFiber&) = delete;
    TransactionFiber(TransactionFiber&& other)
        : mTxRunner(std::move(other.mTxRunner))
    {}
    TransactionFiber& operator=(const TransactionFiber&) = delete;
    TransactionFiber& operator=(TransactionFiber&& other) {
        mTxRunner = std::move(other.mTxRunner);
    }
public:
    /**
     * @brief Wait until the transaction completes or blocks
     *
     * Must only be called from outside the transaction thread.
     *
     * @return True if the transaction completed, false if it blocked
     */
    bool wait() {
        return mTxRunner->wait();
    }
    /**
     * @brief Blocks the transaction (and notifies the issuing thread)
     *
     * Must only be called from inside the transaction.
     */
    void block() {
        mTxRunner->block();
    }
    /**
     * @brief Unblocks the blocked transaction
     *
     * Must only be called from outside the transaction.
     *
     * @return Whether the transaction was unblocked
     */
    bool unblock() {
        return mTxRunner->unblock();
    }
};

/**
 * @brief ClientManager is the main class. It should be instantiated only once.
 *
 * This class is used to initialize the network, set up the cashes, start the
 * OS threads, opens connections to TellStore and the commit manager etc. It
 * prepares everything so that the user can run transactions. This class takes
 * a context class as template argument (which can be void if it is not needed).
 * This Context type will be created for each OS thread and passed to all fibers.
 * It will act like a thread local variable. This context will be passed to all
 * callback functions.
 */
template<class Context>
class ClientManager {
private:
    tell::store::ClientManager<impl::FiberContext<Context>> mClientManager;
    impl::ClientTable mClientTable;
    std::unique_ptr<store::ScanMemoryManager> mScanMemoryManager;
public:
    /**
     * @brief Constructor
     *
     * This will construct a new ClientManager.
     *
     * @param[in] clientConfig
     * @parblock
     * The Configuration of the client. It contains information TellDB needs in
     * order to connect to TellStore and the commit manager.
     * @endparblock
     * @param[in] args
     * @parblock
     * For each processor (hardware level thread), the client manager will create a
     * new context object. To do so, it will pass the arguments to the Context class.
     * @endparblock
     */
    template<class... Args>
    ClientManager(tell::store::ClientConfig& clientConfig, Args... args)
        : mClientManager(clientConfig, &mClientTable, args...)
    {
        store::TransactionRunner::executeBlocking(mClientManager,
                [this](store::ClientHandle &handle, impl::FiberContext<Context>&){
            mClientTable.init(handle);
        });
    }

    ~ClientManager() {
        store::TransactionRunner::executeBlocking(mClientManager,
                [this](store::ClientHandle &handle, impl::FiberContext<Context>&){
            mClientTable.destroy(handle);
        });
    }

    /**
     * @brief starts a new transaction and executes fun within its context
     *
     * This function will create a new fiber which runs in the context of a new Transaction.
     * The function will get a context (if the context is not void) and a transaction object.
     *
     * @param[in] type The type of the transaction
     * @param[in] fun
     * @parblock
     * The function to call. This function will run in the new fiber. Therefore
     * it must not use any posix mutexes or other kinds of blocks. Using blocking within a
     * fiber might result in a dead lock, since blocking on a lock will not result in a
     * fiber rescheduling (since the underlying thread will block).
     * Furthermore it should not:
     *   - Call into system calls (yield).
     *   - Call any blocking operations.
     * In general it should not do anything that will schedule the underlying OS thread.
     * @endparblock
     */
    template<class Fun>
    TransactionFiber<Context> startTransaction(
            Fun& fun,
            tell::store::TransactionType type = tell::store::TransactionType::READ_WRITE)
    {
        TransactionFiber<Context> fiber(mClientManager, type);
        fiber.exec(fun);
        return fiber;
    }

    /**
     * @brief allocates scan memomry. Be cautious with this call as it is extremely expensive!
     *
     * A typcial use of this function would be to allocate as many chunks as there are
     * connections to tellstore nodes times the number of scans that could possibly be
     * processed in parallel plus some extra buffer.
     *
     * @param chunkCount
     * @param chunkLength
     */
    void allocateScanMemory(size_t chunkCount, size_t chunkLength) {
        mScanMemoryManager = mClientManager.allocateScanMemory(chunkCount, chunkLength);
    }

    /**
     * @brief returns a reference to the scanMemoryManager, which must be initialized first!
     *
     * Initialization works with a call to callocateScanMemory().
     *
     * @return
     */
    store::ScanMemoryManager *getScanMemoryManager() {
        return mScanMemoryManager.get();
    }

    /**
     * @brief Shutdown everything
     *
     * This functions has to get called before the client application quits. After
     * calling shutdown, no other functionality within TellDB can be used - otherwise
     * the results will be non-deterministic (and it might crash).
     */
    void shutdown() {
        mClientManager->shutdown();
    }
};

} // namespace db
} // namespace tell

<|endoftext|>