<commit_before>/*
 *
 * RippleLikeTransactionApi
 *
 * Created by El Khalil Bellakrid on 06/01/2019.
 *
 * The MIT License (MIT)
 *
 * Copyright (c) 2019 Ledger
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to deal
 * in the Software without restriction, including without limitation the rights
 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 * copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in all
 * copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 *
 */

#include "RippleLikeTransactionApi.h"
#include <wallet/common/Amount.h>
#include <wallet/common/AbstractAccount.hpp>
#include <wallet/common/AbstractWallet.hpp>
#include <ripple/RippleLikeAddress.h>
#include <bytes/BytesWriter.h>
#include <bytes/BytesReader.h>
#include <bytes/RLP/RLPListEncoder.h>
#include <bytes/RLP/RLPStringEncoder.h>
#include <utils/hex.h>
#include <api_impl/BigIntImpl.hpp>

namespace ledger {
    namespace core {
        // a helper to write VLE fields
        bool writeLengthPrefix(BytesWriter& writer, uint64_t size) {
            // encoding here: https://developers.ripple.com/serialization.html#length-prefixing
            if (size <= 192) {
                writer.writeByte(size & 0xFF);

                return true;
            } else if (size <= 12480) {
                //     l       = 193 + ((byte1 - 193) * 256) + byte2
                // <=> l - 193 = ((byte1 - 193) * 256) + byte2
                // <=> byte2 = (l - 193) & 0xFF
                // <=> byte1 = ((l - 193) >> 8) & 0xFF
                size -= 193;
                writer.writeByte((size >> 8) & 0xFF);
                writer.writeByte(size & 0xFF);

                return true;
            } else if (size <= 918744) {
                //     l         = 12481 + ((byte1 - 241) * 65536) + (byte2 * 256) + byte3
                // <=> l - 12481 = ((byte1 - 241) * 65536) + (byte2 * 256) + byte3
                // <=> byte3 = (l - 12481) & 0xFF
                // <=> byte2 = ((l - 12481) >> 8) & 0xFF
                // <=> byte1 = ((l - 12481) >> 16) & 0xFF
                size -= 12481;
                writer.writeByte((size >> 16) & 0xFF);
                writer.writeByte((size >> 8) & 0xFF);
                writer.writeByte(size);
            }

            // cannot have more bytes
            return false;
        }

        RippleLikeTransactionApi::RippleLikeTransactionApi(const api::Currency &currency) {
            _currency = currency;
        }

        RippleLikeTransactionApi::RippleLikeTransactionApi(const std::shared_ptr<OperationApi> &operation) {
            auto &tx = operation->getBackend().rippleTransaction.getValue();
            _time = tx.receivedAt;

            if (tx.block.nonEmpty()) {
                _block = std::make_shared<RippleLikeBlockApi>(tx.block.getValue());
            } else {
                _block = nullptr;
            }

            _hash = tx.hash;

            _currency = operation->getAccount()->getWallet()->getCurrency();

            _fees = std::make_shared<Amount>(_currency, 0, tx.fees);
            _value = std::make_shared<Amount>(_currency, 0, tx.value);

            _receiver = RippleLikeAddress::fromBase58(tx.receiver, _currency);
            _sender = RippleLikeAddress::fromBase58(tx.sender, _currency);

        }

        std::string RippleLikeTransactionApi::getHash() {
            return _hash;
        }

        std::shared_ptr<api::Amount> RippleLikeTransactionApi::getFees() {
            return _fees;
        }

        std::shared_ptr<api::RippleLikeAddress> RippleLikeTransactionApi::getReceiver() {
            return _receiver;
        }

        std::shared_ptr<api::RippleLikeAddress> RippleLikeTransactionApi::getSender() {
            return _sender;
        }

        std::shared_ptr<api::Amount> RippleLikeTransactionApi::getValue() {
            return _value;
        }

        std::chrono::system_clock::time_point RippleLikeTransactionApi::getDate() {
            return _time;
        }

        std::shared_ptr<api::BigInt> RippleLikeTransactionApi::getLedgerSequence() {
            return _ledgerSequence;
        }

        std::shared_ptr<api::BigInt> RippleLikeTransactionApi::getSequence() {
            return _sequence;
        }

        std::vector<uint8_t> RippleLikeTransactionApi::getSigningPubKey() {
            return _signingPubKey;
        }

        void RippleLikeTransactionApi::setSignature(const std::vector<uint8_t> &rSignature,
                                                    const std::vector<uint8_t> &sSignature) {
            _rSignature = rSignature;
            _sSignature = sSignature;
        }

        void RippleLikeTransactionApi::setDERSignature(const std::vector<uint8_t> &signature) {
            BytesReader reader(signature);
            //DER prefix
            reader.readNextByte();
            //Total length
            reader.readNextVarInt();
            //Nb of elements for R
            reader.readNextByte();
            //R length
            auto rSize = reader.readNextVarInt();
            if (rSize > 0 && reader.peek() == 0x00) {
                reader.readNextByte();
                _rSignature = reader.read(rSize - 1);
            } else {
                _rSignature = reader.read(rSize);
            }
            //Nb of elements for S
            reader.readNextByte();
            //S length
            auto sSize = reader.readNextVarInt();
            if (sSize > 0 && reader.peek() == 0x00) {
                reader.readNextByte();
                _sSignature = reader.read(sSize - 1);
            } else {
                _sSignature = reader.read(sSize);
            }
        }

        //Field ID References:
        // https://github.com/ripple/rippled/blob/master/src/ripple/protocol/SField.h#L57-L74
        // and https://github.com/ripple/rippled/blob/72e6005f562a8f0818bc94803d222ac9345e1e40/src/ripple/protocol/impl/SField.cpp#L72-L266
        std::vector<uint8_t> RippleLikeTransactionApi::serialize() {
            BytesWriter writer;

            //1 byte TransactionType Field ID:   Type Code = 1, Field Code = 2
            writer.writeByte(0x12);
            //2 bytes TransactionType ("Payment")
            writer.writeByteArray({0x00, 0x00});

            //1 byte Flags Field ID:   Type Code = 2, Field Code = 2
            writer.writeByte(0x22);
            //4 bytes Flags (tfFullyCanonicalSig ?)
            writer.writeByteArray({0x80, 0x00, 0x00, 0x00});

            //1 byte Sequence Field ID:   Type Code = 2, Field Code = 4
            writer.writeByte(0x24);
            //4 bytes Sequence
            auto sequence = _sequence->toString(16);
            writer.writeBeValue<uint32_t>(static_cast<const uint32_t>(_sequence->intValue()));

            //2 bytes LastLedgerSequence Field ID:   Type Code = 2, Field Code = 27
            writer.writeByteArray({0x20, 0x1B});
            //LastLedgerSequence
            writer.writeBeValue<uint32_t>(static_cast<const uint32_t>(_ledgerSequence->intValue()));

            auto maxAmountBound = std::vector<uint8_t>({0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00});
            auto bigIntMax = BigInt::fromHex(hex::toString(maxAmountBound));

            //1 byte Amount Field ID:   Type Code = 6, Field Code = 1
            writer.writeByte(0x61);
            //8 bytes Amount (with bitwise OR with 0x4000000000000000)
            auto finalValue = BigInt::fromHex(_value->toBigInt()->toString(16)) + bigIntMax;
            writer.writeBeValue<uint64_t>(static_cast<const uint64_t>(finalValue.toUint64()));

            //1 byte Fee Field ID:   Type Code = 6, Field Code = 8
            writer.writeByte(0x68);
            //8 bytes Fees (with bitwise OR with 0x4000000000000000)
            auto finalFees = BigInt::fromHex(_fees->toBigInt()->toString(16)) + bigIntMax;
            writer.writeBeValue<uint64_t>(static_cast<const uint64_t>(finalFees.toUint64()));

            //TODO: !!!find out if this is included in raw unsigned tx or not
            //1 byte Signing pubKey Field ID:   Type Code = 7, Field Code = 3 (STI_VL = 7 type)
            writer.writeByte(0x73);
            //Var bytes Signing pubKey (prefix length)
            writer.writeVarInt(_signingPubKey.size());
            writer.writeByteArray(_signingPubKey);

            if (!_rSignature.empty() && !_sSignature.empty()) {
                //1 byte Signature Field ID:   Type Code = 7, Field Code = 4 (STI_VL = 7 type, and TxnSignature = 4)
                writer.writeByte(0x74);
                //Var bytes Signature (prefix length)
                //Get length of VarInt representing length of R
                BytesWriter rLength;
                rLength.writeVarInt(_rSignature.size());
                //Get length of VarInt representing length of R
                BytesWriter sLength;
                sLength.writeVarInt(_sSignature.size());
                //Get length of VarInt representing length of R and S (plus their stack sizes)
                auto sAndRLengthInt = 1 + rLength.toByteArray().size() + _rSignature.size() + 1 + sLength.toByteArray().size() + _sSignature.size();
                BytesWriter sAndRLength;
                sAndRLength.writeVarInt(sAndRLengthInt);
                //DER Signature = Total Size | DER prefix | Size(S+R) | R StackSize | R Length | R | S StackSize | S Length | S
                auto totalSigLength = 1 + sAndRLength.toByteArray().size() + sAndRLengthInt;
                writer.writeVarInt(totalSigLength);
                //DER prefix
                BytesWriter sigWriter;
                writer.writeByte(0x30);
                //Size of DER signature minus DER prefix | Size(S+R)
                writer.writeVarInt(sAndRLengthInt);
                //R field
                writer.writeByte(0x02); //Nb of stack elements
                writer.writeVarInt(_rSignature.size());
                writer.writeByteArray(_rSignature);
                //S field
                writer.writeByte(0x02); //Nb of stack elements
                writer.writeVarInt(_sSignature.size());
                writer.writeByteArray(_sSignature);
            }

            //1 byte Account Field ID: Type Code = 8, Field Code = 1 (STI_ACCOUNT = 8 type, and Account = 1)
            writer.writeByte(0x81);
            //20 bytes Acount (hash160 of pubKey without 0x00 prefix)
            auto senderHash = _sender->getHash160();
            writer.writeVarInt(senderHash.size());
            writer.writeByteArray(senderHash);
            //1 byte Destination Field ID: Type Code = 8, Field Code = 3 (STI_ACCOUNT = 8 type, and Destination = 3)
            writer.writeByte(0x83);
            //20 bytes Destination (hash160 of pubKey without 0x00 prefix)
            auto receiverHash = _receiver->getHash160();
            writer.writeVarInt(receiverHash.size());
            writer.writeByteArray(receiverHash);

            if (!_memos.empty()) {
                writer.writeByte(0xF9); // STI_ARRAY (Memos); Type Code = 15, Memos; Field ID = 9

                for (auto& memo : _memos) {
                    writer.writeByte(0xEA); // STI_OBJECT (Memo); Type Code = 10

                    if (!memo.ty.empty()) {
                        writer.writeByte(0x7C); // STI_VL (MemoType), 12
                        auto written = writeLengthPrefix(writer, memo.ty.size());
                        assert(written);
                        writer.writeString(memo.ty);
                    }

                    if (!memo.data.empty()) {
                        writer.writeByte(0x7D); // STI_VL (MemoData), 13
                        auto written = writeLengthPrefix(writer, memo.data.size());
                        assert(written);
                        writer.writeString(memo.data);
                    }

                    if (!memo.fmt.empty()) {
                        writer.writeByte(0x7D); // STI_VL (MemoFormat), 14
                        auto written = writeLengthPrefix(writer, memo.fmt.size());
                        assert(written);
                        writer.writeString(memo.fmt);
                    }

                    writer.writeByte(0xE1); // end of object
                }

                writer.writeByte(0xF1); // end of array
            }

            return writer.toByteArray();
        }

        RippleLikeTransactionApi &RippleLikeTransactionApi::setFees(const std::shared_ptr<BigInt> &fees) {
            if (!fees) {
                throw make_exception(api::ErrorCode::INVALID_ARGUMENT,
                                     "RippleLikeTransactionApi::setFees: Invalid Fees");
            }
            _fees = std::make_shared<Amount>(_currency, 0, *fees);
            return *this;
        }

        RippleLikeTransactionApi &RippleLikeTransactionApi::setValue(const std::shared_ptr<BigInt> &value) {
            if (!value) {
                throw make_exception(api::ErrorCode::INVALID_ARGUMENT,
                                     "RippleLikeTransactionApi::setValue: Invalid Value");
            }
            _value = std::make_shared<Amount>(_currency, 0, *value);
            return *this;
        }

        RippleLikeTransactionApi &RippleLikeTransactionApi::setSequence(const BigInt& sequence) {
            _sequence = std::make_shared<api::BigIntImpl>(sequence);
            return *this;
        }

        RippleLikeTransactionApi & RippleLikeTransactionApi::setLedgerSequence(const BigInt& ledgerSequence) {
            _ledgerSequence = std::make_shared<api::BigIntImpl>(ledgerSequence);
            return *this;
        }

        RippleLikeTransactionApi &RippleLikeTransactionApi::setSender(const std::shared_ptr<api::RippleLikeAddress> &sender) {
            _sender = sender;
            return *this;
        }

        RippleLikeTransactionApi &RippleLikeTransactionApi::setReceiver(const std::shared_ptr<api::RippleLikeAddress> &receiver) {
            _receiver = receiver;
            return *this;
        }

        RippleLikeTransactionApi &RippleLikeTransactionApi::setSigningPubKey(const std::vector<uint8_t> &pubKey) {
            _signingPubKey = pubKey;
            return *this;
        }

        RippleLikeTransactionApi &RippleLikeTransactionApi::setHash(const std::string &hash) {
            _hash = hash;
            return *this;
        }

        std::vector<api::RippleLikeMemo> RippleLikeTransactionApi::getMemos() {
            return _memos;
        }

        void RippleLikeTransactionApi::addMemo(api::RippleLikeMemo const& memo) {
            _memos.push_back(memo);
        }
    }
}
<commit_msg>Fix Ripple TX VLE encoding (again!).<commit_after>/*
 *
 * RippleLikeTransactionApi
 *
 * Created by El Khalil Bellakrid on 06/01/2019.
 *
 * The MIT License (MIT)
 *
 * Copyright (c) 2019 Ledger
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to deal
 * in the Software without restriction, including without limitation the rights
 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 * copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in all
 * copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 *
 */

#include "RippleLikeTransactionApi.h"
#include <wallet/common/Amount.h>
#include <wallet/common/AbstractAccount.hpp>
#include <wallet/common/AbstractWallet.hpp>
#include <ripple/RippleLikeAddress.h>
#include <bytes/BytesWriter.h>
#include <bytes/BytesReader.h>
#include <bytes/RLP/RLPListEncoder.h>
#include <bytes/RLP/RLPStringEncoder.h>
#include <utils/hex.h>
#include <api_impl/BigIntImpl.hpp>

namespace ledger {
    namespace core {
        // a helper to write VLE fields
        bool writeLengthPrefix(BytesWriter& writer, uint64_t size) {
            // encoding here: https://developers.ripple.com/serialization.html#length-prefixing
            if (size <= 192) {
                writer.writeByte(size & 0xFF);

                return true;
            } else if (size <= 12480) {
                //     l       = 193 + ((byte1 - 193) * 256) + byte2
                // <=> l - 193 = ((byte1 - 193) * 256) + byte2
                // <=> byte2 = (l - 193) & 0xFF
                // <=> byte1 = 193 + ((l - 193) >> 8) & 0xFF
                size -= 193;
                writer.writeByte(193 + (size >> 8) & 0xFF);
                writer.writeByte(size & 0xFF);

                return true;
            } else if (size <= 918744) {
                //     l         = 12481 + ((byte1 - 241) * 65536) + (byte2 * 256) + byte3
                // <=> l - 12481 = ((byte1 - 241) * 65536) + (byte2 * 256) + byte3
                // <=> byte3 = (l - 12481) & 0xFF
                // <=> byte2 = ((l - 12481) >> 8) & 0xFF
                // <=> byte1 = 241 + ((l - 12481) >> 16) & 0xFF
                size -= 12481;
                writer.writeByte(241 + (size >> 16) & 0xFF);
                writer.writeByte((size >> 8) & 0xFF);
                writer.writeByte(size & 0xFF);
            }

            // cannot have more bytes
            return false;
        }

        RippleLikeTransactionApi::RippleLikeTransactionApi(const api::Currency &currency) {
            _currency = currency;
        }

        RippleLikeTransactionApi::RippleLikeTransactionApi(const std::shared_ptr<OperationApi> &operation) {
            auto &tx = operation->getBackend().rippleTransaction.getValue();
            _time = tx.receivedAt;

            if (tx.block.nonEmpty()) {
                _block = std::make_shared<RippleLikeBlockApi>(tx.block.getValue());
            } else {
                _block = nullptr;
            }

            _hash = tx.hash;

            _currency = operation->getAccount()->getWallet()->getCurrency();

            _fees = std::make_shared<Amount>(_currency, 0, tx.fees);
            _value = std::make_shared<Amount>(_currency, 0, tx.value);

            _receiver = RippleLikeAddress::fromBase58(tx.receiver, _currency);
            _sender = RippleLikeAddress::fromBase58(tx.sender, _currency);

        }

        std::string RippleLikeTransactionApi::getHash() {
            return _hash;
        }

        std::shared_ptr<api::Amount> RippleLikeTransactionApi::getFees() {
            return _fees;
        }

        std::shared_ptr<api::RippleLikeAddress> RippleLikeTransactionApi::getReceiver() {
            return _receiver;
        }

        std::shared_ptr<api::RippleLikeAddress> RippleLikeTransactionApi::getSender() {
            return _sender;
        }

        std::shared_ptr<api::Amount> RippleLikeTransactionApi::getValue() {
            return _value;
        }

        std::chrono::system_clock::time_point RippleLikeTransactionApi::getDate() {
            return _time;
        }

        std::shared_ptr<api::BigInt> RippleLikeTransactionApi::getLedgerSequence() {
            return _ledgerSequence;
        }

        std::shared_ptr<api::BigInt> RippleLikeTransactionApi::getSequence() {
            return _sequence;
        }

        std::vector<uint8_t> RippleLikeTransactionApi::getSigningPubKey() {
            return _signingPubKey;
        }

        void RippleLikeTransactionApi::setSignature(const std::vector<uint8_t> &rSignature,
                                                    const std::vector<uint8_t> &sSignature) {
            _rSignature = rSignature;
            _sSignature = sSignature;
        }

        void RippleLikeTransactionApi::setDERSignature(const std::vector<uint8_t> &signature) {
            BytesReader reader(signature);
            //DER prefix
            reader.readNextByte();
            //Total length
            reader.readNextVarInt();
            //Nb of elements for R
            reader.readNextByte();
            //R length
            auto rSize = reader.readNextVarInt();
            if (rSize > 0 && reader.peek() == 0x00) {
                reader.readNextByte();
                _rSignature = reader.read(rSize - 1);
            } else {
                _rSignature = reader.read(rSize);
            }
            //Nb of elements for S
            reader.readNextByte();
            //S length
            auto sSize = reader.readNextVarInt();
            if (sSize > 0 && reader.peek() == 0x00) {
                reader.readNextByte();
                _sSignature = reader.read(sSize - 1);
            } else {
                _sSignature = reader.read(sSize);
            }
        }

        //Field ID References:
        // https://github.com/ripple/rippled/blob/master/src/ripple/protocol/SField.h#L57-L74
        // and https://github.com/ripple/rippled/blob/72e6005f562a8f0818bc94803d222ac9345e1e40/src/ripple/protocol/impl/SField.cpp#L72-L266
        std::vector<uint8_t> RippleLikeTransactionApi::serialize() {
            BytesWriter writer;

            //1 byte TransactionType Field ID:   Type Code = 1, Field Code = 2
            writer.writeByte(0x12);
            //2 bytes TransactionType ("Payment")
            writer.writeByteArray({0x00, 0x00});

            //1 byte Flags Field ID:   Type Code = 2, Field Code = 2
            writer.writeByte(0x22);
            //4 bytes Flags (tfFullyCanonicalSig ?)
            writer.writeByteArray({0x80, 0x00, 0x00, 0x00});

            //1 byte Sequence Field ID:   Type Code = 2, Field Code = 4
            writer.writeByte(0x24);
            //4 bytes Sequence
            auto sequence = _sequence->toString(16);
            writer.writeBeValue<uint32_t>(static_cast<const uint32_t>(_sequence->intValue()));

            //2 bytes LastLedgerSequence Field ID:   Type Code = 2, Field Code = 27
            writer.writeByteArray({0x20, 0x1B});
            //LastLedgerSequence
            writer.writeBeValue<uint32_t>(static_cast<const uint32_t>(_ledgerSequence->intValue()));

            auto maxAmountBound = std::vector<uint8_t>({0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00});
            auto bigIntMax = BigInt::fromHex(hex::toString(maxAmountBound));

            //1 byte Amount Field ID:   Type Code = 6, Field Code = 1
            writer.writeByte(0x61);
            //8 bytes Amount (with bitwise OR with 0x4000000000000000)
            auto finalValue = BigInt::fromHex(_value->toBigInt()->toString(16)) + bigIntMax;
            writer.writeBeValue<uint64_t>(static_cast<const uint64_t>(finalValue.toUint64()));

            //1 byte Fee Field ID:   Type Code = 6, Field Code = 8
            writer.writeByte(0x68);
            //8 bytes Fees (with bitwise OR with 0x4000000000000000)
            auto finalFees = BigInt::fromHex(_fees->toBigInt()->toString(16)) + bigIntMax;
            writer.writeBeValue<uint64_t>(static_cast<const uint64_t>(finalFees.toUint64()));

            //TODO: !!!find out if this is included in raw unsigned tx or not
            //1 byte Signing pubKey Field ID:   Type Code = 7, Field Code = 3 (STI_VL = 7 type)
            writer.writeByte(0x73);
            //Var bytes Signing pubKey (prefix length)
            writer.writeVarInt(_signingPubKey.size());
            writer.writeByteArray(_signingPubKey);

            if (!_rSignature.empty() && !_sSignature.empty()) {
                //1 byte Signature Field ID:   Type Code = 7, Field Code = 4 (STI_VL = 7 type, and TxnSignature = 4)
                writer.writeByte(0x74);
                //Var bytes Signature (prefix length)
                //Get length of VarInt representing length of R
                BytesWriter rLength;
                rLength.writeVarInt(_rSignature.size());
                //Get length of VarInt representing length of R
                BytesWriter sLength;
                sLength.writeVarInt(_sSignature.size());
                //Get length of VarInt representing length of R and S (plus their stack sizes)
                auto sAndRLengthInt = 1 + rLength.toByteArray().size() + _rSignature.size() + 1 + sLength.toByteArray().size() + _sSignature.size();
                BytesWriter sAndRLength;
                sAndRLength.writeVarInt(sAndRLengthInt);
                //DER Signature = Total Size | DER prefix | Size(S+R) | R StackSize | R Length | R | S StackSize | S Length | S
                auto totalSigLength = 1 + sAndRLength.toByteArray().size() + sAndRLengthInt;
                writer.writeVarInt(totalSigLength);
                //DER prefix
                BytesWriter sigWriter;
                writer.writeByte(0x30);
                //Size of DER signature minus DER prefix | Size(S+R)
                writer.writeVarInt(sAndRLengthInt);
                //R field
                writer.writeByte(0x02); //Nb of stack elements
                writer.writeVarInt(_rSignature.size());
                writer.writeByteArray(_rSignature);
                //S field
                writer.writeByte(0x02); //Nb of stack elements
                writer.writeVarInt(_sSignature.size());
                writer.writeByteArray(_sSignature);
            }

            //1 byte Account Field ID: Type Code = 8, Field Code = 1 (STI_ACCOUNT = 8 type, and Account = 1)
            writer.writeByte(0x81);
            //20 bytes Acount (hash160 of pubKey without 0x00 prefix)
            auto senderHash = _sender->getHash160();
            writer.writeVarInt(senderHash.size());
            writer.writeByteArray(senderHash);
            //1 byte Destination Field ID: Type Code = 8, Field Code = 3 (STI_ACCOUNT = 8 type, and Destination = 3)
            writer.writeByte(0x83);
            //20 bytes Destination (hash160 of pubKey without 0x00 prefix)
            auto receiverHash = _receiver->getHash160();
            writer.writeVarInt(receiverHash.size());
            writer.writeByteArray(receiverHash);

            if (!_memos.empty()) {
                writer.writeByte(0xF9); // STI_ARRAY (Memos); Type Code = 15, Memos; Field ID = 9

                for (auto& memo : _memos) {
                    writer.writeByte(0xEA); // STI_OBJECT (Memo); Type Code = 10

                    if (!memo.ty.empty()) {
                        writer.writeByte(0x7C); // STI_VL (MemoType), 12
                        auto written = writeLengthPrefix(writer, memo.ty.size());
                        assert(written);
                        writer.writeString(memo.ty);
                    }

                    if (!memo.data.empty()) {
                        writer.writeByte(0x7D); // STI_VL (MemoData), 13
                        auto written = writeLengthPrefix(writer, memo.data.size());
                        assert(written);
                        writer.writeString(memo.data);
                    }

                    if (!memo.fmt.empty()) {
                        writer.writeByte(0x7D); // STI_VL (MemoFormat), 14
                        auto written = writeLengthPrefix(writer, memo.fmt.size());
                        assert(written);
                        writer.writeString(memo.fmt);
                    }

                    writer.writeByte(0xE1); // end of object
                }

                writer.writeByte(0xF1); // end of array
            }

            return writer.toByteArray();
        }

        RippleLikeTransactionApi &RippleLikeTransactionApi::setFees(const std::shared_ptr<BigInt> &fees) {
            if (!fees) {
                throw make_exception(api::ErrorCode::INVALID_ARGUMENT,
                                     "RippleLikeTransactionApi::setFees: Invalid Fees");
            }
            _fees = std::make_shared<Amount>(_currency, 0, *fees);
            return *this;
        }

        RippleLikeTransactionApi &RippleLikeTransactionApi::setValue(const std::shared_ptr<BigInt> &value) {
            if (!value) {
                throw make_exception(api::ErrorCode::INVALID_ARGUMENT,
                                     "RippleLikeTransactionApi::setValue: Invalid Value");
            }
            _value = std::make_shared<Amount>(_currency, 0, *value);
            return *this;
        }

        RippleLikeTransactionApi &RippleLikeTransactionApi::setSequence(const BigInt& sequence) {
            _sequence = std::make_shared<api::BigIntImpl>(sequence);
            return *this;
        }

        RippleLikeTransactionApi & RippleLikeTransactionApi::setLedgerSequence(const BigInt& ledgerSequence) {
            _ledgerSequence = std::make_shared<api::BigIntImpl>(ledgerSequence);
            return *this;
        }

        RippleLikeTransactionApi &RippleLikeTransactionApi::setSender(const std::shared_ptr<api::RippleLikeAddress> &sender) {
            _sender = sender;
            return *this;
        }

        RippleLikeTransactionApi &RippleLikeTransactionApi::setReceiver(const std::shared_ptr<api::RippleLikeAddress> &receiver) {
            _receiver = receiver;
            return *this;
        }

        RippleLikeTransactionApi &RippleLikeTransactionApi::setSigningPubKey(const std::vector<uint8_t> &pubKey) {
            _signingPubKey = pubKey;
            return *this;
        }

        RippleLikeTransactionApi &RippleLikeTransactionApi::setHash(const std::string &hash) {
            _hash = hash;
            return *this;
        }

        std::vector<api::RippleLikeMemo> RippleLikeTransactionApi::getMemos() {
            return _memos;
        }

        void RippleLikeTransactionApi::addMemo(api::RippleLikeMemo const& memo) {
            _memos.push_back(memo);
        }
    }
}
<|endoftext|>


<commit_before>/**
\description The is the main for the new version of the mert algorithm developed during the 2nd MT marathon
*/

#include <limits>
#include <unistd.h>
#include <cstdlib>
#include <iostream>
#include <fstream>
#include <cmath>
#include <ctime>

#include <getopt.h>

#include "Data.h"
#include "Point.h"
#include "Scorer.h"
#include "ScorerFactory.h"
#include "ScoreData.h"
#include "FeatureData.h"
#include "Optimizer.h"
#include "Types.h"
#include "Timer.h"
#include "Util.h"

#include "../moses/src/ThreadPool.h"

float min_interval = 1e-3;

using namespace std;

void usage(void)
{
  cerr<<"usage: mert -d <dimensions> (mandatory )"<<endl;
  cerr<<"[-n] retry ntimes (default 1)"<<endl;
  cerr<<"[-m] number of random directions in powell (default 0)"<<endl;
  cerr<<"[-o] the indexes to optimize(default all)"<<endl;
  cerr<<"[-t] the optimizer(default powell)"<<endl;
  cerr<<"[-r] the random seed (defaults to system clock)"<<endl;
	cerr<<"[-p] only create data for paired ranked optimizer"<<endl;
  cerr<<"[--sctype|-s] the scorer type (default BLEU)"<<endl;
  cerr<<"[--scconfig|-c] configuration string passed to scorer"<<endl;
  cerr<<"[--scfile|-S] comma separated list of scorer data files (default score.data)"<<endl;
  cerr<<"[--ffile|-F] comma separated list of feature data files (default feature.data)"<<endl;
  cerr<<"[--ifile|-i] the starting point data file (default init.opt)"<<endl;
#ifdef WITH_THREADS
  cerr<<"[--threads|-T] use multiple threads for random restart (default 1)"<<endl;
#endif
  cerr<<"[-v] verbose level"<<endl;
  cerr<<"[--help|-h] print this message and exit"<<endl;
  exit(1);
}

static struct option long_options[] = {
  {"pdim", 1, 0, 'd'},
  {"ntry",1,0,'n'},
  {"nrandom",1,0,'m'},
  {"rseed",required_argument,0,'r'},
  {"optimize",1,0,'o'},
	{"pro",required_argument,0,'p'},
  {"type",1,0,'t'},
  {"sctype",1,0,'s'},
  {"scconfig",required_argument,0,'c'},
  {"scfile",1,0,'S'},
  {"ffile",1,0,'F'},
  {"ifile",1,0,'i'},
#ifdef WITH_THREADS
  {"threads", required_argument,0,'T'},
#endif
  {"verbose",1,0,'v'},
  {"help",no_argument,0,'h'},
  {0, 0, 0, 0}
};
int option_index;

/**
  * Runs an optimisation, or a random restart.
**/
class OptimizationTask : public Moses::Task
{
  public:
    OptimizationTask(Optimizer* optimizer, const Point& point) :
       m_optimizer(optimizer), m_point(point) {}

    bool DeleteAfterExecution() {
      return false;
    }

    void Run() {
      m_score = m_optimizer->Run(m_point);
    }

    statscore_t getScore() const {
      return m_score;
    }

    const Point& getPoint() const  {
      return m_point;
    }

  private:
    Optimizer* m_optimizer;
    Point m_point;
    statscore_t m_score;
};

int main (int argc, char **argv)
{

  ResetUserTime();

  /*
   Timer timer;
   timer.start("Starting...");
  */

  int c,pdim,i;
  pdim=-1;
  int ntry=1;
  int nrandom=0;
  int seed=0;
  bool hasSeed = false;
#ifdef WITH_THREADS
  size_t threads=1;
#endif
  string type("powell");
  string scorertype("BLEU");
  string scorerconfig("");
  string scorerfile("statscore.data");
  string featurefile("features.data");
  string initfile("init.opt");
	string pairedrankfile("");

  string tooptimizestr("");
  vector<unsigned> tooptimize;
  vector<vector<parameter_t> > start_list;
  vector<parameter_t> min;
  vector<parameter_t> max;
  //note: those mins and max are the bound for the starting points of the algorithm, not strict bound on the result!

  while ((c=getopt_long (argc, argv, "o:r:d:n:m:t:s:S:F:v:p:", long_options, &option_index)) != -1) {
    switch (c) {
    case 'o':
      tooptimizestr = string(optarg);
      break;
		case 'p':
			pairedrankfile = string(optarg);
			break;
    case 'd':
      pdim = strtol(optarg, NULL, 10);
      break;
    case 'n':
      ntry=strtol(optarg, NULL, 10);
      break;
    case 'm':
      nrandom=strtol(optarg, NULL, 10);
      break;
    case 'r':
      seed=strtol(optarg, NULL, 10);
      hasSeed = true;
      break;
    case 't':
      type=string(optarg);
      break;
    case's':
      scorertype=string(optarg);
      break;
    case 'c':
      scorerconfig = string(optarg);
      break;
    case 'S':
      scorerfile=string(optarg);
      break;
    case 'F':
      featurefile=string(optarg);
      break;
    case 'i':
      initfile=string(optarg);
      break;
    case 'v':
      setverboselevel(strtol(optarg,NULL,10));
      break;
#ifdef WITH_THREADS
    case 'T':
      threads = strtol(optarg, NULL, 10);
      if (threads < 1) threads = 1;
      break;
#endif
    default:
      usage();
    }
  }
  if (pdim < 0)
    usage();

  if (hasSeed) {
    cerr << "Seeding random numbers with " << seed << endl;
    srandom(seed);
  } else {
    cerr << "Seeding random numbers with system clock " << endl;
    srandom(time(NULL));
  }

  // read in starting points
  std::string onefile;
  while (!initfile.empty()) {
    getNextPound(initfile, onefile, ",");
    vector<parameter_t> start;
    ifstream opt(onefile.c_str());
    if(opt.fail()) {
      cerr<<"could not open initfile: " << initfile << endl;
      exit(3);
    }
    start.resize(pdim);//to do:read from file
    int j;
    for( j=0; j<pdim&&!opt.fail(); j++)
      opt>>start[j];
    if(j<pdim) {
      cerr<<initfile<<":Too few starting weights." << endl;
      exit(3);
    }
    start_list.push_back(start);
    // for the first time, also read in the min/max values for scores
    if (start_list.size() == 1) {
      min.resize(pdim);
      for( j=0; j<pdim&&!opt.fail(); j++)
        opt>>min[j];
      if(j<pdim) {
        cerr<<initfile<<":Too few minimum weights." << endl;
        cerr<<"error could not initialize start point with " << initfile << endl;
        exit(3);
      }
      max.resize(pdim);
      for( j=0; j<pdim&&!opt.fail(); j++)
        opt>>max[j];
      if(j<pdim) {
        cerr<<initfile<<":Too few maximum weights." << endl;
        exit(3);
      }
    }
    opt.close();
  }

  vector<string> ScoreDataFiles;
  if (scorerfile.length() > 0) {
    std::string substring;
    while (!scorerfile.empty()) {
      getNextPound(scorerfile, substring, ",");
      ScoreDataFiles.push_back(substring);
    }
  }

  vector<string> FeatureDataFiles;
  if (featurefile.length() > 0) {
    std::string substring;
    while (!featurefile.empty()) {
      getNextPound(featurefile, substring, ",");
      FeatureDataFiles.push_back(substring);
    }
  }

  if (ScoreDataFiles.size() != FeatureDataFiles.size()) {
    throw runtime_error("Error: there is a different number of previous score and feature files");
  }

  //it make sense to know what parameter set were used to generate the nbest
  ScorerFactory SF;
  Scorer *TheScorer=SF.getScorer(scorertype,scorerconfig);

  //load data
  Data D(*TheScorer);
  for (size_t i=0; i < ScoreDataFiles.size(); i++) {
    cerr<<"Loading Data from: "<< ScoreDataFiles.at(i)  << " and " << FeatureDataFiles.at(i) << endl;
    D.load(FeatureDataFiles.at(i), ScoreDataFiles.at(i));
  }

  PrintUserTime("Data loaded");

  if (tooptimizestr.length() > 0) {
    cerr << "Weights to optimize: " << tooptimizestr << endl;

    //parse string to get weights to optimize
    //and set them as active
    std::string substring;
    int index;
    while (!tooptimizestr.empty()) {
      getNextPound(tooptimizestr, substring, ",");
      index = D.getFeatureIndex(substring);
      cerr << "FeatNameIndex:" << index << " to insert" << endl;
      //index = strtol(substring.c_str(), NULL, 10);
      if (index >= 0 && index < pdim) {
        tooptimize.push_back(index);
      } else {
        cerr << "Index " << index << " is out of bounds. Allowed indexes are [0," << (pdim-1) << "]." << endl;
      }
    }
  } else {
    //set all weights as active
    tooptimize.resize(pdim);//We'll optimize on everything
    for(int  i=0; i<pdim; i++) {
      tooptimize[i]=1;
    }
  }

	if (pairedrankfile.compare("") != 0) {
		D.sample_ranked_pairs(pairedrankfile);
		PrintUserTime("Stopping...");
		exit(0);
	}

  Optimizer *O=OptimizerFactory::BuildOptimizer(pdim,tooptimize,start_list[0],type,nrandom);
  O->SetScorer(TheScorer);
  O->SetFData(D.getFeatureData());

#ifdef WITH_THREADS
  cerr << "Creating a pool of " << threads << " threads" << endl;
  Moses::ThreadPool pool(threads);
#endif

  vector<OptimizationTask*> tasks;

  // run with specified starting points
  for(size_t i=0;i<start_list.size();i++) {
    //Generate from the full feature set. Warning: must be done after Optimizer initialization
    Point P(start_list[i], min, max);
    OptimizationTask* task = new OptimizationTask(O,P);
    tasks.push_back(task);
#ifdef WITH_THREADS
    pool.Submit(task);
#else
    task->Run();
#endif
  }

  //run with random starting points
  for (int i = 0; i < ntry; ++i) {
    Point P(start_list[0], min, max);
    P.Randomize(); // randomize within min and max as given to the constructor
    OptimizationTask* task = new OptimizationTask(O,P);
    tasks.push_back(task);
#ifdef WITH_THREADS
    pool.Submit(task);
#else
    task->Run();
#endif
  }

  //wait for all threads to finish
#ifdef WITH_THREADS
  pool.Stop(true);
#endif

  //collect results
  statscore_t best=0, mean=0, var=0;
  Point bestP;
  for (vector<OptimizationTask*>::const_iterator i = tasks.begin(); i != tasks.end(); ++i) {
    statscore_t score = (*i)->getScore();
    mean += score;
    var += score*score;
    if (score > best) {
      bestP = (*i)->getPoint();
      best = score;
    }
  }

  mean/=(float)ntry;
  var/=(float)ntry;
  var=sqrt(abs(var-mean*mean));
  if (verboselevel()>1)
    cerr<<"best score: "<< best << " variance of the score (for "<<ntry<<" try): "<<var<<endl;

  // L1-Normalization of the best Point
  if ((int)tooptimize.size() == pdim)
    bestP.NormalizeL1();

  cerr << "Best point: " << bestP << " => " << best << endl;
  ofstream res("weights.txt");
  res<<bestP<<endl;

  PrintUserTime("Stopping...");
}
<commit_msg>clean-up<commit_after>/**
\description The is the main for the new version of the mert algorithm developed during the 2nd MT marathon
*/

#include <limits>
#include <unistd.h>
#include <cstdlib>
#include <iostream>
#include <fstream>
#include <cmath>
#include <ctime>

#include <getopt.h>

#include "Data.h"
#include "Point.h"
#include "Scorer.h"
#include "ScorerFactory.h"
#include "ScoreData.h"
#include "FeatureData.h"
#include "Optimizer.h"
#include "Types.h"
#include "Timer.h"
#include "Util.h"

#include "../moses/src/ThreadPool.h"

float min_interval = 1e-3;

using namespace std;

void usage(void)
{
  cerr<<"usage: mert -d <dimensions> (mandatory )"<<endl;
  cerr<<"[-n] retry ntimes (default 1)"<<endl;
  cerr<<"[-m] number of random directions in powell (default 0)"<<endl;
  cerr<<"[-o] the indexes to optimize(default all)"<<endl;
  cerr<<"[-t] the optimizer(default powell)"<<endl;
  cerr<<"[-r] the random seed (defaults to system clock)"<<endl;
	cerr<<"[-p] only create data for paired ranked optimizer"<<endl;
  cerr<<"[--sctype|-s] the scorer type (default BLEU)"<<endl;
  cerr<<"[--scconfig|-c] configuration string passed to scorer"<<endl;
  cerr<<"[--scfile|-S] comma separated list of scorer data files (default score.data)"<<endl;
  cerr<<"[--ffile|-F] comma separated list of feature data files (default feature.data)"<<endl;
  cerr<<"[--ifile|-i] the starting point data file (default init.opt)"<<endl;
#ifdef WITH_THREADS
  cerr<<"[--threads|-T] use multiple threads for random restart (default 1)"<<endl;
#endif
  cerr<<"[-v] verbose level"<<endl;
  cerr<<"[--help|-h] print this message and exit"<<endl;
  exit(1);
}

static struct option long_options[] = {
  {"pdim", 1, 0, 'd'},
  {"ntry",1,0,'n'},
  {"nrandom",1,0,'m'},
  {"rseed",required_argument,0,'r'},
  {"optimize",1,0,'o'},
	{"pro",required_argument,0,'p'},
  {"type",1,0,'t'},
  {"sctype",1,0,'s'},
  {"scconfig",required_argument,0,'c'},
  {"scfile",1,0,'S'},
  {"ffile",1,0,'F'},
  {"ifile",1,0,'i'},
#ifdef WITH_THREADS
  {"threads", required_argument,0,'T'},
#endif
  {"verbose",1,0,'v'},
  {"help",no_argument,0,'h'},
  {0, 0, 0, 0}
};
int option_index;

/**
  * Runs an optimisation, or a random restart.
**/
class OptimizationTask : public Moses::Task
{
  public:
    OptimizationTask(Optimizer* optimizer, const Point& point) :
       m_optimizer(optimizer), m_point(point) {}

    bool DeleteAfterExecution() {
      return false;
    }

    void Run() {
      m_score = m_optimizer->Run(m_point);
    }

    statscore_t getScore() const {
      return m_score;
    }

    const Point& getPoint() const  {
      return m_point;
    }

  private:
    Optimizer* m_optimizer;
    Point m_point;
    statscore_t m_score;
};

int main (int argc, char **argv)
{

  ResetUserTime();

  /*
   Timer timer;
   timer.start("Starting...");
  */

  int c,pdim,i;
  pdim=-1;
  int ntry=1;
  int nrandom=0;
  int seed=0;
  bool hasSeed = false;
#ifdef WITH_THREADS
  size_t threads=1;
#endif
  string type("powell");
  string scorertype("BLEU");
  string scorerconfig("");
  string scorerfile("statscore.data");
  string featurefile("features.data");
  string initfile("init.opt");
	string pairedrankfile("");

  string tooptimizestr("");
  vector<unsigned> tooptimize;
  vector<vector<parameter_t> > start_list;
  vector<parameter_t> min;
  vector<parameter_t> max;
  //note: those mins and max are the bound for the starting points of the algorithm, not strict bound on the result!

  while ((c=getopt_long (argc, argv, "o:r:d:n:m:t:s:S:F:v:p:", long_options, &option_index)) != -1) {
    switch (c) {
    case 'o':
      tooptimizestr = string(optarg);
      break;
		case 'p':
			pairedrankfile = string(optarg);
			break;
    case 'd':
      pdim = strtol(optarg, NULL, 10);
      break;
    case 'n':
      ntry=strtol(optarg, NULL, 10);
      break;
    case 'm':
      nrandom=strtol(optarg, NULL, 10);
      break;
    case 'r':
      seed=strtol(optarg, NULL, 10);
      hasSeed = true;
      break;
    case 't':
      type=string(optarg);
      break;
    case's':
      scorertype=string(optarg);
      break;
    case 'c':
      scorerconfig = string(optarg);
      break;
    case 'S':
      scorerfile=string(optarg);
      break;
    case 'F':
      featurefile=string(optarg);
      break;
    case 'i':
      initfile=string(optarg);
      break;
    case 'v':
      setverboselevel(strtol(optarg,NULL,10));
      break;
#ifdef WITH_THREADS
    case 'T':
      threads = strtol(optarg, NULL, 10);
      if (threads < 1) threads = 1;
      break;
#endif
    default:
      usage();
    }
  }
  if (pdim < 0)
    usage();

  if (hasSeed) {
    cerr << "Seeding random numbers with " << seed << endl;
    srandom(seed);
  } else {
    cerr << "Seeding random numbers with system clock " << endl;
    srandom(time(NULL));
  }

  // read in starting points
  std::string onefile;
  while (!initfile.empty()) {
    getNextPound(initfile, onefile, ",");
    vector<parameter_t> start;
    ifstream opt(onefile.c_str());
    if(opt.fail()) {
      cerr<<"could not open initfile: " << initfile << endl;
      exit(3);
    }
    start.resize(pdim);//to do:read from file
    int j;
    for( j=0; j<pdim&&!opt.fail(); j++)
      opt>>start[j];
    if(j<pdim) {
      cerr<<initfile<<":Too few starting weights." << endl;
      exit(3);
    }
    start_list.push_back(start);
    // for the first time, also read in the min/max values for scores
    if (start_list.size() == 1) {
      min.resize(pdim);
      for( j=0; j<pdim&&!opt.fail(); j++)
        opt>>min[j];
      if(j<pdim) {
        cerr<<initfile<<":Too few minimum weights." << endl;
        cerr<<"error could not initialize start point with " << initfile << endl;
        exit(3);
      }
      max.resize(pdim);
      for( j=0; j<pdim&&!opt.fail(); j++)
        opt>>max[j];
      if(j<pdim) {
        cerr<<initfile<<":Too few maximum weights." << endl;
        exit(3);
      }
    }
    opt.close();
  }

  vector<string> ScoreDataFiles;
  if (scorerfile.length() > 0) {
    std::string substring;
    while (!scorerfile.empty()) {
      getNextPound(scorerfile, substring, ",");
      ScoreDataFiles.push_back(substring);
    }
  }

  vector<string> FeatureDataFiles;
  if (featurefile.length() > 0) {
    std::string substring;
    while (!featurefile.empty()) {
      getNextPound(featurefile, substring, ",");
      FeatureDataFiles.push_back(substring);
    }
  }

  if (ScoreDataFiles.size() != FeatureDataFiles.size()) {
    throw runtime_error("Error: there is a different number of previous score and feature files");
  }

  //it make sense to know what parameter set were used to generate the nbest
  ScorerFactory SF;
  Scorer *TheScorer=SF.getScorer(scorertype,scorerconfig);

  //load data
  Data D(*TheScorer);
  for (size_t i=0; i < ScoreDataFiles.size(); i++) {
    cerr<<"Loading Data from: "<< ScoreDataFiles.at(i)  << " and " << FeatureDataFiles.at(i) << endl;
    D.load(FeatureDataFiles.at(i), ScoreDataFiles.at(i));
  }

  PrintUserTime("Data loaded");

  if (tooptimizestr.length() > 0) {
    cerr << "Weights to optimize: " << tooptimizestr << endl;

    //parse string to get weights to optimize
    //and set them as active
    std::string substring;
    int index;
    while (!tooptimizestr.empty()) {
      getNextPound(tooptimizestr, substring, ",");
      index = D.getFeatureIndex(substring);
      cerr << "FeatNameIndex:" << index << " to insert" << endl;
      //index = strtol(substring.c_str(), NULL, 10);
      if (index >= 0 && index < pdim) {
        tooptimize.push_back(index);
      } else {
        cerr << "Index " << index << " is out of bounds. Allowed indexes are [0," << (pdim-1) << "]." << endl;
      }
    }
  } else {
    //set all weights as active
    tooptimize.resize(pdim);//We'll optimize on everything
    for(int  i=0; i<pdim; i++) {
      tooptimize[i]=1;
    }
  }

	if (pairedrankfile.compare("") != 0) {
		D.sample_ranked_pairs(pairedrankfile);
		PrintUserTime("Stopping...");
		exit(0);
	}

  Optimizer *O=OptimizerFactory::BuildOptimizer(pdim,tooptimize,start_list[0],type,nrandom);
  O->SetScorer(TheScorer);
  O->SetFData(D.getFeatureData());

#ifdef WITH_THREADS
  cerr << "Creating a pool of " << threads << " threads" << endl;
  Moses::ThreadPool pool(threads);
#endif

  vector<OptimizationTask*> tasks;

  // run with specified starting points
  for(size_t i=0;i<start_list.size();i++) {
    //Generate from the full feature set. Warning: must be done after Optimizer initialization
    Point P(start_list[i], min, max);
    OptimizationTask* task = new OptimizationTask(O,P);
    tasks.push_back(task);
#ifdef WITH_THREADS
    pool.Submit(task);
#else
    task->Run();
#endif
  }

  //run with random starting points
  for (int i = 0; i < ntry; ++i) {
    Point P(start_list[0], min, max);
    P.Randomize(); // randomize within min and max as given to the constructor
    OptimizationTask* task = new OptimizationTask(O,P);
    tasks.push_back(task);
#ifdef WITH_THREADS
    pool.Submit(task);
#else
    task->Run();
#endif
  }

  //wait for all threads to finish
#ifdef WITH_THREADS
  pool.Stop(true);
#endif

  //collect results
  statscore_t best=0, mean=0, var=0;
  Point bestP;
  for (vector<OptimizationTask*>::const_iterator i = tasks.begin(); i != tasks.end(); ++i) {
    statscore_t score = (*i)->getScore();
    mean += score;
    var += score*score;
    if (score > best) {
      bestP = (*i)->getPoint();
      best = score;
    }
    delete *i;
  }

  mean/=(float)ntry;
  var/=(float)ntry;
  var=sqrt(abs(var-mean*mean));
  if (verboselevel()>1)
    cerr<<"best score: "<< best << " variance of the score (for "<<ntry<<" try): "<<var<<endl;

  // L1-Normalization of the best Point
  if ((int)tooptimize.size() == pdim)
    bestP.NormalizeL1();

  cerr << "Best point: " << bestP << " => " << best << endl;
  ofstream res("weights.txt");
  res<<bestP<<endl;

  PrintUserTime("Stopping...");
}
<|endoftext|>


<commit_before>/*
Copyright(c) 2016-2018 Panos Karabelas

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and / or sell
copies of the Software, and to permit persons to whom the Software is furnished
to do so, subject to the following conditions :

The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.IN NO EVENT SHALL THE AUTHORS OR
COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/

//= INCLUDES =================================
#include "Widget_Toolbar.h"
#include "../../ImGui/Source/imgui.h"
#include "../../ImGui/Source/imgui_internal.h"
#include "../IconProvider.h"
#include "../EditorHelper.h"
#include "Rendering/Renderer.h"
#include "../EditorHelper.h"
//============================================

//= NAMESPACES ==========
using namespace Directus;
//=======================

namespace Toolbar_Options
{
	static float g_buttonSize = 20.0f;
	static bool g_showRendererOptions	= false;
	static bool g_physics = true;
	static bool g_aabb = false;
	static bool g_gizmos = true;
	static bool g_pickingRay = false;
	static bool g_grid = true;
	static bool g_performanceMetrics = false;
	std::vector<char*> g_rendererViews = {
		"Default",
		"Albedo",
		"Normal",
		"Specular",
		"Depth"
	};
	static int g_rendererViewInt = 0;
	static const char* g_rendererView = g_rendererViews[g_rendererViewInt];
}

Widget_Toolbar::Widget_Toolbar()
{

}

void Widget_Toolbar::Initialize(Context* context)
{
	Widget::Initialize(context);
	m_title = "Toolbar";
	m_windowFlags = ImGuiWindowFlags_NoCollapse |
		ImGuiWindowFlags_NoResize				|
		ImGuiWindowFlags_NoMove					|
		ImGuiWindowFlags_NoSavedSettings		|
		ImGuiWindowFlags_NoScrollbar			|
		ImGuiWindowFlags_NoTitleBar;

	Engine::EngineMode_Disable(Engine_Game);
}

void Widget_Toolbar::Begin()
{
	ImGuiContext& g = *GImGui;
	float width		= g.IO.DisplaySize.x;
	float height	= g.FontBaseSize + g.Style.FramePadding.y * 2.0f - 1.0f;
	ImGui::SetNextWindowPos(ImVec2(0.0f, height - 1.0f));
	ImGui::SetNextWindowSize(ImVec2(width, height + 16));
	ImGui::PushStyleVar(ImGuiStyleVar_WindowPadding, ImVec2(0, 5));
	ImGui::Begin(m_title.c_str(), &m_isVisible, m_windowFlags);
}

void Widget_Toolbar::Update(float deltaTime)
{
	// Play button
	ImGui::SameLine();
	ImGui::PushStyleColor(ImGuiCol_Button, Engine::EngineMode_IsSet(Engine_Game) ? ImGui::GetStyle().Colors[ImGuiCol_ButtonActive] : ImGui::GetStyle().Colors[ImGuiCol_Button]);
	if (THUMBNAIL_BUTTON_BY_TYPE(Icon_Button_Play, Toolbar_Options::g_buttonSize))
	{
		Engine::EngineMode_Toggle(Engine_Game);
	}
	ImGui::PopStyleColor();

	// Renderer options button
	ImGui::SameLine();
	ImGui::PushStyleColor(ImGuiCol_Button, Toolbar_Options::g_showRendererOptions ? ImGui::GetStyle().Colors[ImGuiCol_ButtonActive] : ImGui::GetStyle().Colors[ImGuiCol_Button]);
	if (THUMBNAIL_BUTTON_BY_TYPE(Icon_Component_Options, Toolbar_Options::g_buttonSize))
	{
		Toolbar_Options::g_showRendererOptions = true;
	}
	ImGui::PopStyleColor();

	ImGui::PopStyleVar();

	// Visibility
	if (Toolbar_Options::g_showRendererOptions)	ShowRendererOptions();
}

void Widget_Toolbar::ShowRendererOptions()
{
	ImGui::Begin("Renderer Options", &Toolbar_Options::g_showRendererOptions, ImGuiWindowFlags_AlwaysAutoResize);

	// G-Buffer Visualization
	{
		if (ImGui::BeginCombo("G-Buffer", Toolbar_Options::g_rendererView))
		{
			for (auto i = 0; i < Toolbar_Options::g_rendererViews.size(); i++)
			{
				bool is_selected = (Toolbar_Options::g_rendererView == Toolbar_Options::g_rendererViews[i]);
				if (ImGui::Selectable(Toolbar_Options::g_rendererViews[i], is_selected))
				{
					Toolbar_Options::g_rendererView		= Toolbar_Options::g_rendererViews[i];
					Toolbar_Options::g_rendererViewInt	= i;
				}
				if (is_selected)
				{
					ImGui::SetItemDefaultFocus();
				}
			}
			ImGui::EndCombo();
		}

		if (Toolbar_Options::g_rendererViewInt == 0) // Combined
		{
			Renderer::RenderFlags_Disable(Render_Albedo);
			Renderer::RenderFlags_Disable(Render_Normal);
			Renderer::RenderFlags_Disable(Render_Specular);
			Renderer::RenderFlags_Disable(Render_Depth);
		}
		else if (Toolbar_Options::g_rendererViewInt == 1) // Albedo
		{
			Renderer::RenderFlags_Enable(Render_Albedo);
			Renderer::RenderFlags_Disable(Render_Normal);
			Renderer::RenderFlags_Disable(Render_Specular);
			Renderer::RenderFlags_Disable(Render_Depth);
		}
		else if (Toolbar_Options::g_rendererViewInt == 2) // Normal
		{
			Renderer::RenderFlags_Disable(Render_Albedo);
			Renderer::RenderFlags_Enable(Render_Normal);
			Renderer::RenderFlags_Disable(Render_Specular);
			Renderer::RenderFlags_Disable(Render_Depth);
		}
		else if (Toolbar_Options::g_rendererViewInt == 3) // Specular
		{
			Renderer::RenderFlags_Disable(Render_Albedo);
			Renderer::RenderFlags_Disable(Render_Normal);
			Renderer::RenderFlags_Enable(Render_Specular);
			Renderer::RenderFlags_Disable(Render_Depth);
		}
		else if (Toolbar_Options::g_rendererViewInt == 4) // Depth
		{
			Renderer::RenderFlags_Disable(Render_Albedo);
			Renderer::RenderFlags_Disable(Render_Normal);
			Renderer::RenderFlags_Disable(Render_Specular);
			Renderer::RenderFlags_Enable(Render_Depth);
		}
	}

	ImGui::Separator();

	// Effects
	{
		bool bloom					= Renderer::RenderFlags_IsSet(Render_Bloom);
		bool correction				= Renderer::RenderFlags_IsSet(Render_Correction);
		bool fxaa					= Renderer::RenderFlags_IsSet(Render_FXAA);
		bool sharpening				= Renderer::RenderFlags_IsSet(Render_Sharpening);
		bool chromaticAberration	= Renderer::RenderFlags_IsSet(Render_ChromaticAberration);

		ImGui::Checkbox("Bloom", &bloom);
		ImGui::Checkbox("Tone-mapping & Gamma correction", &correction);
		ImGui::Checkbox("FXAA", &fxaa);
		ImGui::Checkbox("Chromatic Aberration", &chromaticAberration);
		ImGui::Checkbox("Sharpening", &sharpening);

		bloom				? Renderer::RenderFlags_Enable(Render_Bloom)				: Renderer::RenderFlags_Disable(Render_Bloom);
		correction			? Renderer::RenderFlags_Enable(Render_Correction)			: Renderer::RenderFlags_Disable(Render_Correction);
		fxaa				? Renderer::RenderFlags_Enable(Render_FXAA)					: Renderer::RenderFlags_Disable(Render_FXAA);
		sharpening			? Renderer::RenderFlags_Enable(Render_Sharpening)			: Renderer::RenderFlags_Disable(Render_Sharpening);
		chromaticAberration	? Renderer::RenderFlags_Enable(Render_ChromaticAberration)	: Renderer::RenderFlags_Disable(Render_ChromaticAberration);
	}

	ImGui::Separator();

	// Misc
	{
		ImGui::Checkbox("Physics", &Toolbar_Options::g_physics);
		ImGui::Checkbox("AABB", &Toolbar_Options::g_aabb);
		ImGui::Checkbox("Gizmos", &Toolbar_Options::g_gizmos);
		ImGui::Checkbox("Picking Ray", &Toolbar_Options::g_pickingRay);
		ImGui::Checkbox("Scene Grid", &Toolbar_Options::g_grid);
		ImGui::Checkbox("Performance Metrics", &Toolbar_Options::g_performanceMetrics);

		Toolbar_Options::g_physics				? Renderer::RenderFlags_Enable(Render_Physics)				: Renderer::RenderFlags_Disable(Render_Physics);
		Toolbar_Options::g_aabb					? Renderer::RenderFlags_Enable(Render_AABB)					: Renderer::RenderFlags_Disable(Render_AABB);
		Toolbar_Options::g_gizmos				? Renderer::RenderFlags_Enable(Render_Light)				: Renderer::RenderFlags_Disable(Render_Light);
		Toolbar_Options::g_pickingRay			? Renderer::RenderFlags_Enable(Render_PickingRay)			: Renderer::RenderFlags_Disable(Render_PickingRay);
		Toolbar_Options::g_grid					? Renderer::RenderFlags_Enable(Render_SceneGrid)			: Renderer::RenderFlags_Disable(Render_SceneGrid);
		Toolbar_Options::g_performanceMetrics	? Renderer::RenderFlags_Enable(Render_PerformanceMetrics)	: Renderer::RenderFlags_Disable(Render_PerformanceMetrics);
	}

	ImGui::End();
}
<commit_msg>Revert "Replaced array with vector and wrapped all variables inside a namespace"<commit_after>/*
Copyright(c) 2016-2018 Panos Karabelas

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and / or sell
copies of the Software, and to permit persons to whom the Software is furnished
to do so, subject to the following conditions :

The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.IN NO EVENT SHALL THE AUTHORS OR
COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/

//= INCLUDES =================================
#include "Widget_Toolbar.h"
#include "../../ImGui/Source/imgui.h"
#include "../../ImGui/Source/imgui_internal.h"
#include "../IconProvider.h"
#include "../EditorHelper.h"
#include "Rendering/Renderer.h"
#include "../EditorHelper.h"
//============================================

//= NAMESPACES ==========
using namespace Directus;
//=======================

static float g_buttonSize = 20.0f;

static bool g_showRendererOptions	= false;
static bool g_physics = true;
static bool g_aabb = false;
static bool g_gizmos = true;
static bool g_pickingRay = false;
static bool g_grid = true;
static bool g_performanceMetrics = false;
const char* g_rendererViews[] =
{
	"Default",
	"Albedo",
	"Normal",
	"Specular",
	"Depth"
};
static int g_rendererViewInt = 0;
static const char* g_rendererView = g_rendererViews[g_rendererViewInt];

Widget_Toolbar::Widget_Toolbar()
{

}

void Widget_Toolbar::Initialize(Context* context)
{
	Widget::Initialize(context);
	m_title = "Toolbar";
	m_windowFlags = ImGuiWindowFlags_NoCollapse |
		ImGuiWindowFlags_NoResize				|
		ImGuiWindowFlags_NoMove					|
		ImGuiWindowFlags_NoSavedSettings		|
		ImGuiWindowFlags_NoScrollbar			|
		ImGuiWindowFlags_NoTitleBar;

	Engine::EngineMode_Disable(Engine_Game);
}

void Widget_Toolbar::Begin()
{
	ImGuiContext& g = *GImGui;
	float width		= g.IO.DisplaySize.x;
	float height	= g.FontBaseSize + g.Style.FramePadding.y * 2.0f - 1.0f;
	ImGui::SetNextWindowPos(ImVec2(0.0f, height - 1.0f));
	ImGui::SetNextWindowSize(ImVec2(width, height + 16));
	ImGui::PushStyleVar(ImGuiStyleVar_WindowPadding, ImVec2(0, 5));
	ImGui::Begin(m_title.c_str(), &m_isVisible, m_windowFlags);
}

void Widget_Toolbar::Update(float deltaTime)
{
	// Play button
	ImGui::SameLine();
	ImGui::PushStyleColor(ImGuiCol_Button, Engine::EngineMode_IsSet(Engine_Game) ? ImGui::GetStyle().Colors[ImGuiCol_ButtonActive] : ImGui::GetStyle().Colors[ImGuiCol_Button]);
	if (THUMBNAIL_BUTTON_BY_TYPE(Icon_Button_Play, g_buttonSize))
	{
		Engine::EngineMode_Toggle(Engine_Game);
	}
	ImGui::PopStyleColor();

	// Renderer options button
	ImGui::SameLine();
	ImGui::PushStyleColor(ImGuiCol_Button, g_showRendererOptions ? ImGui::GetStyle().Colors[ImGuiCol_ButtonActive] : ImGui::GetStyle().Colors[ImGuiCol_Button]);
	if (THUMBNAIL_BUTTON_BY_TYPE(Icon_Component_Options, g_buttonSize))
	{
		g_showRendererOptions = true;
	}
	ImGui::PopStyleColor();

	ImGui::PopStyleVar();

	// Visibility
	if (g_showRendererOptions)	ShowRendererOptions();
}

void Widget_Toolbar::ShowRendererOptions()
{
	ImGui::Begin("Renderer Options", &g_showRendererOptions, ImGuiWindowFlags_AlwaysAutoResize);

	// G-Buffer Visualization
	{
		if (ImGui::BeginCombo("G-Buffer", g_rendererView))
		{
			for (int i = 0; i < IM_ARRAYSIZE(g_rendererViews); i++)
			{
				bool is_selected = (g_rendererView == g_rendererViews[i]);
				if (ImGui::Selectable(g_rendererViews[i], is_selected))
				{
					g_rendererView		= g_rendererViews[i];
					g_rendererViewInt	= i;
				}
				if (is_selected)
				{
					ImGui::SetItemDefaultFocus();
				}
			}
			ImGui::EndCombo();
		}

		if (g_rendererViewInt == 0) // Combined
		{
			Renderer::RenderFlags_Disable(Render_Albedo);
			Renderer::RenderFlags_Disable(Render_Normal);
			Renderer::RenderFlags_Disable(Render_Specular);
			Renderer::RenderFlags_Disable(Render_Depth);
		}
		else if (g_rendererViewInt == 1) // Albedo
		{
			Renderer::RenderFlags_Enable(Render_Albedo);
			Renderer::RenderFlags_Disable(Render_Normal);
			Renderer::RenderFlags_Disable(Render_Specular);
			Renderer::RenderFlags_Disable(Render_Depth);
		}
		else if (g_rendererViewInt == 2) // Normal
		{
			Renderer::RenderFlags_Disable(Render_Albedo);
			Renderer::RenderFlags_Enable(Render_Normal);
			Renderer::RenderFlags_Disable(Render_Specular);
			Renderer::RenderFlags_Disable(Render_Depth);
		}
		else if (g_rendererViewInt == 3) // Specular
		{
			Renderer::RenderFlags_Disable(Render_Albedo);
			Renderer::RenderFlags_Disable(Render_Normal);
			Renderer::RenderFlags_Enable(Render_Specular);
			Renderer::RenderFlags_Disable(Render_Depth);
		}
		else if (g_rendererViewInt == 4) // Depth
		{
			Renderer::RenderFlags_Disable(Render_Albedo);
			Renderer::RenderFlags_Disable(Render_Normal);
			Renderer::RenderFlags_Disable(Render_Specular);
			Renderer::RenderFlags_Enable(Render_Depth);
		}
	}

	ImGui::Separator();

	// Effects
	{
		bool bloom					= Renderer::RenderFlags_IsSet(Render_Bloom);
		bool correction				= Renderer::RenderFlags_IsSet(Render_Correction);
		bool fxaa					= Renderer::RenderFlags_IsSet(Render_FXAA);
		bool sharpening				= Renderer::RenderFlags_IsSet(Render_Sharpening);
		bool chromaticAberration	= Renderer::RenderFlags_IsSet(Render_ChromaticAberration);

		ImGui::Checkbox("Bloom", &bloom);
		ImGui::Checkbox("Tone-mapping & Gamma correction", &correction);
		ImGui::Checkbox("FXAA", &fxaa);
		ImGui::Checkbox("Chromatic Aberration", &chromaticAberration);
		ImGui::Checkbox("Sharpening", &sharpening);

		bloom				? Renderer::RenderFlags_Enable(Render_Bloom)				: Renderer::RenderFlags_Disable(Render_Bloom);
		correction			? Renderer::RenderFlags_Enable(Render_Correction)			: Renderer::RenderFlags_Disable(Render_Correction);
		fxaa				? Renderer::RenderFlags_Enable(Render_FXAA)					: Renderer::RenderFlags_Disable(Render_FXAA);
		sharpening			? Renderer::RenderFlags_Enable(Render_Sharpening)			: Renderer::RenderFlags_Disable(Render_Sharpening);
		chromaticAberration	? Renderer::RenderFlags_Enable(Render_ChromaticAberration)	: Renderer::RenderFlags_Disable(Render_ChromaticAberration);
	}

	ImGui::Separator();

	// Misc
	{
		ImGui::Checkbox("Physics", &g_physics);
		ImGui::Checkbox("AABB", &g_aabb);
		ImGui::Checkbox("Gizmos", &g_gizmos);
		ImGui::Checkbox("Picking Ray", &g_pickingRay);
		ImGui::Checkbox("Scene Grid", &g_grid);
		ImGui::Checkbox("Performance Metrics", &g_performanceMetrics);

		g_physics				? Renderer::RenderFlags_Enable(Render_Physics)				: Renderer::RenderFlags_Disable(Render_Physics);
		g_aabb					? Renderer::RenderFlags_Enable(Render_AABB)					: Renderer::RenderFlags_Disable(Render_AABB);
		g_gizmos				? Renderer::RenderFlags_Enable(Render_Light)				: Renderer::RenderFlags_Disable(Render_Light);
		g_pickingRay			? Renderer::RenderFlags_Enable(Render_PickingRay)			: Renderer::RenderFlags_Disable(Render_PickingRay);
		g_grid					? Renderer::RenderFlags_Enable(Render_SceneGrid)			: Renderer::RenderFlags_Disable(Render_SceneGrid);
		g_performanceMetrics	? Renderer::RenderFlags_Enable(Render_PerformanceMetrics)	: Renderer::RenderFlags_Disable(Render_PerformanceMetrics);
	}

	ImGui::End();
}
<|endoftext|>


<commit_before>#include <stdio.h>

#include "messaging.h"
#include "container.cpp"

void io_request::assign(io_request &req)
{
	this->offset = req.offset;
	this->io = req.io;
	this->priv = req.priv;
	this->access_method = req.access_method;
	this->num_bufs = req.num_bufs;
	this->vec_capacity = req.vec_capacity;
	this->partial = req.partial;
	this->completed_size = req.completed_size;
	this->orig = req.orig;
	this->refcnt = 0;
	/*
	 * If the request uses embedded vector, then the new request
	 * should point to its own embedded vector. Otherwise,
	 * the new request steals the vector from the old one.
	 */
	if (req.use_embedded())
		this->vec_pointer = this->embedded_vecs;
	else
		this->vec_pointer = req.vec_pointer;
	this->next = req.next;
	this->completed_size = req.completed_size;
	memcpy(this->embedded_vecs, req.embedded_vecs,
			sizeof(req.embedded_vecs[0]) * NUM_EMBEDDED_IOVECS);
	req.vec_pointer = req.embedded_vecs;
	req.vec_capacity = NUM_EMBEDDED_IOVECS;
	req.clear();
}

void io_request::add_buf(char *buf, int size)
{
	if (num_bufs >= vec_capacity) {
		if (vec_pointer == embedded_vecs) {
			vec_capacity = MIN_NUM_ALLOC_IOVECS;
			vec_pointer = new struct iovec[vec_capacity];
			memcpy(vec_pointer, embedded_vecs,
					sizeof(embedded_vecs[0]) * NUM_EMBEDDED_IOVECS);
		}
		else {
			vec_capacity *= 2;
			struct iovec *tmp = new struct iovec[vec_capacity];
			memcpy(tmp, vec_pointer,
					sizeof(vec_pointer[0]) * vec_capacity / 2);
			delete [] vec_pointer;
			vec_pointer = tmp;
		}
	}
	assert(num_bufs < vec_capacity);
	vec_pointer[num_bufs].iov_base = buf;
	vec_pointer[num_bufs].iov_len = size;
	num_bufs++;
}

void io_request::add_buf_front(char *buf, int size)
{
	if (num_bufs >= vec_capacity) {
		if (vec_pointer == embedded_vecs) {
			vec_capacity = MIN_NUM_ALLOC_IOVECS;
			vec_pointer = new struct iovec[vec_capacity];
			memcpy(vec_pointer + 1, embedded_vecs,
					sizeof(embedded_vecs[0]) * NUM_EMBEDDED_IOVECS);
		}
		else {
			vec_capacity *= 2;
			struct iovec *tmp = new struct iovec[vec_capacity];
			memcpy(tmp + 1, vec_pointer,
					sizeof(vec_pointer[0]) * vec_capacity / 2);
			delete [] vec_pointer;
			vec_pointer = tmp;
		}
	}
	else {
		memmove(vec_pointer + 1, vec_pointer,
				sizeof(vec_pointer[0]) * num_bufs);
	}
	assert(num_bufs < vec_capacity);
	vec_pointer[0].iov_base = buf;
	vec_pointer[0].iov_len = size;
	num_bufs++;
}

template<class T>
msg_sender<T>::msg_sender(int buf_size, thread_safe_FIFO_queue<T> **queues,
		int num_queues) {
	buf = (T *) numa_alloc_local(sizeof(T) * buf_size);
	this->buf_size = buf_size;
	num_current = 0;
	dest_queues = (thread_safe_FIFO_queue<T> **) numa_alloc_local(
			sizeof(thread_safe_FIFO_queue<T> *) * num_queues);
	memcpy(dest_queues, queues, sizeof(thread_safe_FIFO_queue<T> *) * num_queues);
	this->num_queues = num_queues;
}

/**
 * flush the entries in the buffer to the queues.
 * A queue is randomly picked. If the queue is full, pick the next queue
 * until all queues are tried or all entries in the buffer is flushed.
 * return the number of entries that have been flushed.
 */
template<class T>
int msg_sender<T>::flush() {
	if (num_current == 0)
		return 0;

	int base_idx;
	if (num_queues == 1)
		base_idx = 0;
	else
		base_idx = random() % num_queues;
	int num_sent = 0;
	T *tmp = buf;
	for (int i = 0; num_current > 0 && i < num_queues; i++) {
		thread_safe_FIFO_queue<T> *q = dest_queues[(base_idx + i) % num_queues];
		assert(q);

		// TODO the thread might be blocked if it's full.
		// it might hurt performance. We should try other
		// queues first before being blocked.
		int ret = q->add(tmp, num_current);
		tmp += ret;
		num_current -= ret;
		num_sent += ret;
	}

	/* move the remaining entries to the beginning of the buffer. */
	if (num_current)
		memmove(buf, tmp, num_current * sizeof(T));

	return num_sent;
}

template<class T>
int msg_sender<T>::send_cached(T *msg) {
	/*
	 * if the buffer is full, and we can't flush
	 * any messages, there is nothing we can do.
	 */
	if (num_current == buf_size && flush() == 0)
		return 0;

	buf[num_current++] = *msg;
	if (num_current == buf_size)
		flush();
	/* one message has been cached. */
	return 1;
}

/**
 * these are to force to instantiate the templates
 * for io_request and io_reply.
 */
template class thread_safe_FIFO_queue<io_request>;
template class thread_safe_FIFO_queue<io_reply>;
template class msg_sender<io_request>;
template class msg_sender<io_reply>;
<commit_msg>Fix a bug in msg_sender.<commit_after>#include <stdio.h>

#include "messaging.h"
#include "container.cpp"

void io_request::assign(io_request &req)
{
	this->offset = req.offset;
	this->io = req.io;
	this->priv = req.priv;
	this->access_method = req.access_method;
	this->num_bufs = req.num_bufs;
	this->vec_capacity = req.vec_capacity;
	this->partial = req.partial;
	this->completed_size = req.completed_size;
	this->orig = req.orig;
	this->refcnt = 0;
	/*
	 * If the request uses embedded vector, then the new request
	 * should point to its own embedded vector. Otherwise,
	 * the new request steals the vector from the old one.
	 */
	if (req.use_embedded())
		this->vec_pointer = this->embedded_vecs;
	else
		this->vec_pointer = req.vec_pointer;
	this->next = req.next;
	this->completed_size = req.completed_size;
	memcpy(this->embedded_vecs, req.embedded_vecs,
			sizeof(req.embedded_vecs[0]) * NUM_EMBEDDED_IOVECS);
	req.vec_pointer = req.embedded_vecs;
	req.vec_capacity = NUM_EMBEDDED_IOVECS;
	req.clear();
}

void io_request::add_buf(char *buf, int size)
{
	if (num_bufs >= vec_capacity) {
		if (vec_pointer == embedded_vecs) {
			vec_capacity = MIN_NUM_ALLOC_IOVECS;
			vec_pointer = new struct iovec[vec_capacity];
			memcpy(vec_pointer, embedded_vecs,
					sizeof(embedded_vecs[0]) * NUM_EMBEDDED_IOVECS);
		}
		else {
			vec_capacity *= 2;
			struct iovec *tmp = new struct iovec[vec_capacity];
			memcpy(tmp, vec_pointer,
					sizeof(vec_pointer[0]) * vec_capacity / 2);
			delete [] vec_pointer;
			vec_pointer = tmp;
		}
	}
	assert(num_bufs < vec_capacity);
	vec_pointer[num_bufs].iov_base = buf;
	vec_pointer[num_bufs].iov_len = size;
	num_bufs++;
}

void io_request::add_buf_front(char *buf, int size)
{
	if (num_bufs >= vec_capacity) {
		if (vec_pointer == embedded_vecs) {
			vec_capacity = MIN_NUM_ALLOC_IOVECS;
			vec_pointer = new struct iovec[vec_capacity];
			memcpy(vec_pointer + 1, embedded_vecs,
					sizeof(embedded_vecs[0]) * NUM_EMBEDDED_IOVECS);
		}
		else {
			vec_capacity *= 2;
			struct iovec *tmp = new struct iovec[vec_capacity];
			memcpy(tmp + 1, vec_pointer,
					sizeof(vec_pointer[0]) * vec_capacity / 2);
			delete [] vec_pointer;
			vec_pointer = tmp;
		}
	}
	else {
		memmove(vec_pointer + 1, vec_pointer,
				sizeof(vec_pointer[0]) * num_bufs);
	}
	assert(num_bufs < vec_capacity);
	vec_pointer[0].iov_base = buf;
	vec_pointer[0].iov_len = size;
	num_bufs++;
}

template<class T>
msg_sender<T>::msg_sender(int buf_size, thread_safe_FIFO_queue<T> **queues,
		int num_queues) {
	buf = (T *) numa_alloc_local(sizeof(T) * buf_size);
	this->buf_size = buf_size;
	num_current = 0;
	dest_queues = (thread_safe_FIFO_queue<T> **) numa_alloc_local(
			sizeof(thread_safe_FIFO_queue<T> *) * num_queues);
	memcpy(dest_queues, queues, sizeof(thread_safe_FIFO_queue<T> *) * num_queues);
	this->num_queues = num_queues;
}

/**
 * flush the entries in the buffer to the queues.
 * A queue is randomly picked. If the queue is full, pick the next queue
 * until all queues are tried or all entries in the buffer is flushed.
 * return the number of entries that have been flushed.
 */
template<class T>
int msg_sender<T>::flush() {
	if (num_current == 0)
		return 0;

	int base_idx;
	if (num_queues == 1)
		base_idx = 0;
	else
		base_idx = random() % num_queues;
	int num_sent = 0;
	T *tmp = buf;
	for (int i = 0; num_current > 0 && i < num_queues; i++) {
		thread_safe_FIFO_queue<T> *q = dest_queues[(base_idx + i) % num_queues];
		assert(q);

		// TODO the thread might be blocked if it's full.
		// it might hurt performance. We should try other
		// queues first before being blocked.
		int ret = q->add(tmp, num_current);
		tmp += ret;
		num_current -= ret;
		num_sent += ret;
	}

	/* move the remaining entries to the beginning of the buffer. */
	if (num_current && buf != tmp) {
		for (int i = 0; i < num_current; i++) {
			assert(tmp[i].get_offset() >= 0);
			buf[i] = tmp[i];
			assert(buf[i].get_offset() >= 0);
		}
	}

	return num_sent;
}

template<class T>
int msg_sender<T>::send_cached(T *msg) {
	/*
	 * if the buffer is full, and we can't flush
	 * any messages, there is nothing we can do.
	 */
	if (num_current == buf_size && flush() == 0)
		return 0;

	buf[num_current++] = *msg;
	if (num_current == buf_size)
		flush();
	/* one message has been cached. */
	return 1;
}

/**
 * these are to force to instantiate the templates
 * for io_request and io_reply.
 */
template class thread_safe_FIFO_queue<io_request>;
template class thread_safe_FIFO_queue<io_reply>;
template class msg_sender<io_request>;
template class msg_sender<io_reply>;
<|endoftext|>


<commit_before>/**
 * @file
 * @brief Example of using try, catch, and throw statements
 * @date 16.07.12
 * @author Ilia Vaprol
 */

#include <framework/example/self.h>

#include <exception>
#include <cstdio>

EMBOX_EXAMPLE(run);

class MyException : public std::exception
{
public:
	MyException() { }
	virtual ~MyException() throw() { }
	virtual const char* what() const throw() { return "MyException"; }
};

static int run(int argc, char **argv) {
	int num = 1;
	{
		std::printf("%d -- throw an exception and catch it\n", num);
		try {
			std::printf("throw.. ");
			throw MyException();
		}
		catch (MyException e) {
			std::printf("catch: %s\n", e.what());
		}
		std::printf("%d -- done\n", num++);
	}
	{
		std::printf("%d -- throw an exception and will not catch it\n", num);
		throw MyException();
		std::printf("it will never be printed\n");
	}

	return 0;
}

<commit_msg>tests: cxx: Fix exceptions test<commit_after>/**
 * @file
 * @brief Test of using try, catch, and throw statements
 * @date 16.07.12
 * @author Ilia Vaprol
 */

#include <exception>
#include <embox/test.h>
#include "test_cxx.h"

EMBOX_TEST_SUITE_EXT("c++ exception test", NULL, NULL, NULL, NULL);

namespace {

class MyException : public std::exception
{
public:
	MyException() { }
	virtual ~MyException() throw() { }
	virtual const char* what() const throw() { return "MyException"; }
};

TEST_CASE("Throw/catch exception") {
	try {
		test_emit('a');
		throw MyException();
	}
	catch (MyException e) {
		test_emit('b');
	}
	test_assert_emitted("ab");
}
} /* namespace */
<|endoftext|>


<commit_before>/**
 * Copyright (c) 2015 Damien Tardy-Panis
 *
 * This file is subject to the terms and conditions defined in
 * file 'LICENSE', which is part of this source code package.
 **/

#include "ComicDatabaseResource.h"

#include <QDebug>
#include <QStandardPaths>
#include <QDir>
#include <QFile>
#include <QSqlQuery>

#include "Comic.h"

const QString ComicDatabaseResource::_databaseName = "dailycomics.sqlite";
const QString ComicDatabaseResource::_comicsTableName = "comics";

ComicDatabaseResource* ComicDatabaseResource::m_instance = NULL;

ComicDatabaseResource::ComicDatabaseResource(QObject *parent) :
    QObject(parent),
    m_dbDirPath(""),
    m_dbFilePath("")
{
    db = QSqlDatabase::addDatabase("QSQLITE");
}

ComicDatabaseResource* ComicDatabaseResource::instance()
{
    if (!m_instance) {
        m_instance = new ComicDatabaseResource();
        m_instance->openDb();
        m_instance->checkStructure();
    }

    return m_instance;
}

bool ComicDatabaseResource::openDb()
{
    if (!QFile::exists(dbFilePath()) && !createDbFile())
        return false;
    db.setDatabaseName(dbFilePath());

    return db.open();
}

bool ComicDatabaseResource::load(Comic *comic)
{
    QSqlQuery query("SELECT time, url, read, favorite FROM " + _comicsTableName +  " WHERE id='" + comic->id() + "'");

    if (!query.first())
        return false;

    QDateTime time = query.value(0).toDateTime();
    QUrl url       = query.value(1).toUrl();
    bool read      = query.value(2).toBool();
    bool favorite  = query.value(3).toBool();

    comic->setFetchTime(time);
    comic->setStripImageUrl(url);
    comic->setNewStrip(!read);
    comic->setFavorite(favorite);

    return true;
}

bool ComicDatabaseResource::save(Comic *comic)
{
    QSqlQuery query(db);
    query.prepare("REPLACE INTO " + _comicsTableName + " (id, time, url, read, favorite) \n"
                  "VALUES (:id, :time, :url, :read, :favorite)");

    query.bindValue(":time",     comic->fetchTime());
    query.bindValue(":url",      comic->stripImageUrl());
    query.bindValue(":read",     !comic->newStrip());
    query.bindValue(":id",       comic->id());
    query.bindValue(":favorite", comic->favorite());

    return query.exec();
}

QStringList ComicDatabaseResource::favoriteIds()
{
    QSqlQuery query("SELECT id FROM " + _comicsTableName +  " WHERE favorite <> 0");
    QStringList favoriteIds;

    while (query.next()) {
        favoriteIds.append(query.value(0).toString());
    }

    return favoriteIds;
}

bool ComicDatabaseResource::saveFavorites(QStringList favoriteIds)
{
    QSqlQuery query(db);
    bool result;

    if (!db.transaction())
        return false;

    result = query.exec("UPDATE " + _comicsTableName + " SET favorite = 0");

    if (!result) {
        db.rollback();
        return false;
    }

    if (!favoriteIds.isEmpty())
    {
        result = query.exec("UPDATE " + _comicsTableName + " SET favorite = 1 \n"
                            "WHERE id IN (\"" + favoriteIds.join("\",\"") + "\")");

        if (!result) {
            db.rollback();
            return false;
        }
    }

    return db.commit();
}

QString ComicDatabaseResource::dbDirPath()
{
    if (m_dbDirPath.isEmpty()) {
        m_dbDirPath = QStandardPaths::writableLocation(QStandardPaths::DataLocation);
    }
    return m_dbDirPath;
}

QString ComicDatabaseResource::dbFilePath()
{
    if (m_dbFilePath.isEmpty()) {
        m_dbFilePath = dbDirPath().append(QDir::separator()).append(_databaseName);
    }

    return m_dbFilePath;
}

bool ComicDatabaseResource::createDbFile()
{
    QDir dbDir;
    QFile dbFile(dbFilePath());

    return dbDir.mkpath(dbDirPath()) && dbFile.open(QIODevice::ReadWrite);
}

void ComicDatabaseResource::checkStructure()
{
    if (!db.isOpen())
        return;

    if (!db.tables().contains(_comicsTableName)) {
        createStructure();
    }
}

bool ComicDatabaseResource::createStructure()
{
    if (!db.isOpen())
        return false;

    QSqlQuery query("CREATE TABLE " + _comicsTableName + " (\n"
                    "id       VARCHAR(50)  PRIMARY KEY,  -- comic id \n"
                    "time     DATETIME     DEFAULT NULL, -- last successful fetch time \n"
                    "url      VARCHAR(300) DEFAULT NULL, -- last retrieved image url \n"
                    "read     INTEGER(1)   DEFAULT 0,    -- 0 if not read \n"
                    "favorite INTEGER(1)   DEFAULT 0     -- 0 if not favorite \n"
                    ")", db);

    return query.exec();
}

<commit_msg>fix addition of new comics (never selected before)<commit_after>/**
 * Copyright (c) 2015 Damien Tardy-Panis
 *
 * This file is subject to the terms and conditions defined in
 * file 'LICENSE', which is part of this source code package.
 **/

#include "ComicDatabaseResource.h"

#include <QDebug>
#include <QStandardPaths>
#include <QDir>
#include <QFile>
#include <QSqlQuery>

#include "Comic.h"

const QString ComicDatabaseResource::_databaseName = "dailycomics.sqlite";
const QString ComicDatabaseResource::_comicsTableName = "comics";

ComicDatabaseResource* ComicDatabaseResource::m_instance = NULL;

ComicDatabaseResource::ComicDatabaseResource(QObject *parent) :
    QObject(parent),
    m_dbDirPath(""),
    m_dbFilePath("")
{
    db = QSqlDatabase::addDatabase("QSQLITE");
}

ComicDatabaseResource* ComicDatabaseResource::instance()
{
    if (!m_instance) {
        m_instance = new ComicDatabaseResource();
        m_instance->openDb();
        m_instance->checkStructure();
    }

    return m_instance;
}

bool ComicDatabaseResource::openDb()
{
    if (!QFile::exists(dbFilePath()) && !createDbFile())
        return false;
    db.setDatabaseName(dbFilePath());

    return db.open();
}

bool ComicDatabaseResource::load(Comic *comic)
{
    QSqlQuery query("SELECT time, url, read, favorite FROM " + _comicsTableName +  " WHERE id='" + comic->id() + "'");

    if (!query.first())
        return false;

    QDateTime time = query.value(0).toDateTime();
    QUrl url       = query.value(1).toUrl();
    bool read      = query.value(2).toBool();
    bool favorite  = query.value(3).toBool();

    comic->setFetchTime(time);
    comic->setStripImageUrl(url);
    comic->setNewStrip(!read);
    comic->setFavorite(favorite);

    return true;
}

bool ComicDatabaseResource::save(Comic *comic)
{
    QSqlQuery query(db);
    query.prepare("REPLACE INTO " + _comicsTableName + " (id, time, url, read, favorite) \n"
                  "VALUES (:id, :time, :url, :read, :favorite)");

    query.bindValue(":time",     comic->fetchTime());
    query.bindValue(":url",      comic->stripImageUrl());
    query.bindValue(":read",     !comic->newStrip());
    query.bindValue(":id",       comic->id());
    query.bindValue(":favorite", comic->favorite());

    return query.exec();
}

QStringList ComicDatabaseResource::favoriteIds()
{
    QSqlQuery query("SELECT id FROM " + _comicsTableName +  " WHERE favorite <> 0");
    QStringList favoriteIds;

    while (query.next()) {
        favoriteIds.append(query.value(0).toString());
    }

    return favoriteIds;
}

bool ComicDatabaseResource::saveFavorites(QStringList favoriteIds)
{
    QSqlQuery query(db);
    bool result;

    if (!db.transaction())
        return false;

    result = query.exec("UPDATE " + _comicsTableName + " SET favorite = 0");

    if (!result) {
        db.rollback();
        return false;
    }

    if (!favoriteIds.isEmpty())
    {
        QStringList values;
        for (int i = 0; i < favoriteIds.size(); ++i) {
            values.append("('" + favoriteIds.at(i) + "', '1')");
        }
        result = query.exec("REPLACE INTO " + _comicsTableName + " (id, favorite) \n"
                            "VALUES " + values.join(','));

        if (!result) {
            db.rollback();
            return false;
        }
    }

    return db.commit();
}

QString ComicDatabaseResource::dbDirPath()
{
    if (m_dbDirPath.isEmpty()) {
        m_dbDirPath = QStandardPaths::writableLocation(QStandardPaths::DataLocation);
    }
    return m_dbDirPath;
}

QString ComicDatabaseResource::dbFilePath()
{
    if (m_dbFilePath.isEmpty()) {
        m_dbFilePath = dbDirPath().append(QDir::separator()).append(_databaseName);
    }

    return m_dbFilePath;
}

bool ComicDatabaseResource::createDbFile()
{
    QDir dbDir;
    QFile dbFile(dbFilePath());

    return dbDir.mkpath(dbDirPath()) && dbFile.open(QIODevice::ReadWrite);
}

void ComicDatabaseResource::checkStructure()
{
    if (!db.isOpen())
        return;

    if (!db.tables().contains(_comicsTableName)) {
        createStructure();
    }
}

bool ComicDatabaseResource::createStructure()
{
    if (!db.isOpen())
        return false;

    QSqlQuery query("CREATE TABLE " + _comicsTableName + " (\n"
                    "id       VARCHAR(50)  PRIMARY KEY,  -- comic id \n"
                    "time     DATETIME     DEFAULT NULL, -- last successful fetch time \n"
                    "url      VARCHAR(300) DEFAULT NULL, -- last retrieved image url \n"
                    "read     INTEGER(1)   DEFAULT 0,    -- 0 if not read \n"
                    "favorite INTEGER(1)   DEFAULT 0     -- 0 if not favorite \n"
                    ")", db);

    return query.exec();
}

<|endoftext|>