<commit_before>/*
 * Copyright (C) 2017-2018 Red Hat, Inc.
 *
 * Licensed under the GNU Lesser General Public License Version 2.1
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2.1 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with this library; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
 */

#include <algorithm>
#include <map>
#include <sstream>

#include "../hy-subject.h"
#include "../nevra.hpp"

#include "RPMItem.hpp"

namespace libdnf {

static const std::map< TransactionItemReason, int > reasonPriorities = {
    {TransactionItemReason::UNKNOWN, 0},
    {TransactionItemReason::CLEAN, 1},
    {TransactionItemReason::WEAK_DEPENDENCY, 2},
    {TransactionItemReason::DEPENDENCY, 3},
    {TransactionItemReason::GROUP, 4},
    {TransactionItemReason::USER, 5}};

RPMItem::RPMItem(SQLite3Ptr conn)
  : Item{conn}
{
}

RPMItem::RPMItem(SQLite3Ptr conn, int64_t pk)
  : Item{conn}
{
    dbSelect(pk);
}

void
RPMItem::save()
{
    if (getId() == 0) {
        dbSelectOrInsert();
    } else {
        // TODO: dbUpdate() ?
    }
}

void
RPMItem::dbSelect(int64_t pk)
{
    const char *sql =
        "SELECT "
        "  name, "
        "  epoch, "
        "  version, "
        "  release, "
        "  arch "
        "FROM "
        "  rpm "
        "WHERE "
        "  item_id = ?";
    SQLite3::Statement query(*conn.get(), sql);
    query.bindv(pk);
    query.step();

    setId(pk);
    setName(query.get< std::string >(0));
    setEpoch(query.get< int >(1));
    setVersion(query.get< std::string >(2));
    setRelease(query.get< std::string >(3));
    setArch(query.get< std::string >(4));
}

void
RPMItem::dbInsert()
{
    // populates this->id
    Item::save();

    const char *sql =
        "INSERT INTO "
        "  rpm "
        "VALUES "
        "  (?, ?, ?, ?, ?, ?)";
    SQLite3::Statement query(*conn.get(), sql);
    query.bindv(getId(), getName(), getEpoch(), getVersion(), getRelease(), getArch());
    query.step();
}

static TransactionItemPtr
transactionItemFromQuery(SQLite3Ptr conn, SQLite3::Query &query, int64_t transID)
{
    auto trans_item = std::make_shared< TransactionItem >(conn, transID);
    auto item = std::make_shared< RPMItem >(conn);
    trans_item->setItem(item);
    trans_item->setId(query.get< int >("id"));
    trans_item->setAction(static_cast< TransactionItemAction >(query.get< int >("action")));
    trans_item->setReason(static_cast< TransactionItemReason >(query.get< int >("reason")));
    trans_item->setRepoid(query.get< std::string >("repoid"));
    trans_item->setState(static_cast< TransactionItemState >(query.get< int >("state")));
    item->setId(query.get< int >("item_id"));
    item->setName(query.get< std::string >("name"));
    item->setEpoch(query.get< int >("epoch"));
    item->setVersion(query.get< std::string >("version"));
    item->setRelease(query.get< std::string >("release"));
    item->setArch(query.get< std::string >("arch"));
    return trans_item;
}

std::vector< TransactionItemPtr >
RPMItem::getTransactionItems(SQLite3Ptr conn, int64_t transaction_id)
{
    std::vector< TransactionItemPtr > result;

    const char *sql =
        "SELECT "
        // trans_item
        "  ti.id, "
        "  ti.action, "
        "  ti.reason, "
        "  ti.state, "
        // repo
        "  r.repoid, "
        // rpm
        "  i.item_id, "
        "  i.name, "
        "  i.epoch, "
        "  i.version, "
        "  i.release, "
        "  i.arch "
        "FROM "
        "  trans_item ti, "
        "  repo r, "
        "  rpm i "
        "WHERE "
        "  ti.trans_id = ? "
        "  AND ti.repo_id = r.id "
        "  AND ti.item_id = i.item_id";
    SQLite3::Query query(*conn.get(), sql);
    query.bindv(transaction_id);

    while (query.step() == SQLite3::Statement::StepResult::ROW) {
        result.push_back(transactionItemFromQuery(conn, query, transaction_id));
    }
    return result;
}

std::string
RPMItem::getNEVRA() const
{
    // TODO: use string formatting
    if (epoch > 0) {
        return name + "-" + std::to_string(epoch) + ":" + version + "-" + release + "." + arch;
    }
    return name + "-" + version + "-" + release + "." + arch;
}

std::string
RPMItem::toStr() const
{
    return getNEVRA();
}

void
RPMItem::dbSelectOrInsert()
{
    const char *sql =
        "SELECT "
        "  item_id "
        "FROM "
        "  rpm "
        "WHERE "
        "  name = ? "
        "  AND epoch = ? "
        "  AND version = ? "
        "  AND release = ? "
        "  AND arch = ?";

    SQLite3::Statement query(*conn.get(), sql);

    query.bindv(getName(), getEpoch(), getVersion(), getRelease(), getArch());
    SQLite3::Statement::StepResult result = query.step();

    if (result == SQLite3::Statement::StepResult::ROW) {
        setId(query.get< int >(0));
    } else {
        // insert and get the ID back
        dbInsert();
    }
}

TransactionItemPtr
RPMItem::getTransactionItem(SQLite3Ptr conn, const std::string &nevra)
{
    Nevra nevraObject;
    if (!nevraObject.parse(nevra.c_str(), HY_FORM_NEVRA)) {
        return nullptr;
    }
    // TODO: hy_nevra_possibility should set epoch to 0 if epoch is not specified and HY_FORM_NEVRA
    // is used
    if (nevraObject.getEpoch() < 0) {
        nevraObject.setEpoch(0);
    }

    const char *sql = R"**(
        SELECT
            ti.trans_id,
            ti.id,
            ti.action,
            ti.reason,
            ti.state,
            r.repoid,
            i.item_id,
            i.name,
            i.epoch,
            i.version,
            i.release,
            i.arch
        FROM
            trans_item ti,
            repo r,
            rpm i
        WHERE
            ti.repo_id = r.id
            AND ti.item_id = i.item_id
            AND i.name = ?
            AND i.epoch = ?
            AND i.version = ?
            AND i.release = ?
            AND i.arch = ?
        ORDER BY
           ti.id DESC
        LIMIT 1
    )**";
    SQLite3::Query query(*conn, sql);
    query.bindv(nevraObject.getName(),
                nevraObject.getEpoch(),
                nevraObject.getVersion(),
                nevraObject.getRelease(),
                nevraObject.getArch());
    if (query.step() == SQLite3::Statement::StepResult::ROW) {
        return transactionItemFromQuery(conn, query, query.get< int64_t >("trans_id"));
    }
    return nullptr;
}

TransactionItemReason
RPMItem::resolveTransactionItemReason(SQLite3Ptr conn,
                                      const std::string &name,
                                      const std::string &arch,
                                      int64_t maxTransactionId)
{
    const char *sql = R"**(
        SELECT
            ti.action as action,
            ti.reason as reason
        FROM
            trans_item ti
        JOIN
            trans t ON ti.trans_id = t.id
        JOIN
            rpm i USING (item_id)
        WHERE
            t.state = 1
            /* see comment in TransactionItem.hpp - TransactionItemAction */
            AND ti.action not in (3, 5, 7, 10)
            AND i.name = ?
            AND i.arch = ?
        ORDER BY
            ti.trans_id DESC
        LIMIT 1
    )**";

    if (arch != "") {
        SQLite3::Query query(*conn, sql);
        query.bindv(name, arch);

        if (query.step() == SQLite3::Statement::StepResult::ROW) {
            auto action = static_cast< TransactionItemAction >(query.get< int64_t >("action"));
            if (action == TransactionItemAction::REMOVE) {
                return TransactionItemReason::UNKNOWN;
            }
            auto reason = static_cast< TransactionItemReason >(query.get< int64_t >("reason"));
            return reason;
        }
    } else {
        const char *arch_sql = R"**(
            SELECT DISTINCT
                arch
            FROM
                rpm
            WHERE
                name = ?
        )**";

        SQLite3::Query arch_query(*conn, arch_sql);
        arch_query.bindv(name);

        TransactionItemReason result = TransactionItemReason::UNKNOWN;

        while (arch_query.step() == SQLite3::Statement::StepResult::ROW) {
            auto rpm_arch = arch_query.get< std::string >("arch");

            SQLite3::Query query(*conn, sql);
            query.bindv(name, rpm_arch);
            while (query.step() == SQLite3::Statement::StepResult::ROW) {
                auto action = static_cast< TransactionItemAction >(query.get< int64_t >("action"));
                if (action == TransactionItemAction::REMOVE) {
                    continue;
                }
                auto reason = static_cast< TransactionItemReason >(query.get< int64_t >("reason"));
                if (reasonPriorities.at(reason) > reasonPriorities.at(result)) {
                    result = reason;
                }
            }
        }
        return result;
    }
    return TransactionItemReason::UNKNOWN;
}

/**
 * Compare RPM packages
 * This method doesn't care about compare package names
 * \param other RPMItem to compare with
 * \return true if other package is newer (has higher version and/or epoch)
 */
bool
RPMItem::operator<(const RPMItem &other) const
{
    // compare epochs
    int32_t epochDif = other.getEpoch() - getEpoch();
    if (epochDif > 0) {
        return true;
    } else if (epoch < 0) {
        return false;
    }

    // compare versions
    std::stringstream versionThis(getVersion());
    std::stringstream versionOther(other.getVersion());

    std::string bufferThis;
    std::string bufferOther;
    while (std::getline(versionThis, bufferThis, '.') &&
           std::getline(versionOther, bufferOther, '.')) {
        int subVersionThis = std::stoi(bufferThis);
        int subVersionOther = std::stoi(bufferOther);
        if (subVersionThis == subVersionOther) {
            continue;
        }
        return subVersionOther > subVersionThis;
    }
    return false;
}

std::vector< int64_t >
RPMItem::searchTransactions(SQLite3Ptr conn, const std::vector< std::string > &patterns)
{
    std::vector< int64_t > result;

    const char *sql = R"**(
        SELECT DISTINCT
            t.id
        FROM
            trans t,
            trans_item ti,
            rpm i
        WHERE
            t.state = 1
            AND ti.item_id = i.item_id
            AND (
                i.name = ?
                OR i.epoch = ?
                OR i.version = ?
                OR i.release = ?
                OR i.arch = ?
            )
        ORDER BY
           trans_id DESC
    )**";
    SQLite3::Query query(*conn, sql);
    for (auto pattern : patterns) {
        query.bindv(pattern, pattern, pattern, pattern, pattern);
        while (query.step() == SQLite3::Statement::StepResult::ROW) {
            result.push_back(query.get< int64_t >("id"));
        }
    }
    std::sort(result.begin(), result.end());
    auto last = std::unique(result.begin(), result.end());
    result.erase(last, result.end());
    return result;
}

} // namespace libdnf
<commit_msg>[transaction] Migrate RPMItem::resolveTransactionItemReason to reason comparison.<commit_after>/*
 * Copyright (C) 2017-2018 Red Hat, Inc.
 *
 * Licensed under the GNU Lesser General Public License Version 2.1
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2.1 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with this library; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
 */

#include <algorithm>
#include <map>
#include <sstream>

#include "../hy-subject.h"
#include "../nevra.hpp"

#include "RPMItem.hpp"

namespace libdnf {

RPMItem::RPMItem(SQLite3Ptr conn)
  : Item{conn}
{
}

RPMItem::RPMItem(SQLite3Ptr conn, int64_t pk)
  : Item{conn}
{
    dbSelect(pk);
}

void
RPMItem::save()
{
    if (getId() == 0) {
        dbSelectOrInsert();
    } else {
        // TODO: dbUpdate() ?
    }
}

void
RPMItem::dbSelect(int64_t pk)
{
    const char *sql =
        "SELECT "
        "  name, "
        "  epoch, "
        "  version, "
        "  release, "
        "  arch "
        "FROM "
        "  rpm "
        "WHERE "
        "  item_id = ?";
    SQLite3::Statement query(*conn.get(), sql);
    query.bindv(pk);
    query.step();

    setId(pk);
    setName(query.get< std::string >(0));
    setEpoch(query.get< int >(1));
    setVersion(query.get< std::string >(2));
    setRelease(query.get< std::string >(3));
    setArch(query.get< std::string >(4));
}

void
RPMItem::dbInsert()
{
    // populates this->id
    Item::save();

    const char *sql =
        "INSERT INTO "
        "  rpm "
        "VALUES "
        "  (?, ?, ?, ?, ?, ?)";
    SQLite3::Statement query(*conn.get(), sql);
    query.bindv(getId(), getName(), getEpoch(), getVersion(), getRelease(), getArch());
    query.step();
}

static TransactionItemPtr
transactionItemFromQuery(SQLite3Ptr conn, SQLite3::Query &query, int64_t transID)
{
    auto trans_item = std::make_shared< TransactionItem >(conn, transID);
    auto item = std::make_shared< RPMItem >(conn);
    trans_item->setItem(item);
    trans_item->setId(query.get< int >("id"));
    trans_item->setAction(static_cast< TransactionItemAction >(query.get< int >("action")));
    trans_item->setReason(static_cast< TransactionItemReason >(query.get< int >("reason")));
    trans_item->setRepoid(query.get< std::string >("repoid"));
    trans_item->setState(static_cast< TransactionItemState >(query.get< int >("state")));
    item->setId(query.get< int >("item_id"));
    item->setName(query.get< std::string >("name"));
    item->setEpoch(query.get< int >("epoch"));
    item->setVersion(query.get< std::string >("version"));
    item->setRelease(query.get< std::string >("release"));
    item->setArch(query.get< std::string >("arch"));
    return trans_item;
}

std::vector< TransactionItemPtr >
RPMItem::getTransactionItems(SQLite3Ptr conn, int64_t transaction_id)
{
    std::vector< TransactionItemPtr > result;

    const char *sql =
        "SELECT "
        // trans_item
        "  ti.id, "
        "  ti.action, "
        "  ti.reason, "
        "  ti.state, "
        // repo
        "  r.repoid, "
        // rpm
        "  i.item_id, "
        "  i.name, "
        "  i.epoch, "
        "  i.version, "
        "  i.release, "
        "  i.arch "
        "FROM "
        "  trans_item ti, "
        "  repo r, "
        "  rpm i "
        "WHERE "
        "  ti.trans_id = ? "
        "  AND ti.repo_id = r.id "
        "  AND ti.item_id = i.item_id";
    SQLite3::Query query(*conn.get(), sql);
    query.bindv(transaction_id);

    while (query.step() == SQLite3::Statement::StepResult::ROW) {
        result.push_back(transactionItemFromQuery(conn, query, transaction_id));
    }
    return result;
}

std::string
RPMItem::getNEVRA() const
{
    // TODO: use string formatting
    if (epoch > 0) {
        return name + "-" + std::to_string(epoch) + ":" + version + "-" + release + "." + arch;
    }
    return name + "-" + version + "-" + release + "." + arch;
}

std::string
RPMItem::toStr() const
{
    return getNEVRA();
}

void
RPMItem::dbSelectOrInsert()
{
    const char *sql =
        "SELECT "
        "  item_id "
        "FROM "
        "  rpm "
        "WHERE "
        "  name = ? "
        "  AND epoch = ? "
        "  AND version = ? "
        "  AND release = ? "
        "  AND arch = ?";

    SQLite3::Statement query(*conn.get(), sql);

    query.bindv(getName(), getEpoch(), getVersion(), getRelease(), getArch());
    SQLite3::Statement::StepResult result = query.step();

    if (result == SQLite3::Statement::StepResult::ROW) {
        setId(query.get< int >(0));
    } else {
        // insert and get the ID back
        dbInsert();
    }
}

TransactionItemPtr
RPMItem::getTransactionItem(SQLite3Ptr conn, const std::string &nevra)
{
    Nevra nevraObject;
    if (!nevraObject.parse(nevra.c_str(), HY_FORM_NEVRA)) {
        return nullptr;
    }
    // TODO: hy_nevra_possibility should set epoch to 0 if epoch is not specified and HY_FORM_NEVRA
    // is used
    if (nevraObject.getEpoch() < 0) {
        nevraObject.setEpoch(0);
    }

    const char *sql = R"**(
        SELECT
            ti.trans_id,
            ti.id,
            ti.action,
            ti.reason,
            ti.state,
            r.repoid,
            i.item_id,
            i.name,
            i.epoch,
            i.version,
            i.release,
            i.arch
        FROM
            trans_item ti,
            repo r,
            rpm i
        WHERE
            ti.repo_id = r.id
            AND ti.item_id = i.item_id
            AND i.name = ?
            AND i.epoch = ?
            AND i.version = ?
            AND i.release = ?
            AND i.arch = ?
        ORDER BY
           ti.id DESC
        LIMIT 1
    )**";
    SQLite3::Query query(*conn, sql);
    query.bindv(nevraObject.getName(),
                nevraObject.getEpoch(),
                nevraObject.getVersion(),
                nevraObject.getRelease(),
                nevraObject.getArch());
    if (query.step() == SQLite3::Statement::StepResult::ROW) {
        return transactionItemFromQuery(conn, query, query.get< int64_t >("trans_id"));
    }
    return nullptr;
}

TransactionItemReason
RPMItem::resolveTransactionItemReason(SQLite3Ptr conn,
                                      const std::string &name,
                                      const std::string &arch,
                                      int64_t maxTransactionId)
{
    const char *sql = R"**(
        SELECT
            ti.action as action,
            ti.reason as reason
        FROM
            trans_item ti
        JOIN
            trans t ON ti.trans_id = t.id
        JOIN
            rpm i USING (item_id)
        WHERE
            t.state = 1
            /* see comment in TransactionItem.hpp - TransactionItemAction */
            AND ti.action not in (3, 5, 7, 10)
            AND i.name = ?
            AND i.arch = ?
        ORDER BY
            ti.trans_id DESC
        LIMIT 1
    )**";

    if (arch != "") {
        SQLite3::Query query(*conn, sql);
        query.bindv(name, arch);

        if (query.step() == SQLite3::Statement::StepResult::ROW) {
            auto action = static_cast< TransactionItemAction >(query.get< int64_t >("action"));
            if (action == TransactionItemAction::REMOVE) {
                return TransactionItemReason::UNKNOWN;
            }
            auto reason = static_cast< TransactionItemReason >(query.get< int64_t >("reason"));
            return reason;
        }
    } else {
        const char *arch_sql = R"**(
            SELECT DISTINCT
                arch
            FROM
                rpm
            WHERE
                name = ?
        )**";

        SQLite3::Query arch_query(*conn, arch_sql);
        arch_query.bindv(name);

        TransactionItemReason result = TransactionItemReason::UNKNOWN;

        while (arch_query.step() == SQLite3::Statement::StepResult::ROW) {
            auto rpm_arch = arch_query.get< std::string >("arch");

            SQLite3::Query query(*conn, sql);
            query.bindv(name, rpm_arch);
            while (query.step() == SQLite3::Statement::StepResult::ROW) {
                auto action = static_cast< TransactionItemAction >(query.get< int64_t >("action"));
                if (action == TransactionItemAction::REMOVE) {
                    continue;
                }
                auto reason = static_cast< TransactionItemReason >(query.get< int64_t >("reason"));
                if (reason > result) {
                    result = reason;
                }
            }
        }
        return result;
    }
    return TransactionItemReason::UNKNOWN;
}

/**
 * Compare RPM packages
 * This method doesn't care about compare package names
 * \param other RPMItem to compare with
 * \return true if other package is newer (has higher version and/or epoch)
 */
bool
RPMItem::operator<(const RPMItem &other) const
{
    // compare epochs
    int32_t epochDif = other.getEpoch() - getEpoch();
    if (epochDif > 0) {
        return true;
    } else if (epoch < 0) {
        return false;
    }

    // compare versions
    std::stringstream versionThis(getVersion());
    std::stringstream versionOther(other.getVersion());

    std::string bufferThis;
    std::string bufferOther;
    while (std::getline(versionThis, bufferThis, '.') &&
           std::getline(versionOther, bufferOther, '.')) {
        int subVersionThis = std::stoi(bufferThis);
        int subVersionOther = std::stoi(bufferOther);
        if (subVersionThis == subVersionOther) {
            continue;
        }
        return subVersionOther > subVersionThis;
    }
    return false;
}

std::vector< int64_t >
RPMItem::searchTransactions(SQLite3Ptr conn, const std::vector< std::string > &patterns)
{
    std::vector< int64_t > result;

    const char *sql = R"**(
        SELECT DISTINCT
            t.id
        FROM
            trans t,
            trans_item ti,
            rpm i
        WHERE
            t.state = 1
            AND ti.item_id = i.item_id
            AND (
                i.name = ?
                OR i.epoch = ?
                OR i.version = ?
                OR i.release = ?
                OR i.arch = ?
            )
        ORDER BY
           trans_id DESC
    )**";
    SQLite3::Query query(*conn, sql);
    for (auto pattern : patterns) {
        query.bindv(pattern, pattern, pattern, pattern, pattern);
        while (query.step() == SQLite3::Statement::StepResult::ROW) {
            result.push_back(query.get< int64_t >("id"));
        }
    }
    std::sort(result.begin(), result.end());
    auto last = std::unique(result.begin(), result.end());
    result.erase(last, result.end());
    return result;
}

} // namespace libdnf
<|endoftext|>


<commit_before>//
// Created by David Sere on 25/07/16.
//

#include <tidings/plumbing.capnp.h>
#include "TestConfig.h"
#include "../src/CSAOptMessageQueue.h"
#include "../src/kj/KjStringPipe.h"

void setData(Plumbing::Builder p, std::string id, std::time_t timestamp, std::string sender, Plumbing::Type type){
    p.setId(id); p.setTimestamp(timestamp); p.setSender(sender); p.setType(type);
}

SCENARIO("Workers want to register and unregister with the Messagequeue", "") {
    int tidingsPort = 12345;
    int plumbingsPort = 12378;

    GIVEN("A messagequeue") {
        CSAOpt::MessageQueue messageQueue(tidingsPort, plumbingsPort);

        zmqpp::context context;
        zmqpp::socket_type type = zmqpp::socket_type::req;
        zmqpp::socket socket(context, type);

        const std::string endpoint = "tcp://localhost:" + std::to_string(plumbingsPort);

        socket.connect(endpoint);

        WHEN("A worker wants to join"){
            zmqpp::message req;

            ::capnp::MallocMessageBuilder message;
            Plumbing::Builder plumbing = message.initRoot<Plumbing>();

            std::time_t t = std::time(0);
            setData(plumbing, "1234", t, "worker1", Plumbing::Type::REGISTER);

            kj::std::StringPipe pipe;
            writePackedMessage(pipe, message);

            req << pipe.getData();
            socket.send(req);

            THEN("The server responds with OK"){
                zmqpp::message resp;
                socket.receive(resp);

                std::string recvdata;
                resp >> recvdata;

                kj::std::StringPipe newPipe(recvdata);
                ::capnp::PackedMessageReader recvMessage(newPipe);

                Plumbing::Reader recvPlumbing = recvMessage.getRoot<Plumbing>();

                REQUIRE(strcmp(plumbing.getId().cStr(), recvPlumbing.getId().cStr()) == 0);
                REQUIRE(recvPlumbing.getType() == Plumbing::Type::ACK);
            }
        }

        WHEN("A worker want's to leave after it jointed"){
            zmqpp::message req;

            ::capnp::MallocMessageBuilder message;
            Plumbing::Builder plumbing = message.initRoot<Plumbing>();

            std::time_t t = std::time(0);
            setData(plumbing, "1234", t, "worker1", Plumbing::Type::REGISTER);

            kj::std::StringPipe pipe;
            writePackedMessage(pipe, message);

            // send 'register'
            req << pipe.getData();
            socket.send(req);

            // ignore response
            zmqpp::message resp;
            socket.receive(resp);

            // send 'unregister'
            ::capnp::MallocMessageBuilder message2;
            Plumbing::Builder plumbing2 = message2.initRoot<Plumbing>();

            std::time_t t2 = std::time(0);
            setData(plumbing2, "1234", t2, "worker1", Plumbing::Type::UNREGISTER);
            pipe.clear();
            writePackedMessage(pipe, message2);

            req << pipe.getData();
            socket.send(req);

            THEN("The server responds with OK"){
                zmqpp::message resp;
                socket.receive(resp);

                std::string recvdata;
                resp >> recvdata;

                kj::std::StringPipe newPipe(recvdata);
                ::capnp::PackedMessageReader recvMessage(newPipe);

                Plumbing::Reader recvPlumbing = recvMessage.getRoot<Plumbing>();

                REQUIRE(strcmp(plumbing.getId().cStr(), recvPlumbing.getId().cStr()) == 0);
                REQUIRE(recvPlumbing.getType() == Plumbing::Type::ACK);
            }
        }

        WHEN("A worker want's to leave but never joined"){
            zmqpp::message req;

            ::capnp::MallocMessageBuilder message;
            Plumbing::Builder plumbing = message.initRoot<Plumbing>();

            std::time_t t = std::time(0);
            setData(plumbing, "neverJoinedId", t, "worker1", Plumbing::Type::UNREGISTER);

            kj::std::StringPipe pipe;
            writePackedMessage(pipe, message);

            req << pipe.getData();
            socket.send(req);
            THEN("The server responds with Error"){
                zmqpp::message resp;
                socket.receive(resp);

                std::string recvdata;
                resp >> recvdata;

                kj::std::StringPipe newPipe(recvdata);
                ::capnp::PackedMessageReader recvMessage(newPipe);

                Plumbing::Reader recvPlumbing = recvMessage.getRoot<Plumbing>();

                REQUIRE(strcmp(plumbing.getId().cStr(), recvPlumbing.getId().cStr()) == 0);
                REQUIRE(recvPlumbing.getType() == Plumbing::Type::ERROR);
            }
        }

    }
}<commit_msg>Refactored variable names<commit_after>//
// Created by David Sere on 25/07/16.
//

#include <tidings/plumbing.capnp.h>
#include "TestConfig.h"
#include "../src/CSAOptMessageQueue.h"
#include "../src/kj/KjStringPipe.h"

void setData(Plumbing::Builder p, std::string id, std::time_t timestamp, std::string sender, Plumbing::Type type){
    p.setId(id); p.setTimestamp(timestamp); p.setSender(sender); p.setType(type);
}

SCENARIO("Workers want to register and unregister with the Messagequeue", "") {
    int tidingsPort = 12345;
    int plumbingsPort = 12378;

    GIVEN("A messagequeue") {
        CSAOpt::MessageQueue* messageQueue = new CSAOpt::MessageQueue(tidingsPort, plumbingsPort);

        zmqpp::context context;
        zmqpp::socket_type type = zmqpp::socket_type::req;
        zmqpp::socket socket(context, type);

        const std::string endpoint = "tcp://localhost:" + std::to_string(plumbingsPort);

        socket.connect(endpoint);

        WHEN("A worker wants to join"){
            zmqpp::message req;

            ::capnp::MallocMessageBuilder message;
            Plumbing::Builder plumbing = message.initRoot<Plumbing>();

            std::time_t t = std::time(0);
            setData(plumbing, "1234", t, "worker1", Plumbing::Type::REGISTER);

            kj::std::StringPipe pipe;
            writePackedMessage(pipe, message);

            req << pipe.getData();

            socket.send(req);

            THEN("The server responds with OK"){
                zmqpp::message resp;
                socket.receive(resp);

                std::string recvdata;
                resp >> recvdata;

                kj::std::StringPipe newPipe(recvdata);
                ::capnp::PackedMessageReader recvMessage(newPipe);
                Plumbing::Reader recvPlumbing = recvMessage.getRoot<Plumbing>();

                REQUIRE(strcmp(plumbing.getId().cStr(), recvPlumbing.getId().cStr()) == 0);
                REQUIRE(recvPlumbing.getType() == Plumbing::Type::ACK);
            }
        }

        WHEN("A worker wants to leave after it jointed"){
            zmqpp::message req;

            ::capnp::MallocMessageBuilder message;
            Plumbing::Builder plumbing = message.initRoot<Plumbing>();

            std::time_t t = std::time(0);
            setData(plumbing, "1234", t, "worker1", Plumbing::Type::REGISTER);

            kj::std::StringPipe pipe;
            writePackedMessage(pipe, message);

            // send 'register'
            req << pipe.getData();
            socket.send(req);

            // ignore response
            zmqpp::message resp;
            socket.receive(resp);

            // send 'unregister'
            ::capnp::MallocMessageBuilder message2;
            Plumbing::Builder plumbing2 = message2.initRoot<Plumbing>();

            std::time_t t2 = std::time(0);
            setData(plumbing2, "1234", t2, "worker1", Plumbing::Type::UNREGISTER);
            pipe.clear();
            writePackedMessage(pipe, message2);

            req << pipe.getData();
            socket.send(req);

            THEN("The server responds with OK"){
                zmqpp::message resp;
                socket.receive(resp);

                std::string recvdata;
                resp >> recvdata;

                kj::std::StringPipe newPipe(recvdata);
                ::capnp::PackedMessageReader recvMessage(newPipe);

                Plumbing::Reader recvPlumbing = recvMessage.getRoot<Plumbing>();

                REQUIRE(strcmp(plumbing.getId().cStr(), recvPlumbing.getId().cStr()) == 0);
                REQUIRE(recvPlumbing.getType() == Plumbing::Type::ACK);
            }
        }

        WHEN("A worker want's to leave but never joined"){
            zmqpp::message reqBad;

            ::capnp::MallocMessageBuilder messageBad;
            Plumbing::Builder plumbingBad = messageBad.initRoot<Plumbing>();

            std::time_t tBad = std::time(0);
            setData(plumbingBad, "1239", tBad, "worker7", Plumbing::Type::UNREGISTER);

            kj::std::StringPipe pipeBad;
            writePackedMessage(pipeBad, messageBad);

            reqBad << pipeBad.getData();

            socket.send(reqBad);

            THEN("The server responds with Error"){
                zmqpp::message respBad;
                socket.receive(respBad);

                std::string recvdataBad;
                respBad >> recvdataBad;

                kj::std::StringPipe newPipeBad(recvdataBad);
                ::capnp::PackedMessageReader recvMessageBad(newPipeBad);
                Plumbing::Reader recvPlumbingBad = recvMessageBad.getRoot<Plumbing>();

                REQUIRE(strcmp(plumbingBad.getId().cStr(), recvPlumbingBad.getId().cStr()) == 0);
                REQUIRE(recvPlumbingBad.getType() == Plumbing::Type::ERROR);
            }
        }

        delete messageQueue;
    }
}<|endoftext|>


<commit_before>/**
 * The MIT License (MIT)
 *
 * Copyright © 2017-2019 Ruben Van Boxem
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to deal
 * in the Software without restriction, including without limitation the rights
 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 * copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 * THE SOFTWARE.
 **/

#include "gui/window.h++"

namespace skui::gui
{
  const window_flags window::default_flags = window_flag::exit_on_close | window_flag::anti_alias;
}
<commit_msg>Use OpenGL by default on Windows.<commit_after>/**
 * The MIT License (MIT)
 *
 * Copyright © 2017-2019 Ruben Van Boxem
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to deal
 * in the Software without restriction, including without limitation the rights
 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 * copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 * THE SOFTWARE.
 **/

#include "gui/window.h++"

namespace skui::gui
{
  const window_flags window::default_flags = window_flag::exit_on_close | window_flag::anti_alias | window_flag::opengl;
}
<|endoftext|>


<commit_before>/*!
 * Copyright (c) 2016 by Contributors
 * \file q_fully_connected.cc
 * \brief Quantized CONV operator
 * \author HPI-DeepLearning
*/

#include "./q_convolution-inl.h"
#include <mshadow/base.h>
#include <mshadow/tensor.h>
#include <memory>
#include "./binary_layer.h"
#include "./xnor_cpu.h"
#if MXNET_USE_MKL2017 == 1
#include <mkl_memory.h>
#include "../../src/operator/mkl/mkl_memory-inl.h"
#include "../../src/operator/mkl/mkl_convolution-inl.h"
#endif  // MXNET_USE_MKL2017
#if MXNET_USE_NNPACK == 1
#include "../../src/operator/nnpack/nnpack_convolution-inl.h"
#endif  // MXNET_USE_NNPACK
# include <chrono>
using ns = std::chrono::nanoseconds;
using get_time = std::chrono::steady_clock ;

namespace mshadow {
	inline BINARY_WORD concatenate(float* array)
	{
		BINARY_WORD rvalue=0;
		BINARY_WORD sign;

		#pragma omp parallel for
		for (int i = 0; i < BITS_PER_BINARY_WORD; i++)
		{
			sign = (array[i]>=0);
			rvalue = rvalue | (sign<< (i));
		}

		return rvalue;
	}
	inline void get_binary_row(float* row, BINARY_WORD * b_row, int size){

		for (int i = 0; i < size; i+=BITS_PER_BINARY_WORD) {
			float * array = new float[BITS_PER_BINARY_WORD];

			#pragma omp parallel for
			for (int j = 0;j < BITS_PER_BINARY_WORD; ++j) {
				array[j] = row[i+j];
			}

			b_row[i/BITS_PER_BINARY_WORD] = concatenate(array);
			delete[] array;
		}
	}

	inline void get_binary_col(float* col, BINARY_WORD * b_col, int n, int k){

		for(int x=0; x < k; ++x){
			for(int y=0; y<(n/BITS_PER_BINARY_WORD); y++){
				float * array = new float[BITS_PER_BINARY_WORD];

				#pragma omp parallel for
				for(int b=0; b<BITS_PER_BINARY_WORD; ++b){
					array[b] = col[(y*BITS_PER_BINARY_WORD+b)*k + x];
				}

				b_col[y*k + x]=concatenate(array);
				delete[] array;
			}
		}
	}

	inline void xnor_gemm(int M, int K, int N,
							BINARY_WORD *A, int lda,
							BINARY_WORD *B, int ldb,
							float *C, int ldc){
	    int i,n,k;

		#pragma omp parallel for
	    for(i = 0; i < M; ++i){
			#pragma omp parallel for
	        for(n = 0; n < N; ++n){
	        	BINARY_WORD A_PART = A[i*lda+n];
				#pragma omp parallel for
	            for(k = 0; k < K; ++k){
	            	C[i*ldc+k] += (float)__builtin_popcount(~(A_PART ^ B[n*ldb+k]));
	            }
	        }
	    }
	}

	inline void baseline_gemm(int M, int K, int N,
							float *A, int lda,
							float *B, int ldb,
							float *C, int ldc){
	    int i,n,k;
	    for(i = 0; i < M; ++i){
	        for(n = 0; n < N; ++n){
	        	float A_PART = A[i*lda+n];
	            for(k = 0; k < K; ++k){
	            	C[i*ldc+k] += A_PART * B[n*ldb+k];
	            }
	        }
	    }
	}

    inline void QConvolutionForward(const Tensor<cpu, 4, float> &data,
                                    const Tensor<cpu, 2, float> &wmat,
                                    const Tensor<cpu, 2, float> &in_col,
                                    const Tensor<cpu, 2, float> &temp_dst,
                                    const Tensor<cpu, 4, float> &out,
                                    const mxnet::op::QConvolutionParam &param) {

      CHECK_EQ(param.stride[0], 1) << "binary convolution currently only supported with stride==1";
      CHECK_EQ(param.stride[1], 1) << "binary convolution currently only supported with stride==1";

      ///*
      int m = wmat.size(0);
      int n = wmat.size(1);
      int k = in_col.size(1);
      BINARY_WORD* binary_row = (BINARY_WORD*)malloc(m * n/BITS_PER_BINARY_WORD * sizeof(BINARY_WORD));
      BINARY_WORD* binary_col = (BINARY_WORD*)malloc(n * k/BITS_PER_BINARY_WORD * sizeof(BINARY_WORD));

	  get_binary_row(wmat.dptr_, binary_row, m*n);
	  get_binary_col(in_col.dptr_, binary_col, n, k);

	  auto start = std::chrono::high_resolution_clock::now();
	  ///*
	  xnor_gemm(m, k, n/BITS_PER_BINARY_WORD,
			  binary_row, n/BITS_PER_BINARY_WORD,
			  binary_col, k,
			  temp_dst.dptr_, k);
	  //*/

	  /*
	  //test using baseline gemm kernel
	  baseline_gemm(m, k, n,
			  	  	wmat.dptr_, n,
					in_col.dptr_, k,
					temp_dst.dptr_, k);
	  */
	  auto finish = std::chrono::high_resolution_clock::now();
	  std::chrono::duration<double> elapsed = finish - start;
	  std::cout << "xnor Elapsed time: " << elapsed.count() << " s\n";
	  free(binary_row);
	  free(binary_col);
      //*/

	  /*
	  auto binary_layer = std::unique_ptr<mxnet::op::BinaryLayer>(
		  new mxnet::op::BinaryLayer(data.size(1), //   input depth
								  data.size(2), //    input x
								  data.size(3), //    input y
								  param.num_filter,// number filters
								  param.kernel[0], // weight x
								  param.kernel[1],//  weight y
								  param.pad[0],//     padding
								  param.pad[1],//     padding
          wmat.shape_[0], // m*n with m=num_filter
          wmat.shape_[1], // m*n with n=weight_x * weight_y * input depth
          //in_col.shape_[0], // n*k with n=weight_x * weight_y * input depth
          //in_col.shape_[1], // n*k with k=output_x * output_y * batch_size
          //temp_dst.shape_[1], // m*k  with m=num_filter
          temp_dst.shape_[1]));// m*k with k=output_x * output_y * batch_size

	  auto start = std::chrono::high_resolution_clock::now();

      binary_layer->set_input_as_col(in_col);
      binary_layer->set_weights(wmat);

      //LOG(INFO) << "\n" << binary_layer->weights_as_string();

      mxnet::op::xnor_cpu::binary_gemm(binary_layer->binary_weights,
                                       binary_layer->binary_input,
                                       binary_layer->output,
                                       binary_layer->m,
                                       binary_layer->n,
                                       binary_layer->k);

	  auto finish = std::chrono::high_resolution_clock::now();
	  std::chrono::duration<double> elapsed = finish - start;
	  std::cout << "Elapsed time: " << elapsed.count() << " s\n";

      binary_layer->get_output(temp_dst); //convert back binary output and copy into float for next layer
      */

    }

    template<typename DType>
    inline void QConvolutionForward(const Tensor<cpu, 4, DType> &data,
                                    const Tensor<cpu, 2, DType> &wmat,
                                    const Tensor<cpu, 2, DType> &in_col,
                                    const Tensor<cpu, 2, DType> &temp_dst,
                                    const Tensor<cpu, 4, DType> &out,
                                    const mxnet::op::QConvolutionParam &param) {
      CHECK(false) << "only float supported";
    }
}

namespace mxnet {
namespace op {

DMLC_REGISTER_PARAMETER(QConvolutionParam);

template<>
Operator* CreateOp<cpu>(QConvolutionParam param, int dtype,
                        std::vector<TShape> *in_shape,
                        std::vector<TShape> *out_shape,
                        Context ctx) {
  Operator *op = NULL;
#if MXNET_USE_MKL2017 == 1
  if ((param.dilate[0] == 1 && param.dilate[1] == 1)
      && param.kernel.ndim() == 2) {
      LOG(FATAL) << "QConvolution not supported with MKL";
    switch (dtype) {
    case mshadow::kFloat32:
      return new MKLConvolutionOp<cpu, float>(param);
    case mshadow::kFloat64:
      return new MKLConvolutionOp<cpu, double>(param);
    default:
      break;
    }
  }
  LOG(INFO) << MKLConvolutionOp<cpu, float>::getName() << " Skip MKL optimization";
#endif
#if MXNET_USE_NNPACK == 1
  const size_t batch_size = (*in_shape)[0][0];
  if ((param.dilate[0] == 1 && param.dilate[1] == 1)
      && param.kernel.ndim() == 2 && (!param.no_bias)
      && param.num_group == 1 && (batch_size == 1 ||
      ((batch_size > 1) && (param.stride[0] == 1) &&
      (param.stride[1] == 1)))) {
      LOG(FATAL) << "QConvolution not supported with NNPACK";
    switch (dtype) {
    case mshadow::kFloat32:
      return new NNPACKConvolutionOp<cpu, float>(param);
    default:
      break;
    }
  }
#endif
  MSHADOW_REAL_TYPE_SWITCH(dtype, DType, {
    op = new QConvolutionOp<cpu, DType>(param);
  })
  return op;
}

// DO_BIND_DISPATCH comes from operator_common.h
Operator *QConvolutionProp::CreateOperatorEx(Context ctx,
                                            std::vector<TShape> *in_shape,
                                            std::vector<int> *in_type) const {
  std::vector<TShape> out_shape, aux_shape;
  std::vector<int> out_type, aux_type;
  CHECK(InferType(in_type, &out_type, &aux_type));
  CHECK(InferShape(in_shape, &out_shape, &aux_shape));
  DO_BIND_DISPATCH(CreateOp, param_, (*in_type)[0], in_shape, &out_shape, ctx);
}

MXNET_REGISTER_OP_PROPERTY(QConvolution, QConvolutionProp)
.add_argument("data", "Symbol", "Input data to the ConvolutionOp.")
.add_argument("weight", "Symbol", "Weight matrix.")
.add_argument("bias", "Symbol", "Bias parameter.")
.add_arguments(QConvolutionParam::__FIELDS__())
.describe("Apply convolution to input then add a bias.");

}  // namespace op
}  // namespace mxnet
<commit_msg>remove binary_layer code<commit_after>/*!
 * Copyright (c) 2016 by Contributors
 * \file q_fully_connected.cc
 * \brief Quantized CONV operator
 * \author HPI-DeepLearning
*/

#include "./q_convolution-inl.h"
#include <mshadow/base.h>
#include <mshadow/tensor.h>
#include <memory>
#include "./binary_layer.h"
#include "./xnor_cpu.h"
#if MXNET_USE_MKL2017 == 1
#include <mkl_memory.h>
#include "../../src/operator/mkl/mkl_memory-inl.h"
#include "../../src/operator/mkl/mkl_convolution-inl.h"
#endif  // MXNET_USE_MKL2017
#if MXNET_USE_NNPACK == 1
#include "../../src/operator/nnpack/nnpack_convolution-inl.h"
#endif  // MXNET_USE_NNPACK
# include <chrono>
using ns = std::chrono::nanoseconds;
using get_time = std::chrono::steady_clock ;

namespace mshadow {
	inline BINARY_WORD concatenate(float* array)
	{
		BINARY_WORD rvalue=0;
		BINARY_WORD sign;

		#pragma omp parallel for
		for (int i = 0; i < BITS_PER_BINARY_WORD; i++)
		{
			sign = (array[i]>=0);
			rvalue = rvalue | (sign<< (i));
		}

		return rvalue;
	}
	inline void get_binary_row(float* row, BINARY_WORD * b_row, int size){

		for (int i = 0; i < size; i+=BITS_PER_BINARY_WORD) {
			float * array = new float[BITS_PER_BINARY_WORD];

			#pragma omp parallel for
			for (int j = 0;j < BITS_PER_BINARY_WORD; ++j) {
				array[j] = row[i+j];
			}

			b_row[i/BITS_PER_BINARY_WORD] = concatenate(array);
			delete[] array;
		}
	}

	inline void get_binary_col(float* col, BINARY_WORD * b_col, int n, int k){

		for(int x=0; x < k; ++x){
			for(int y=0; y<(n/BITS_PER_BINARY_WORD); y++){
				float * array = new float[BITS_PER_BINARY_WORD];

				#pragma omp parallel for
				for(int b=0; b<BITS_PER_BINARY_WORD; ++b){
					array[b] = col[(y*BITS_PER_BINARY_WORD+b)*k + x];
				}

				b_col[y*k + x]=concatenate(array);
				delete[] array;
			}
		}
	}

	inline void xnor_gemm(int M, int K, int N,
							BINARY_WORD *A, int lda,
							BINARY_WORD *B, int ldb,
							float *C, int ldc){
	    int i,n,k;

		#pragma omp parallel for
	    for(i = 0; i < M; ++i){
			#pragma omp parallel for
	        for(n = 0; n < N; ++n){
	        	BINARY_WORD A_PART = A[i*lda+n];
				#pragma omp parallel for
	            for(k = 0; k < K; ++k){
	            	C[i*ldc+k] += (float)__builtin_popcount(~(A_PART ^ B[n*ldb+k]));
	            }
	        }
	    }
	}

	inline void baseline_gemm(int M, int K, int N,
							float *A, int lda,
							float *B, int ldb,
							float *C, int ldc){
	    int i,n,k;
	    for(i = 0; i < M; ++i){
	        for(n = 0; n < N; ++n){
	        	float A_PART = A[i*lda+n];
	            for(k = 0; k < K; ++k){
	            	C[i*ldc+k] += A_PART * B[n*ldb+k];
	            }
	        }
	    }
	}

    inline void QConvolutionForward(const Tensor<cpu, 4, float> &data,
                                    const Tensor<cpu, 2, float> &wmat,
                                    const Tensor<cpu, 2, float> &in_col,
                                    const Tensor<cpu, 2, float> &temp_dst,
                                    const Tensor<cpu, 4, float> &out,
                                    const mxnet::op::QConvolutionParam &param) {

      CHECK_EQ(param.stride[0], 1) << "binary convolution currently only supported with stride==1";
      CHECK_EQ(param.stride[1], 1) << "binary convolution currently only supported with stride==1";

      ///*
      int m = wmat.size(0);
      int n = wmat.size(1);
      int k = in_col.size(1);
      BINARY_WORD* binary_row = (BINARY_WORD*)malloc(m * n/BITS_PER_BINARY_WORD * sizeof(BINARY_WORD));
      BINARY_WORD* binary_col = (BINARY_WORD*)malloc(n * k/BITS_PER_BINARY_WORD * sizeof(BINARY_WORD));

	  get_binary_row(wmat.dptr_, binary_row, m*n);
	  get_binary_col(in_col.dptr_, binary_col, n, k);

	  auto start = std::chrono::high_resolution_clock::now();
	  ///*
	  xnor_gemm(m, k, n/BITS_PER_BINARY_WORD,
			  binary_row, n/BITS_PER_BINARY_WORD,
			  binary_col, k,
			  temp_dst.dptr_, k);
	  //*/

	  /*
	  //test using baseline gemm kernel
	  baseline_gemm(m, k, n,
			  	  	wmat.dptr_, n,
					in_col.dptr_, k,
					temp_dst.dptr_, k);
	  */
	  auto finish = std::chrono::high_resolution_clock::now();
	  std::chrono::duration<double> elapsed = finish - start;
	  std::cout << "xnor Elapsed time: " << elapsed.count() << " s\n";
	  free(binary_row);
	  free(binary_col);
    }

    template<typename DType>
    inline void QConvolutionForward(const Tensor<cpu, 4, DType> &data,
                                    const Tensor<cpu, 2, DType> &wmat,
                                    const Tensor<cpu, 2, DType> &in_col,
                                    const Tensor<cpu, 2, DType> &temp_dst,
                                    const Tensor<cpu, 4, DType> &out,
                                    const mxnet::op::QConvolutionParam &param) {
      CHECK(false) << "only float supported";
    }
}

namespace mxnet {
namespace op {

DMLC_REGISTER_PARAMETER(QConvolutionParam);

template<>
Operator* CreateOp<cpu>(QConvolutionParam param, int dtype,
                        std::vector<TShape> *in_shape,
                        std::vector<TShape> *out_shape,
                        Context ctx) {
  Operator *op = NULL;
#if MXNET_USE_MKL2017 == 1
  if ((param.dilate[0] == 1 && param.dilate[1] == 1)
      && param.kernel.ndim() == 2) {
      LOG(FATAL) << "QConvolution not supported with MKL";
    switch (dtype) {
    case mshadow::kFloat32:
      return new MKLConvolutionOp<cpu, float>(param);
    case mshadow::kFloat64:
      return new MKLConvolutionOp<cpu, double>(param);
    default:
      break;
    }
  }
  LOG(INFO) << MKLConvolutionOp<cpu, float>::getName() << " Skip MKL optimization";
#endif
#if MXNET_USE_NNPACK == 1
  const size_t batch_size = (*in_shape)[0][0];
  if ((param.dilate[0] == 1 && param.dilate[1] == 1)
      && param.kernel.ndim() == 2 && (!param.no_bias)
      && param.num_group == 1 && (batch_size == 1 ||
      ((batch_size > 1) && (param.stride[0] == 1) &&
      (param.stride[1] == 1)))) {
      LOG(FATAL) << "QConvolution not supported with NNPACK";
    switch (dtype) {
    case mshadow::kFloat32:
      return new NNPACKConvolutionOp<cpu, float>(param);
    default:
      break;
    }
  }
#endif
  MSHADOW_REAL_TYPE_SWITCH(dtype, DType, {
    op = new QConvolutionOp<cpu, DType>(param);
  })
  return op;
}

// DO_BIND_DISPATCH comes from operator_common.h
Operator *QConvolutionProp::CreateOperatorEx(Context ctx,
                                            std::vector<TShape> *in_shape,
                                            std::vector<int> *in_type) const {
  std::vector<TShape> out_shape, aux_shape;
  std::vector<int> out_type, aux_type;
  CHECK(InferType(in_type, &out_type, &aux_type));
  CHECK(InferShape(in_shape, &out_shape, &aux_shape));
  DO_BIND_DISPATCH(CreateOp, param_, (*in_type)[0], in_shape, &out_shape, ctx);
}

MXNET_REGISTER_OP_PROPERTY(QConvolution, QConvolutionProp)
.add_argument("data", "Symbol", "Input data to the ConvolutionOp.")
.add_argument("weight", "Symbol", "Weight matrix.")
.add_argument("bias", "Symbol", "Bias parameter.")
.add_arguments(QConvolutionParam::__FIELDS__())
.describe("Apply convolution to input then add a bias.");

}  // namespace op
}  // namespace mxnet
<|endoftext|>


<commit_before>#include <QCache>
#include <QDebug>
#include <QDateTime>
#include <QString>
#include <QStringList>
#include <QtConcurrentRun>

#include <algorithm>

#include <AlpinoCorpus/Error.hh>
#include "FilterModel.hh"

namespace ac = alpinocorpus;

FilterModel::FilterModel(CorpusPtr corpus, QObject *parent)
:
    QAbstractTableModel(parent),
    d_corpus(corpus),
    d_hits(0),
    d_entryCache(new EntryCache(1000000)),
    d_timer(new QTimer)
{
    connect(d_timer.data(), SIGNAL(timeout()),
        SLOT(fireDataChanged()));
    connect(this, SIGNAL(queryStopped(int, int)),
        SLOT(lastDataChanged(int, int)));
    connect(this, SIGNAL(queryFinished(int, int, bool)),
        SLOT(lastDataChanged(int, int, bool)));
    connect(this, SIGNAL(queryFinished(int, int, bool)),
        SLOT(finalizeQuery(int, int, bool)));
}

FilterModel::~FilterModel()
{
    cancelQuery();
}

int FilterModel::columnCount(QModelIndex const &index) const
{
    return 3;
}

int FilterModel::rowCount(QModelIndex const &index) const
{
    QMutexLocker locker(&d_resultsMutex);
    return d_results.size();
}

QVariant FilterModel::data(QModelIndex const &index, int role) const
{
    QMutexLocker locker(&d_resultsMutex);

    if (role == Qt::TextAlignmentRole)
        if (index.column() == 1)
            return (Qt::AlignTop + Qt::AlignHCenter);
        else
            return Qt::AlignTop;

    if (!index.isValid()
        || index.row() >= d_results.size()
        || index.row() < 0
        || !(role == Qt::DisplayRole || role == Qt::UserRole))
        return QVariant();

    switch (index.column())
    {
        case 0:
            return d_results.at(index.row()).name;
        case 1:
            return d_results.at(index.row()).hits;
        case 2:
            return d_results.at(index.row()).data;
        default:
            return QVariant();
    }
}

QString FilterModel::asXML() const
{
    QStringList docList;
    docList.append("<entries>");

    docList.append("<entriesinfo>");
    docList.append(QString("<corpus>%1</corpus>")
        .arg(d_corpus->name().c_str()));
    docList.append(QString("<filter>%1</filter>").arg(d_query));
    QString date(QDateTime::currentDateTime().toLocalTime().toString());
    docList.append(QString("<date>%1</date>").arg(date));
    docList.append("</entriesinfo>");

    int count = rowCount(QModelIndex());
    for (size_t i = 0; i < count; i++) {
        QString filename = data(index(i, 0), Qt::DisplayRole).toString();
        QString count = data(index(i, 1), Qt::DisplayRole).toString();
        QString xmlData = data(index(i, 2), Qt::DisplayRole).toString().trimmed()
            .replace("<?xml version=\"1.0\" encoding=\"UTF-8\"?>", "");

        QString line = QString("<entry><filename>%1</filename><count>%2</count>%3</entry>")
            .arg(filename)
            .arg(count)
            .arg(xmlData);

        docList.append(line);
    }
    docList.append("</entries>");

    QString doc = docList.join("\n");

    return doc;
}

void FilterModel::finalizeQuery(int n, int totalEntries, bool cached)
{
    if (!cached) {
        // Cache the query
        d_entryCache->insert(d_query, new CacheItem(d_hits, d_results));
    }
}

QVariant FilterModel::headerData(int column, Qt::Orientation orientation, int role) const
{
    if (orientation != Qt::Horizontal
        || role != Qt::DisplayRole)
        return QVariant();

    switch (column)
    {
        case 0:
            return tr("File");
        case 1:
            return tr("Hits");
        case 2:
            return tr("Data");
        default:
            return QVariant();
    }
}

QModelIndex FilterModel::indexOfFile(QString const &filename) const
{
    QMutexLocker locker(&d_resultsMutex);
    int index = -1;
    for (int i = 0, size = d_results.size(); i < size; ++i)
    {
        if (d_results.at(i).name == filename)
        {
            index = i;
            break;
        }
    }

    return createIndex(index, 0);
}

void FilterModel::fireDataChanged()
{
    // @TODO make this more robust so no assumption is required.

    int rows;
    {
      QMutexLocker locker(&d_resultsMutex);
      rows = d_results.size();
    }

    emit layoutAboutToBeChanged();
    emit dataChanged(index(d_lastRow, 0), index(rows - 1, 2));
    emit nEntriesFound(rows, d_hits);
    emit layoutChanged();

    d_lastRow = rows - 1;
}

void FilterModel::lastDataChanged(int n, int totalEntries)
{
  Q_UNUSED(n);
  Q_UNUSED(totalEntries);

  fireDataChanged();

}

void FilterModel::lastDataChanged(int n, int totalEntries, bool cached)
{
  if (cached)
    return;

  lastDataChanged(n, totalEntries);

  d_timer->stop();
}

void FilterModel::runQuery(QString const &query, QString const &stylesheet)
{
    cancelQuery(); // just in case

    int size;
    {
      QMutexLocker locker(&d_resultsMutex);
      size = d_results.size();
      d_results.clear();
    }

    emit dataChanged(index(0, 0), index(size, 0));

    d_query = query;

    // Do nothing if this is a dummy filter model with a stupid null pointer
    if (!d_corpus)
        return;

    d_timer->setInterval(100);
    d_timer->setSingleShot(false);
    d_lastRow = 0;
    d_timer->start();

    if (!d_query.isEmpty())
        d_entriesFuture = QtConcurrent::run(this,
            &FilterModel::getEntriesWithQuery, d_query, stylesheet);
    else {
        if (stylesheet.isNull())
            d_entriesFuture = QtConcurrent::run(this, &FilterModel::getEntries,
                d_corpus->begin(), d_corpus->end(), false);
        else
            d_entriesFuture = QtConcurrent::run(this, &FilterModel::getEntries,
                d_corpus->beginWithStylesheet(stylesheet.toUtf8().constData()),
                d_corpus->end(), true);

    }
}

QString const &FilterModel::lastQuery() const
{
    return d_query;
}

void FilterModel::cancelQuery()
{
    d_cancelled = true;
    d_entryIterator.interrupt();
    d_entriesFuture.waitForFinished();
    d_timer->stop();
}

// run async, because query() starts searching immediately
void FilterModel::getEntriesWithQuery(QString const &query,
    QString const &stylesheet)
{
    if (d_entryCache->contains(query)) {
        {
          QMutexLocker locker(&d_resultsMutex);
          d_results = d_entryCache->object(query)->entries;
        }

        d_hits = d_entryCache->object(query)->hits;

        emit queryFinished(d_results.size(), d_results.size(), true);
        return;
    }

    std::string cQuery = query.toUtf8().constData();

    try {
        if (stylesheet.isNull())
            FilterModel::getEntries(
                d_corpus->query(alpinocorpus::CorpusReader::XPATH,
                    cQuery),
                d_corpus->end(),
                false);
        else
            FilterModel::getEntries(
                d_corpus->queryWithStylesheet(alpinocorpus::CorpusReader::XPATH,
                    query.toUtf8().constData(), stylesheet.toUtf8().constData(),
                    std::list<ac::CorpusReader::MarkerQuery>(
                        1, ac::CorpusReader::MarkerQuery(cQuery, "active", "1"))),
                d_corpus->end(),
                true);

    } catch (alpinocorpus::Error const &e) {
        qDebug() << "Alpino Error in FilterModel::getEntries: " << e.what();
        emit queryFailed(e.what());
    } catch (std::exception const &e) {
        qDebug() << "Error in FilterModel::getEntries: " << e.what();
        emit queryFailed(e.what());
    }
}

// run async
void FilterModel::getEntries(EntryIterator const &begin, EntryIterator const &end,
    bool withStylesheet)
{
    try {
        emit queryStarted(0); // we don't know how many entries will be found

        d_cancelled = false;
        d_hits = 0;
        d_entryIterator = begin;

        for (; !d_cancelled && d_entryIterator != end;
          ++d_entryIterator)
        {
            ++d_hits;

            QString entry(QString::fromUtf8((*d_entryIterator).c_str()));

            // Lock the results list.
            QMutexLocker locker(&d_resultsMutex);

            /*
             * WARNING: This assumes all the hits per result only occur right after
             * each other, never shuffled. Otherwise we might want to change to QHash
             * or QMap for fast lookup.
             */
            int row = d_results.size() - 1;
            if (row >= 0 && d_results[row].name == entry)
                ++d_results[row].hits;
            // Add found file to the list
            else
            {
                ++row;
                if (withStylesheet)
                    d_results.append(Entry(entry, 1,
                        QString::fromUtf8((d_entryIterator.contents(*d_corpus)).c_str())));
                else
                    d_results.append(Entry(entry, 1, QString::null));
            }
        }

        if (d_cancelled)
            emit queryStopped(d_results.size(), d_results.size());
        else
            emit queryFinished(d_results.size(), d_results.size(), false);
    }
    // When d_entryIterator.interrupt() is called by cancelQuery():
    catch (alpinocorpus::IterationInterrupted const &e) {
        emit queryStopped(d_results.size(), d_results.size());
    }
    // When something goes terribly terribly wrong, like entering a query
    // that doesn't yield nodes but strings.
    catch (alpinocorpus::Error const &e) {
        qDebug() << "Error in FilterModel::getEntries: " << e.what();
        emit queryFailed(e.what());
    }
}
<commit_msg>More fine-grained locking in FilterModel.<commit_after>#include <QCache>
#include <QDebug>
#include <QDateTime>
#include <QString>
#include <QStringList>
#include <QtConcurrentRun>

#include <algorithm>

#include <AlpinoCorpus/Error.hh>
#include "FilterModel.hh"

namespace ac = alpinocorpus;

FilterModel::FilterModel(CorpusPtr corpus, QObject *parent)
:
    QAbstractTableModel(parent),
    d_corpus(corpus),
    d_hits(0),
    d_entryCache(new EntryCache(1000000)),
    d_timer(new QTimer)
{
    connect(d_timer.data(), SIGNAL(timeout()),
        SLOT(fireDataChanged()));
    connect(this, SIGNAL(queryStopped(int, int)),
        SLOT(lastDataChanged(int, int)));
    connect(this, SIGNAL(queryFinished(int, int, bool)),
        SLOT(lastDataChanged(int, int, bool)));
    connect(this, SIGNAL(queryFinished(int, int, bool)),
        SLOT(finalizeQuery(int, int, bool)));
}

FilterModel::~FilterModel()
{
    cancelQuery();
}

int FilterModel::columnCount(QModelIndex const &index) const
{
    return 3;
}

int FilterModel::rowCount(QModelIndex const &index) const
{
    QMutexLocker locker(&d_resultsMutex);
    return d_results.size();
}

QVariant FilterModel::data(QModelIndex const &index, int role) const
{
    if (role == Qt::TextAlignmentRole)
        if (index.column() == 1)
            return (Qt::AlignTop + Qt::AlignHCenter);
        else
            return Qt::AlignTop;

    size_t nResults;
    {
        QMutexLocker locker(&d_resultsMutex);
        nResults = d_results.size();
    }

    if (!index.isValid()
        || index.row() >= nResults
        || index.row() < 0
        || !(role == Qt::DisplayRole || role == Qt::UserRole))
        return QVariant();

    QMutexLocker locker(&d_resultsMutex);
    switch (index.column())
    {
        QMutexLocker locker(&d_resultsMutex);
        case 0:
            return d_results.at(index.row()).name;
        case 1:
            return d_results.at(index.row()).hits;
        case 2:
            return d_results.at(index.row()).data;
        default:
            return QVariant();
    }
}

QString FilterModel::asXML() const
{
    QStringList docList;
    docList.append("<entries>");

    docList.append("<entriesinfo>");
    docList.append(QString("<corpus>%1</corpus>")
        .arg(d_corpus->name().c_str()));
    docList.append(QString("<filter>%1</filter>").arg(d_query));
    QString date(QDateTime::currentDateTime().toLocalTime().toString());
    docList.append(QString("<date>%1</date>").arg(date));
    docList.append("</entriesinfo>");

    int count = rowCount(QModelIndex());
    for (size_t i = 0; i < count; i++) {
        QString filename = data(index(i, 0), Qt::DisplayRole).toString();
        QString count = data(index(i, 1), Qt::DisplayRole).toString();
        QString xmlData = data(index(i, 2), Qt::DisplayRole).toString().trimmed()
            .replace("<?xml version=\"1.0\" encoding=\"UTF-8\"?>", "");

        QString line = QString("<entry><filename>%1</filename><count>%2</count>%3</entry>")
            .arg(filename)
            .arg(count)
            .arg(xmlData);

        docList.append(line);
    }
    docList.append("</entries>");

    QString doc = docList.join("\n");

    return doc;
}

void FilterModel::finalizeQuery(int n, int totalEntries, bool cached)
{
    if (!cached) {
        // Cache the query
        d_entryCache->insert(d_query, new CacheItem(d_hits, d_results));
    }
}

QVariant FilterModel::headerData(int column, Qt::Orientation orientation, int role) const
{
    if (orientation != Qt::Horizontal
        || role != Qt::DisplayRole)
        return QVariant();

    switch (column)
    {
        case 0:
            return tr("File");
        case 1:
            return tr("Hits");
        case 2:
            return tr("Data");
        default:
            return QVariant();
    }
}

QModelIndex FilterModel::indexOfFile(QString const &filename) const
{
    QMutexLocker locker(&d_resultsMutex);
    int index = -1;
    for (int i = 0, size = d_results.size(); i < size; ++i)
    {
        if (d_results.at(i).name == filename)
        {
            index = i;
            break;
        }
    }

    return createIndex(index, 0);
}

void FilterModel::fireDataChanged()
{
    // @TODO make this more robust so no assumption is required.

    int rows;
    {
      QMutexLocker locker(&d_resultsMutex);
      rows = d_results.size();
    }

    emit layoutAboutToBeChanged();
    emit dataChanged(index(d_lastRow, 0), index(rows - 1, 2));
    emit nEntriesFound(rows, d_hits);
    emit layoutChanged();

    d_lastRow = rows - 1;
}

void FilterModel::lastDataChanged(int n, int totalEntries)
{
  Q_UNUSED(n);
  Q_UNUSED(totalEntries);

  fireDataChanged();

}

void FilterModel::lastDataChanged(int n, int totalEntries, bool cached)
{
  if (cached)
    return;

  lastDataChanged(n, totalEntries);

  d_timer->stop();
}

void FilterModel::runQuery(QString const &query, QString const &stylesheet)
{
    cancelQuery(); // just in case

    int size;
    {
      QMutexLocker locker(&d_resultsMutex);
      size = d_results.size();
      d_results.clear();
    }

    emit dataChanged(index(0, 0), index(size, 0));

    d_query = query;

    // Do nothing if this is a dummy filter model with a stupid null pointer
    if (!d_corpus)
        return;

    d_timer->setInterval(100);
    d_timer->setSingleShot(false);
    d_lastRow = 0;
    d_timer->start();

    if (!d_query.isEmpty())
        d_entriesFuture = QtConcurrent::run(this,
            &FilterModel::getEntriesWithQuery, d_query, stylesheet);
    else {
        if (stylesheet.isNull())
            d_entriesFuture = QtConcurrent::run(this, &FilterModel::getEntries,
                d_corpus->begin(), d_corpus->end(), false);
        else
            d_entriesFuture = QtConcurrent::run(this, &FilterModel::getEntries,
                d_corpus->beginWithStylesheet(stylesheet.toUtf8().constData()),
                d_corpus->end(), true);

    }
}

QString const &FilterModel::lastQuery() const
{
    return d_query;
}

void FilterModel::cancelQuery()
{
    d_cancelled = true;
    d_entryIterator.interrupt();
    d_entriesFuture.waitForFinished();
    d_timer->stop();
}

// run async, because query() starts searching immediately
void FilterModel::getEntriesWithQuery(QString const &query,
    QString const &stylesheet)
{
    if (d_entryCache->contains(query)) {
        {
          QMutexLocker locker(&d_resultsMutex);
          d_results = d_entryCache->object(query)->entries;
        }

        d_hits = d_entryCache->object(query)->hits;

        emit queryFinished(d_results.size(), d_results.size(), true);
        return;
    }

    std::string cQuery = query.toUtf8().constData();

    try {
        if (stylesheet.isNull())
            FilterModel::getEntries(
                d_corpus->query(alpinocorpus::CorpusReader::XPATH,
                    cQuery),
                d_corpus->end(),
                false);
        else
            FilterModel::getEntries(
                d_corpus->queryWithStylesheet(alpinocorpus::CorpusReader::XPATH,
                    query.toUtf8().constData(), stylesheet.toUtf8().constData(),
                    std::list<ac::CorpusReader::MarkerQuery>(
                        1, ac::CorpusReader::MarkerQuery(cQuery, "active", "1"))),
                d_corpus->end(),
                true);

    } catch (alpinocorpus::Error const &e) {
        qDebug() << "Alpino Error in FilterModel::getEntries: " << e.what();
        emit queryFailed(e.what());
    } catch (std::exception const &e) {
        qDebug() << "Error in FilterModel::getEntries: " << e.what();
        emit queryFailed(e.what());
    }
}

// run async
void FilterModel::getEntries(EntryIterator const &begin, EntryIterator const &end,
    bool withStylesheet)
{
    try {
        emit queryStarted(0); // we don't know how many entries will be found

        d_cancelled = false;
        d_hits = 0;
        d_entryIterator = begin;

        for (; !d_cancelled && d_entryIterator != end;
          ++d_entryIterator)
        {
            ++d_hits;

            QString entry(QString::fromUtf8((*d_entryIterator).c_str()));

            /*
             * WARNING: This assumes all the hits per result only occur right after
             * each other, never shuffled. Otherwise we might want to change to QHash
             * or QMap for fast lookup.
             */
            int row = d_results.size() - 1;
            if (row >= 0 && d_results[row].name == entry) {
                QMutexLocker locker(&d_resultsMutex);
                ++d_results[row].hits;
            }
            // Add found file to the list
            else
            {
                ++row;
                if (withStylesheet) {
                    QString contents =
                        QString::fromUtf8((d_entryIterator.contents(*d_corpus)).c_str());
                    d_results.append(Entry(entry, 1, contents));
                }
                else {
                    QMutexLocker locker(&d_resultsMutex);
                    d_results.append(Entry(entry, 1, QString::null));
                }
            }
        }

        if (d_cancelled)
            emit queryStopped(d_results.size(), d_results.size());
        else
            emit queryFinished(d_results.size(), d_results.size(), false);
    }
    // When d_entryIterator.interrupt() is called by cancelQuery():
    catch (alpinocorpus::IterationInterrupted const &e) {
        emit queryStopped(d_results.size(), d_results.size());
    }
    // When something goes terribly terribly wrong, like entering a query
    // that doesn't yield nodes but strings.
    catch (alpinocorpus::Error const &e) {
        qDebug() << "Error in FilterModel::getEntries: " << e.what();
        emit queryFailed(e.what());
    }
}
<|endoftext|>


<commit_before>
#include <algorithm>
#include <iostream>
#include <vector>
#include <cstdio>
#include <cstring>
#include <cassert>

#include "GetF0/get_f0.h"

namespace {

template <class SourceFormat, class DestFormat>
struct StaticCaster {
  DestFormat operator()(SourceFormat from)
  {
    return static_cast<DestFormat>(from);
  }
};

template <class SourceFormat, class DestFormat>
int readFile(const std::string& fileName, std::vector<DestFormat>& output)
{
  std::FILE* inputFile = std::fopen(fileName.c_str(), "rb");
  if (inputFile == nullptr) {
    std::perror("Cannot open input");
    return 1;
  }

  if (std::fseek(inputFile, 0, SEEK_END) == -1) {
    std::perror("Cannot seek to end");
    return 1;
  }

  auto endPosition = std::ftell(inputFile);
  if (endPosition == -1) {
    std::perror("Cannot get end position");
    return 1;
  }

  if (std::fseek(inputFile, 0, SEEK_SET) == -1) {
    std::perror("Cannot seek to beginning");
    return 1;
  }

  if (endPosition % sizeof(SourceFormat) != 0) {
    std::cerr << "Assertion failed: File size not a multiple of format type"
              << std::endl;
    return 1;
  }

  auto numberSamples = endPosition / sizeof(SourceFormat);

  output.reserve(numberSamples);

  enum { BUFFER_SIZE = 4096 };
  SourceFormat buffer[BUFFER_SIZE];
  size_t readSize;

  do {
    readSize = std::fread(buffer, sizeof(SourceFormat), BUFFER_SIZE, inputFile);

    std::transform(buffer, buffer + readSize, std::back_inserter(output),
                   StaticCaster<SourceFormat, DestFormat>());

  } while (readSize == BUFFER_SIZE);

  if (std::ferror(inputFile) != 0) {
    std::perror("Error reading file");
    return 1;
  }

  if (std::fclose(inputFile) != 0) {
    std::perror("Error closing file");
    return 1;
  }

  return 0;
}

class GetF0_impl : public GetF0::GetF0 {
public:

  typedef std::vector<Sample> SampleVector;
  typedef std::vector<float> OutputVector;

  GetF0_impl(SampleFrequency sampleFrequency,
	     SampleVector& samples);

protected:

  long read_samples(float** buffer, long num_records) override;

  long read_samples_overlap(float** buffer, long num_records,
                            long step) override;

  void write_output_reversed(float* f0p, float* vuvp, float* rms_speech,
                             float* acpkp, int vecsize) override;

private:

  SampleVector& m_samples;

  unsigned m_position;

public:
  OutputVector m_outputVector;
};

GetF0_impl::GetF0_impl(SampleFrequency sampleFrequency, SampleVector& samples)
    : GetF0(sampleFrequency), m_samples(samples), m_position(0)
{
}

long GetF0_impl::read_samples(float** buffer, long num_records) {
  auto range = std::min<long>(
      num_records, m_samples.size() - m_position);
  *buffer = &m_samples[m_position];
  m_position += range;
  return range;
}

long GetF0_impl::read_samples_overlap(float** buffer, long num_records,
                                      long step)
{
  // (num_records - step) old records
  // (step) new records

  m_position -= (num_records - step);
  return read_samples(buffer, num_records);
}

void GetF0_impl::write_output_reversed(float* f0p, float* vuvp,
                                       float* rms_speech, float* acpkp,
                                       int vecsize)
{
  std::reverse_copy(f0p, f0p + vecsize, std::back_inserter(m_outputVector));
}

} // namespace anonymous

int main(int argc, char* argv[])
{
  typedef short DiskSample;

  GetF0_impl::SampleVector samples;
  if (readFile<DiskSample, GetF0_impl::Sample>(argv[1], samples) != 0) {
    return 1;
  }

  std::cout << "Read " << samples.size() << " samples." << std::endl;

  GetF0_impl::SampleFrequency freq = 16000;
  GetF0_impl f0(freq, samples);
  f0.init();
  f0.run();

  std::cout << "Returned " << f0.m_outputVector.size() << " data points."
            << std::endl;
  std::cout << "streamBufferSize " << f0.streamBufferSize() << std::endl;
  std::cout << "streamOverlapSize " << f0.streamOverlapSize() << std::endl;

  std::cout << std::endl;

  for (int i = 0; i < f0.m_outputVector.size(); ++i) {
    if (i > 0 && i % 10 == 0) {
      std::cout << std::endl;
    }

    std::cout << f0.m_outputVector[i] << " ";
  }
  std::cout << std::endl;

  return 0;
}
<commit_msg>Remove the vector reserve<commit_after>
#include <algorithm>
#include <iostream>
#include <vector>
#include <cstdio>
#include <cstring>
#include <cassert>

#include "GetF0/get_f0.h"

namespace {

template <class SourceFormat, class DestFormat>
struct StaticCaster {
  DestFormat operator()(SourceFormat from)
  {
    return static_cast<DestFormat>(from);
  }
};

template <class SourceFormat, class DestFormat>
int readFile(const std::string& fileName, std::vector<DestFormat>& output)
{
  std::FILE* inputFile = std::fopen(fileName.c_str(), "rb");
  if (inputFile == nullptr) {
    std::perror("Cannot open input");
    return 1;
  }

  enum { BUFFER_SIZE = 4096 };
  SourceFormat buffer[BUFFER_SIZE];
  size_t readSize;

  do {
    readSize = std::fread(buffer, sizeof(SourceFormat), BUFFER_SIZE, inputFile);

    std::transform(buffer, buffer + readSize, std::back_inserter(output),
                   StaticCaster<SourceFormat, DestFormat>());

  } while (readSize == BUFFER_SIZE);

  if (std::ferror(inputFile) != 0) {
    std::perror("Error reading file");
    return 1;
  }

  if (std::fclose(inputFile) != 0) {
    std::perror("Error closing file");
    return 1;
  }

  return 0;
}

class GetF0_impl : public GetF0::GetF0 {
public:

  typedef std::vector<Sample> SampleVector;
  typedef std::vector<float> OutputVector;

  GetF0_impl(SampleFrequency sampleFrequency,
	     SampleVector& samples);

protected:

  long read_samples(float** buffer, long num_records) override;

  long read_samples_overlap(float** buffer, long num_records,
                            long step) override;

  void write_output_reversed(float* f0p, float* vuvp, float* rms_speech,
                             float* acpkp, int vecsize) override;

private:

  SampleVector& m_samples;

  unsigned m_position;

public:
  OutputVector m_outputVector;
};

GetF0_impl::GetF0_impl(SampleFrequency sampleFrequency, SampleVector& samples)
    : GetF0(sampleFrequency), m_samples(samples), m_position(0)
{
}

long GetF0_impl::read_samples(float** buffer, long num_records) {
  auto range = std::min<long>(
      num_records, m_samples.size() - m_position);
  *buffer = &m_samples[m_position];
  m_position += range;
  return range;
}

long GetF0_impl::read_samples_overlap(float** buffer, long num_records,
                                      long step)
{
  // (num_records - step) old records
  // (step) new records

  m_position -= (num_records - step);
  return read_samples(buffer, num_records);
}

void GetF0_impl::write_output_reversed(float* f0p, float* vuvp,
                                       float* rms_speech, float* acpkp,
                                       int vecsize)
{
  std::reverse_copy(f0p, f0p + vecsize, std::back_inserter(m_outputVector));
}

} // namespace anonymous

int main(int argc, char* argv[])
{
  typedef short DiskSample;

  GetF0_impl::SampleVector samples;
  if (readFile<DiskSample, GetF0_impl::Sample>(argv[1], samples) != 0) {
    return 1;
  }

  std::cout << "Read " << samples.size() << " samples." << std::endl;

  GetF0_impl::SampleFrequency freq = 16000;
  GetF0_impl f0(freq, samples);
  f0.init();
  f0.run();

  std::cout << "Returned " << f0.m_outputVector.size() << " data points."
            << std::endl;
  std::cout << "streamBufferSize " << f0.streamBufferSize() << std::endl;
  std::cout << "streamOverlapSize " << f0.streamOverlapSize() << std::endl;

  std::cout << std::endl;

  for (int i = 0; i < f0.m_outputVector.size(); ++i) {
    if (i > 0 && i % 10 == 0) {
      std::cout << std::endl;
    }

    std::cout << f0.m_outputVector[i] << " ";
  }
  std::cout << std::endl;

  return 0;
}
<|endoftext|>


<commit_before>/*
    tooltipeditdialog.cpp  -  Kopete Tooltip Editor

    Copyright (c) 2004 by Stefan Gehn <metz AT gehn.net>
    Kopete    (c) 2004 by the Kopete developers  <kopete-devel@kde.org>

    *************************************************************************
    *                                                                       *
    * This program is free software; you can redistribute it and/or modify  *
    * it under the terms of the GNU General Public License as published by  *
    * the Free Software Foundation; either version 2 of the License, or     *
    * (at your option) any later version.                                   *
    *                                                                       *
    *************************************************************************
*/

#include "tooltipeditdialog.h"
#include "tooltipeditwidget.h"

#include "kopetecontactproperty.h"
#include "kopeteglobal.h"
#include "kopeteprefs.h"

#include <qapplication.h>
#include <qtoolbutton.h>
#include <qstringlist.h>

#include <kiconloader.h>
#include <klistview.h>
#include <klocale.h>

class TooltipItem : public KListViewItem
{
	public:
		TooltipItem(KListView *parent, const QString& label, const QString& propertyName)
			: KListViewItem(parent, label),
				mPropName(propertyName)
		{
		}

		TooltipItem(KListView *parent, Q3ListViewItem *item, const QString& label, const QString& propertyName)
			: KListViewItem(parent, item, label),
				mPropName(propertyName)
		{
		}

		QString propertyName() const { return mPropName; }
	private:
		QString mPropName;
};

TooltipEditDialog::TooltipEditDialog(QWidget *parent, const char* name)
	: KDialogBase(parent, name, true, i18n("Tooltip Editor"), Ok|Cancel, Ok, true)
{
	mMainWidget = new TooltipEditWidget(this, "TooltipEditDialog::mMainWidget");
	setMainWidget(mMainWidget);
	mMainWidget->lstUsedItems->header()->hide();
	mMainWidget->lstUnusedItems->header()->hide();
	mMainWidget->lstUsedItems->setSorting( -1 );
	mMainWidget->lstUnusedItems->setSorting( 0 );

	const Kopete::ContactPropertyTmpl::Map propmap(
		Kopete::Global::Properties::self()->templateMap());
	QStringList usedKeys = KopetePrefs::prefs()->toolTipContents();

	// first fill the "used" list
	QStringList::Iterator usedIt=usedKeys.end();
	do
	{
		usedIt--;
		// only add if that property key is really known
		if(propmap.contains(*usedIt) && !propmap[*usedIt].isPrivate())
		{
			new TooltipItem(mMainWidget->lstUsedItems,
				propmap[*usedIt].label(), *usedIt);
		}
	} while(usedIt != usedKeys.begin());

	// then iterate over all known properties and insert the remaining ones
	// into the "unused" list
	Kopete::ContactPropertyTmpl::Map::ConstIterator it;
	for(it = propmap.begin(); it != propmap.end(); ++it)
	{
		if((usedKeys.contains(it.key())==0) && (!it.data().isPrivate()))
			new TooltipItem(mMainWidget->lstUnusedItems, it.data().label(), it.key());
	}

	connect(mMainWidget->lstUnusedItems, SIGNAL(selectionChanged(Q3ListViewItem *)),
		this, SLOT(slotUnusedSelected(Q3ListViewItem *)));
	connect(mMainWidget->lstUsedItems, SIGNAL(selectionChanged(Q3ListViewItem *)),
		this, SLOT(slotUsedSelected(Q3ListViewItem *)));

	QIcon iconSet;
	iconSet = SmallIconSet("up");
	mMainWidget->tbUp->setIconSet(iconSet);
	mMainWidget->tbUp->setEnabled(false);
	mMainWidget->tbUp->setAutoRepeat(true);
	connect(mMainWidget->tbUp, SIGNAL(clicked()), SLOT(slotUpButton()));

	iconSet = SmallIconSet("down");
	mMainWidget->tbDown->setIconSet(iconSet);
	mMainWidget->tbDown->setEnabled(false);
	mMainWidget->tbDown->setAutoRepeat(true);
	connect(mMainWidget->tbDown, SIGNAL(clicked()), SLOT(slotDownButton()));

	iconSet = QApplication::isRightToLeft() ? SmallIconSet("back") : SmallIconSet("forward");
	mMainWidget->tbAdd->setIconSet(iconSet);
	mMainWidget->tbAdd->setEnabled(false);
	connect(mMainWidget->tbAdd, SIGNAL(clicked()), SLOT(slotAddButton()));

	iconSet = QApplication::isRightToLeft() ? SmallIconSet("forward") : SmallIconSet("back");
	mMainWidget->tbRemove->setIconSet(iconSet);
	mMainWidget->tbRemove->setEnabled(false);
	connect(mMainWidget->tbRemove, SIGNAL(clicked()), SLOT(slotRemoveButton()));

	connect(this, SIGNAL(okClicked()), this, SLOT(slotOkClicked()));

	resize(QSize(450, 450));
}

void TooltipEditDialog::slotOkClicked()
{
	QStringList oldList = KopetePrefs::prefs()->toolTipContents();
	QStringList newList;
	Q3ListViewItemIterator it(mMainWidget->lstUsedItems);
	QString keyname;

	while(it.current())
	{
		keyname = static_cast<TooltipItem *>(it.current())->propertyName();
		newList += keyname;
		kdDebug(14000) << k_funcinfo <<
			"Adding key '" << keyname << "' to tooltip list" << endl;
		++it;
	}

	if(oldList != newList)
	{
		KopetePrefs::prefs()->setToolTipContents(newList);
		emit changed(true);
		kdDebug(14000) << k_funcinfo << "tooltip fields changed, emitting changed()" << endl;
	}
}

void TooltipEditDialog::slotUnusedSelected(Q3ListViewItem *item)
{
	//mMainWidget->tbRemove->setEnabled(false);
	mMainWidget->tbAdd->setEnabled(item!=0);
}

void TooltipEditDialog::slotUsedSelected(Q3ListViewItem *item)
{
	mMainWidget->tbRemove->setEnabled(item!=0);
	//mMainWidget->tbAdd->setEnabled(false);
	if (item)
	{
		mMainWidget->tbUp->setEnabled(item->itemAbove() != 0);
		mMainWidget->tbDown->setEnabled(item->itemBelow() != 0);
	}
	else
	{
		mMainWidget->tbUp->setEnabled(false);
		mMainWidget->tbDown->setEnabled(false);
	}
}

void TooltipEditDialog::slotUpButton()
{
	Q3ListViewItem *item = mMainWidget->lstUsedItems->currentItem();
	Q3ListViewItem *prev = item->itemAbove();
	if(prev == 0) // we are first item already
		return;

	prev->moveItem(item);
	slotUsedSelected(item);
}

void TooltipEditDialog::slotDownButton()
{
	Q3ListViewItem *item = mMainWidget->lstUsedItems->currentItem();
	Q3ListViewItem *next = item->itemBelow();
	if(next == 0) // we are last item already
		return;

	item->moveItem(next);
	slotUsedSelected(item);
}

void TooltipEditDialog::slotAddButton()
{
	TooltipItem *item = static_cast<TooltipItem *>(mMainWidget->lstUnusedItems->currentItem());
	if(!item)
		return;
	//kdDebug(14000) << k_funcinfo << endl;

	// build a new one in the "used" list
	new TooltipItem(mMainWidget->lstUsedItems, item->text(0), item->propertyName());

	// remove the old one from "unused" list
	mMainWidget->lstUnusedItems->takeItem(item);
	delete item;
}

void TooltipEditDialog::slotRemoveButton()
{
	TooltipItem *item = static_cast<TooltipItem *>(mMainWidget->lstUsedItems->currentItem());
	if(!item)
		return;
	//kdDebug(14000) << k_funcinfo << endl;

	// build a new one in the "unused" list
	new TooltipItem(mMainWidget->lstUnusedItems, item->text(0), item->propertyName());

	// remove the old one from "used" list
	mMainWidget->lstUsedItems->takeItem(item);
	delete item;
}

#include "tooltipeditdialog.moc"
<commit_msg>Backport use doubleclick to add/remove<commit_after>/*
    tooltipeditdialog.cpp  -  Kopete Tooltip Editor

    Copyright (c) 2004 by Stefan Gehn <metz AT gehn.net>
    Kopete    (c) 2004 by the Kopete developers  <kopete-devel@kde.org>

    *************************************************************************
    *                                                                       *
    * This program is free software; you can redistribute it and/or modify  *
    * it under the terms of the GNU General Public License as published by  *
    * the Free Software Foundation; either version 2 of the License, or     *
    * (at your option) any later version.                                   *
    *                                                                       *
    *************************************************************************
*/

#include "tooltipeditdialog.h"
#include "tooltipeditwidget.h"

#include "kopetecontactproperty.h"
#include "kopeteglobal.h"
#include "kopeteprefs.h"

#include <qapplication.h>
#include <qtoolbutton.h>
#include <qstringlist.h>

#include <kiconloader.h>
#include <klistview.h>
#include <klocale.h>

class TooltipItem : public KListViewItem
{
	public:
		TooltipItem(KListView *parent, const QString& label, const QString& propertyName)
			: KListViewItem(parent, label),
				mPropName(propertyName)
		{
		}

		TooltipItem(KListView *parent, Q3ListViewItem *item, const QString& label, const QString& propertyName)
			: KListViewItem(parent, item, label),
				mPropName(propertyName)
		{
		}

		QString propertyName() const { return mPropName; }
	private:
		QString mPropName;
};

TooltipEditDialog::TooltipEditDialog(QWidget *parent, const char* name)
	: KDialogBase(parent, name, true, i18n("Tooltip Editor"), Ok|Cancel, Ok, true)
{
	mMainWidget = new TooltipEditWidget(this, "TooltipEditDialog::mMainWidget");
	setMainWidget(mMainWidget);
	mMainWidget->lstUsedItems->header()->hide();
	mMainWidget->lstUnusedItems->header()->hide();
	mMainWidget->lstUsedItems->setSorting( -1 );
	mMainWidget->lstUnusedItems->setSorting( 0 );

	const Kopete::ContactPropertyTmpl::Map propmap(
		Kopete::Global::Properties::self()->templateMap());
	QStringList usedKeys = KopetePrefs::prefs()->toolTipContents();

#warning "kde4: test if it works with qt4"
    connect(mMainWidget->lstUnusedItems, SIGNAL(doubleClicked ( QListViewItem *, const QPoint &, int )), this, SLOT(slotAddButton()));
    connect(mMainWidget->lstUsedItems, SIGNAL(doubleClicked ( QListViewItem *, const QPoint &, int )), this, SLOT(slotRemoveButton()));

	// first fill the "used" list
	QStringList::Iterator usedIt=usedKeys.end();
	do
	{
		usedIt--;
		// only add if that property key is really known
		if(propmap.contains(*usedIt) && !propmap[*usedIt].isPrivate())
		{
			new TooltipItem(mMainWidget->lstUsedItems,
				propmap[*usedIt].label(), *usedIt);
		}
	} while(usedIt != usedKeys.begin());

	// then iterate over all known properties and insert the remaining ones
	// into the "unused" list
	Kopete::ContactPropertyTmpl::Map::ConstIterator it;
	for(it = propmap.begin(); it != propmap.end(); ++it)
	{
		if((usedKeys.contains(it.key())==0) && (!it.data().isPrivate()))
			new TooltipItem(mMainWidget->lstUnusedItems, it.data().label(), it.key());
	}

	connect(mMainWidget->lstUnusedItems, SIGNAL(selectionChanged(Q3ListViewItem *)),
		this, SLOT(slotUnusedSelected(Q3ListViewItem *)));
	connect(mMainWidget->lstUsedItems, SIGNAL(selectionChanged(Q3ListViewItem *)),
		this, SLOT(slotUsedSelected(Q3ListViewItem *)));

	QIcon iconSet;
	iconSet = SmallIconSet("up");
	mMainWidget->tbUp->setIconSet(iconSet);
	mMainWidget->tbUp->setEnabled(false);
	mMainWidget->tbUp->setAutoRepeat(true);
	connect(mMainWidget->tbUp, SIGNAL(clicked()), SLOT(slotUpButton()));

	iconSet = SmallIconSet("down");
	mMainWidget->tbDown->setIconSet(iconSet);
	mMainWidget->tbDown->setEnabled(false);
	mMainWidget->tbDown->setAutoRepeat(true);
	connect(mMainWidget->tbDown, SIGNAL(clicked()), SLOT(slotDownButton()));

	iconSet = QApplication::isRightToLeft() ? SmallIconSet("back") : SmallIconSet("forward");
	mMainWidget->tbAdd->setIconSet(iconSet);
	mMainWidget->tbAdd->setEnabled(false);
	connect(mMainWidget->tbAdd, SIGNAL(clicked()), SLOT(slotAddButton()));

	iconSet = QApplication::isRightToLeft() ? SmallIconSet("forward") : SmallIconSet("back");
	mMainWidget->tbRemove->setIconSet(iconSet);
	mMainWidget->tbRemove->setEnabled(false);
	connect(mMainWidget->tbRemove, SIGNAL(clicked()), SLOT(slotRemoveButton()));

	connect(this, SIGNAL(okClicked()), this, SLOT(slotOkClicked()));

	resize(QSize(450, 450));
}

void TooltipEditDialog::slotOkClicked()
{
	QStringList oldList = KopetePrefs::prefs()->toolTipContents();
	QStringList newList;
	Q3ListViewItemIterator it(mMainWidget->lstUsedItems);
	QString keyname;

	while(it.current())
	{
		keyname = static_cast<TooltipItem *>(it.current())->propertyName();
		newList += keyname;
		kdDebug(14000) << k_funcinfo <<
			"Adding key '" << keyname << "' to tooltip list" << endl;
		++it;
	}

	if(oldList != newList)
	{
		KopetePrefs::prefs()->setToolTipContents(newList);
		emit changed(true);
		kdDebug(14000) << k_funcinfo << "tooltip fields changed, emitting changed()" << endl;
	}
}

void TooltipEditDialog::slotUnusedSelected(Q3ListViewItem *item)
{
	//mMainWidget->tbRemove->setEnabled(false);
	mMainWidget->tbAdd->setEnabled(item!=0);
}

void TooltipEditDialog::slotUsedSelected(Q3ListViewItem *item)
{
	mMainWidget->tbRemove->setEnabled(item!=0);
	//mMainWidget->tbAdd->setEnabled(false);
	if (item)
	{
		mMainWidget->tbUp->setEnabled(item->itemAbove() != 0);
		mMainWidget->tbDown->setEnabled(item->itemBelow() != 0);
	}
	else
	{
		mMainWidget->tbUp->setEnabled(false);
		mMainWidget->tbDown->setEnabled(false);
	}
}

void TooltipEditDialog::slotUpButton()
{
	Q3ListViewItem *item = mMainWidget->lstUsedItems->currentItem();
	Q3ListViewItem *prev = item->itemAbove();
	if(prev == 0) // we are first item already
		return;

	prev->moveItem(item);
	slotUsedSelected(item);
}

void TooltipEditDialog::slotDownButton()
{
	Q3ListViewItem *item = mMainWidget->lstUsedItems->currentItem();
	Q3ListViewItem *next = item->itemBelow();
	if(next == 0) // we are last item already
		return;

	item->moveItem(next);
	slotUsedSelected(item);
}

void TooltipEditDialog::slotAddButton()
{
	TooltipItem *item = static_cast<TooltipItem *>(mMainWidget->lstUnusedItems->currentItem());
	if(!item)
		return;
	//kdDebug(14000) << k_funcinfo << endl;

	// build a new one in the "used" list
	new TooltipItem(mMainWidget->lstUsedItems, item->text(0), item->propertyName());

	// remove the old one from "unused" list
	mMainWidget->lstUnusedItems->takeItem(item);
	delete item;
}

void TooltipEditDialog::slotRemoveButton()
{
	TooltipItem *item = static_cast<TooltipItem *>(mMainWidget->lstUsedItems->currentItem());
	if(!item)
		return;
	//kdDebug(14000) << k_funcinfo << endl;

	// build a new one in the "unused" list
	new TooltipItem(mMainWidget->lstUnusedItems, item->text(0), item->propertyName());

	// remove the old one from "used" list
	mMainWidget->lstUsedItems->takeItem(item);
	delete item;
}

#include "tooltipeditdialog.moc"
<|endoftext|>