<commit_before>// define this macro if using secp384r1
// #define MCLBN_FP_UNIT_SIZE 6
#include <mcl/she.hpp>
#include <cybozu/option.hpp>
#include <cybozu/benchmark.hpp>
#include <sstream>
#include <vector>
#include <omp.h>

using namespace mcl::she;

const int maxMsg = 10000;

typedef std::vector<CipherTextG1> CipherTextG1Vec;

void loadSave(const SecretKey& sec, const PublicKey& pub, bool compress)
{
	printf("loadSave compress=%d\n", compress);
	int m = 123;
	std::string s;
	{
		CipherTextG1 c;
		pub.enc(c, m);
		std::ostringstream oss;
		if (compress) {
			cybozu::save(oss, c);
		} else {
			oss.write((const char*)&c, sizeof(c));
		}
		s = oss.str();
		printf("s.size()=%zd\n", s.size());
	}
	{
		std::istringstream iss(s);
		CipherTextG1 c;
		if (compress) {
			cybozu::load(c, iss);
		} else {
			iss.read((char*)&c, sizeof(c));
		}
		int m2 = sec.dec(c);
		printf("m2=%d\n", m2);
	}
}

void benchEnc(const PrecomputedPublicKey& ppub, int vecN)
{
	cybozu::CpuClock clk;
	CipherTextG1Vec cv;
	cv.resize(vecN);
	const int C = 10;
	for (int k = 0; k < C; k++) {
		clk.begin();
		#pragma omp parallel for
		for (int i = 0; i < vecN; i++) {
			ppub.enc(cv[i], i);
		}
		clk.end();
	}
	clk.put("enc");
}

void benchAdd(const PrecomputedPublicKey& ppub, int addN, int vecN)
{
	cybozu::CpuClock clk;
	CipherTextG1Vec sumv, cv;
	sumv.resize(vecN);
	cv.resize(addN);
	for (int i = 0; i < addN; i++) {
		ppub.enc(cv[i], i);
	}
	const int C = 10;
	for (int k = 0; k < C; k++) {
		clk.begin();
		#pragma omp parallel for
		for (int j = 0; j < vecN; j++) {
			sumv[j] = cv[0];
			for (int i = 1; i < addN; i++) {
				sumv[j].add(cv[i]);
			}
		}
		clk.end();
	}
	clk.put("add");
}

void benchDec(const PrecomputedPublicKey& ppub, const SecretKey& sec, int vecN)
{
	cybozu::CpuClock clk;
	CipherTextG1Vec cv;
	cv.resize(vecN);
	for (int i = 0; i < vecN; i++) {
		ppub.enc(cv[i], i % maxMsg);
	}
	const int C = 10;
	for (int k = 0; k < C; k++) {
		clk.begin();
		#pragma omp parallel for
		for (int i = 0; i < vecN; i++) {
			sec.dec(cv[i]);
		}
		clk.end();
	}
	clk.put("dec");
}

void exec(const std::string& mode, int addN, int vecN)
{
	SecretKey sec;
	sec.setByCSPRNG();

	PublicKey pub;
	sec.getPublicKey(pub);
	PrecomputedPublicKey ppub;
	ppub.init(pub);

	if (mode == "enc") {
		benchEnc(ppub, vecN);
		return;
	}
	if (mode == "add") {
		benchAdd(ppub, addN, vecN);
		return;
	}
	if (mode == "dec") {
		benchDec(ppub, sec, vecN);
		return;
	}
	if (mode == "loadsave") {
		loadSave(sec, pub, false);
		loadSave(sec, pub, true);
	}
	printf("not supported mode=%s\n", mode.c_str());
}

int main(int argc, char *argv[])
	try
{
	// initialize system
	const size_t hashSize = maxMsg;
#if 1
	const mcl::EcParam& param = mcl::ecparam::secp256k1;
	initG1only(param, hashSize);
#else
	init();
	setRangeForDLP(hashSize);
#endif

	cybozu::Option opt;
	int cpuN;
	int addN;
	int vecN;
	std::string mode;
	opt.appendOpt(&cpuN, 1, "cpu", "# of cpus");
	opt.appendOpt(&addN, 128, "add", "# of add");
	opt.appendOpt(&vecN, 1024, "n", "# of elements");
	opt.appendParam(&mode, "mode", "enc|add|dec|loadsave");
	opt.appendHelp("h");
	if (opt.parse(argc, argv)) {
		opt.put();
	} else {
		opt.usage();
		return 1;
	}
	omp_set_num_threads(cpuN);
	exec(mode, addN, vecN);
} catch (std::exception& e) {
	printf("ERR %s\n", e.what());
	return 1;
}

<commit_msg>[she] fix she sample code for loadsave<commit_after>// define this macro if using secp384r1
// #define MCLBN_FP_UNIT_SIZE 6
#include <mcl/she.hpp>
#include <cybozu/option.hpp>
#include <cybozu/benchmark.hpp>
#include <sstream>
#include <vector>
#include <omp.h>

using namespace mcl::she;

const int maxMsg = 10000;

typedef std::vector<CipherTextG1> CipherTextG1Vec;

void loadSave(const SecretKey& sec, const PublicKey& pub, bool compress)
{
	printf("loadSave compress=%d\n", compress);
	int m = 123;
	std::string s;
	{
		CipherTextG1 c;
		pub.enc(c, m);
		std::ostringstream oss;
		if (compress) {
			cybozu::save(oss, c);
		} else {
			oss.write((const char*)&c, sizeof(c));
		}
		s = oss.str();
		printf("s.size()=%zd\n", s.size());
	}
	{
		std::istringstream iss(s);
		CipherTextG1 c;
		if (compress) {
			cybozu::load(c, iss);
		} else {
			iss.read((char*)&c, sizeof(c));
		}
		int m2 = sec.dec(c);
		printf("m2=%d\n", m2);
	}
}

void benchEnc(const PrecomputedPublicKey& ppub, int vecN)
{
	cybozu::CpuClock clk;
	CipherTextG1Vec cv;
	cv.resize(vecN);
	const int C = 10;
	for (int k = 0; k < C; k++) {
		clk.begin();
		#pragma omp parallel for
		for (int i = 0; i < vecN; i++) {
			ppub.enc(cv[i], i);
		}
		clk.end();
	}
	clk.put("enc");
}

void benchAdd(const PrecomputedPublicKey& ppub, int addN, int vecN)
{
	cybozu::CpuClock clk;
	CipherTextG1Vec sumv, cv;
	sumv.resize(vecN);
	cv.resize(addN);
	for (int i = 0; i < addN; i++) {
		ppub.enc(cv[i], i);
	}
	const int C = 10;
	for (int k = 0; k < C; k++) {
		clk.begin();
		#pragma omp parallel for
		for (int j = 0; j < vecN; j++) {
			sumv[j] = cv[0];
			for (int i = 1; i < addN; i++) {
				sumv[j].add(cv[i]);
			}
		}
		clk.end();
	}
	clk.put("add");
}

void benchDec(const PrecomputedPublicKey& ppub, const SecretKey& sec, int vecN)
{
	cybozu::CpuClock clk;
	CipherTextG1Vec cv;
	cv.resize(vecN);
	for (int i = 0; i < vecN; i++) {
		ppub.enc(cv[i], i % maxMsg);
	}
	const int C = 10;
	for (int k = 0; k < C; k++) {
		clk.begin();
		#pragma omp parallel for
		for (int i = 0; i < vecN; i++) {
			sec.dec(cv[i]);
		}
		clk.end();
	}
	clk.put("dec");
}

void exec(const std::string& mode, int addN, int vecN)
{
	SecretKey sec;
	sec.setByCSPRNG();

	PublicKey pub;
	sec.getPublicKey(pub);
	PrecomputedPublicKey ppub;
	ppub.init(pub);

	if (mode == "enc") {
		benchEnc(ppub, vecN);
		return;
	}
	if (mode == "add") {
		benchAdd(ppub, addN, vecN);
		return;
	}
	if (mode == "dec") {
		benchDec(ppub, sec, vecN);
		return;
	}
	if (mode == "loadsave") {
		loadSave(sec, pub, false);
		loadSave(sec, pub, true);
		return;
	}
	printf("not supported mode=%s\n", mode.c_str());
}

int main(int argc, char *argv[])
	try
{
	// initialize system
	const size_t hashSize = maxMsg;
#if 1
	const mcl::EcParam& param = mcl::ecparam::secp256k1;
	initG1only(param, hashSize);
#else
	init();
	setRangeForDLP(hashSize);
#endif

	cybozu::Option opt;
	int cpuN;
	int addN;
	int vecN;
	std::string mode;
	opt.appendOpt(&cpuN, 1, "cpu", "# of cpus");
	opt.appendOpt(&addN, 128, "add", "# of add");
	opt.appendOpt(&vecN, 1024, "n", "# of elements");
	opt.appendParam(&mode, "mode", "enc|add|dec|loadsave");
	opt.appendHelp("h");
	if (opt.parse(argc, argv)) {
		opt.put();
	} else {
		opt.usage();
		return 1;
	}
	omp_set_num_threads(cpuN);
	exec(mode, addN, vecN);
} catch (std::exception& e) {
	printf("ERR %s\n", e.what());
	return 1;
}

<|endoftext|>


<commit_before>#pragma warning(push)
#pragma warning(disable: 4273)
#pragma warning(disable: 4351)
#include <cstdio>
#include "context.h"
#include "callable.h"
#include "point.h"
#include <iostream>
#define _INC_STDIO
#include "include/3rdparty/tpunit++/tpunit++.hpp"
#pragma warning(pop)

/*! A default callable
 */
class MockCallable: public dukpp03::Callable<dukpp03::context::Context>
{
public:
    MockCallable()
    {

    }

    Callable* clone()
    {
        return new MockCallable();
    }

    /*! Returns count of required arguments
        \return count of required arguments
     */
    virtual int requiredArguments()
    {
        return 0;
    }
    /*! Performs call of object, using specified context
        \param[in] c context
        \return count of values on stack, placed by functions
     */
    virtual int call(dukpp03::context::Context* c)
    {
        dukpp03::PushValue<int, dukpp03::context::Context>::perform(c, 1, false);
        return 1;
    }

};

struct ContextTest : tpunit::TestFixture
{
public:
    ContextTest() : tpunit::TestFixture(
       TEST(ContextTest::testInitGet),
       TEST(ContextTest::testPushGet),
       TEST(ContextTest::testEvalNormal),
       TEST(ContextTest::testEvalFail),
       TEST(ContextTest::testEvalTimeout),
       TEST(ContextTest::testClean),
       TEST(ContextTest::testReset),
       TEST(ContextTest::testThrow),
       TEST(ContextTest::testRegisterGlobal),
       TEST(ContextTest::testRegisterCallable)
    ) {}

    /*! Tests getting and setting reference data
     */
    void testInitGet()
    {
        dukpp03::context::Context ctx;
        ASSERT_TRUE( dukpp03::context::Context::getContext(ctx.context()) == &ctx );
        ASSERT_TRUE( dukpp03::context::Context::getContext(ctx.context()) == &ctx );
    }

    /*! Tests pushing and getting values
     */
    void testPushGet()
    {
        dukpp03::context::Context ctx;

        // Common case
        int test_number = 0;
        {
            Point pts2d(3, 4);
            dukpp03::PushValue<Point, dukpp03::context::Context>::perform(&ctx, pts2d, false);
            dukpp03::Maybe<Point> mbpts2d =
                dukpp03::GetValue<Point, dukpp03::context::Context>::perform(&ctx, test_number++);
            ASSERT_TRUE( mbpts2d.exists() );
            ASSERT_TRUE( is_fuzzy_equal(mbpts2d.value().x(), 3));
            ASSERT_TRUE( is_fuzzy_equal(mbpts2d.value().y(), 4));
        }

        // char
        {
            char c = 121;
            dukpp03::PushValue<char, dukpp03::context::Context>::perform(&ctx, c, false);
            dukpp03::Maybe<char> maybev = dukpp03::GetValue<char, dukpp03::context::Context>::perform(&ctx, test_number++);
            ASSERT_TRUE( maybev.exists());
            ASSERT_TRUE( maybev.value() == 121);
        }

        // unsigned char
        {
            unsigned char c = 121;
            dukpp03::PushValue<unsigned char, dukpp03::context::Context>::perform(&ctx, c, false);
            dukpp03::Maybe<unsigned char> maybev = dukpp03::GetValue<unsigned char, dukpp03::context::Context>::perform(&ctx, test_number++);
            ASSERT_TRUE( maybev.exists());
            ASSERT_TRUE( maybev.value() == 121);
        }

        // int
        {
            int c = 121;
            dukpp03::PushValue<int, dukpp03::context::Context>::perform(&ctx, c, false);
            dukpp03::Maybe<int> maybev = dukpp03::GetValue<int, dukpp03::context::Context>::perform(&ctx, test_number++);
            ASSERT_TRUE( maybev.exists());
            ASSERT_TRUE( maybev.value() == 121);
        }

        // unsigned int
        {
            unsigned int c = 121;
            dukpp03::PushValue<unsigned int, dukpp03::context::Context>::perform(&ctx, c, false);
            dukpp03::Maybe<unsigned int> maybev = dukpp03::GetValue<unsigned int, dukpp03::context::Context>::perform(&ctx, test_number++);
            ASSERT_TRUE( maybev.exists());
            ASSERT_TRUE( maybev.value() == 121);
        }

        // long
        {
            long c = 121;
            dukpp03::PushValue<long, dukpp03::context::Context>::perform(&ctx, c, false);
            dukpp03::Maybe<long> maybev = dukpp03::GetValue<long, dukpp03::context::Context>::perform(&ctx, test_number++);
            ASSERT_TRUE( maybev.exists());
            ASSERT_TRUE( maybev.value() == 121);
        }

        // unsigned long
        {
            unsigned long c = 121;
            dukpp03::PushValue<unsigned long, dukpp03::context::Context>::perform(&ctx, c, false);
            dukpp03::Maybe<unsigned long> maybev = dukpp03::GetValue<unsigned long, dukpp03::context::Context>::perform(&ctx, test_number++);
            ASSERT_TRUE( maybev.exists());
            ASSERT_TRUE( maybev.value() == 121);
        }

        // long long
        {
            long long c = 121;
            dukpp03::PushValue<long long, dukpp03::context::Context>::perform(&ctx, c, false);
            dukpp03::Maybe<long long> maybev = dukpp03::GetValue<long long, dukpp03::context::Context>::perform(&ctx, test_number++);
            ASSERT_TRUE( maybev.exists());
            ASSERT_TRUE( maybev.value() == 121);
        }

        // unsigned long
        {
            unsigned long long c = 121;
            dukpp03::PushValue<unsigned long long, dukpp03::context::Context>::perform(&ctx, c, false);
            dukpp03::Maybe<unsigned long long> maybev = dukpp03::GetValue<unsigned long long, dukpp03::context::Context>::perform(&ctx, test_number++);
            ASSERT_TRUE( maybev.exists());
            ASSERT_TRUE( maybev.value() == 121);
        }

        // bool
        {
            bool c = false;
            dukpp03::PushValue<bool, dukpp03::context::Context>::perform(&ctx, c, false);
            dukpp03::Maybe<bool> maybev = dukpp03::GetValue<bool, dukpp03::context::Context>::perform(&ctx, test_number++);
            ASSERT_TRUE( maybev.exists());
            ASSERT_TRUE( maybev.value() == false);
        }

        // float
        {
            float c = 1.5;
            dukpp03::PushValue<float, dukpp03::context::Context>::perform(&ctx, c, false);
            dukpp03::Maybe<float> maybev = dukpp03::GetValue<float, dukpp03::context::Context>::perform(&ctx, test_number++);
            ASSERT_TRUE( maybev.exists());
            ASSERT_TRUE( is_fuzzy_equal(maybev.value(), c));
        }

        // double
        {
            double c = 1.5;
            dukpp03::PushValue<double, dukpp03::context::Context>::perform(&ctx, c, false);
            dukpp03::Maybe<double> maybev = dukpp03::GetValue<double, dukpp03::context::Context>::perform(&ctx, test_number++);
            ASSERT_TRUE( maybev.exists());
            ASSERT_TRUE( is_fuzzy_equal(maybev.value(), c));
        }

        // long double
        {
            long double c = 1.5;
            dukpp03::PushValue<long double, dukpp03::context::Context>::perform(&ctx, c, false);
            dukpp03::Maybe<long double> maybev = dukpp03::GetValue<long double, dukpp03::context::Context>::perform(&ctx, test_number++);
            ASSERT_TRUE( maybev.exists());
            ASSERT_TRUE( is_fuzzy_equal(maybev.value(), c));
        }

        // std::string
        {
            const char* c = "23";
            dukpp03::PushValue<std::string, dukpp03::context::Context>::perform(&ctx, c, false);
            dukpp03::Maybe<std::string> maybev = dukpp03::GetValue<std::string, dukpp03::context::Context>::perform(&ctx, test_number++);
            ASSERT_TRUE( maybev.exists());
            ASSERT_TRUE( maybev.value() == "23");
        }
    }
    /*! Test for normal evaluation process
     */
    void testEvalNormal()
    {
        std::string error;
        dukpp03::context::Context ctx;
        bool eval_result = ctx.eval("1 + 1", false, &error);
        ASSERT_TRUE( eval_result );
        ASSERT_TRUE( error.size() == 0 );
        dukpp03::Maybe<int> result = dukpp03::GetValue<int, dukpp03::context::Context>::perform(&ctx, -1);
        ASSERT_TRUE( result.exists() );
        ASSERT_TRUE( result.value() == 2);
    }
    /*! Test for non-compilable code
     */
    void testEvalFail()
    {
        std::string error;
        dukpp03::context::Context ctx;
        bool eval_result = ctx.eval("1 + ;", true, &error);
        ASSERT_TRUE( !eval_result );
        ASSERT_TRUE( error.size() != 0 );
    }
    /*! Test for timeout
     */
    void testEvalTimeout()
    {
        std::string error;
        dukpp03::context::Context ctx;
        ctx.setMaximumExecutionTime(1000);
        bool eval_result = ctx.eval("while(true) {}", true, &error);
        ASSERT_TRUE( !eval_result );
        ASSERT_TRUE( error.size() != 0 );
    }

    /*! Test cleaning  of a pool
     */
    void testClean()
    {
        dukpp03::context::Context ctx;
        Point pts2d(3, 4);
        dukpp03::PushValue<Point, dukpp03::context::Context>::perform(&ctx, pts2d, false);
        dukpp03::Maybe<Point> mbpts2d =
            dukpp03::GetValue<Point, dukpp03::context::Context>::perform(&ctx, -1);
        ASSERT_TRUE( mbpts2d.exists() );

        ctx.clean();

        mbpts2d = dukpp03::GetValue<Point, dukpp03::context::Context>::perform(&ctx, -1);
        ASSERT_TRUE( mbpts2d.exists() == false );
    }
    /*! Test cleaning both pools and full reset of context
     */
    void testReset()
    {
        dukpp03::context::Context ctx;
        Point pts2d(3, 4);
        dukpp03::PushValue<Point, dukpp03::context::Context>::perform(&ctx, pts2d, true);
        dukpp03::Maybe<Point> mbpts2d =
            dukpp03::GetValue<Point, dukpp03::context::Context>::perform(&ctx, -1);
        ASSERT_TRUE( mbpts2d.exists() );

        ctx.reset();

        mbpts2d = dukpp03::GetValue<Point, dukpp03::context::Context>::perform(&ctx, -1);
        ASSERT_TRUE( mbpts2d.exists() == false );
        ASSERT_TRUE( dukpp03::context::Context::getContext(ctx.context()) == &ctx );
    }
    /*! Tests throwing for object
     */
    void testThrow()
    {
        dukpp03::context::Context ctx;
        ctx.throwError("Generic Error!");
        const char* s = duk_to_string(ctx.context(), -1);
        ASSERT_TRUE( s != NULL );
        std::string testvalue = s;
        ASSERT_TRUE(  testvalue.size() != 0 );
    }
    /*! Tests registering value as property of global object
     */
    void testRegisterGlobal()
    {
        dukpp03::context::Context ctx;
        ctx.registerGlobal("value", true);
        bool eval_result = ctx.eval(" !value ", false);
        ASSERT_TRUE( eval_result );
        dukpp03::Maybe<bool> result = dukpp03::GetValue<bool, dukpp03::context::Context>::perform(&ctx, -1);
        ASSERT_TRUE( result.exists() );
        ASSERT_TRUE( result.value() == false );
    }
    /*! Tests registering callable function
     */
    void testRegisterCallable()
    {
        dukpp03::context::Context ctx;
        ctx.registerCallable("f", new MockCallable());
        bool eval_result = ctx.eval(" (f() + f()) * (f() + f()) ; f() + f()*f() ", false);
        ASSERT_TRUE( eval_result );
        dukpp03::Maybe<int> result = dukpp03::GetValue<int, dukpp03::context::Context>::perform(&ctx, -1);
        ASSERT_TRUE( result.exists() );
        ASSERT_TRUE( result.value() == 2 );
    }

} _context_test;<commit_msg>First attempt to make this callable.<commit_after>#pragma warning(push)
#pragma warning(disable: 4273)
#pragma warning(disable: 4351)
#include <cstdio>
#include "context.h"
#include "callable.h"
#include "point.h"
#include <iostream>
#define _INC_STDIO
#include "include/3rdparty/tpunit++/tpunit++.hpp"
#pragma warning(pop)

/*! A default callable
 */
class MockCallable: public dukpp03::Callable<dukpp03::context::Context>
{
public:
    MockCallable()
    {

    }

    Callable* clone()
    {
        return new MockCallable();
    }

    /*! Returns count of required arguments
        \return count of required arguments
     */
    virtual int requiredArguments()
    {
        return 0;
    }
    /*! Performs call of object, using specified context
        \param[in] c context
        \return count of values on stack, placed by functions
     */
    virtual int call(dukpp03::context::Context* c)
    {
        dukpp03::PushValue<int, dukpp03::context::Context>::perform(c, 1, false);
        return 1;
    }

};

/*! A  callable from this
 */
class ThisMockCallable: public dukpp03::FunctionCallable<dukpp03::context::Context>
{
public:
    ThisMockCallable()
    {

    }

    Callable* clone()
    {
        return new ThisMockCallable();
    }

    /*! Returns count of required arguments
        \return count of required arguments
     */
    virtual int requiredArguments()
    {
        return 0;
    }
    /*! Performs call of object, using specified context
        \param[in] c context
        \return count of values on stack, placed by functions
     */
    virtual int call(dukpp03::context::Context* c)
    {
        duk_context* ctx = c->context();
        duk_push_this(ctx);
        duk_get_prop_string(ctx, -1, "prop");
        std::cout << c->getTop() << "\n";
        dukpp03::Maybe<int> ma = dukpp03::GetValue<int, dukpp03::context::Context>::perform(c, 1);
        duk_pop_2(ctx);
        int result = 1;
        if (ma.exists())
        {
            result += ma.value();
            std::cout << "value exists: " << ma.value() << "\n";
        }
        else
        {
            std::cout << "unable to get value\n";
        }
        dukpp03::PushValue<int, dukpp03::context::Context>::perform(c, result, false);
        return 1;
    }

};

struct ContextTest : tpunit::TestFixture
{
public:
    ContextTest() : tpunit::TestFixture(
       TEST(ContextTest::testInitGet),
       TEST(ContextTest::testPushGet),
       TEST(ContextTest::testEvalNormal),
       TEST(ContextTest::testEvalFail),
       TEST(ContextTest::testEvalTimeout),
       TEST(ContextTest::testClean),
       TEST(ContextTest::testReset),
       TEST(ContextTest::testThrow),
       TEST(ContextTest::testRegisterGlobal),
       TEST(ContextTest::testRegisterCallable),
       TEST(ContextTest::testRegisterThisCallable)
    ) {}

    /*! Tests getting and setting reference data
     */
    void testInitGet()
    {
        dukpp03::context::Context ctx;
        ASSERT_TRUE( dukpp03::context::Context::getContext(ctx.context()) == &ctx );
        ASSERT_TRUE( dukpp03::context::Context::getContext(ctx.context()) == &ctx );
    }

    /*! Tests pushing and getting values
     */
    void testPushGet()
    {
        dukpp03::context::Context ctx;

        // Common case
        int test_number = 0;
        {
            Point pts2d(3, 4);
            dukpp03::PushValue<Point, dukpp03::context::Context>::perform(&ctx, pts2d, false);
            dukpp03::Maybe<Point> mbpts2d =
                dukpp03::GetValue<Point, dukpp03::context::Context>::perform(&ctx, test_number++);
            ASSERT_TRUE( mbpts2d.exists() );
            ASSERT_TRUE( is_fuzzy_equal(mbpts2d.value().x(), 3));
            ASSERT_TRUE( is_fuzzy_equal(mbpts2d.value().y(), 4));
        }

        // char
        {
            char c = 121;
            dukpp03::PushValue<char, dukpp03::context::Context>::perform(&ctx, c, false);
            dukpp03::Maybe<char> maybev = dukpp03::GetValue<char, dukpp03::context::Context>::perform(&ctx, test_number++);
            ASSERT_TRUE( maybev.exists());
            ASSERT_TRUE( maybev.value() == 121);
        }

        // unsigned char
        {
            unsigned char c = 121;
            dukpp03::PushValue<unsigned char, dukpp03::context::Context>::perform(&ctx, c, false);
            dukpp03::Maybe<unsigned char> maybev = dukpp03::GetValue<unsigned char, dukpp03::context::Context>::perform(&ctx, test_number++);
            ASSERT_TRUE( maybev.exists());
            ASSERT_TRUE( maybev.value() == 121);
        }

        // int
        {
            int c = 121;
            dukpp03::PushValue<int, dukpp03::context::Context>::perform(&ctx, c, false);
            dukpp03::Maybe<int> maybev = dukpp03::GetValue<int, dukpp03::context::Context>::perform(&ctx, test_number++);
            ASSERT_TRUE( maybev.exists());
            ASSERT_TRUE( maybev.value() == 121);
        }

        // unsigned int
        {
            unsigned int c = 121;
            dukpp03::PushValue<unsigned int, dukpp03::context::Context>::perform(&ctx, c, false);
            dukpp03::Maybe<unsigned int> maybev = dukpp03::GetValue<unsigned int, dukpp03::context::Context>::perform(&ctx, test_number++);
            ASSERT_TRUE( maybev.exists());
            ASSERT_TRUE( maybev.value() == 121);
        }

        // long
        {
            long c = 121;
            dukpp03::PushValue<long, dukpp03::context::Context>::perform(&ctx, c, false);
            dukpp03::Maybe<long> maybev = dukpp03::GetValue<long, dukpp03::context::Context>::perform(&ctx, test_number++);
            ASSERT_TRUE( maybev.exists());
            ASSERT_TRUE( maybev.value() == 121);
        }

        // unsigned long
        {
            unsigned long c = 121;
            dukpp03::PushValue<unsigned long, dukpp03::context::Context>::perform(&ctx, c, false);
            dukpp03::Maybe<unsigned long> maybev = dukpp03::GetValue<unsigned long, dukpp03::context::Context>::perform(&ctx, test_number++);
            ASSERT_TRUE( maybev.exists());
            ASSERT_TRUE( maybev.value() == 121);
        }

        // long long
        {
            long long c = 121;
            dukpp03::PushValue<long long, dukpp03::context::Context>::perform(&ctx, c, false);
            dukpp03::Maybe<long long> maybev = dukpp03::GetValue<long long, dukpp03::context::Context>::perform(&ctx, test_number++);
            ASSERT_TRUE( maybev.exists());
            ASSERT_TRUE( maybev.value() == 121);
        }

        // unsigned long
        {
            unsigned long long c = 121;
            dukpp03::PushValue<unsigned long long, dukpp03::context::Context>::perform(&ctx, c, false);
            dukpp03::Maybe<unsigned long long> maybev = dukpp03::GetValue<unsigned long long, dukpp03::context::Context>::perform(&ctx, test_number++);
            ASSERT_TRUE( maybev.exists());
            ASSERT_TRUE( maybev.value() == 121);
        }

        // bool
        {
            bool c = false;
            dukpp03::PushValue<bool, dukpp03::context::Context>::perform(&ctx, c, false);
            dukpp03::Maybe<bool> maybev = dukpp03::GetValue<bool, dukpp03::context::Context>::perform(&ctx, test_number++);
            ASSERT_TRUE( maybev.exists());
            ASSERT_TRUE( maybev.value() == false);
        }

        // float
        {
            float c = 1.5;
            dukpp03::PushValue<float, dukpp03::context::Context>::perform(&ctx, c, false);
            dukpp03::Maybe<float> maybev = dukpp03::GetValue<float, dukpp03::context::Context>::perform(&ctx, test_number++);
            ASSERT_TRUE( maybev.exists());
            ASSERT_TRUE( is_fuzzy_equal(maybev.value(), c));
        }

        // double
        {
            double c = 1.5;
            dukpp03::PushValue<double, dukpp03::context::Context>::perform(&ctx, c, false);
            dukpp03::Maybe<double> maybev = dukpp03::GetValue<double, dukpp03::context::Context>::perform(&ctx, test_number++);
            ASSERT_TRUE( maybev.exists());
            ASSERT_TRUE( is_fuzzy_equal(maybev.value(), c));
        }

        // long double
        {
            long double c = 1.5;
            dukpp03::PushValue<long double, dukpp03::context::Context>::perform(&ctx, c, false);
            dukpp03::Maybe<long double> maybev = dukpp03::GetValue<long double, dukpp03::context::Context>::perform(&ctx, test_number++);
            ASSERT_TRUE( maybev.exists());
            ASSERT_TRUE( is_fuzzy_equal(maybev.value(), c));
        }

        // std::string
        {
            const char* c = "23";
            dukpp03::PushValue<std::string, dukpp03::context::Context>::perform(&ctx, c, false);
            dukpp03::Maybe<std::string> maybev = dukpp03::GetValue<std::string, dukpp03::context::Context>::perform(&ctx, test_number++);
            ASSERT_TRUE( maybev.exists());
            ASSERT_TRUE( maybev.value() == "23");
        }
    }
    /*! Test for normal evaluation process
     */
    void testEvalNormal()
    {
        std::string error;
        dukpp03::context::Context ctx;
        bool eval_result = ctx.eval("1 + 1", false, &error);
        ASSERT_TRUE( eval_result );
        ASSERT_TRUE( error.size() == 0 );
        dukpp03::Maybe<int> result = dukpp03::GetValue<int, dukpp03::context::Context>::perform(&ctx, -1);
        ASSERT_TRUE( result.exists() );
        ASSERT_TRUE( result.value() == 2);
    }
    /*! Test for non-compilable code
     */
    void testEvalFail()
    {
        std::string error;
        dukpp03::context::Context ctx;
        bool eval_result = ctx.eval("1 + ;", true, &error);
        ASSERT_TRUE( !eval_result );
        ASSERT_TRUE( error.size() != 0 );
    }
    /*! Test for timeout
     */
    void testEvalTimeout()
    {
        std::string error;
        dukpp03::context::Context ctx;
        ctx.setMaximumExecutionTime(1000);
        bool eval_result = ctx.eval("while(true) {}", true, &error);
        ASSERT_TRUE( !eval_result );
        ASSERT_TRUE( error.size() != 0 );
    }

    /*! Test cleaning  of a pool
     */
    void testClean()
    {
        dukpp03::context::Context ctx;
        Point pts2d(3, 4);
        dukpp03::PushValue<Point, dukpp03::context::Context>::perform(&ctx, pts2d, false);
        dukpp03::Maybe<Point> mbpts2d =
            dukpp03::GetValue<Point, dukpp03::context::Context>::perform(&ctx, -1);
        ASSERT_TRUE( mbpts2d.exists() );

        ctx.clean();

        mbpts2d = dukpp03::GetValue<Point, dukpp03::context::Context>::perform(&ctx, -1);
        ASSERT_TRUE( mbpts2d.exists() == false );
    }
    /*! Test cleaning both pools and full reset of context
     */
    void testReset()
    {
        dukpp03::context::Context ctx;
        Point pts2d(3, 4);
        dukpp03::PushValue<Point, dukpp03::context::Context>::perform(&ctx, pts2d, true);
        dukpp03::Maybe<Point> mbpts2d =
            dukpp03::GetValue<Point, dukpp03::context::Context>::perform(&ctx, -1);
        ASSERT_TRUE( mbpts2d.exists() );

        ctx.reset();

        mbpts2d = dukpp03::GetValue<Point, dukpp03::context::Context>::perform(&ctx, -1);
        ASSERT_TRUE( mbpts2d.exists() == false );
        ASSERT_TRUE( dukpp03::context::Context::getContext(ctx.context()) == &ctx );
    }
    /*! Tests throwing for object
     */
    void testThrow()
    {
        dukpp03::context::Context ctx;
        ctx.throwError("Generic Error!");
        const char* s = duk_to_string(ctx.context(), -1);
        ASSERT_TRUE( s != NULL );
        std::string testvalue = s;
        ASSERT_TRUE(  testvalue.size() != 0 );
    }
    /*! Tests registering value as property of global object
     */
    void testRegisterGlobal()
    {
        dukpp03::context::Context ctx;
        ctx.registerGlobal("value", true);
        bool eval_result = ctx.eval(" !value ", false);
        ASSERT_TRUE( eval_result );
        dukpp03::Maybe<bool> result = dukpp03::GetValue<bool, dukpp03::context::Context>::perform(&ctx, -1);
        ASSERT_TRUE( result.exists() );
        ASSERT_TRUE( result.value() == false );
    }
    /*! Tests registering callable function
     */
    void testRegisterCallable()
    {
        dukpp03::context::Context ctx;
        ctx.registerCallable("f", new MockCallable());
        bool eval_result = ctx.eval(" (f() + f()) * (f() + f()) ; f() + f()*f() ", false);
        ASSERT_TRUE( eval_result );
        dukpp03::Maybe<int> result = dukpp03::GetValue<int, dukpp03::context::Context>::perform(&ctx, -1);
        ASSERT_TRUE( result.exists() );
        ASSERT_TRUE( result.value() == 2 );
    }

    void testRegisterThisCallable()
    {
        std::string error;
        dukpp03::context::Context ctx;
        ctx.registerCallable("f", new ThisMockCallable());
        bool eval_result = ctx.eval("var o  = {\"prop\" : 3}; o.f = f;  o.f()", false, &error);
        if (!eval_result)
        {
            std::cout << error << "\n";
        }
        ASSERT_TRUE( eval_result );
        dukpp03::Maybe<int> result = dukpp03::GetValue<int, dukpp03::context::Context>::perform(&ctx, -1);
        ASSERT_TRUE( result.exists() );
        ASSERT_TRUE( result.value() == 4 );
    }

} _context_test;<|endoftext|>


<commit_before>/*
 * LGParseLink.cc
 *
 * Copyright (C) 2017 Linas Vepstas
 *
 * Author: Linas Vepstas <linasvepstas@gmail.com>
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License v3 as
 * published by the Free Software Foundation and including the exceptions
 * at http://opencog.org/wiki/Licenses
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program; if not, write to:
 * Free Software Foundation, Inc.,
 * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 */

#include "LGParseLink.h"

using namespace opencog;

LGParseLink::LGParseLink(const HandleSeq& oset, Type t)
	: FunctionLink(oset, t)
{
}

LGParseLink::LGParseLink(const Link& l)
	: FunctionLink(l)
{
	// Type must be as expected
	Type tparse = l.getType();
	if (not classserver().isA(tparse, LG_PARSE_LINK))
	{
		const std::string& tname = classserver().getTypeName(tparse);
		throw InvalidParamException(TRACE_INFO,
			"Expecting an LgDictNode, got %s", tname.c_str());
	}
}

Handle LGParseLink::execute(AtomSpace* as) const
{
}

DEFINE_LINK_FACTORY(LGParseLink, LG_PARSE_LINK)

/* ===================== END OF FILE ===================== */
<commit_msg>Start filling in parser internals<commit_after>/*
 * LGParseLink.cc
 *
 * Copyright (C) 2017 Linas Vepstas
 *
 * Author: Linas Vepstas <linasvepstas@gmail.com>
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License v3 as
 * published by the Free Software Foundation and including the exceptions
 * at http://opencog.org/wiki/Licenses
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program; if not, write to:
 * Free Software Foundation, Inc.,
 * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 */

#include <opencog/atoms/base/Node.h>
#include "LGParseLink.h"

using namespace opencog;

LGParseLink::LGParseLink(const HandleSeq& oset, Type t)
	: FunctionLink(oset, t)
{
	size_t osz = oset.size();
	if (2 != osz)
		throw InvalidParamException(TRACE_INFO,
			"LGParseLink: Expecting two arguments, got %lu", osz);

	Type pht = oset[0]->getType();
	if (PHRASE_NODE != pht and VARIABLE_NODE != pht and GLOB_NODE != pht)
		throw InvalidParamException(TRACE_INFO,
			"LGParseLink: Expecting PhraseNode, got %s",
			oset[0]->toString().c_str());

	Type dit = oset[1]->getType();
	if (LG_DICT_NODE != dit and VARIABLE_NODE != dit and GLOB_NODE != dit)
		throw InvalidParamException(TRACE_INFO,
			"LGParseLink: Expecting LgDictNode, got %s",
			oset[1]->toString().c_str());
}

LGParseLink::LGParseLink(const Link& l)
	: FunctionLink(l)
{
	// Type must be as expected
	Type tparse = l.getType();
	if (not classserver().isA(tparse, LG_PARSE_LINK))
	{
		const std::string& tname = classserver().getTypeName(tparse);
		throw InvalidParamException(TRACE_INFO,
			"Expecting an LgParseLink, got %s", tname.c_str());
	}
}

Handle LGParseLink::execute(AtomSpace* as) const
{
	if (PHRASE_NODE != _outgoing[0]->getType()) return Handle();
	if (LG_DICT_NODE != _outgoing[1]->getType()) return Handle();

	return Handle(createNode(SENTENCE_NODE, "foo"));
}

DEFINE_LINK_FACTORY(LGParseLink, LG_PARSE_LINK)

/* ===================== END OF FILE ===================== */
<|endoftext|>


<commit_before>#include <iostream>
#include "semilinSetNdd.h"
#include "../datastructs/sparse_vec.h"
#include "semilinear_util.h"

int static_i = 0;

// insert an offset if it is unique in offsets
std::vector<std::vector<int>>& insert_offset(std::vector<std::vector<int>>& offsets, const std::vector<int>& offset) {
  if(std::find(offsets.begin(), offsets.end(), offset) == offsets.end())
  {
    offsets.push_back(offset);
  }
  return offsets;
}

std::vector<std::vector<int>> multiply_offsets(const std::vector<std::vector<int>>& offsets1, const std::vector<std::vector<int>>& offsets2) {
  std::vector<std::vector<int>> offsets;
  for(auto o1 : offsets1) {
    for(auto o2 : offsets2) {
      assert(o1.size() == o2.size());
      std::vector<int> o(o1.size());
      for(int i=0; i<o1.size(); ++i) {
        o.at(i) = o1.at(i) + o2.at(i);
      }
      offsets = insert_offset(offsets, o);
    }
  }
  return offsets;
}

std::string serialize_offsets(const std::vector<std::vector<int>>& offsets) {
  std::stringstream result;
  result << "[";
  for(const auto& offset : offsets) {
    if(&offset != &offsets.at(0))
      result << ",";
    result << "(";
    for(const auto& o : offset) {
      if(&o != &offset.at(0))
        result << ",";
      result << o;
    }
    result << ")";
  }
  result << "]";
  return result.str();
}

// return all linear independent offsets
std::vector<std::vector<int>> SemilinSetNdd::getIndependentOffsets(const std::vector<std::vector<int>>& offsets) const {

  std::vector<SparseVec<VarId, Counter, DummyDivider>> offset_;
  for(auto offset:offsets) {
    std::vector<std::pair<VarId, Counter>> tmp;
    for(auto var_pos : var_map) {
      VarId varid = var_pos.first;
      Counter value = offset.at(var_pos.second);
      if(value != 0) tmp.push_back({varid, value});
    }
    if(tmp.size() != 0) {
      SparseVec<VarId, Counter, DummyDivider> tmp_(std::move(tmp));
      offset_.push_back(tmp_);
    }
  }
  if(offset_.size() == 0) return {};
  VecSet<SparseVec<VarId, Counter, DummyDivider>> offset_vec_set{std::move(offset_)};

  SimplifySet<SparseVecSimplifier<VarId,Counter,DummyDivider>>(offset_vec_set);

  std::vector<std::vector<std::pair<VarId, Counter>>> temp_result;

  for(auto offset : offset_vec_set) {
    temp_result.push_back(offset.getVector());
  }

  std::vector<std::vector<int>> returned_result;

  for(auto offset : temp_result) {
    std::vector<int> o(k,0);
    for(auto map : offset) {
      o.at( var_map.find(map.first)->second ) = map.second;
    }
    returned_result.push_back(o);
  }

  return returned_result;
}

bool SemilinSetNdd::genepi_init(std::string pluginname, int number_of_variables)
{
  genepi_loader_init();
  genepi_loader_load_default_plugins();
  plugin = genepi_loader_get_plugin(pluginname.c_str());
  std::cout << "Plugin: " << genepi_plugin_get_name(plugin) << std::endl;
  // TODO: use of cache???
  solver = genepi_solver_create(plugin, 0, 0, 0);
  k = number_of_variables;
  return true; // TODO: return correct value!
}

bool SemilinSetNdd::genepi_dealloc() {
  genepi_solver_delete(solver);
  genepi_plugin_del_reference(plugin);
  genepi_loader_terminate();
  return true;
}

SemilinSetNdd::SemilinSetNdd() : set(Genepi(solver, k, false)) {
}

SemilinSetNdd::SemilinSetNdd(int zero) : set(Genepi(solver, k, false)) {
  assert(zero == 0);
}

SemilinSetNdd::SemilinSetNdd(VarId var) : SemilinSetNdd(var, 1) {
}

SemilinSetNdd::SemilinSetNdd(VarId var, int cnt) {
  auto v = var_map.find(var);
  if(v == var_map.end())
  {
    var_map.insert(std::make_pair(var, var_map.size()));
    v = var_map.find(var);
  }

  int position = v->second; // which position is this variable on?
  std::vector<int> alpha(k);
  alpha.at(position) = cnt;

  this->set = Genepi(solver, alpha, false);
  this->offsets.push_back(alpha);
}
SemilinSetNdd::SemilinSetNdd(Genepi set, std::vector<std::vector<int>> offsets) : set(set), offsets(offsets) {
}

SemilinSetNdd::SemilinSetNdd(const SemilinSetNdd& expr) : set(expr.set), offsets(expr.offsets) {
}

SemilinSetNdd::~SemilinSetNdd() {
}

SemilinSetNdd SemilinSetNdd::operator=(const SemilinSetNdd& term) {
  this->set = term.set;
  this->offsets = term.offsets;
  return *this;
}

SemilinSetNdd SemilinSetNdd::operator+=(const SemilinSetNdd& term) {
  this->set = this->set.union_op(term.set);
  for(auto offset : term.offsets)
    insert_offset(this->offsets, offset);

  this->offsets = this->getIndependentOffsets(this->offsets);
  return *this;
}

SemilinSetNdd SemilinSetNdd::operator*=(const SemilinSetNdd& term) {
  std::vector<int> sel_a(3*k, 1);
  std::vector<int> sel_b(3*k, 1);
  for(int i = 0; i < k; i++)
  {
    sel_a[3*i]     = 0;
    sel_b[3*i+1]   = 0;
  }

  // inverse projection from original dimensions to an extended version, which is intersected with the natural numbers
  Genepi a_ext(this->set.invproject(sel_a).intersect(Genepi(solver, 3*k, true)));
  Genepi b_ext(term.set.invproject(sel_b).intersect(Genepi(solver, 3*k, true)));

  std::vector<int> generic_alpha = {1, 1, -1}; // 1*a[i]+1*b[i]-1*c[i]=0
  Genepi generic_sum(solver, generic_alpha, 0);

  std::vector<int> component_selection(3*k, 1); // TODO: maybe we only use k+2 or so elements
  std::vector<int> component_projection(3*k, 0); // we want to project away component 3i and 3i+1

  // initialize the result automaton with (a_i,b_i,N) for i in 0..k-1
  Genepi result(solver, 3*k, true); // natural numbers
  result = result.intersect(a_ext).intersect(b_ext);

  // one run for each variable
  // at the end of each loop run we delete the used a and b component
  // the new sum component will be at position i at the end of the loop run
  // (a,b,a+b) component relevant for current run is at position (i,i+1,i+2) at the begin of a run
  for(int i = 0; i < k; i++)
  {
    component_selection.resize(3*k-2*i);
    component_projection.resize(3*k-2*i);
    // TODO: figure out, if precalculating this and apply inv_project has better complexity
    // than always create a new sum automaton for different components
    component_selection[i]   = 0;
    component_selection[i+1] = 0;
    component_selection[i+2] = 0;
    // inverse projection on generic sum automaton to create automaton for component i
    Genepi component_sum(generic_sum.invproject(component_selection));
    // use this sum automaton on the intermediate result
    result = result.intersect(component_sum);

    component_selection[i]   = 1; // reset component_selection
    component_selection[i+1] = 1;
    component_selection[i+2] = 1;

    component_projection[i]   = 1; // project away the already used a and b component at position (i,i+1)
    component_projection[i+1] = 1;
    result = result.project(component_projection);
    component_projection[i]   = 0;
    component_projection[i+1] = 0;
  }
  this->set = result;
  this->offsets = multiply_offsets(this->offsets, term.offsets);
  this->offsets = this->getIndependentOffsets(this->offsets);
  return *this;

}

bool SemilinSetNdd::operator < (const SemilinSetNdd& term) const {
  return this->set < term.set;
}
bool SemilinSetNdd::operator == (const SemilinSetNdd& term) const {
  return this->set == term.set;
}
SemilinSetNdd SemilinSetNdd::star () const {
  SemilinSetNdd offset_star = one();
  for(auto offset : this->offsets) {
    offset_star *= SemilinSetNdd(Genepi(this->solver, offset, true),{std::vector<int>(k,0)});
  }

  std::cout << serialize_offsets(this->offsets) << std::endl;
  SemilinSetNdd result = one(); // result = 1

  SemilinSetNdd final_result = one();
  SemilinSetNdd final_result_old;

  SemilinSetNdd S_i = one();
  do {
    final_result_old = final_result;

    S_i *= *this;  // = S^i
    result += S_i; // result = sum_{i=0}^n S^i
    final_result = one() + (result * offset_star); // 1 + (sum_{i=0}^n S^i) * offset_star
  } while (final_result != final_result_old);
  return final_result;
}

SemilinSetNdd SemilinSetNdd::null() {
  if(!SemilinSetNdd::elem_null)
    SemilinSetNdd::elem_null = std::shared_ptr<SemilinSetNdd>(new SemilinSetNdd());
  return *SemilinSetNdd::elem_null;
}

SemilinSetNdd SemilinSetNdd::one() {
  if(!SemilinSetNdd::elem_one)
    SemilinSetNdd::elem_one = std::shared_ptr<SemilinSetNdd>(new SemilinSetNdd(Genepi(solver, std::vector<int>(k,0), false), {std::vector<int>(k,0)}));
  return *SemilinSetNdd::elem_one;
}

std::string SemilinSetNdd::string() const {
  std::stringstream result;
  auto calculated_offsets = this->offsets;
  result << "calculated offsets:\t\t" << serialize_offsets(calculated_offsets) << std::endl;
  auto cleaned_calculated_offsets = this->getIndependentOffsets(calculated_offsets);
  result << "calculated offsets (clean):\t" << serialize_offsets(cleaned_calculated_offsets) << std::endl;
  result << "size:\t" << this->set.getSize() << std::endl;
  result << "automaton written:\t" << this->set.output("result", static_i++, "") << std::endl;
  return result.str();
}

const bool SemilinSetNdd::is_idempotent = true;
const bool SemilinSetNdd::is_commutative = true;
std::shared_ptr<SemilinSetNdd> SemilinSetNdd::elem_null;
std::shared_ptr<SemilinSetNdd> SemilinSetNdd::elem_one;
genepi_solver* SemilinSetNdd::solver;
genepi_plugin* SemilinSetNdd::plugin;
std::unordered_map<VarId, int> SemilinSetNdd::var_map;
int SemilinSetNdd::k = 0;
<commit_msg>do not use the simplifier in slsetndd, would introduce a bug<commit_after>#include <iostream>
#include "semilinSetNdd.h"
#include "../datastructs/sparse_vec.h"
#include "semilinear_util.h"

int static_i = 0;

// insert an offset if it is unique in offsets
std::vector<std::vector<int>>& insert_offset(std::vector<std::vector<int>>& offsets, const std::vector<int>& offset) {
  if(std::find(offsets.begin(), offsets.end(), offset) == offsets.end())
  {
    offsets.push_back(offset);
  }
  return offsets;
}

std::vector<std::vector<int>> multiply_offsets(const std::vector<std::vector<int>>& offsets1, const std::vector<std::vector<int>>& offsets2) {
  std::vector<std::vector<int>> offsets;
  for(auto o1 : offsets1) {
    for(auto o2 : offsets2) {
      assert(o1.size() == o2.size());
      std::vector<int> o(o1.size());
      for(int i=0; i<o1.size(); ++i) {
        o.at(i) = o1.at(i) + o2.at(i);
      }
      offsets = insert_offset(offsets, o);
    }
  }
  return offsets;
}

std::string serialize_offsets(const std::vector<std::vector<int>>& offsets) {
  std::stringstream result;
  result << "[";
  for(const auto& offset : offsets) {
    if(&offset != &offsets.at(0))
      result << ",";
    result << "(";
    for(const auto& o : offset) {
      if(&o != &offset.at(0))
        result << ",";
      result << o;
    }
    result << ")";
  }
  result << "]";
  return result.str();
}

// return all linear independent offsets
// FIXME: this kills also the [0,…,0] vector which is needed
std::vector<std::vector<int>> SemilinSetNdd::getIndependentOffsets(const std::vector<std::vector<int>>& offsets) const {

  std::vector<SparseVec<VarId, Counter, DummyDivider>> offset_;
  for(auto offset:offsets) {
    std::vector<std::pair<VarId, Counter>> tmp;
    for(auto var_pos : var_map) {
      VarId varid = var_pos.first;
      Counter value = offset.at(var_pos.second);
      if(value != 0) tmp.push_back({varid, value});
    }
    if(tmp.size() != 0) {
      SparseVec<VarId, Counter, DummyDivider> tmp_(std::move(tmp));
      offset_.push_back(tmp_);
    }
  }
  if(offset_.size() == 0) return {};
  VecSet<SparseVec<VarId, Counter, DummyDivider>> offset_vec_set{std::move(offset_)};

  SimplifySet<SparseVecSimplifier<VarId,Counter,DummyDivider>>(offset_vec_set);

  std::vector<std::vector<std::pair<VarId, Counter>>> temp_result;

  for(auto offset : offset_vec_set) {
    temp_result.push_back(offset.getVector());
  }

  std::vector<std::vector<int>> returned_result;

  for(auto offset : temp_result) {
    std::vector<int> o(k,0);
    for(auto map : offset) {
      o.at( var_map.find(map.first)->second ) = map.second;
    }
    returned_result.push_back(o);
  }

  return returned_result;
}

bool SemilinSetNdd::genepi_init(std::string pluginname, int number_of_variables)
{
  genepi_loader_init();
  genepi_loader_load_default_plugins();
  plugin = genepi_loader_get_plugin(pluginname.c_str());
  std::cout << "Plugin: " << genepi_plugin_get_name(plugin) << std::endl;
  // TODO: use of cache???
  solver = genepi_solver_create(plugin, 0, 0, 0);
  k = number_of_variables;
  return true; // TODO: return correct value!
}

bool SemilinSetNdd::genepi_dealloc() {
  genepi_solver_delete(solver);
  genepi_plugin_del_reference(plugin);
  genepi_loader_terminate();
  return true;
}

SemilinSetNdd::SemilinSetNdd() : set(Genepi(solver, k, false)) {
}

SemilinSetNdd::SemilinSetNdd(int zero) : set(Genepi(solver, k, false)) {
  assert(zero == 0);
}

SemilinSetNdd::SemilinSetNdd(VarId var) : SemilinSetNdd(var, 1) {
}

SemilinSetNdd::SemilinSetNdd(VarId var, int cnt) {
  auto v = var_map.find(var);
  if(v == var_map.end())
  {
    var_map.insert(std::make_pair(var, var_map.size()));
    v = var_map.find(var);
  }

  int position = v->second; // which position is this variable on?
  std::vector<int> alpha(k);
  alpha.at(position) = cnt;

  this->set = Genepi(solver, alpha, false);
  this->offsets.push_back(alpha);
}
SemilinSetNdd::SemilinSetNdd(Genepi set, std::vector<std::vector<int>> offsets) : set(set), offsets(offsets) {
}

SemilinSetNdd::SemilinSetNdd(const SemilinSetNdd& expr) : set(expr.set), offsets(expr.offsets) {
}

SemilinSetNdd::~SemilinSetNdd() {
}

SemilinSetNdd SemilinSetNdd::operator=(const SemilinSetNdd& term) {
  this->set = term.set;
  this->offsets = term.offsets;
  return *this;
}

SemilinSetNdd SemilinSetNdd::operator+=(const SemilinSetNdd& term) {
  this->set = this->set.union_op(term.set);
  for(auto offset : term.offsets)
    insert_offset(this->offsets, offset);

  // TODO: reactivate this once the indepentent offsets are correct
  // this->offsets = this->getIndependentOffsets(this->offsets);
  return *this;
}

SemilinSetNdd SemilinSetNdd::operator*=(const SemilinSetNdd& term) {
  std::vector<int> sel_a(3*k, 1);
  std::vector<int> sel_b(3*k, 1);
  for(int i = 0; i < k; i++)
  {
    sel_a[3*i]     = 0;
    sel_b[3*i+1]   = 0;
  }

  // inverse projection from original dimensions to an extended version, which is intersected with the natural numbers
  Genepi a_ext(this->set.invproject(sel_a).intersect(Genepi(solver, 3*k, true)));
  Genepi b_ext(term.set.invproject(sel_b).intersect(Genepi(solver, 3*k, true)));

  std::vector<int> generic_alpha = {1, 1, -1}; // 1*a[i]+1*b[i]-1*c[i]=0
  Genepi generic_sum(solver, generic_alpha, 0);

  std::vector<int> component_selection(3*k, 1); // TODO: maybe we only use k+2 or so elements
  std::vector<int> component_projection(3*k, 0); // we want to project away component 3i and 3i+1

  // initialize the result automaton with (a_i,b_i,N) for i in 0..k-1
  Genepi result(solver, 3*k, true); // natural numbers
  result = result.intersect(a_ext).intersect(b_ext);

  // one run for each variable
  // at the end of each loop run we delete the used a and b component
  // the new sum component will be at position i at the end of the loop run
  // (a,b,a+b) component relevant for current run is at position (i,i+1,i+2) at the begin of a run
  for(int i = 0; i < k; i++)
  {
    component_selection.resize(3*k-2*i);
    component_projection.resize(3*k-2*i);
    // TODO: figure out, if precalculating this and apply inv_project has better complexity
    // than always create a new sum automaton for different components
    component_selection[i]   = 0;
    component_selection[i+1] = 0;
    component_selection[i+2] = 0;
    // inverse projection on generic sum automaton to create automaton for component i
    Genepi component_sum(generic_sum.invproject(component_selection));
    // use this sum automaton on the intermediate result
    result = result.intersect(component_sum);

    component_selection[i]   = 1; // reset component_selection
    component_selection[i+1] = 1;
    component_selection[i+2] = 1;

    component_projection[i]   = 1; // project away the already used a and b component at position (i,i+1)
    component_projection[i+1] = 1;
    result = result.project(component_projection);
    component_projection[i]   = 0;
    component_projection[i+1] = 0;
  }
  this->set = result;
  this->offsets = multiply_offsets(this->offsets, term.offsets);
  // TODO: reactivate this once the indepentent offsets are correct
  // this->offsets = this->getIndependentOffsets(this->offsets);
  return *this;

}

bool SemilinSetNdd::operator < (const SemilinSetNdd& term) const {
  return this->set < term.set;
}
bool SemilinSetNdd::operator == (const SemilinSetNdd& term) const {
  return this->set == term.set;
}
SemilinSetNdd SemilinSetNdd::star () const {
  SemilinSetNdd offset_star = one();
  // INFO: this is the point where we use the saved (minimal base) of offsets
  // if our simplifier kills null-vectors we loose information (0^*=1...)
  // and our loop will never reach the fixed point
  for(auto offset : this->offsets) {
    offset_star *= SemilinSetNdd(Genepi(this->solver, offset, true),{std::vector<int>(k,0)});
  }

  SemilinSetNdd result = one(); // result = 1

  SemilinSetNdd final_result = one();
  SemilinSetNdd final_result_old;

  SemilinSetNdd S_i = one();
  do {
    final_result_old = final_result;

    S_i *= *this;  // = S^i
    result += S_i; // result = sum_{i=0}^n S^i
    final_result = one() + (result * offset_star); // 1 + (sum_{i=0}^n S^i) * offset_star
  } while (final_result != final_result_old);
  return final_result;
}

SemilinSetNdd SemilinSetNdd::null() {
  if(!SemilinSetNdd::elem_null)
    SemilinSetNdd::elem_null = std::shared_ptr<SemilinSetNdd>(new SemilinSetNdd());
  return *SemilinSetNdd::elem_null;
}

SemilinSetNdd SemilinSetNdd::one() {
  if(!SemilinSetNdd::elem_one)
    SemilinSetNdd::elem_one = std::shared_ptr<SemilinSetNdd>(new SemilinSetNdd(Genepi(solver, std::vector<int>(k,0), false), {std::vector<int>(k,0)}));
  return *SemilinSetNdd::elem_one;
}

std::string SemilinSetNdd::string() const {
  std::stringstream result;
  // auto calculated_offsets = this->offsets;
  // result << "calculated offsets:\t\t" << serialize_offsets(calculated_offsets) << std::endl;
  // auto cleaned_calculated_offsets = this->getIndependentOffsets(calculated_offsets);
  // result << "calculated offsets (clean):\t" << serialize_offsets(cleaned_calculated_offsets) << std::endl;
  result << std::endl;
  result << "size:\t" << this->set.getSize() << std::endl;
  // result << "automaton written:\t" << this->set.output("result", static_i++, "") << std::endl;
  return result.str();
}

const bool SemilinSetNdd::is_idempotent = true;
const bool SemilinSetNdd::is_commutative = true;
std::shared_ptr<SemilinSetNdd> SemilinSetNdd::elem_null;
std::shared_ptr<SemilinSetNdd> SemilinSetNdd::elem_one;
genepi_solver* SemilinSetNdd::solver;
genepi_plugin* SemilinSetNdd::plugin;
std::unordered_map<VarId, int> SemilinSetNdd::var_map;
int SemilinSetNdd::k = 0;
<|endoftext|>


<commit_before>// This file may be redistributed and modified only under the terms of
// the GNU Lesser General Public License (See COPYING for details).
// Copyright (C) 2007 Simon Goodall

#include <cassert>
#include <stdlib.h>
#include <errno.h>
#include <stdio.h>

#include <dirent.h>
#include <sys/stat.h>
#include <sys/types.h>

#include <map>
#include <string>
#include <list>
#include <algorithm>

#include "libwfut/platform.h"

#ifdef _WIN32
#include <direct.h>
#define mkdir(path,mode) _mkdir(path)
#define stat _stat
#endif

namespace WFUT {

FILE *os_create_tmpfile() {
#ifdef _WIN32
  char path[MAX_PATH];
  int ret = GetTempPathA(MAX_PATH, path);
  if(ret > MAX_PATH || (ret == 0))
    strcpy(path, ".\\");
  char filename[MAX_PATH];
  //only 3 prefix characters allowed
  ret = GetTempFileNameA(path,"wfu",0,filename);
  if(ret == 0)
    sprintf(filename,"%swfut%d.tmp",path,rand());
  //There is a special 'D' sign, which means temporary and delete on close
  //http://msdn.microsoft.com/en-us/library/yeby3zcb%28v=vs.71%29.aspx
  return fopen(filename, "w+bD");
#else
  return tmpfile();
#endif
}

void os_free_tmpfile(FILE *fp) {
  assert(fp != 0);
  // tmpfile cleans up after itself
  fclose(fp);
}

int os_mkdir(const std::string &dir) {
  return mkdir(dir.c_str(), 0700);
}

bool os_exists(const std::string &file) {
  struct stat info;
  if (::stat(file.c_str(), &info) == 0) {
    return true;
  } else {
    return false;
  }
}

int os_set_executable(const std::string &file) {
#ifdef _WIN32
  // nothing to do for windows
  return 0;
#else
 struct stat info;
  if (::stat(file.c_str(), &info) == 0) {

    mode_t mode = info.st_mode;
    mode |= S_IXGRP | S_IXOTH | S_IXUSR;

    // TODO: Should we restrict these permissions some more?
    //        E.g., only user?
    return chmod(file.c_str(), mode);
  }
  return 0;
#endif
}

int os_dir_walk(const std::string &path, const std::list<std::string> &excludes, std::list<std::string> &files) {
  DIR *d = opendir(path.c_str());
  if (d != 0) {
    struct dirent *dent = readdir(d);
    while (dent != 0) {
      const std::string d_name(dent->d_name);
      if (d_name != "." && d_name != ".." && std::find(excludes.begin(), excludes.end(), d_name) == excludes.end()) {

      if (dent->d_type == DT_DIR) {
        const std::string &pathname = path + "/" + d_name;
        os_dir_walk(pathname, excludes, files);
      } else if (dent->d_type == DT_REG) {
        const std::string &filename = path + "/" + d_name;
        files.push_back(filename);
      }}
      dent = readdir(d);
    }
  }

  return 0;
}

} /* namespace WFUT */
<commit_msg>Fix compile error with mingw32 dirent.<commit_after>// This file may be redistributed and modified only under the terms of
// the GNU Lesser General Public License (See COPYING for details).
// Copyright (C) 2007 Simon Goodall

#include <cassert>
#include <stdlib.h>
#include <errno.h>
#include <stdio.h>

//get MSVC header here: http://www.softagalleria.net/download/dirent/dirent-1.11.zip
#include <dirent.h>
#include <sys/stat.h>
#include <sys/types.h>

#include <map>
#include <string>
#include <list>
#include <algorithm>

#include "libwfut/platform.h"

#ifdef _WIN32
#include <direct.h>
#define mkdir(path,mode) _mkdir(path)
#define stat _stat
#endif

namespace WFUT {

FILE *os_create_tmpfile() {
#ifdef _MSC_VER
  char path[MAX_PATH];
  int ret = GetTempPathA(MAX_PATH, path);
  if(ret > MAX_PATH || (ret == 0)){
    strcpy(path, ".\\");
  }
  char filename[MAX_PATH];
  //only 3 prefix characters allowed
  ret = GetTempFileNameA(path,"wfu",0,filename);
  if(ret == 0){
    sprintf(filename,"%swfut%d.tmp",path,rand());
  }
  //There is a special 'D' sign, which means temporary and delete on close
  //http://msdn.microsoft.com/en-us/library/yeby3zcb%28v=vs.71%29.aspx
  return fopen(filename, "w+bD");
#else
  return tmpfile();
#endif
}

void os_free_tmpfile(FILE *fp) {
  assert(fp != 0);
  // tmpfile cleans up after itself
  fclose(fp);
}

int os_mkdir(const std::string &dir) {
  return mkdir(dir.c_str(), 0700);
}

bool os_exists(const std::string &file) {
  struct stat info;
  if (::stat(file.c_str(), &info) == 0) {
    return true;
  } else {
    return false;
  }
}

int os_set_executable(const std::string &file) {
#ifdef _WIN32
  // nothing to do for windows
  return 0;
#else
 struct stat info;
  if (::stat(file.c_str(), &info) == 0) {

    mode_t mode = info.st_mode;
    mode |= S_IXGRP | S_IXOTH | S_IXUSR;

    // TODO: Should we restrict these permissions some more?
    //        E.g., only user?
    return chmod(file.c_str(), mode);
  }
  return 0;
#endif
}

int os_dir_walk(const std::string &path, const std::list<std::string> &excludes, std::list<std::string> &files) {
  DIR *d = opendir(path.c_str());
  if (d != 0) {
    struct dirent *dent = readdir(d);
    while (dent != 0) {
      const std::string d_name(dent->d_name);
      if (d_name != "." && d_name != ".." && std::find(excludes.begin(), excludes.end(), d_name) == excludes.end()) {
      //NOTE: dent->d_type member is not existing in mingw32, so we need to use stat.
      struct stat filestat;
      stat(d_name.c_str(), &filestat);
      if (S_ISDIR(filestat.st_mode)) {
        const std::string &pathname = path + "/" + d_name;
        os_dir_walk(pathname, excludes, files);
      } else if (S_ISREG(filestat.st_mode)) {
        const std::string &filename = path + "/" + d_name;
        files.push_back(filename);
      }}
      dent = readdir(d);
    }
  }

  return 0;
}

} /* namespace WFUT */<|endoftext|>


<commit_before>// -*- mode:C++; tab-width:4; c-basic-offset:2; indent-tabs-mode:t -*-
/*
 * Ceph - scalable distributed file system
 *
 * Copyright (C) 2004-2006 Sage Weil <sage@newdream.net>
 *
 * This is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License version 2.1, as published by the Free Software
 * Foundation.  See file COPYING.
 *
 */

#include "LogStream.h"
#include "MDS.h"
#include "LogEvent.h"

#include "osdc/Filer.h"

#include "common/Logger.h"

#include "events/EString.h"
#include "events/EInodeUpdate.h"
#include "events/EDirUpdate.h"
#include "events/EUnlink.h"
#include "events/EAlloc.h"

#include <iostream>
using namespace std;

#include "config.h"
#undef dout
#define  dout(l)    if (l<=g_conf.debug || l<=g_conf.debug_mds_log) cout << "mds" << mds->get_nodeid() << ".logstream "

LogStream::LogStream(MDS *mds, Filer *filer, inodeno_t log_ino, Logger *l)
{
  this->mds = mds;
  this->filer = filer;
  this->logger = l;

  // inode
  memset(&log_inode, 0, sizeof(log_inode));
  log_inode.ino = log_ino;
  log_inode.layout = g_OSD_MDLogLayout;

  if (g_conf.mds_local_osd) {
	log_inode.layout.object_layout = OBJECT_LAYOUT_STARTOSD;
	log_inode.layout.osd = mds->get_nodeid() + 10000;   // hack
  }

  // wr
  sync_pos = flush_pos = append_pos = 0;
  autoflush = true;

  // rd
  read_pos = 0;
  reading = false;
}

// ----------------------------
// writing

class C_LS_Append : public Context {
  LogStream *ls;
  off_t off;
public:
  C_LS_Append(LogStream *ls, off_t off) {
	this->ls = ls;
	this->off = off;
  }
  void finish(int r) {
	ls->_append_2(off);
  }
};

off_t LogStream::append(LogEvent *e)
{
  // serialize
  bufferlist bl;
  e->encode(bl);

  size_t elen = bl.length();

  // append
  dout(15) << "append event type " << e->get_type() << " size " << elen << " at log offset " << append_pos << endl;

  off_t off = append_pos;
  append_pos += elen;

  //dout(15) << "write buf was " << write_buf.length() << " bl " << write_buf << endl;
  write_buf.claim_append(bl);
  //dout(15) << "write buf now " << write_buf.length() << " bl " << write_buf << endl;

  return off;
}

void LogStream::_append_2(off_t off)
{
  dout(15) << g_clock.now() << " sync_pos now " << off << " skew " << off % g_conf.mds_log_pad_entry << endl;
  sync_pos = off;

  // discard written bufferlist
  assert(writing_buffers.count(off) == 1);
  delete writing_buffers[off];
  writing_buffers.erase(off);

  utime_t now = g_clock.now();
  now -= writing_latency[off];
  writing_latency.erase(off);
  logger->finc("lsum", (double)now);
  logger->inc("lnum", 1);

  // wake up waiters
  map< off_t, list<Context*> >::iterator it = waiting_for_sync.begin();
  while (it != waiting_for_sync.end()) {
	if (it->first > sync_pos) break;

	// wake them up!
	dout(15) << it->second.size() << " waiters at offset " << it->first << " <= " << sync_pos << endl;
	for (list<Context*>::iterator cit = it->second.begin();
		 cit != it->second.end();
		 cit++)
	  mds->finished_queue.push_back(*cit);

	// continue
	waiting_for_sync.erase(it++);
  }
}

void LogStream::wait_for_sync(Context *c, off_t off)
{
  if (off == 0) off = append_pos;
  assert(off > sync_pos);

  dout(15) << "sync " << c << " waiting for " << off << "  (sync_pos currently " << sync_pos << ")" << endl;
  waiting_for_sync[off].push_back(c);

  // initiate flush now?  (since we have a waiter...)
  if (autoflush) flush();
}

void LogStream::flush()
{
  // write to disk
  if (write_buf.length()) {
	dout(15) << "flush flush_pos " << flush_pos << " < append_pos " << append_pos << ", writing " << write_buf.length() << " bytes" << endl;

	assert(write_buf.length() == append_pos - flush_pos);

	// tuck writing buffer away until write finishes
	writing_buffers[append_pos] = new bufferlist;
	writing_buffers[append_pos]->claim(write_buf);

	writing_latency[append_pos] = g_clock.now();

	// write it
	mds->filer->write(log_inode,
					  flush_pos, writing_buffers[append_pos]->length(),
					  *writing_buffers[append_pos],
					  0,
					  new C_LS_Append(this, append_pos), // on ack
					  NULL);                             // on safe

	flush_pos = append_pos;
  } else {
	dout(15) << "flush flush_pos " << flush_pos << " == append_pos " << append_pos << ", nothing to do" << endl;
  }

}

// -------------------------------------------------
// reading

LogEvent *LogStream::get_next_event()
{
  if (read_buf.length() < 2*sizeof(__uint32_t))
	return 0;

  // parse type, length
  int off = 0;
  __uint32_t type, length;
  read_buf.copy(off, sizeof(__uint32_t), (char*)&type);
  off += sizeof(__uint32_t);
  read_buf.copy(off, sizeof(__uint32_t), (char*)&length);
  off += sizeof(__uint32_t);

  dout(15) << "getting next event from " << read_pos << ", type " << type << ", size " << length << endl;

  assert(type > 0);

  if (read_buf.length() < off + length)
	return 0;

  // create event
  LogEvent *le;
  switch (type) {
  case EVENT_STRING:  // string
	le = new EString();
	break;

  case EVENT_INODEUPDATE:
	le = new EInodeUpdate();
	break;

  case EVENT_DIRUPDATE:
	le = new EDirUpdate();
	break;

  case EVENT_UNLINK:
	le = new EUnlink();
	break;

  case EVENT_ALLOC:
	le = new EAlloc();
	break;

  default:
	dout(1) << "uh oh, unknown event type " << type << endl;
	assert(0);
  }

  // decode
  le->decode_payload(read_buf, off);
  off = sizeof(type) + sizeof(length) + length;  // advance past any padding that wasn't decoded..

  // discard front of read_buf
  read_pos += off;
  read_buf.splice(0, off);

  dout(15) << "get_next_event got event, read_pos now " << read_pos << " (append_pos is " << append_pos << ")" << endl;

  // ok!
  return le;
}

class C_LS_ReadChunk : public Context {
public:
  bufferlist bl;
  LogStream *ls;

  C_LS_ReadChunk(LogStream *ls) {
	this->ls = ls;
  }
  void finish(int result) {
	ls->_did_read(bl);
  }
};

void LogStream::wait_for_next_event(Context *c)
{
  // add waiter
  if (c) waiting_for_read.push_back(c);

  // issue read
  off_t tail = read_pos + read_buf.length();
  size_t size = g_conf.mds_log_read_inc;
  if (tail + (off_t)size > sync_pos) {
	size = sync_pos - tail;
	dout(15) << "wait_for_next_event ugh.. read_pos is " << read_pos << ", tail is " << tail << ", sync_pos only " << sync_pos << ", flush_pos " << flush_pos << ", append_pos " << append_pos << endl;

	if (size == 0) {
	  //	assert(size > 0);   // bleh, wait for sync, etc.
	  // just do it.  communication is ordered, right?   FIXME SOMEDAY this is totally gross blech
	  //size = flush_pos - tail;
	  // read tiny bit, kill some time
	  assert(flush_pos > sync_pos);
	  size = 1;
	}
  }

  dout(15) << "wait_for_next_event reading from pos " << tail << " len " << size << endl;
  C_LS_ReadChunk *readc = new C_LS_ReadChunk(this);
  mds->filer->read(log_inode,
				   tail, g_conf.mds_log_read_inc,
				   &readc->bl,
				   readc);
}

void LogStream::_did_read(bufferlist& blist)
{
  dout(15) << "_did_read got " << blist.length() << " bytes at offset " << (read_pos + read_buf.length()) << endl;
  read_buf.claim_append(blist);

  list<Context*> finished;
  finished.splice(finished.begin(), waiting_for_read);
  finish_contexts(finished, 0);
}

<commit_msg>wait for sync before read.  untested.<commit_after>// -*- mode:C++; tab-width:4; c-basic-offset:2; indent-tabs-mode:t -*-
/*
 * Ceph - scalable distributed file system
 *
 * Copyright (C) 2004-2006 Sage Weil <sage@newdream.net>
 *
 * This is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License version 2.1, as published by the Free Software
 * Foundation.  See file COPYING.
 *
 */

#include "LogStream.h"
#include "MDS.h"
#include "LogEvent.h"

#include "osdc/Filer.h"

#include "common/Logger.h"

#include "events/EString.h"
#include "events/EInodeUpdate.h"
#include "events/EDirUpdate.h"
#include "events/EUnlink.h"
#include "events/EAlloc.h"

#include <iostream>
using namespace std;

#include "config.h"
#undef dout
#define  dout(l)    if (l<=g_conf.debug || l<=g_conf.debug_mds_log) cout << "mds" << mds->get_nodeid() << ".logstream "

LogStream::LogStream(MDS *mds, Filer *filer, inodeno_t log_ino, Logger *l)
{
  this->mds = mds;
  this->filer = filer;
  this->logger = l;

  // inode
  memset(&log_inode, 0, sizeof(log_inode));
  log_inode.ino = log_ino;
  log_inode.layout = g_OSD_MDLogLayout;

  if (g_conf.mds_local_osd) {
	log_inode.layout.object_layout = OBJECT_LAYOUT_STARTOSD;
	log_inode.layout.osd = mds->get_nodeid() + 10000;   // hack
  }

  // wr
  sync_pos = flush_pos = append_pos = 0;
  autoflush = true;

  // rd
  read_pos = 0;
  reading = false;
}

// ----------------------------
// writing

class C_LS_Append : public Context {
  LogStream *ls;
  off_t off;
public:
  C_LS_Append(LogStream *ls, off_t off) {
	this->ls = ls;
	this->off = off;
  }
  void finish(int r) {
	ls->_append_2(off);
  }
};

off_t LogStream::append(LogEvent *e)
{
  // serialize
  bufferlist bl;
  e->encode(bl);

  size_t elen = bl.length();

  // append
  dout(15) << "append event type " << e->get_type() << " size " << elen << " at log offset " << append_pos << endl;

  off_t off = append_pos;
  append_pos += elen;

  //dout(15) << "write buf was " << write_buf.length() << " bl " << write_buf << endl;
  write_buf.claim_append(bl);
  //dout(15) << "write buf now " << write_buf.length() << " bl " << write_buf << endl;

  return off;
}

void LogStream::_append_2(off_t off)
{
  dout(15) << g_clock.now() << " sync_pos now " << off << " skew " << off % g_conf.mds_log_pad_entry << endl;
  sync_pos = off;

  // discard written bufferlist
  assert(writing_buffers.count(off) == 1);
  delete writing_buffers[off];
  writing_buffers.erase(off);

  utime_t now = g_clock.now();
  now -= writing_latency[off];
  writing_latency.erase(off);
  logger->finc("lsum", (double)now);
  logger->inc("lnum", 1);

  // wake up waiters
  map< off_t, list<Context*> >::iterator it = waiting_for_sync.begin();
  while (it != waiting_for_sync.end()) {
	if (it->first > sync_pos) break;

	// wake them up!
	dout(15) << it->second.size() << " waiters at offset " << it->first << " <= " << sync_pos << endl;
	for (list<Context*>::iterator cit = it->second.begin();
		 cit != it->second.end();
		 cit++)
	  mds->finished_queue.push_back(*cit);

	// continue
	waiting_for_sync.erase(it++);
  }
}

void LogStream::wait_for_sync(Context *c, off_t off)
{
  if (off == 0) off = append_pos;
  assert(off > sync_pos);

  dout(15) << "sync " << c << " waiting for " << off << "  (sync_pos currently " << sync_pos << ")" << endl;
  waiting_for_sync[off].push_back(c);

  // initiate flush now?  (since we have a waiter...)
  if (autoflush) flush();
}

void LogStream::flush()
{
  // write to disk
  if (write_buf.length()) {
	dout(15) << "flush flush_pos " << flush_pos << " < append_pos " << append_pos << ", writing " << write_buf.length() << " bytes" << endl;

	assert(write_buf.length() == append_pos - flush_pos);

	// tuck writing buffer away until write finishes
	writing_buffers[append_pos] = new bufferlist;
	writing_buffers[append_pos]->claim(write_buf);

	writing_latency[append_pos] = g_clock.now();

	// write it
	mds->filer->write(log_inode,
					  flush_pos, writing_buffers[append_pos]->length(),
					  *writing_buffers[append_pos],
					  0,
					  new C_LS_Append(this, append_pos), // on ack
					  NULL);                             // on safe

	flush_pos = append_pos;
  } else {
	dout(15) << "flush flush_pos " << flush_pos << " == append_pos " << append_pos << ", nothing to do" << endl;
  }

}

// -------------------------------------------------
// reading

LogEvent *LogStream::get_next_event()
{
  if (read_buf.length() < 2*sizeof(__uint32_t))
	return 0;

  // parse type, length
  int off = 0;
  __uint32_t type, length;
  read_buf.copy(off, sizeof(__uint32_t), (char*)&type);
  off += sizeof(__uint32_t);
  read_buf.copy(off, sizeof(__uint32_t), (char*)&length);
  off += sizeof(__uint32_t);

  dout(15) << "getting next event from " << read_pos << ", type " << type << ", size " << length << endl;

  assert(type > 0);

  if (read_buf.length() < off + length)
	return 0;

  // create event
  LogEvent *le;
  switch (type) {
  case EVENT_STRING:  // string
	le = new EString();
	break;

  case EVENT_INODEUPDATE:
	le = new EInodeUpdate();
	break;

  case EVENT_DIRUPDATE:
	le = new EDirUpdate();
	break;

  case EVENT_UNLINK:
	le = new EUnlink();
	break;

  case EVENT_ALLOC:
	le = new EAlloc();
	break;

  default:
	dout(1) << "uh oh, unknown event type " << type << endl;
	assert(0);
  }

  // decode
  le->decode_payload(read_buf, off);
  off = sizeof(type) + sizeof(length) + length;  // advance past any padding that wasn't decoded..

  // discard front of read_buf
  read_pos += off;
  read_buf.splice(0, off);

  dout(15) << "get_next_event got event, read_pos now " << read_pos << " (append_pos is " << append_pos << ")" << endl;

  // ok!
  return le;
}

class C_LS_ReadAfterSync : public Context {
public:
  LogStream *ls;
  Context *waitfornext;
  C_LS_ReadAfterSync(LogStream *l, Context *c) : ls(l), waitfornext(c) {}
  void finish(int) {
	ls->wait_for_next_event(waitfornext);
  }
};

class C_LS_ReadChunk : public Context {
public:
  bufferlist bl;
  LogStream *ls;

  C_LS_ReadChunk(LogStream *ls) {
	this->ls = ls;
  }
  void finish(int result) {
	ls->_did_read(bl);
  }
};

void LogStream::wait_for_next_event(Context *c)
{
  if (read_pos == sync_pos) {
	dout(-15) << "waiting for sync before initiating read at " << read_pos << endl;
	wait_for_sync(new C_LS_ReadAfterSync(this, c));
	return;
  }

  // add waiter
  if (c) waiting_for_read.push_back(c);

  // issue read
  off_t tail = read_pos + read_buf.length();
  size_t size = g_conf.mds_log_read_inc;
  if (tail + (off_t)size > sync_pos) {
	size = sync_pos - tail;
	dout(15) << "wait_for_next_event ugh.. read_pos is " << read_pos << ", tail is " << tail << ", sync_pos only " << sync_pos << ", flush_pos " << flush_pos << ", append_pos " << append_pos << endl;

	if (size == 0) {
	  //	assert(size > 0);   // bleh, wait for sync, etc.
	  // just do it.  communication is ordered, right?   FIXME SOMEDAY this is totally gross blech
	  //size = flush_pos - tail;
	  // read tiny bit, kill some time
	  assert(flush_pos > sync_pos);
	  size = 1;
	}
  }

  dout(15) << "wait_for_next_event reading from pos " << tail << " len " << size << endl;
  C_LS_ReadChunk *readc = new C_LS_ReadChunk(this);
  mds->filer->read(log_inode,
				   tail, g_conf.mds_log_read_inc,
				   &readc->bl,
				   readc);
}

void LogStream::_did_read(bufferlist& blist)
{
  dout(15) << "_did_read got " << blist.length() << " bytes at offset " << (read_pos + read_buf.length()) << endl;
  read_buf.claim_append(blist);

  list<Context*> finished;
  finished.splice(finished.begin(), waiting_for_read);
  finish_contexts(finished, 0);
}

<|endoftext|>


<commit_before>#include "SysControl.h"
#include "ControlsApp.h"
#include "ControlsXPad360.h"

//ƽַ̨
#ifdef _WIN32
#pragma execution_character_set("utf-8")
#endif

using namespace MathLib;

class CSysControlLocal : public CSysControl
{
public:
	CSysControlLocal();
    virtual ~CSysControlLocal();

	virtual void Init();	//ɫҪʼɫҪʼ
	virtual void Update(float ifps);
	virtual void Shutdown();

	virtual int GetState(int state);		//״̬״̬״̬ӵ״̬ȵȡ
	virtual int ClearState(int state);

	virtual float GetMouseDX();
	virtual float GetMouseDY();

	virtual void SetMouseGrab(int g);		//Ƿʾ
	virtual int GetMouseGrab();			//ȡ״̬ʾػǲʾ״̬

	virtual void SetControlMode(ControlMode mode);			//ģʽ
	virtual ControlMode GetControlMode() const;				//ȡģʽ

private:
	void Update_Mouse(float ifps);
	void Update_Keyboard(float ifps);
	void Update_XPad360(float ifps);

	CControlsApp        *m_pControlsApp;		//Ϸƶ
	CControlsXPad360    *m_pControlsXPad360;

}<commit_msg>Signed-off-by: mrlitong <litongtongxue@gmail.com><commit_after>#include "SysControl.h"
#include "ControlsApp.h"
#include "ControlsXPad360.h"

//ƽַ̨
#ifdef _WIN32
#pragma execution_character_set("utf-8")
#endif

using namespace MathLib;

class CSysControlLocal : public CSysControl
{
public:
	CSysControlLocal();
    virtual ~CSysControlLocal();

	virtual void Init();	//ɫҪʼɫҪʼ
	virtual void Update(float ifps);
	virtual void Shutdown();

	virtual int GetState(int state);		//״̬״̬״̬ӵ״̬ȵȡ
	virtual int ClearState(int state);

	virtual float GetMouseDX();
	virtual float GetMouseDY();

	virtual void SetMouseGrab(int g);		//Ƿʾ
	virtual int GetMouseGrab();			//ȡ״̬ʾػǲʾ״̬

	virtual void SetControlMode(ControlMode mode);			//ģʽ
	virtual ControlMode GetControlMode() const;				//ȡģʽ

private:
	void Update_Mouse(float ifps);
	void Update_Keyboard(float ifps);
	void Update_XPad360(float ifps);

	CControlsApp        *m_pControlsApp;		//Ϸƶ
	CControlsXPad360    *m_pControlsXPad360;

	int                 m_nOldMouseX;			//һX
	int                 m_nOldMouseY;			//һY

}<|endoftext|>


<commit_before>// util.cpp
// Author: Allen Porter <allen@thebends.org>
#include "util.h"

#include <ythread/callback-inl.h>
#include <arpa/inet.h>
#include <netinet/in.h>
#include <sys/types.h>
#include <sys/socket.h>
#include <iostream>
#include <vector>
#include <yhttp/util.h>

using namespace std;

namespace btunnel {

bool GetHostPort(const string& host_port, string* host, uint16_t* port) {
  size_t c = host_port.find(":");
  if (c == string::npos) {
    return false;
  }
  *host = host_port.substr(0, c);
  *port = (uint16_t)strtoll(host_port.substr(c + 1).c_str(), NULL, 10);
  return (!host->empty() || *port > 0);
}

bool GetMap(const string& str, map<string, string>* records) {
  records->clear();
  vector<string> parts;
  yhttpserver::Split(str, ',', &parts);
  int len = parts.size();
  for (int i = 0; i < len; i++) {
    vector<string> kv;
    yhttpserver::Split(parts[i], '=', &kv);
    if (kv.size() != 2) {
      records->clear();
      return false;
    }
    (*records)[kv[0]] = kv[1];
  }
  return true;
}

}  // namespace btunnel
<commit_msg>Fix Split() breakage.<commit_after>// util.cpp
// Author: Allen Porter <allen@thebends.org>
#include "util.h"

#include <ythread/callback-inl.h>
#include <arpa/inet.h>
#include <netinet/in.h>
#include <sys/types.h>
#include <sys/socket.h>
#include <iostream>
#include <vector>
#include <yhttp/util.h>

using namespace std;

namespace btunnel {

void Split(const std::string& message, char c,
           std::vector<std::string>* parts) {
  parts->clear();
  const char *last = message.c_str();
  const char *end = last + message.size();
  const char *cur = message.c_str();
  while (cur < end) {
    if (*cur == c) {
      if ((cur - last) != 0) {
        parts->push_back(std::string(last, cur - last));
      }
      last = cur + 1;
    }
    cur++;
  }
  if (last != end) {
    parts->push_back(std::string(last, cur - last));
  }
}

bool GetHostPort(const string& host_port, string* host, uint16_t* port) {
  size_t c = host_port.find(":");
  if (c == string::npos) {
    return false;
  }
  *host = host_port.substr(0, c);
  *port = (uint16_t)strtoll(host_port.substr(c + 1).c_str(), NULL, 10);
  return (!host->empty() || *port > 0);
}

bool GetMap(const string& str, map<string, string>* records) {
  records->clear();
  vector<string> parts;
  Split(str, ',', &parts);
  int len = parts.size();
  for (int i = 0; i < len; i++) {
    vector<string> kv;
    Split(parts[i], '=', &kv);
    if (kv.size() != 2) {
      records->clear();
      return false;
    }
    (*records)[kv[0]] = kv[1];
  }
  return true;
}

}  // namespace btunnel
<|endoftext|>


<commit_before>#include "util/file_piece.hh"

#include "util/exception.hh"
#include "util/file.hh"
#include "util/mmap.hh"
#ifdef WIN32
#include <io.h>
#endif // WIN32

#include <iostream>
#include <string>
#include <limits>

#include <assert.h>
#include <ctype.h>
#include <fcntl.h>
#include <stdlib.h>
#include <sys/types.h>
#include <sys/stat.h>

#ifdef HAVE_ZLIB
#include <zlib.h>
#endif

namespace util {

ParseNumberException::ParseNumberException(StringPiece value) throw() {
  *this << "Could not parse \"" << value << "\" into a number";
}

GZException::GZException(void *file) {
#ifdef HAVE_ZLIB
  int num;
  *this << gzerror( (gzFile_s*) file, &num) << " from zlib";
#endif // HAVE_ZLIB
}

// Sigh this is the only way I could come up with to do a _const_ bool.  It has ' ', '\f', '\n', '\r', '\t', and '\v' (same as isspace on C locale).
const bool kSpaces[256] = {0,0,0,0,0,0,0,0,0,1,1,1,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0};

FilePiece::FilePiece(const char *name, std::ostream *show_progress, std::size_t min_buffer) :
  file_(OpenReadOrThrow(name)), total_size_(SizeFile(file_.get())), page_(SizePage()),
  progress_(total_size_ == kBadSize ? NULL : show_progress, std::string("Reading ") + name, total_size_) {
  Initialize(name, show_progress, min_buffer);
}

FilePiece::FilePiece(int fd, const char *name, std::ostream *show_progress, std::size_t min_buffer)  :
  file_(fd), total_size_(SizeFile(file_.get())), page_(SizePage()),
  progress_(total_size_ == kBadSize ? NULL : show_progress, std::string("Reading ") + name, total_size_) {
  Initialize(name, show_progress, min_buffer);
}

FilePiece::~FilePiece() {
#ifdef HAVE_ZLIB
  if (gz_file_) {
    // zlib took ownership
    file_.release();
    int ret;
    if (Z_OK != (ret = gzclose( (gzFile_s*) gz_file_))) {
      std::cerr << "could not close file " << file_name_ << " using zlib" << std::endl;
      abort();
    }
  }
#endif
}

StringPiece FilePiece::ReadLine(char delim) {
  std::size_t skip = 0;
  while (true) {
    for (const char *i = position_ + skip; i < position_end_; ++i) {
      if (*i == delim) {
        StringPiece ret(position_, i - position_);
        position_ = i + 1;
        return ret;
      }
    }
    if (at_end_) {
      if (position_ == position_end_) Shift();
      return Consume(position_end_);
    }
    skip = position_end_ - position_;
    Shift();
  }
}

float FilePiece::ReadFloat() {
  return ReadNumber<float>();
}
double FilePiece::ReadDouble() {
  return ReadNumber<double>();
}
long int FilePiece::ReadLong() {
  return ReadNumber<long int>();
}
unsigned long int FilePiece::ReadULong() {
  return ReadNumber<unsigned long int>();
}

void FilePiece::Initialize(const char *name, std::ostream *show_progress, std::size_t min_buffer)  {
#ifdef HAVE_ZLIB
  gz_file_ = NULL;
#endif
  file_name_ = name;

  default_map_size_ = page_ * std::max<std::size_t>((min_buffer / page_ + 1), 2);
  position_ = NULL;
  position_end_ = NULL;
  mapped_offset_ = 0;
  at_end_ = false;

  if (total_size_ == kBadSize) {
    // So the assertion passes.
    fallback_to_read_ = false;
    if (show_progress)
      *show_progress << "File " << name << " isn't normal.  Using slower read() instead of mmap().  No progress bar." << std::endl;
    TransitionToRead();
  } else {
    fallback_to_read_ = false;
  }
  Shift();
  // gzip detect.
  if ((position_end_ - position_) > 2 && *position_ == 0x1f && static_cast<unsigned char>(*(position_ + 1)) == 0x8b) {
#ifndef HAVE_ZLIB
    UTIL_THROW(GZException, "Looks like a gzip file but support was not compiled in.");
#endif
    if (!fallback_to_read_) {
      at_end_ = false;
      TransitionToRead();
    }
  }
}

namespace {
void ParseNumber(const char *begin, char *&end, float &out) {
#if defined(sun) || defined(WIN32)
  out = static_cast<float>(strtod(begin, &end));
#else
  out = strtof(begin, &end);
#endif
}
void ParseNumber(const char *begin, char *&end, double &out) {
  out = strtod(begin, &end);
}
void ParseNumber(const char *begin, char *&end, long int &out) {
  out = strtol(begin, &end, 10);
}
void ParseNumber(const char *begin, char *&end, unsigned long int &out) {
  out = strtoul(begin, &end, 10);
}
} // namespace

template <class T> T FilePiece::ReadNumber() {
  SkipSpaces();
  while (last_space_ < position_) {
    if (at_end_) {
      // Hallucinate a null off the end of the file.
      std::string buffer(position_, position_end_);
      char *end;
      T ret;
      ParseNumber(buffer.c_str(), end, ret);
      if (buffer.c_str() == end) throw ParseNumberException(buffer);
      position_ += end - buffer.c_str();
      return ret;
    }
    Shift();
  }
  char *end;
  T ret;
  ParseNumber(position_, end, ret);
  if (end == position_) throw ParseNumberException(ReadDelimited());
  position_ = end;
  return ret;
}

const char *FilePiece::FindDelimiterOrEOF(const bool *delim)  {
  std::size_t skip = 0;
  while (true) {
    for (const char *i = position_ + skip; i < position_end_; ++i) {
      if (delim[static_cast<unsigned char>(*i)]) return i;
    }
    if (at_end_) {
      if (position_ == position_end_) Shift();
      return position_end_;
    }
    skip = position_end_ - position_;
    Shift();
  }
}

void FilePiece::Shift() {
  if (at_end_) {
    progress_.Finished();
    throw EndOfFileException();
  }
  uint64_t desired_begin = position_ - data_.begin() + mapped_offset_;

  if (!fallback_to_read_) MMapShift(desired_begin);
  // Notice an mmap failure might set the fallback.
  if (fallback_to_read_) ReadShift();

  for (last_space_ = position_end_ - 1; last_space_ >= position_; --last_space_) {
    if (isspace(*last_space_))  break;
  }
}

void FilePiece::MMapShift(uint64_t desired_begin) {
  // Use mmap.
  uint64_t ignore = desired_begin % page_;
  // Duplicate request for Shift means give more data.
  if (position_ == data_.begin() + ignore) {
    default_map_size_ *= 2;
  }
  // Local version so that in case of failure it doesn't overwrite the class variable.
  uint64_t mapped_offset = desired_begin - ignore;

  uint64_t mapped_size;
  if (default_map_size_ >= static_cast<std::size_t>(total_size_ - mapped_offset)) {
    at_end_ = true;
    mapped_size = total_size_ - mapped_offset;
  } else {
    mapped_size = default_map_size_;
  }

  // Forcibly clear the existing mmap first.
  data_.reset();
  try {
    MapRead(POPULATE_OR_LAZY, *file_, mapped_offset, mapped_size, data_);
  } catch (const util::ErrnoException &e) {
    if (desired_begin) {
      SeekOrThrow(*file_, desired_begin);
    }
    // The mmap was scheduled to end the file, but now we're going to read it.
    at_end_ = false;
    TransitionToRead();
    return;
  }
  mapped_offset_ = mapped_offset;
  position_ = data_.begin() + ignore;
  position_end_ = data_.begin() + mapped_size;

  progress_.Set(desired_begin);
}

void FilePiece::TransitionToRead() {
  assert(!fallback_to_read_);
  fallback_to_read_ = true;
  data_.reset();
  data_.reset(malloc(default_map_size_), default_map_size_, scoped_memory::MALLOC_ALLOCATED);
  UTIL_THROW_IF(!data_.get(), ErrnoException, "malloc failed for " << default_map_size_);
  position_ = data_.begin();
  position_end_ = position_;

#ifdef HAVE_ZLIB
  assert(!gz_file_);
  gz_file_ = gzdopen(file_.get(), "r");
  UTIL_THROW_IF(!gz_file_, GZException, "zlib failed to open " << file_name_);
#endif
}

#ifdef WIN32
typedef int ssize_t;
#endif

void FilePiece::ReadShift() {
  assert(fallback_to_read_);
  // Bytes [data_.begin(), position_) have been consumed.
  // Bytes [position_, position_end_) have been read into the buffer.

  // Start at the beginning of the buffer if there's nothing useful in it.
  if (position_ == position_end_) {
    mapped_offset_ += (position_end_ - data_.begin());
    position_ = data_.begin();
    position_end_ = position_;
  }

  std::size_t already_read = position_end_ - data_.begin();

  if (already_read == default_map_size_) {
    if (position_ == data_.begin()) {
      // Buffer too small.
      std::size_t valid_length = position_end_ - position_;
      default_map_size_ *= 2;
      data_.call_realloc(default_map_size_);
      UTIL_THROW_IF(!data_.get(), ErrnoException, "realloc failed for " << default_map_size_);
      position_ = data_.begin();
      position_end_ = position_ + valid_length;
    } else {
      size_t moving = position_end_ - position_;
      memmove(data_.get(), position_, moving);
      position_ = data_.begin();
      position_end_ = position_ + moving;
      already_read = moving;
    }
  }

  ssize_t read_return;
#ifdef HAVE_ZLIB
  read_return = gzread( (gzFile_s*) gz_file_, static_cast<char*>(data_.get()) + already_read, default_map_size_ - already_read);
  if (read_return == -1) throw GZException(gz_file_);
  if (total_size_ != kBadSize) {
    // Just get the position, don't actually seek.  Apparently this is how you do it. . .
    off_t ret = lseek(file_.get(), 0, SEEK_CUR);
    if (ret != -1) progress_.Set(ret);
  }
#else
  read_return = read(file_.get(), static_cast<char*>(data_.get()) + already_read, default_map_size_ - already_read);
  UTIL_THROW_IF(read_return == -1, ErrnoException, "read failed");
  progress_.Set(mapped_offset_);
#endif
  if (read_return == 0) {
    at_end_ = true;
  }
  position_end_ += read_return;
}

} // namespace util
<commit_msg>zlib changes<commit_after>#include "util/file_piece.hh"

#include "util/exception.hh"
#include "util/file.hh"
#include "util/mmap.hh"
#ifdef WIN32
#include <io.h>
#endif // WIN32

#include <iostream>
#include <string>
#include <limits>

#include <assert.h>
#include <ctype.h>
#include <fcntl.h>
#include <stdlib.h>
#include <sys/types.h>
#include <sys/stat.h>

#ifdef HAVE_ZLIB
#include <zlib.h>
#endif

namespace util {

ParseNumberException::ParseNumberException(StringPiece value) throw() {
  *this << "Could not parse \"" << value << "\" into a number";
}

GZException::GZException(void *file) {
#ifdef HAVE_ZLIB
  int num;
  *this << gzerror( (gzFile) file, &num) << " from zlib";
#endif // HAVE_ZLIB
}

// Sigh this is the only way I could come up with to do a _const_ bool.  It has ' ', '\f', '\n', '\r', '\t', and '\v' (same as isspace on C locale).
const bool kSpaces[256] = {0,0,0,0,0,0,0,0,0,1,1,1,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0};

FilePiece::FilePiece(const char *name, std::ostream *show_progress, std::size_t min_buffer) :
  file_(OpenReadOrThrow(name)), total_size_(SizeFile(file_.get())), page_(SizePage()),
  progress_(total_size_ == kBadSize ? NULL : show_progress, std::string("Reading ") + name, total_size_) {
  Initialize(name, show_progress, min_buffer);
}

FilePiece::FilePiece(int fd, const char *name, std::ostream *show_progress, std::size_t min_buffer)  :
  file_(fd), total_size_(SizeFile(file_.get())), page_(SizePage()),
  progress_(total_size_ == kBadSize ? NULL : show_progress, std::string("Reading ") + name, total_size_) {
  Initialize(name, show_progress, min_buffer);
}

FilePiece::~FilePiece() {
#ifdef HAVE_ZLIB
  if (gz_file_) {
    // zlib took ownership
    file_.release();
    int ret;
    if (Z_OK != (ret = gzclose( (gzFile) gz_file_))) {
      std::cerr << "could not close file " << file_name_ << " using zlib" << std::endl;
      abort();
    }
  }
#endif
}

StringPiece FilePiece::ReadLine(char delim) {
  std::size_t skip = 0;
  while (true) {
    for (const char *i = position_ + skip; i < position_end_; ++i) {
      if (*i == delim) {
        StringPiece ret(position_, i - position_);
        position_ = i + 1;
        return ret;
      }
    }
    if (at_end_) {
      if (position_ == position_end_) Shift();
      return Consume(position_end_);
    }
    skip = position_end_ - position_;
    Shift();
  }
}

float FilePiece::ReadFloat() {
  return ReadNumber<float>();
}
double FilePiece::ReadDouble() {
  return ReadNumber<double>();
}
long int FilePiece::ReadLong() {
  return ReadNumber<long int>();
}
unsigned long int FilePiece::ReadULong() {
  return ReadNumber<unsigned long int>();
}

void FilePiece::Initialize(const char *name, std::ostream *show_progress, std::size_t min_buffer)  {
#ifdef HAVE_ZLIB
  gz_file_ = NULL;
#endif
  file_name_ = name;

  default_map_size_ = page_ * std::max<std::size_t>((min_buffer / page_ + 1), 2);
  position_ = NULL;
  position_end_ = NULL;
  mapped_offset_ = 0;
  at_end_ = false;

  if (total_size_ == kBadSize) {
    // So the assertion passes.
    fallback_to_read_ = false;
    if (show_progress)
      *show_progress << "File " << name << " isn't normal.  Using slower read() instead of mmap().  No progress bar." << std::endl;
    TransitionToRead();
  } else {
    fallback_to_read_ = false;
  }
  Shift();
  // gzip detect.
  if ((position_end_ - position_) > 2 && *position_ == 0x1f && static_cast<unsigned char>(*(position_ + 1)) == 0x8b) {
#ifndef HAVE_ZLIB
    UTIL_THROW(GZException, "Looks like a gzip file but support was not compiled in.");
#endif
    if (!fallback_to_read_) {
      at_end_ = false;
      TransitionToRead();
    }
  }
}

namespace {
void ParseNumber(const char *begin, char *&end, float &out) {
#if defined(sun) || defined(WIN32)
  out = static_cast<float>(strtod(begin, &end));
#else
  out = strtof(begin, &end);
#endif
}
void ParseNumber(const char *begin, char *&end, double &out) {
  out = strtod(begin, &end);
}
void ParseNumber(const char *begin, char *&end, long int &out) {
  out = strtol(begin, &end, 10);
}
void ParseNumber(const char *begin, char *&end, unsigned long int &out) {
  out = strtoul(begin, &end, 10);
}
} // namespace

template <class T> T FilePiece::ReadNumber() {
  SkipSpaces();
  while (last_space_ < position_) {
    if (at_end_) {
      // Hallucinate a null off the end of the file.
      std::string buffer(position_, position_end_);
      char *end;
      T ret;
      ParseNumber(buffer.c_str(), end, ret);
      if (buffer.c_str() == end) throw ParseNumberException(buffer);
      position_ += end - buffer.c_str();
      return ret;
    }
    Shift();
  }
  char *end;
  T ret;
  ParseNumber(position_, end, ret);
  if (end == position_) throw ParseNumberException(ReadDelimited());
  position_ = end;
  return ret;
}

const char *FilePiece::FindDelimiterOrEOF(const bool *delim)  {
  std::size_t skip = 0;
  while (true) {
    for (const char *i = position_ + skip; i < position_end_; ++i) {
      if (delim[static_cast<unsigned char>(*i)]) return i;
    }
    if (at_end_) {
      if (position_ == position_end_) Shift();
      return position_end_;
    }
    skip = position_end_ - position_;
    Shift();
  }
}

void FilePiece::Shift() {
  if (at_end_) {
    progress_.Finished();
    throw EndOfFileException();
  }
  uint64_t desired_begin = position_ - data_.begin() + mapped_offset_;

  if (!fallback_to_read_) MMapShift(desired_begin);
  // Notice an mmap failure might set the fallback.
  if (fallback_to_read_) ReadShift();

  for (last_space_ = position_end_ - 1; last_space_ >= position_; --last_space_) {
    if (isspace(*last_space_))  break;
  }
}

void FilePiece::MMapShift(uint64_t desired_begin) {
  // Use mmap.
  uint64_t ignore = desired_begin % page_;
  // Duplicate request for Shift means give more data.
  if (position_ == data_.begin() + ignore) {
    default_map_size_ *= 2;
  }
  // Local version so that in case of failure it doesn't overwrite the class variable.
  uint64_t mapped_offset = desired_begin - ignore;

  uint64_t mapped_size;
  if (default_map_size_ >= static_cast<std::size_t>(total_size_ - mapped_offset)) {
    at_end_ = true;
    mapped_size = total_size_ - mapped_offset;
  } else {
    mapped_size = default_map_size_;
  }

  // Forcibly clear the existing mmap first.
  data_.reset();
  try {
    MapRead(POPULATE_OR_LAZY, *file_, mapped_offset, mapped_size, data_);
  } catch (const util::ErrnoException &e) {
    if (desired_begin) {
      SeekOrThrow(*file_, desired_begin);
    }
    // The mmap was scheduled to end the file, but now we're going to read it.
    at_end_ = false;
    TransitionToRead();
    return;
  }
  mapped_offset_ = mapped_offset;
  position_ = data_.begin() + ignore;
  position_end_ = data_.begin() + mapped_size;

  progress_.Set(desired_begin);
}

void FilePiece::TransitionToRead() {
  assert(!fallback_to_read_);
  fallback_to_read_ = true;
  data_.reset();
  data_.reset(malloc(default_map_size_), default_map_size_, scoped_memory::MALLOC_ALLOCATED);
  UTIL_THROW_IF(!data_.get(), ErrnoException, "malloc failed for " << default_map_size_);
  position_ = data_.begin();
  position_end_ = position_;

#ifdef HAVE_ZLIB
  assert(!gz_file_);
  gz_file_ = gzdopen(file_.get(), "r");
  UTIL_THROW_IF(!gz_file_, GZException, "zlib failed to open " << file_name_);
#endif
}

#ifdef WIN32
typedef int ssize_t;
#endif

void FilePiece::ReadShift() {
  assert(fallback_to_read_);
  // Bytes [data_.begin(), position_) have been consumed.
  // Bytes [position_, position_end_) have been read into the buffer.

  // Start at the beginning of the buffer if there's nothing useful in it.
  if (position_ == position_end_) {
    mapped_offset_ += (position_end_ - data_.begin());
    position_ = data_.begin();
    position_end_ = position_;
  }

  std::size_t already_read = position_end_ - data_.begin();

  if (already_read == default_map_size_) {
    if (position_ == data_.begin()) {
      // Buffer too small.
      std::size_t valid_length = position_end_ - position_;
      default_map_size_ *= 2;
      data_.call_realloc(default_map_size_);
      UTIL_THROW_IF(!data_.get(), ErrnoException, "realloc failed for " << default_map_size_);
      position_ = data_.begin();
      position_end_ = position_ + valid_length;
    } else {
      size_t moving = position_end_ - position_;
      memmove(data_.get(), position_, moving);
      position_ = data_.begin();
      position_end_ = position_ + moving;
      already_read = moving;
    }
  }

  ssize_t read_return;
#ifdef HAVE_ZLIB
  read_return = gzread( (gzFile) gz_file_, static_cast<char*>(data_.get()) + already_read, default_map_size_ - already_read);
  if (read_return == -1) throw GZException(gz_file_);
  if (total_size_ != kBadSize) {
    // Just get the position, don't actually seek.  Apparently this is how you do it. . .
    off_t ret = lseek(file_.get(), 0, SEEK_CUR);
    if (ret != -1) progress_.Set(ret);
  }
#else
  read_return = read(file_.get(), static_cast<char*>(data_.get()) + already_read, default_map_size_ - already_read);
  UTIL_THROW_IF(read_return == -1, ErrnoException, "read failed");
  progress_.Set(mapped_offset_);
#endif
  if (read_return == 0) {
    at_end_ = true;
  }
  position_end_ += read_return;
}

} // namespace util
<|endoftext|>