<commit_before>/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

#include "InactivityMonitor.h"

#include "ReadChecker.h"
#include "WriteChecker.h"

#include <activemq/threads/CompositeTask.h>
#include <activemq/threads/CompositeTaskRunner.h>
#include <activemq/commands/WireFormatInfo.h>
#include <activemq/commands/KeepAliveInfo.h>

#include <decaf/lang/Math.h>
#include <decaf/lang/Thread.h>
#include <decaf/lang/Runnable.h>
#include <decaf/lang/Boolean.h>

using namespace std;
using namespace activemq;
using namespace activemq::commands;
using namespace activemq::threads;
using namespace activemq::transport;
using namespace activemq::transport::inactivity;
using namespace activemq::exceptions;
using namespace activemq::wireformat;
using namespace decaf;
using namespace decaf::io;
using namespace decaf::util;
using namespace decaf::util::concurrent;
using namespace decaf::util::concurrent::atomic;
using namespace decaf::lang;
using namespace decaf::lang::exceptions;

////////////////////////////////////////////////////////////////////////////////
namespace activemq{
namespace transport{
namespace inactivity{

    class InactivityMonitorData {
    public:

        // The configured WireFormat for the Transport Chain.
        Pointer<WireFormat> wireFormat;

        // Local and Remote WireFormat information.
        Pointer<WireFormatInfo> localWireFormatInfo;
        Pointer<WireFormatInfo> remoteWireFormatInfo;

        Pointer<ReadChecker> readCheckerTask;
        Pointer<WriteChecker> writeCheckerTask;

        Timer readCheckTimer;
        Timer writeCheckTimer;

        Pointer<CompositeTaskRunner> asyncTasks;

        Pointer<AsyncSignalReadErrorkTask> asyncReadTask;
        Pointer<AsyncWriteTask> asyncWriteTask;

        AtomicBoolean monitorStarted;

        AtomicBoolean commandSent;
        AtomicBoolean commandReceived;

        AtomicBoolean failed;
        AtomicBoolean inRead;
        AtomicBoolean inWrite;

        Mutex inWriteMutex;
        Mutex monitor;

        long long readCheckTime;
        long long writeCheckTime;
        long long initialDelayTime;

        bool keepAliveResponseRequired;
    };

    // Task that fires when the TaskRunner is signaled by the ReadCheck Timer Task.
    class AsyncSignalReadErrorkTask : public CompositeTask {
    private:

        InactivityMonitor* parent;
        std::string remote;
        AtomicBoolean failed;

    public:

        AsyncSignalReadErrorkTask( InactivityMonitor* parent, const std::string& remote ) {
            this->parent = parent;
            this->remote = remote;
        }

        void setFailed( bool failed ) {
            this->failed.set( failed );
        }

        virtual bool isPending() const {
            return this->failed.get();
        }

        virtual bool iterate() {

            if( this->failed.compareAndSet( true, false ) ) {

                IOException ex (
                    __FILE__, __LINE__,
                    ( std::string( "Channel was inactive for too long: " ) + remote ).c_str() );

                this->parent->onException( ex );
            }

            return this->failed.get();
        }
    };

    // Task that fires when the TaskRunner is signaled by the WriteCheck Timer Task.
    class AsyncWriteTask : public CompositeTask {
    private:

        InactivityMonitor* parent;
        AtomicBoolean write;

    public:

        AsyncWriteTask( InactivityMonitor* parent ) : parent( parent ) {
        }

        void setWrite( bool write ) {
            this->write.set( write );
        }

        virtual bool isPending() const {
            return this->write.get();
        }

        virtual bool iterate() {

            if( this->write.compareAndSet( true, false ) &&
                this->parent->members->monitorStarted.get() ) {

                try {
                    Pointer<KeepAliveInfo> info( new KeepAliveInfo() );
                    info->setResponseRequired( this->parent->members->keepAliveResponseRequired );
                    this->parent->oneway( info );
                } catch( IOException& e ) {
                    this->parent->onException( e );
                }
            }

            return this->write.get();
        }
    };

}}}

////////////////////////////////////////////////////////////////////////////////
InactivityMonitor::InactivityMonitor( const Pointer<Transport>& next, const Pointer<WireFormat>& wireFormat )
:   TransportFilter( next ), members( new InactivityMonitorData() ) {

    this->members->wireFormat = wireFormat;
    this->members->monitorStarted = false;
    this->members->commandSent = false;
    this->members->commandReceived = true;
    this->members->failed = false;
    this->members->inRead = false;
    this->members->inWrite = false;
    this->members->readCheckTime = 0;
    this->members->writeCheckTime = 0;
    this->members->initialDelayTime = 0;
    this->members->keepAliveResponseRequired = false;
}

////////////////////////////////////////////////////////////////////////////////
InactivityMonitor::InactivityMonitor( const Pointer<Transport>& next,
                                      const decaf::util::Properties& properties,
                                      const Pointer<wireformat::WireFormat>& wireFormat )
:   TransportFilter( next ), members( new InactivityMonitorData() ) {

    this->members->wireFormat = wireFormat;
    this->members->monitorStarted = false;
    this->members->commandSent = false;
    this->members->commandReceived = true;
    this->members->failed = false;
    this->members->inRead = false;
    this->members->inWrite = false;
    this->members->readCheckTime = 0;
    this->members->writeCheckTime = 0;
    this->members->initialDelayTime = 0;
    this->members->keepAliveResponseRequired =
        Boolean::parseBoolean( properties.getProperty( "keepAliveResponseRequired", false ) );
}

////////////////////////////////////////////////////////////////////////////////
InactivityMonitor::~InactivityMonitor() {
    try{
        this->stopMonitorThreads();
    }
    AMQ_CATCHALL_NOTHROW()
}

////////////////////////////////////////////////////////////////////////////////
long long InactivityMonitor::getReadCheckTime() const {
    return this->members->readCheckTime;
}

////////////////////////////////////////////////////////////////////////////////
void InactivityMonitor::setReadCheckTime( long long value ) {
    this->members->readCheckTime = value;
}

////////////////////////////////////////////////////////////////////////////////
long long InactivityMonitor::getWriteCheckTime() const {
    return this->members->writeCheckTime;
}

////////////////////////////////////////////////////////////////////////////////
void InactivityMonitor::setWriteCheckTime( long long value ) {
    this->members->writeCheckTime = value;
}

////////////////////////////////////////////////////////////////////////////////
long long InactivityMonitor::getInitialDelayTime() const {
    return this->members->initialDelayTime;
}

////////////////////////////////////////////////////////////////////////////////
void InactivityMonitor::setInitialDelayTime( long long value ) const {
    this->members->initialDelayTime = value;
}

////////////////////////////////////////////////////////////////////////////////
bool InactivityMonitor::isKeepAliveResponseRequired() const {
    return this->members->keepAliveResponseRequired;
}

////////////////////////////////////////////////////////////////////////////////
void InactivityMonitor::setKeepAliveResponseRequired( bool value ) {
    this->members->keepAliveResponseRequired = value;
}

////////////////////////////////////////////////////////////////////////////////
void InactivityMonitor::close() throw( decaf::io::IOException ) {
    try{
        stopMonitorThreads();
        TransportFilter::close();
    }
    AMQ_CATCH_RETHROW( IOException )
    AMQ_CATCH_EXCEPTION_CONVERT( Exception, IOException )
    AMQ_CATCHALL_THROW( IOException )
}

////////////////////////////////////////////////////////////////////////////////
void InactivityMonitor::onException( const decaf::lang::Exception& ex ) {

    if( this->members->failed.compareAndSet( false, true ) ) {
        stopMonitorThreads();
        TransportFilter::onException( ex );
    }
}

////////////////////////////////////////////////////////////////////////////////
void InactivityMonitor::onCommand( const Pointer<Command>& command ) {

    TransportFilter::onCommand( command );

    this->members->commandReceived.set( true );
    this->members->inRead.set( true );

    try {

        if( command->isWireFormatInfo() ) {
            synchronized( &this->members->monitor ) {

                this->members->remoteWireFormatInfo = command.dynamicCast<WireFormatInfo>();
                try {
                    startMonitorThreads();
                } catch( IOException& e ) {
                    onException( e );
                }
            }
        }

        TransportFilter::onCommand( command );

        this->members->inRead.set( false );

    } catch( Exception& ex ) {
        this->members->inRead.set( false );
        ex.setMark( __FILE__, __LINE__ );
        throw ex;
    }
}

////////////////////////////////////////////////////////////////////////////////
void InactivityMonitor::oneway( const Pointer<Command>& command )
    throw( decaf::io::IOException, decaf::lang::exceptions::UnsupportedOperationException ) {

    try{
        // Disable inactivity monitoring while processing a command.  Synchronize this
        // method - its not synchronized
        // further down the transport stack and gets called by more
        // than one thread  by this class
        synchronized( &this->members->inWriteMutex ) {
            this->members->inWrite.set( true );
            try {

                if( this->members->failed.get() ) {
                    throw IOException(
                        __FILE__, __LINE__,
                        ( std::string( "Channel was inactive for too long: " ) + next->getRemoteAddress() ).c_str() );
                }

                if( command->isWireFormatInfo() ) {
                    synchronized( &this->members->monitor ) {
                        this->members->localWireFormatInfo = command.dynamicCast<WireFormatInfo>();
                        startMonitorThreads();
                    }
                }

                this->next->oneway( command );

                this->members->commandSent.set( true );
                this->members->inWrite.set( false );

            } catch( Exception& ex ) {
                this->members->commandSent.set( true );
                this->members->inWrite.set( false );

                ex.setMark( __FILE__, __LINE__ );
                throw ex;
            }
        }
    }
    AMQ_CATCH_RETHROW( IOException )
    AMQ_CATCH_RETHROW( UnsupportedOperationException )
    AMQ_CATCH_EXCEPTION_CONVERT( Exception, IOException )
    AMQ_CATCHALL_THROW( IOException )
}

////////////////////////////////////////////////////////////////////////////////
bool InactivityMonitor::allowReadCheck( long long elapsed ) {
    return elapsed > ( this->members->readCheckTime * 9 / 10 );
}

////////////////////////////////////////////////////////////////////////////////
void InactivityMonitor::readCheck() {

    if( this->members->inRead.get() || this->members->wireFormat->inReceive() ) {
        return;
    }

    if( !this->members->commandReceived.get() ) {

        // Set the failed state on our async Read Failure Task and wakeup its runner.
        this->members->asyncReadTask->setFailed( true );
        this->members->asyncTasks->wakeup();
    }

    this->members->commandReceived.set( false );
}

////////////////////////////////////////////////////////////////////////////////
void InactivityMonitor::writeCheck() {

    if( this->members->inWrite.get() ) {
        return;
    }

    if(! this->members->commandSent.get() ) {

        this->members->asyncWriteTask->setWrite( true );
        this->members->asyncTasks->wakeup();
    }

    this->members->commandSent.set( false );
}

////////////////////////////////////////////////////////////////////////////////
void InactivityMonitor::startMonitorThreads() {

    synchronized( &this->members->monitor ) {

        if( this->members->monitorStarted.get() ) {
            return;
        }
        if( this->members->localWireFormatInfo == NULL ) {
            return;
        }
        if( this->members->remoteWireFormatInfo == NULL ) {
            return;
        }

        this->members->asyncTasks.reset( new CompositeTaskRunner() );
        this->members->asyncReadTask.reset( new AsyncSignalReadErrorkTask( this, this->getRemoteAddress() ) );
        this->members->asyncWriteTask.reset( new AsyncWriteTask( this ) );

        this->members->asyncTasks->addTask( this->members->asyncReadTask.get() );
        this->members->asyncTasks->addTask( this->members->asyncWriteTask.get() );

        this->members->readCheckTime =
            Math::min( this->members->localWireFormatInfo->getMaxInactivityDuration(),
                        this->members->remoteWireFormatInfo->getMaxInactivityDuration() );

        this->members->initialDelayTime =
            Math::min( this->members->localWireFormatInfo->getMaxInactivityDurationInitalDelay(),
                       this->members->remoteWireFormatInfo->getMaxInactivityDurationInitalDelay() );

        if( this->members->readCheckTime > 0 ) {

            this->members->monitorStarted.set( true );
            this->members->writeCheckerTask.reset( new WriteChecker( this ) );
            this->members->readCheckerTask.reset( new ReadChecker( this ) );
            this->members->writeCheckTime = this->members->readCheckTime > 3 ?
                                                this->members->readCheckTime / 3 : this->members->readCheckTime;

            this->members->writeCheckTimer.scheduleAtFixedRate(
                this->members->writeCheckerTask, this->members->initialDelayTime, this->members->writeCheckTime );
            this->members->readCheckTimer.scheduleAtFixedRate(
                this->members->readCheckerTask, this->members->initialDelayTime, this->members->readCheckTime );
        }
    }
}

////////////////////////////////////////////////////////////////////////////////
void InactivityMonitor::stopMonitorThreads() {

    synchronized( &this->members->monitor ) {

        if( this->members->monitorStarted.compareAndSet( true, false ) ) {

            this->members->readCheckerTask->cancel();
            this->members->writeCheckerTask->cancel();

            this->members->readCheckTimer.purge();
            this->members->readCheckTimer.cancel();
            this->members->writeCheckTimer.purge();
            this->members->writeCheckTimer.cancel();

            this->members->asyncTasks->shutdown();
        }
    }
}
<commit_msg>https://issues.apache.org/activemq/browse/AMQCPP-250<commit_after>/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

#include "InactivityMonitor.h"

#include "ReadChecker.h"
#include "WriteChecker.h"

#include <activemq/threads/CompositeTask.h>
#include <activemq/threads/CompositeTaskRunner.h>
#include <activemq/commands/WireFormatInfo.h>
#include <activemq/commands/KeepAliveInfo.h>

#include <decaf/lang/Math.h>
#include <decaf/lang/Thread.h>
#include <decaf/lang/Runnable.h>
#include <decaf/lang/Boolean.h>

using namespace std;
using namespace activemq;
using namespace activemq::commands;
using namespace activemq::threads;
using namespace activemq::transport;
using namespace activemq::transport::inactivity;
using namespace activemq::exceptions;
using namespace activemq::wireformat;
using namespace decaf;
using namespace decaf::io;
using namespace decaf::util;
using namespace decaf::util::concurrent;
using namespace decaf::util::concurrent::atomic;
using namespace decaf::lang;
using namespace decaf::lang::exceptions;

////////////////////////////////////////////////////////////////////////////////
namespace activemq{
namespace transport{
namespace inactivity{

    class InactivityMonitorData {
    public:

        // The configured WireFormat for the Transport Chain.
        Pointer<WireFormat> wireFormat;

        // Local and Remote WireFormat information.
        Pointer<WireFormatInfo> localWireFormatInfo;
        Pointer<WireFormatInfo> remoteWireFormatInfo;

        Pointer<ReadChecker> readCheckerTask;
        Pointer<WriteChecker> writeCheckerTask;

        Timer readCheckTimer;
        Timer writeCheckTimer;

        Pointer<CompositeTaskRunner> asyncTasks;

        Pointer<AsyncSignalReadErrorkTask> asyncReadTask;
        Pointer<AsyncWriteTask> asyncWriteTask;

        AtomicBoolean monitorStarted;

        AtomicBoolean commandSent;
        AtomicBoolean commandReceived;

        AtomicBoolean failed;
        AtomicBoolean inRead;
        AtomicBoolean inWrite;

        Mutex inWriteMutex;
        Mutex monitor;

        long long readCheckTime;
        long long writeCheckTime;
        long long initialDelayTime;

        bool keepAliveResponseRequired;
    };

    // Task that fires when the TaskRunner is signaled by the ReadCheck Timer Task.
    class AsyncSignalReadErrorkTask : public CompositeTask {
    private:

        InactivityMonitor* parent;
        std::string remote;
        AtomicBoolean failed;

    public:

        AsyncSignalReadErrorkTask( InactivityMonitor* parent, const std::string& remote ) {
            this->parent = parent;
            this->remote = remote;
        }

        void setFailed( bool failed ) {
            this->failed.set( failed );
        }

        virtual bool isPending() const {
            return this->failed.get();
        }

        virtual bool iterate() {

            if( this->failed.compareAndSet( true, false ) ) {

                IOException ex (
                    __FILE__, __LINE__,
                    ( std::string( "Channel was inactive for too long: " ) + remote ).c_str() );

                this->parent->onException( ex );
            }

            return this->failed.get();
        }
    };

    // Task that fires when the TaskRunner is signaled by the WriteCheck Timer Task.
    class AsyncWriteTask : public CompositeTask {
    private:

        InactivityMonitor* parent;
        AtomicBoolean write;

    public:

        AsyncWriteTask( InactivityMonitor* parent ) : parent( parent ) {
        }

        void setWrite( bool write ) {
            this->write.set( write );
        }

        virtual bool isPending() const {
            return this->write.get();
        }

        virtual bool iterate() {

            if( this->write.compareAndSet( true, false ) &&
                this->parent->members->monitorStarted.get() ) {

                try {
                    Pointer<KeepAliveInfo> info( new KeepAliveInfo() );
                    info->setResponseRequired( this->parent->members->keepAliveResponseRequired );
                    this->parent->oneway( info );
                } catch( IOException& e ) {
                    this->parent->onException( e );
                }
            }

            return this->write.get();
        }
    };

}}}

////////////////////////////////////////////////////////////////////////////////
InactivityMonitor::InactivityMonitor( const Pointer<Transport>& next, const Pointer<WireFormat>& wireFormat )
:   TransportFilter( next ), members( new InactivityMonitorData() ) {

    this->members->wireFormat = wireFormat;
    this->members->monitorStarted = false;
    this->members->commandSent = false;
    this->members->commandReceived = true;
    this->members->failed = false;
    this->members->inRead = false;
    this->members->inWrite = false;
    this->members->readCheckTime = 0;
    this->members->writeCheckTime = 0;
    this->members->initialDelayTime = 0;
    this->members->keepAliveResponseRequired = false;
}

////////////////////////////////////////////////////////////////////////////////
InactivityMonitor::InactivityMonitor( const Pointer<Transport>& next,
                                      const decaf::util::Properties& properties,
                                      const Pointer<wireformat::WireFormat>& wireFormat )
:   TransportFilter( next ), members( new InactivityMonitorData() ) {

    this->members->wireFormat = wireFormat;
    this->members->monitorStarted = false;
    this->members->commandSent = false;
    this->members->commandReceived = true;
    this->members->failed = false;
    this->members->inRead = false;
    this->members->inWrite = false;
    this->members->readCheckTime = 0;
    this->members->writeCheckTime = 0;
    this->members->initialDelayTime = 0;
    this->members->keepAliveResponseRequired =
        Boolean::parseBoolean( properties.getProperty( "keepAliveResponseRequired", "false" ) );
}

////////////////////////////////////////////////////////////////////////////////
InactivityMonitor::~InactivityMonitor() {
    try{
        this->stopMonitorThreads();
    }
    AMQ_CATCHALL_NOTHROW()
}

////////////////////////////////////////////////////////////////////////////////
long long InactivityMonitor::getReadCheckTime() const {
    return this->members->readCheckTime;
}

////////////////////////////////////////////////////////////////////////////////
void InactivityMonitor::setReadCheckTime( long long value ) {
    this->members->readCheckTime = value;
}

////////////////////////////////////////////////////////////////////////////////
long long InactivityMonitor::getWriteCheckTime() const {
    return this->members->writeCheckTime;
}

////////////////////////////////////////////////////////////////////////////////
void InactivityMonitor::setWriteCheckTime( long long value ) {
    this->members->writeCheckTime = value;
}

////////////////////////////////////////////////////////////////////////////////
long long InactivityMonitor::getInitialDelayTime() const {
    return this->members->initialDelayTime;
}

////////////////////////////////////////////////////////////////////////////////
void InactivityMonitor::setInitialDelayTime( long long value ) const {
    this->members->initialDelayTime = value;
}

////////////////////////////////////////////////////////////////////////////////
bool InactivityMonitor::isKeepAliveResponseRequired() const {
    return this->members->keepAliveResponseRequired;
}

////////////////////////////////////////////////////////////////////////////////
void InactivityMonitor::setKeepAliveResponseRequired( bool value ) {
    this->members->keepAliveResponseRequired = value;
}

////////////////////////////////////////////////////////////////////////////////
void InactivityMonitor::close() throw( decaf::io::IOException ) {
    try{
        stopMonitorThreads();
        TransportFilter::close();
    }
    AMQ_CATCH_RETHROW( IOException )
    AMQ_CATCH_EXCEPTION_CONVERT( Exception, IOException )
    AMQ_CATCHALL_THROW( IOException )
}

////////////////////////////////////////////////////////////////////////////////
void InactivityMonitor::onException( const decaf::lang::Exception& ex ) {

    if( this->members->failed.compareAndSet( false, true ) ) {
        stopMonitorThreads();
        TransportFilter::onException( ex );
    }
}

////////////////////////////////////////////////////////////////////////////////
void InactivityMonitor::onCommand( const Pointer<Command>& command ) {

    TransportFilter::onCommand( command );

    this->members->commandReceived.set( true );
    this->members->inRead.set( true );

    try {

        if( command->isWireFormatInfo() ) {
            synchronized( &this->members->monitor ) {

                this->members->remoteWireFormatInfo = command.dynamicCast<WireFormatInfo>();
                try {
                    startMonitorThreads();
                } catch( IOException& e ) {
                    onException( e );
                }
            }
        }

        TransportFilter::onCommand( command );

        this->members->inRead.set( false );

    } catch( Exception& ex ) {
        this->members->inRead.set( false );
        ex.setMark( __FILE__, __LINE__ );
        throw ex;
    }
}

////////////////////////////////////////////////////////////////////////////////
void InactivityMonitor::oneway( const Pointer<Command>& command )
    throw( decaf::io::IOException, decaf::lang::exceptions::UnsupportedOperationException ) {

    try{
        // Disable inactivity monitoring while processing a command.  Synchronize this
        // method - its not synchronized
        // further down the transport stack and gets called by more
        // than one thread  by this class
        synchronized( &this->members->inWriteMutex ) {
            this->members->inWrite.set( true );
            try {

                if( this->members->failed.get() ) {
                    throw IOException(
                        __FILE__, __LINE__,
                        ( std::string( "Channel was inactive for too long: " ) + next->getRemoteAddress() ).c_str() );
                }

                if( command->isWireFormatInfo() ) {
                    synchronized( &this->members->monitor ) {
                        this->members->localWireFormatInfo = command.dynamicCast<WireFormatInfo>();
                        startMonitorThreads();
                    }
                }

                this->next->oneway( command );

                this->members->commandSent.set( true );
                this->members->inWrite.set( false );

            } catch( Exception& ex ) {
                this->members->commandSent.set( true );
                this->members->inWrite.set( false );

                ex.setMark( __FILE__, __LINE__ );
                throw ex;
            }
        }
    }
    AMQ_CATCH_RETHROW( IOException )
    AMQ_CATCH_RETHROW( UnsupportedOperationException )
    AMQ_CATCH_EXCEPTION_CONVERT( Exception, IOException )
    AMQ_CATCHALL_THROW( IOException )
}

////////////////////////////////////////////////////////////////////////////////
bool InactivityMonitor::allowReadCheck( long long elapsed ) {
    return elapsed > ( this->members->readCheckTime * 9 / 10 );
}

////////////////////////////////////////////////////////////////////////////////
void InactivityMonitor::readCheck() {

    if( this->members->inRead.get() || this->members->wireFormat->inReceive() ) {
        return;
    }

    if( !this->members->commandReceived.get() ) {

        // Set the failed state on our async Read Failure Task and wakeup its runner.
        this->members->asyncReadTask->setFailed( true );
        this->members->asyncTasks->wakeup();
    }

    this->members->commandReceived.set( false );
}

////////////////////////////////////////////////////////////////////////////////
void InactivityMonitor::writeCheck() {

    if( this->members->inWrite.get() ) {
        return;
    }

    if(! this->members->commandSent.get() ) {

        this->members->asyncWriteTask->setWrite( true );
        this->members->asyncTasks->wakeup();
    }

    this->members->commandSent.set( false );
}

////////////////////////////////////////////////////////////////////////////////
void InactivityMonitor::startMonitorThreads() {

    synchronized( &this->members->monitor ) {

        if( this->members->monitorStarted.get() ) {
            return;
        }
        if( this->members->localWireFormatInfo == NULL ) {
            return;
        }
        if( this->members->remoteWireFormatInfo == NULL ) {
            return;
        }

        this->members->asyncTasks.reset( new CompositeTaskRunner() );
        this->members->asyncReadTask.reset( new AsyncSignalReadErrorkTask( this, this->getRemoteAddress() ) );
        this->members->asyncWriteTask.reset( new AsyncWriteTask( this ) );

        this->members->asyncTasks->addTask( this->members->asyncReadTask.get() );
        this->members->asyncTasks->addTask( this->members->asyncWriteTask.get() );

        this->members->readCheckTime =
            Math::min( this->members->localWireFormatInfo->getMaxInactivityDuration(),
                        this->members->remoteWireFormatInfo->getMaxInactivityDuration() );

        this->members->initialDelayTime =
            Math::min( this->members->localWireFormatInfo->getMaxInactivityDurationInitalDelay(),
                       this->members->remoteWireFormatInfo->getMaxInactivityDurationInitalDelay() );

        if( this->members->readCheckTime > 0 ) {

            this->members->monitorStarted.set( true );
            this->members->writeCheckerTask.reset( new WriteChecker( this ) );
            this->members->readCheckerTask.reset( new ReadChecker( this ) );
            this->members->writeCheckTime = this->members->readCheckTime > 3 ?
                                                this->members->readCheckTime / 3 : this->members->readCheckTime;

            this->members->writeCheckTimer.scheduleAtFixedRate(
                this->members->writeCheckerTask, this->members->initialDelayTime, this->members->writeCheckTime );
            this->members->readCheckTimer.scheduleAtFixedRate(
                this->members->readCheckerTask, this->members->initialDelayTime, this->members->readCheckTime );
        }
    }
}

////////////////////////////////////////////////////////////////////////////////
void InactivityMonitor::stopMonitorThreads() {

    synchronized( &this->members->monitor ) {

        if( this->members->monitorStarted.compareAndSet( true, false ) ) {

            this->members->readCheckerTask->cancel();
            this->members->writeCheckerTask->cancel();

            this->members->readCheckTimer.purge();
            this->members->readCheckTimer.cancel();
            this->members->writeCheckTimer.purge();
            this->members->writeCheckTimer.cancel();

            this->members->asyncTasks->shutdown();
        }
    }
}
<|endoftext|>


<commit_before>/*!
 *  Copyright (c) 2016 by Contributors
 * \file plan_memory.cc
 * \brief Assign memory tag to each of the data entries.
 */
#include <nnvm/graph.h>
#include <nnvm/pass.h>
#include <nnvm/graph_attr_types.h>
#include <nnvm/op_attr_types.h>
#include <memory>
#include "./graph_algorithm.h"

namespace nnvm {
namespace pass {
namespace {

// simple graph based allocator.
class GraphAllocator {
 public:
  // storage id equals integer.
  using StorageID = int;
  // bad storage id
  static const StorageID kBadStorageID = -1;
  // request a free storage
  StorageID Request(int dev_id, int dtype, TShape shape, uint32_t node_id) {
    if (shape.ndim() == 0) return kBadStorageID;
    // search memory block in [size / match_range_, size * match_range_)
    // TODO(tqchen) add size of the dtype, assume 4 bytes for now
    size_t size = shape.Size() * 4;
    if (match_range_ == 0) return this->Alloc(dev_id, size);
    auto begin = free_.lower_bound(size / match_range_);
    auto mid = free_.lower_bound(size);
    auto end = free_.upper_bound(size * match_range_);
    // search for memory blocks larger than requested
    for (auto it = mid; it != end; ++it) {
      StorageEntry *e = it->second;
      if (e->device_id != dev_id) continue;
      if (node_color_.size() != 0 &&
          node_color_[e->released_by_node] != node_color_[node_id]) continue;
      // Use exect matching strategy
      e->max_bytes = std::max(size, e->max_bytes);
      // find a exact match, erase from map and return
      free_.erase(it);
      return e->id;
    }
    // then search for memory blocks smaller than requested space
    for (auto it = mid; it != begin;) {
      --it;
      StorageEntry *e = it->second;
      if (e->device_id != dev_id) continue;
      if (node_color_.size() != 0 &&
          node_color_[e->released_by_node] != node_color_[node_id]) continue;
      // Use exect matching strategy
      e->max_bytes = std::max(size, e->max_bytes);
      // find a exact match, erase from map and return
      free_.erase(it);
      return e->id;
    }
    // cannot find anything return a new one.
    return this->Alloc(dev_id, size);
  }
  // release a memory space.
  void Release(StorageID id, uint32_t node_id) {
    CHECK_NE(id, kBadStorageID);
    StorageEntry *e = data_[id].get();
    e->released_by_node = node_id;
    free_.insert({e->max_bytes, e});
  }
  // totoal number of bytes allocated
  size_t TotalAllocBytes() const {
    size_t total = 0;
    for (auto &p : data_) {
      total += p->max_bytes;
    }
    return total;
  }

  // constructor
  explicit GraphAllocator(const IndexedGraph* idx) : idx_(idx) {
    this->Init(dmlc::GetEnv("NNVM_EXEC_MATCH_RANGE", 16),
               dmlc::GetEnv("NNVM_EXEC_NUM_TEMP", 1));
  }

 private:
  // initialize the graph allocator
  void Init(size_t match_range, uint32_t num_match_color) {
    match_range_ = match_range;
    num_match_color_ = num_match_color;
    if (num_match_color_ > 1) {
      std::vector<uint32_t> importance(idx_->num_nodes(), 0);
      for (uint32_t nid = 0; nid < idx_->num_nodes(); ++nid) {
        if ((*idx_)[nid].source->is_variable()) continue;
        importance[nid] = 1;
      }
      num_match_color_ = pass::ColorNodeGroup(
          *idx_, importance, num_match_color_, &node_color_);
    }
  }

  StorageID Alloc(int dev_id, size_t size) {
    StorageID id = static_cast<StorageID>(data_.size());
    std::unique_ptr<StorageEntry> ptr(new StorageEntry());
    ptr->id = id;
    ptr->device_id = dev_id;
    ptr->max_bytes = size;
    data_.emplace_back(std::move(ptr));
    return id;
  }
  // internal storage entry
  struct StorageEntry {
    // the id of the entry.
    StorageID id;
    // the device id of the storage.
    int device_id;
    // maximum size of storage requested.
    size_t max_bytes{0};
    // node index that released it last time
    uint32_t released_by_node{0};
  };
  // scale used for rough match
  size_t match_range_;
  // whether use color based match algorithm
  uint32_t num_match_color_{1};
  // the size of each dtype
  std::vector<size_t> dtype_size_dict_;
  // free list of storage entry
  std::multimap<size_t, StorageEntry*> free_;
  // all the storage resources available
  std::vector<std::unique_ptr<StorageEntry> > data_;
  // color of nodes in the graph, used for auxiliary policy making.
  std::vector<uint32_t> node_color_;
  // internal indexed graph
  const IndexedGraph* idx_;
};

// function to plan memory
Graph PlanMemory(Graph ret) {
  // setup ref counter
  const IndexedGraph& idx = ret.indexed_graph();
  // reference counter of each node
  std::vector<uint32_t> ref_count(idx.num_node_entries(), 0);
  // step 1: initialize reference count
  for (uint32_t nid = 0; nid < idx.num_nodes(); ++nid) {
    for (const auto& e : idx[nid].inputs) {
      ++ref_count[idx.entry_id(e)];
    }
  }
  for (const auto& e : idx.outputs()) {
    ++ref_count[idx.entry_id(e)];
  }
  // step 2: allocate memory.
  StorageVector storage(idx.num_node_entries(), -1);
  std::vector<int> storage_inplace_index(idx.num_node_entries(), -1);
  const ShapeVector& shape_vec = ret.GetAttr<ShapeVector>("shape");
  const DTypeVector& dtype_vec = ret.GetAttr<DTypeVector>("dtype");
  const DeviceVector* device_vec = nullptr;
  static auto& finplace_option = Op::GetAttr<FInplaceOption>("FInplaceOption");

  if (ret.attrs.count("device") != 0) {
    device_vec = &(ret.GetAttr<DeviceVector>("device"));
  }
  // the allocator.
  GraphAllocator allocator(&idx);
  // number of entries that are not statically allocated.
  size_t num_not_allocated = 0;

  for (uint32_t nid = 0; nid < idx.num_nodes(); ++nid) {
    const auto& inode = idx[nid];
    if (inode.source->is_variable()) continue;
    // check inplace option
    if (finplace_option.count(inode.source->op()) != 0) {
      auto inplace_pairs = finplace_option[inode.source->op()](inode.source->attrs);
      for (auto& kv : inplace_pairs) {
        uint32_t eid_out = idx.entry_id(nid, kv.second);
        uint32_t eid_in = idx.entry_id(inode.inputs[kv.first]);
        if (ref_count[eid_in] == 1 &&
            storage[eid_out] == GraphAllocator::kBadStorageID &&
            storage[eid_in] != GraphAllocator::kBadStorageID &&
            shape_vec[eid_out].Size() == shape_vec[eid_in].Size() &&
            dtype_vec[eid_out] == dtype_vec[eid_in]) {
          // inplace optimization
          storage[eid_out] = storage[eid_in];
          ref_count[eid_in] = 0;
          storage_inplace_index[eid_out] = kv.first;
        }
      }
    }
    // normal allocation
    const int dev_id = (device_vec != nullptr) ? device_vec->at(nid) : 0;
    // allocate output
    for (uint32_t index = 0; index < inode.source->num_outputs(); ++index) {
      uint32_t eid = idx.entry_id(nid, index);
      if (storage[eid] == GraphAllocator::kBadStorageID) {
        storage[eid] = allocator.Request(dev_id, dtype_vec[eid], shape_vec[eid], nid);
      }
    }
    // then free inputs
    for (const auto& e : inode.inputs) {
      uint32_t eid = idx.entry_id(e);
      // temp_ref_count == 0 means it is taken by inplace op
      if (ref_count[eid] == 0) continue;
      // if we decrease it to zero, means we are ready to relase
      --ref_count[eid];
      if (ref_count[eid] == 0 && storage[eid] != GraphAllocator::kBadStorageID) {
        allocator.Release(storage[eid], nid);
      }
    }
    // check if there are outputs that can be freeded immediately
    // these output are not referenced by any operator.
    for (uint32_t index = 0; index < inode.source->num_outputs(); ++index) {
      uint32_t eid = idx.entry_id(nid, index);
      if (ref_count[eid] == 0 && storage[eid] != GraphAllocator::kBadStorageID) {
        allocator.Release(storage[eid], nid);
        // use -2 to indicate that the node was never touched.
        storage_inplace_index[eid] = -2;
      }
      if (storage[eid] == GraphAllocator::kBadStorageID) {
        ++num_not_allocated;
      }
    }
  }
  ret.attrs["storage_id"] = std::make_shared<any>(std::move(storage));
  ret.attrs["storage_inplace_index"] = std::make_shared<any>(std::move(storage_inplace_index));
  ret.attrs["storage_allocated_bytes"] = std::make_shared<any>(allocator.TotalAllocBytes());
  ret.attrs["storage_num_not_allocated"] = std::make_shared<any>(num_not_allocated);
  return ret;
}

NNVM_REGISTER_PASS(PlanMemory)
.describe("Plan the memory allocation of each node entries.")
.set_body(PlanMemory)
.set_change_graph(false)
.depend_graph_attr("dtype")
.depend_graph_attr("shape")
.provide_graph_attr("storage_id")
.provide_graph_attr("storage_inplace_index");

}  // namespace
}  // namespace pass
}  // namespace nnvm
<commit_msg>[PlanMemory]Avoid inplace for kNullop requests (#68)<commit_after>/*!
 *  Copyright (c) 2016 by Contributors
 * \file plan_memory.cc
 * \brief Assign memory tag to each of the data entries.
 */
#include <nnvm/graph.h>
#include <nnvm/pass.h>
#include <nnvm/graph_attr_types.h>
#include <nnvm/op_attr_types.h>
#include <memory>
#include "./graph_algorithm.h"

namespace nnvm {
namespace pass {
namespace {

// simple graph based allocator.
class GraphAllocator {
 public:
  // storage id equals integer.
  using StorageID = int;
  // bad storage id
  static const StorageID kBadStorageID = -1;
  // request a free storage
  StorageID Request(int dev_id, int dtype, TShape shape, uint32_t node_id) {
    if (shape.ndim() == 0) return kBadStorageID;
    // search memory block in [size / match_range_, size * match_range_)
    // TODO(tqchen) add size of the dtype, assume 4 bytes for now
    size_t size = shape.Size() * 4;
    if (match_range_ == 0) return this->Alloc(dev_id, size);
    auto begin = free_.lower_bound(size / match_range_);
    auto mid = free_.lower_bound(size);
    auto end = free_.upper_bound(size * match_range_);
    // search for memory blocks larger than requested
    for (auto it = mid; it != end; ++it) {
      StorageEntry *e = it->second;
      if (e->device_id != dev_id) continue;
      if (node_color_.size() != 0 &&
          node_color_[e->released_by_node] != node_color_[node_id]) continue;
      // Use exect matching strategy
      e->max_bytes = std::max(size, e->max_bytes);
      // find a exact match, erase from map and return
      free_.erase(it);
      return e->id;
    }
    // then search for memory blocks smaller than requested space
    for (auto it = mid; it != begin;) {
      --it;
      StorageEntry *e = it->second;
      if (e->device_id != dev_id) continue;
      if (node_color_.size() != 0 &&
          node_color_[e->released_by_node] != node_color_[node_id]) continue;
      // Use exect matching strategy
      e->max_bytes = std::max(size, e->max_bytes);
      // find a exact match, erase from map and return
      free_.erase(it);
      return e->id;
    }
    // cannot find anything return a new one.
    return this->Alloc(dev_id, size);
  }
  // release a memory space.
  void Release(StorageID id, uint32_t node_id) {
    CHECK_NE(id, kBadStorageID);
    StorageEntry *e = data_[id].get();
    e->released_by_node = node_id;
    free_.insert({e->max_bytes, e});
  }
  // totoal number of bytes allocated
  size_t TotalAllocBytes() const {
    size_t total = 0;
    for (auto &p : data_) {
      total += p->max_bytes;
    }
    return total;
  }

  // constructor
  explicit GraphAllocator(const IndexedGraph* idx) : idx_(idx) {
    this->Init(dmlc::GetEnv("NNVM_EXEC_MATCH_RANGE", 16),
               dmlc::GetEnv("NNVM_EXEC_NUM_TEMP", 1));
  }

 private:
  // initialize the graph allocator
  void Init(size_t match_range, uint32_t num_match_color) {
    match_range_ = match_range;
    num_match_color_ = num_match_color;
    if (num_match_color_ > 1) {
      std::vector<uint32_t> importance(idx_->num_nodes(), 0);
      for (uint32_t nid = 0; nid < idx_->num_nodes(); ++nid) {
        if ((*idx_)[nid].source->is_variable()) continue;
        importance[nid] = 1;
      }
      num_match_color_ = pass::ColorNodeGroup(
          *idx_, importance, num_match_color_, &node_color_);
    }
  }

  StorageID Alloc(int dev_id, size_t size) {
    StorageID id = static_cast<StorageID>(data_.size());
    std::unique_ptr<StorageEntry> ptr(new StorageEntry());
    ptr->id = id;
    ptr->device_id = dev_id;
    ptr->max_bytes = size;
    data_.emplace_back(std::move(ptr));
    return id;
  }
  // internal storage entry
  struct StorageEntry {
    // the id of the entry.
    StorageID id;
    // the device id of the storage.
    int device_id;
    // maximum size of storage requested.
    size_t max_bytes{0};
    // node index that released it last time
    uint32_t released_by_node{0};
  };
  // scale used for rough match
  size_t match_range_;
  // whether use color based match algorithm
  uint32_t num_match_color_{1};
  // the size of each dtype
  std::vector<size_t> dtype_size_dict_;
  // free list of storage entry
  std::multimap<size_t, StorageEntry*> free_;
  // all the storage resources available
  std::vector<std::unique_ptr<StorageEntry> > data_;
  // color of nodes in the graph, used for auxiliary policy making.
  std::vector<uint32_t> node_color_;
  // internal indexed graph
  const IndexedGraph* idx_;
};

// function to plan memory
Graph PlanMemory(Graph ret) {
  // setup ref counter
  const IndexedGraph& idx = ret.indexed_graph();
  // reference counter of each node
  std::vector<uint32_t> ref_count(idx.num_node_entries(), 0);
  // step 1: initialize reference count
  for (uint32_t nid = 0; nid < idx.num_nodes(); ++nid) {
    for (const auto& e : idx[nid].inputs) {
      ++ref_count[idx.entry_id(e)];
    }
  }
  for (const auto& e : idx.outputs()) {
    ++ref_count[idx.entry_id(e)];
  }
  // step 2: allocate memory.
  StorageVector storage(idx.num_node_entries(), -1);
  std::vector<int> storage_inplace_index(idx.num_node_entries(), -1);
  const ShapeVector& shape_vec = ret.GetAttr<ShapeVector>("shape");
  const DTypeVector& dtype_vec = ret.GetAttr<DTypeVector>("dtype");
  const DeviceVector* device_vec = nullptr;
  static auto& finplace_option = Op::GetAttr<FInplaceOption>("FInplaceOption");

  if (ret.attrs.count("device") != 0) {
    device_vec = &(ret.GetAttr<DeviceVector>("device"));
  }
  // the allocator.
  GraphAllocator allocator(&idx);
  // number of entries that are not statically allocated.
  size_t num_not_allocated = 0;

  for (uint32_t nid = 0; nid < idx.num_nodes(); ++nid) {
    const auto& inode = idx[nid];
    if (inode.source->is_variable()) continue;
    // check inplace option
    if (finplace_option.count(inode.source->op()) != 0) {
      auto inplace_pairs = finplace_option[inode.source->op()](inode.source->attrs);
      for (auto& kv : inplace_pairs) {
        uint32_t eid_out = idx.entry_id(nid, kv.second);
        uint32_t eid_in = idx.entry_id(inode.inputs[kv.first]);
        if (ref_count[eid_in] == 1 &&
            ref_count[eid_out] != 0 &&
            storage[eid_out] == GraphAllocator::kBadStorageID &&
            storage[eid_in] != GraphAllocator::kBadStorageID &&
            shape_vec[eid_out].Size() == shape_vec[eid_in].Size() &&
            dtype_vec[eid_out] == dtype_vec[eid_in]) {
          // inplace optimization
          storage[eid_out] = storage[eid_in];
          ref_count[eid_in] = 0;
          storage_inplace_index[eid_out] = kv.first;
        }
      }
    }
    // normal allocation
    const int dev_id = (device_vec != nullptr) ? device_vec->at(nid) : 0;
    // allocate output
    for (uint32_t index = 0; index < inode.source->num_outputs(); ++index) {
      uint32_t eid = idx.entry_id(nid, index);
      if (storage[eid] == GraphAllocator::kBadStorageID) {
        storage[eid] = allocator.Request(dev_id, dtype_vec[eid], shape_vec[eid], nid);
      }
    }
    // then free inputs
    for (const auto& e : inode.inputs) {
      uint32_t eid = idx.entry_id(e);
      // temp_ref_count == 0 means it is taken by inplace op
      if (ref_count[eid] == 0) continue;
      // if we decrease it to zero, means we are ready to relase
      --ref_count[eid];
      if (ref_count[eid] == 0 && storage[eid] != GraphAllocator::kBadStorageID) {
        allocator.Release(storage[eid], nid);
      }
    }
    // check if there are outputs that can be freeded immediately
    // these output are not referenced by any operator.
    for (uint32_t index = 0; index < inode.source->num_outputs(); ++index) {
      uint32_t eid = idx.entry_id(nid, index);
      if (ref_count[eid] == 0 && storage[eid] != GraphAllocator::kBadStorageID) {
        allocator.Release(storage[eid], nid);
        // use -2 to indicate that the node was never touched.
        storage_inplace_index[eid] = -2;
      }
      if (storage[eid] == GraphAllocator::kBadStorageID) {
        ++num_not_allocated;
      }
    }
  }
  ret.attrs["storage_id"] = std::make_shared<any>(std::move(storage));
  ret.attrs["storage_inplace_index"] = std::make_shared<any>(std::move(storage_inplace_index));
  ret.attrs["storage_allocated_bytes"] = std::make_shared<any>(allocator.TotalAllocBytes());
  ret.attrs["storage_num_not_allocated"] = std::make_shared<any>(num_not_allocated);
  return ret;
}

NNVM_REGISTER_PASS(PlanMemory)
.describe("Plan the memory allocation of each node entries.")
.set_body(PlanMemory)
.set_change_graph(false)
.depend_graph_attr("dtype")
.depend_graph_attr("shape")
.provide_graph_attr("storage_id")
.provide_graph_attr("storage_inplace_index");

}  // namespace
}  // namespace pass
}  // namespace nnvm
<|endoftext|>


<commit_before><commit_msg>add support for tables<commit_after><|endoftext|>


<commit_before>const int mod = 1000000007;

struct Mod {
  int n;
  Mod () : n(0) {;}
  Mod (int n) : n(n) { if (n >= mod) n %= mod; }
  operator int() { return n; }
};

bool operator==(Mod a, Mod b) { return a.n == b.n; }
Mod operator+=(Mod &a, Mod b) { a.n += b.n; if (a.n >= mod) a.n -= mod; return a; }
Mod operator-=(Mod &a, Mod b) { a.n -= b.n; if (a.n < 0) a.n += mod; return a; }
Mod operator*=(Mod &a, Mod b) { a.n = ((long long)a.n * b.n) % mod; return a; }
Mod operator+(Mod a, Mod b) { return a += b; }
Mod operator-(Mod a, Mod b) { return a -= b; }
Mod operator*(Mod a, Mod b) { return a *= b; }
Mod operator^(Mod a, int n) {
  if (n == 0) return Mod(1);
  Mod res = (a * a) ^ (n / 2);
  if (n % 2) res = res * a;
  return res;
}

int inv(int a, int p) {
  return (a == 1 ? 1 : (1 - p * inv(p%a, a)) / a + p);
}
Mod operator/(Mod a, Mod b) { return a * Mod(inv(b, mod)); }

#define MAX_N 1024000

Mod fact[MAX_N], factinv[MAX_N];
void init() {
  fact[0] = Mod(1); factinv[0] = 1;
  REP(i,MAX_N-1) {
    fact[i+1] = fact[i] * Mod(i+1);
    factinv[i+1] = factinv[i] / Mod(i+1);
  }
}
Mod comb(int a, int b) {
  return fact[a] * factinv[b] * factinv[a-b];
}
<commit_msg>fix mod<commit_after>const int mod = 1000000007;

struct Mod {
  int n;
  Mod () : n(0) {;}
  Mod (int n) : n(n) { if (n >= mod) n %= mod; }
  operator int() { return n; }
  bool operator==(const Mod &a) { return n == a.n; }
  Mod operator+=(const Mod &a) { n += a.n; if (n >= mod) n -= mod; return *this; }
  Mod operator-=(const Mod &a) { n -= a.n; if (n < 0) n += mod; return *this; }
  Mod operator*=(const Mod &a) { n = ((long long)n * a.n) % mod; return *this; }
};

int inv(int a, int p) { return (a == 1 ? 1 : (1 - p * inv(p%a, a)) / a + p); }
Mod operator+(Mod a, const Mod &b) { return a += b; }
Mod operator-(Mod a, const Mod &b) { return a -= b; }
Mod operator*(Mod a, const Mod &b) { return a *= b; }
Mod operator/(Mod a, Mod b) { return a * Mod(inv(b, mod)); }
Mod operator^(const Mod &a, int n) {
  if (n == 0) return Mod(1);
  Mod res = (a * a) ^ (n / 2);
  if (n % 2) res = res * a;
  return res;
}

#define MAX_N 1024000

Mod fact[MAX_N], factinv[MAX_N];
void init() {
  fact[0] = Mod(1); factinv[0] = 1;
  REP(i,MAX_N-1) {
    fact[i+1] = fact[i] * Mod(i+1);
    factinv[i+1] = factinv[i] / Mod(i+1);
  }
}
Mod comb(int a, int b) {
  return fact[a] * factinv[b] * factinv[a-b];
}
<|endoftext|>


<commit_before>#include <pthread.h>
#include <string.h>

///////////////////// TIMING

typedef unsigned long long LOG_TimeUnit;

static inline LOG_TimeUnit LOG_getTimeStart() {
  unsigned hi, lo;

  __asm__ __volatile__ ("cpuid\n"
                        "rdtsc\n"
         : "=a" (lo), "=d" (hi)
         :: "%rbx", "%rcx");
  return ( (unsigned long long)lo)|( ((unsigned long long)hi)<<32 );
}

static inline LOG_TimeUnit LOG_getTimeStop() {
  unsigned hi, lo;

  __asm__ __volatile__ ("rdtscp\n"
                        "mov %%edx, %0\n\t"
                        "mov %%eax, %1\n\t"
                        "cpuid\n"
          : "=r" (hi), "=r" (lo)
          :: "%rax", "%rbx", "%rcx", "%rdx");
  return ( (unsigned long long)lo)|( ((unsigned long long)hi)<<32 );
}

#ifdef LOGGING
///////////////////// LOGGING

#define LOG_MAX_ENTRIES 1000000

static inline unsigned long LOG_getThread() {
  return pthread_self();
}

typedef struct {
  LOG_TimeUnit start, length;
  char text[64 - 2*sizeof(LOG_TimeUnit)- sizeof(unsigned long)];
  unsigned long thread;
} LOG_entry;

LOG_entry LOG_data[LOG_MAX_ENTRIES];
size_t LOG_ptr;

void LOG_init() {
}

static inline void LOG_add(const char *text, LOG_TimeUnit start, LOG_TimeUnit stop) {
  size_t i = __sync_fetch_and_add(&LOG_ptr, 1);
  strcpy(LOG_data[i].text, text);
  LOG_data[i].start = start;
  LOG_data[i].length = stop-start;
  LOG_data[i].thread = LOG_getThread();
}

static void LOG_dump(const char *filename) {
  FILE *out = fopen(filename, "w");
  fprintf(out, "LOG 2\n");
  LOG_TimeUnit minimum = LOG_data[0].start;
  for (size_t i = 0; i < LOG_ptr; ++i) {
      if (LOG_data[i].start < minimum)
          minimum = LOG_data[i].start;
  }
  for (size_t i = 0; i < LOG_ptr; ++i) {
      fprintf(out, "%lu: %lld %lld %s\n",
              LOG_data[i].thread,
              LOG_data[i].start-minimum,
              LOG_data[i].length,
              LOG_data[i].text);
  }
  fclose(out);
}

#ifdef LOGGING_PERF
///////////////////// PERFORMANCE_COUNTERS

int LOG_fd;

unsigned long long LOG_readCounter() {
    unsigned long long count;
    size_t res = read(LOG_fd, &count, sizeof(unsigned long long));
    if (res != sizeof(unsigned long long)) {
        fprintf(stderr, "read() failed %d", res);
        exit(1);
    }
    return count;
}

void LOG_init_perf(unsigned int type, unsigned long long config) {
    struct perf_event_attr attr = {0};
    attr.type = type;
    attr.config = config;
    attr.inherit = 1;
    attr.disabled = 1;
    attr.size = sizeof(struct perf_event_attr);
    LOG_fd = syscall(__NR_perf_event_open, attr, 0, -1, -1, 0);

    if (LOG_fd < 0) {
        fprintf(stderr, "sys_perf_event_open failed %d", fd);
        exit(1);
    }
}

unsigned long long LOG_getCacheStart() {
    unsigned long long count;
    count = LOG_readCounter();
    ioctl(LOG_fd, PERF_EVENT_IOC_ENABLE);
    return count;
}
unsigned long long LOG_getCacheStop() {
    unsigned long long count;
    ioctl(LOG_fd, PERF_EVENT_IOC_DISABLE);
    count = LOG_readCounter();
    return count;
}

#else // LOGGING_PERF
#define LOG_init_perf(a,b)
#define LOG_getCacheStart() 0
#define LOG_getCacheStop() 0
#endif

#else // LOGGING
#define LOG_add(a,b,c)
#define LOG_dump(a)
#define LOG_init_perf(a,b)
#define LOG_getCacheStart() 0
#define LOG_getCacheStop() 0
#endif
<commit_msg>Fixed a bug: used wrong formatting.<commit_after>#include <pthread.h>
#include <string.h>

///////////////////// TIMING

typedef unsigned long long LOG_TimeUnit;

static inline LOG_TimeUnit LOG_getTimeStart() {
  unsigned hi, lo;

  __asm__ __volatile__ ("cpuid\n"
                        "rdtsc\n"
         : "=a" (lo), "=d" (hi)
         :: "%rbx", "%rcx");
  return ( (unsigned long long)lo)|( ((unsigned long long)hi)<<32 );
}

static inline LOG_TimeUnit LOG_getTimeStop() {
  unsigned hi, lo;

  __asm__ __volatile__ ("rdtscp\n"
                        "mov %%edx, %0\n\t"
                        "mov %%eax, %1\n\t"
                        "cpuid\n"
          : "=r" (hi), "=r" (lo)
          :: "%rax", "%rbx", "%rcx", "%rdx");
  return ( (unsigned long long)lo)|( ((unsigned long long)hi)<<32 );
}

#ifdef LOGGING
///////////////////// LOGGING

#define LOG_MAX_ENTRIES 1000000

static inline unsigned long LOG_getThread() {
  return pthread_self();
}

typedef struct {
  LOG_TimeUnit start, length;
  char text[64 - 2*sizeof(LOG_TimeUnit)- sizeof(unsigned long)];
  unsigned long thread;
} LOG_entry;

LOG_entry LOG_data[LOG_MAX_ENTRIES];
size_t LOG_ptr;

void LOG_init() {
}

static inline void LOG_add(const char *text, LOG_TimeUnit start, LOG_TimeUnit stop) {
  size_t i = __sync_fetch_and_add(&LOG_ptr, 1);
  strcpy(LOG_data[i].text, text);
  LOG_data[i].start = start;
  LOG_data[i].length = stop-start;
  LOG_data[i].thread = LOG_getThread();
}

static void LOG_dump(const char *filename) {
  FILE *out = fopen(filename, "w");
  fprintf(out, "LOG 2\n");
  LOG_TimeUnit minimum = LOG_data[0].start;
  for (size_t i = 0; i < LOG_ptr; ++i) {
      if (LOG_data[i].start < minimum)
          minimum = LOG_data[i].start;
  }
  for (size_t i = 0; i < LOG_ptr; ++i) {
      fprintf(out, "%lu: %llu %llu %s\n",
              LOG_data[i].thread,
              LOG_data[i].start-minimum,
              LOG_data[i].length,
              LOG_data[i].text);
  }
  fclose(out);
}

#ifdef LOGGING_PERF
///////////////////// PERFORMANCE_COUNTERS

int LOG_fd;

unsigned long long LOG_readCounter() {
    unsigned long long count;
    size_t res = read(LOG_fd, &count, sizeof(unsigned long long));
    if (res != sizeof(unsigned long long)) {
        fprintf(stderr, "read() failed %d", res);
        exit(1);
    }
    return count;
}

void LOG_init_perf(unsigned int type, unsigned long long config) {
    struct perf_event_attr attr = {0};
    attr.type = type;
    attr.config = config;
    attr.inherit = 1;
    attr.disabled = 1;
    attr.size = sizeof(struct perf_event_attr);
    LOG_fd = syscall(__NR_perf_event_open, attr, 0, -1, -1, 0);

    if (LOG_fd < 0) {
        fprintf(stderr, "sys_perf_event_open failed %d", fd);
        exit(1);
    }
}

unsigned long long LOG_getCacheStart() {
    unsigned long long count;
    count = LOG_readCounter();
    ioctl(LOG_fd, PERF_EVENT_IOC_ENABLE);
    return count;
}
unsigned long long LOG_getCacheStop() {
    unsigned long long count;
    ioctl(LOG_fd, PERF_EVENT_IOC_DISABLE);
    count = LOG_readCounter();
    return count;
}

#else // LOGGING_PERF
#define LOG_init_perf(a,b)
#define LOG_getCacheStart() 0
#define LOG_getCacheStop() 0
#endif

#else // LOGGING
#define LOG_add(a,b,c)
#define LOG_dump(a)
#define LOG_init_perf(a,b)
#define LOG_getCacheStart() 0
#define LOG_getCacheStop() 0
#endif
<|endoftext|>


<commit_before>
#include <string.h>
#include <vector>
#include <stdio.h>
#include <cassert>
#include <stdexcept>
#include "univalue.h"

using namespace std;

void UniValue::read(const char *raw)
{
    clear();

    bool expectName = false;
    bool expectColon = false;
    vector<UniValue*> stack;

    while (*raw) {
        switch (*raw) {

        case '{':
        case '[': {
            VType utyp = (*raw == '{' ? VOBJ : VARR);
            if (!stack.size()) {
                setObject();
                stack.push_back(this);
            } else {
                UniValue tmpVal(utyp);
                UniValue *top = stack.back();
                top->values.push_back(tmpVal);

                UniValue *newTop = &(top->values.back());
                stack.push_back(newTop);
            }

            if (utyp == VOBJ)
                expectName = true;

            raw++;
            break;
            }

        case '}':
        case ']': {
            if (!stack.size() || expectColon)
                throw runtime_error("json parse: unexpected }]");

            VType utyp = (*raw == '}' ? VOBJ : VARR);
            UniValue *top = stack.back();
            if (utyp != top->getType())
                throw runtime_error("json parse: mismatched }]");

            stack.pop_back();
            expectName = false;

            raw++;
            break;
            }

        case ':': {
            if (!stack.size() || expectName || !expectColon)
                throw runtime_error("json parse: : stack empty or want name");

            UniValue *top = stack.back();
            if (top->getType() != VOBJ)
                throw runtime_error("json parse: : parent not object");

	    expectColon = false;

            raw++;
            break;
            }

        case ',': {
            if (!stack.size() || expectName || expectColon)
                throw runtime_error("json parse: , stack empty or want name");

            UniValue *top = stack.back();
            if (top->getType() == VOBJ)
                expectName = true;

            raw++;
            break;
            }

        case 'n':
        case 't':
        case 'f': {
            if (!stack.size() || expectName || expectColon)
                throw runtime_error("json parse: ntf stack empty or want name");

            VType utyp;
            if (!strncmp(raw, "null", 4)) {
                utyp = VNULL;
                raw += 4;
            } else if (!strncmp(raw, "true", 4)) {
                utyp = VTRUE;
                raw += 4;
            } else if (!strncmp(raw, "false", 5)) {
                utyp = VFALSE;
                raw += 5;
            } else
                throw runtime_error("json parse: ntf unknown keyword");

            UniValue tmpVal(utyp);
            UniValue *top = stack.back();
            top->values.push_back(tmpVal);

            break;
            }

        case '-':
        case '0':
        case '1':
        case '2':
        case '3':
        case '4':
        case '5':
        case '6':
        case '7':
        case '8':
        case '9': {
            if (!stack.size() || expectName || expectColon)
                throw runtime_error("json parse digits: stack empty or want name");

            // part 1: int
            string numStr;

            const char *first = raw;

	    const char *firstDigit = first;
	    if (!isdigit(*firstDigit))
	        firstDigit++;
            if (*firstDigit == '0')
                throw runtime_error("json parse digits 1.5: first digit bad");

            numStr += *raw;                       // copy first char
            raw++;

            if ((*first == '-') && (!isdigit(*raw)))
                throw runtime_error("json parse digits 2: first char bad");

            while ((*raw) && isdigit(*raw)) {     // copy digits
                numStr += *raw;
                raw++;
            }

            // part 2: frac
            if (*raw == '.') {
                numStr += *raw;                   // copy .
                raw++;

                if (!isdigit(*raw))
                    throw runtime_error("json parse digits 3: want digit");
                while ((*raw) && isdigit(*raw)) { // copy digits
                    numStr += *raw;
                    raw++;
                }
            }

            // part 3: exp
            if (*raw == 'e' || *raw == 'E') {
                numStr += *raw;                   // copy E
                raw++;

                if (*raw == '-' || *raw == '+') { // copy +/-
                    numStr += *raw;
                    raw++;
                }

                if (!isdigit(*raw))
                    throw runtime_error("json parse digits 4: want digit");
                while ((*raw) && isdigit(*raw)) { // copy digits
                    numStr += *raw;
                    raw++;
                }
            }

            UniValue tmpVal(VNUM, numStr);
            UniValue *top = stack.back();
            top->values.push_back(tmpVal);

            break;
            }

        case '"': {
            if (!stack.size())
                throw runtime_error("json parse string: stack empty");

            raw++;                                // skip "

            string valStr;

            while (*raw) {
                if (*raw < 0x20)
                    throw runtime_error("json parse string: have ctl chars");

                else if (*raw == '\\') {
                    raw++;                        // skip backslash

                    switch (*raw) {
                    case '"':  valStr += "\""; break;
                    case '\\': valStr += "\\"; break;
                    case '/':  valStr += "/"; break;
                    case 'b':  valStr += "\b"; break;
                    case 'f':  valStr += "\f"; break;
                    case 'n':  valStr += "\n"; break;
                    case 'r':  valStr += "\r"; break;
                    case 't':  valStr += "\t"; break;
                    case 'u':
                        // TODO: not supported yet
                        assert(0);
                        break;
                    default:
                        throw runtime_error("json parse string: unknown escape");

                    }

                    raw++;                        // skip esc'd char
                }

                else if (*raw == '"') {
                    raw++;                        // skip "
                    break;                        // stop scanning
                }

                else {
                    valStr += *raw;
                    raw++;
                }
            }

            UniValue *top = stack.back();

            if (expectName) {
                top->keys.push_back(valStr);
                expectName = false;
		expectColon = true;
            } else {
                UniValue tmpVal(VSTR, valStr);
                top->values.push_back(tmpVal);
            }

            break;
            }

        case ' ':
        case '\t':
        case '\f':
        case '\r':
        case '\n':
            raw++;                                // skip whitespace
            break;

	default:
            throw runtime_error("json parse string: illegal expression");
        }
    }

    if (stack.size() != 0) {
        char msg[64];
        sprintf(msg, "json parse: too many toplevel obj, %lu", stack.size());
        throw runtime_error(msg);
    }
}

<commit_msg>univalue_read: break up into separate tokenizing and parsing steps<commit_after>
#include <string.h>
#include <vector>
#include <stdio.h>
#include <cassert>
#include <stdexcept>
#include "univalue.h"

using namespace std;

enum tokentype {
    TOK_ERR        = -1,
    TOK_NONE       = 0,                           // eof
    TOK_OBJ_OPEN,
    TOK_OBJ_CLOSE,
    TOK_ARR_OPEN,
    TOK_ARR_CLOSE,
    TOK_COLON,
    TOK_COMMA,
    TOK_KW_NULL,
    TOK_KW_TRUE,
    TOK_KW_FALSE,
    TOK_NUMBER,
    TOK_STRING,
};

enum tokentype getJsonToken(string& tokenVal, unsigned int& consumed,
                            const char *raw)
{
    tokenVal.clear();
    consumed = 0;

    const char *rawStart = raw;

    while ((*raw) && (isspace(*raw)))             // skip whitespace
        raw++;

    switch (*raw) {

    case 0:
        return TOK_NONE;

    case '{':
        raw++;
        consumed = (raw - rawStart);
        return TOK_OBJ_OPEN;
    case '}':
        raw++;
        consumed = (raw - rawStart);
        return TOK_OBJ_CLOSE;
    case '[':
        raw++;
        consumed = (raw - rawStart);
        return TOK_ARR_OPEN;
    case ']':
        raw++;
        consumed = (raw - rawStart);
        return TOK_ARR_CLOSE;

    case ':':
        raw++;
        consumed = (raw - rawStart);
        return TOK_COLON;
    case ',':
        raw++;
        consumed = (raw - rawStart);
        return TOK_COMMA;

    case 'n':
    case 't':
    case 'f':
        if (!strncmp(raw, "null", 4)) {
            raw += 4;
            consumed = (raw - rawStart);
            return TOK_KW_NULL;
        } else if (!strncmp(raw, "true", 4)) {
            raw += 4;
            consumed = (raw - rawStart);
            return TOK_KW_TRUE;
        } else if (!strncmp(raw, "false", 5)) {
            raw += 5;
            consumed = (raw - rawStart);
            return TOK_KW_FALSE;
        } else
            return TOK_ERR;

    case '-':
    case '0':
    case '1':
    case '2':
    case '3':
    case '4':
    case '5':
    case '6':
    case '7':
    case '8':
    case '9': {
        // part 1: int
        string numStr;

        const char *first = raw;

        const char *firstDigit = first;
        if (!isdigit(*firstDigit))
            firstDigit++;
        if (*firstDigit == '0')
            return TOK_ERR;

        numStr += *raw;                       // copy first char
        raw++;

        if ((*first == '-') && (!isdigit(*raw)))
            return TOK_ERR;

        while ((*raw) && isdigit(*raw)) {     // copy digits
            numStr += *raw;
            raw++;
        }

        // part 2: frac
        if (*raw == '.') {
            numStr += *raw;                   // copy .
            raw++;

            if (!isdigit(*raw))
                return TOK_ERR;
            while ((*raw) && isdigit(*raw)) { // copy digits
                numStr += *raw;
                raw++;
            }
        }

        // part 3: exp
        if (*raw == 'e' || *raw == 'E') {
            numStr += *raw;                   // copy E
            raw++;

            if (*raw == '-' || *raw == '+') { // copy +/-
                numStr += *raw;
                raw++;
            }

            if (!isdigit(*raw))
                return TOK_ERR;
            while ((*raw) && isdigit(*raw)) { // copy digits
                numStr += *raw;
                raw++;
            }
        }

        tokenVal = numStr;
        consumed = (raw - rawStart);
        return TOK_NUMBER;
        }

    case '"': {
        raw++;                                // skip "

        string valStr;

        while (*raw) {
            if (*raw < 0x20)
                return TOK_ERR;

            else if (*raw == '\\') {
                raw++;                        // skip backslash

                switch (*raw) {
                case '"':  valStr += "\""; break;
                case '\\': valStr += "\\"; break;
                case '/':  valStr += "/"; break;
                case 'b':  valStr += "\b"; break;
                case 'f':  valStr += "\f"; break;
                case 'n':  valStr += "\n"; break;
                case 'r':  valStr += "\r"; break;
                case 't':  valStr += "\t"; break;
                case 'u':
                    // TODO: not supported yet
                    assert(0);
                    break;
                default:
                    return TOK_ERR;

                }

                raw++;                        // skip esc'd char
            }

            else if (*raw == '"') {
                raw++;                        // skip "
                break;                        // stop scanning
            }

            else {
                valStr += *raw;
                raw++;
            }
        }

        tokenVal = valStr;
        consumed = (raw - rawStart);
        return TOK_STRING;
        }

    default:
        return TOK_ERR;
    }
}

void UniValue::read(const char *raw)
{
    clear();

    bool expectName = false;
    bool expectColon = false;
    vector<UniValue*> stack;

    enum tokentype tok = TOK_NONE;
    while (1) {
        string tokenVal;
        unsigned int consumed;
        tok = getJsonToken(tokenVal, consumed, raw);
        if (tok == TOK_NONE || tok == TOK_ERR)
            break;
        raw += consumed;

        switch (tok) {

        case TOK_OBJ_OPEN:
        case TOK_ARR_OPEN: {
            VType utyp = (tok == TOK_OBJ_OPEN ? VOBJ : VARR);
            if (!stack.size()) {
                setObject();
                stack.push_back(this);
            } else {
                UniValue tmpVal(utyp);
                UniValue *top = stack.back();
                top->values.push_back(tmpVal);

                UniValue *newTop = &(top->values.back());
                stack.push_back(newTop);
            }

            if (utyp == VOBJ)
                expectName = true;
            break;
            }

        case TOK_OBJ_CLOSE:
        case TOK_ARR_CLOSE: {
            if (!stack.size() || expectColon)
                throw runtime_error("json parse: unexpected }]");

            VType utyp = (tok == TOK_OBJ_CLOSE ? VOBJ : VARR);
            UniValue *top = stack.back();
            if (utyp != top->getType())
                throw runtime_error("json parse: mismatched }]");

            stack.pop_back();
            expectName = false;
            break;
            }

        case TOK_COLON: {
            if (!stack.size() || expectName || !expectColon)
                throw runtime_error("json parse: : stack empty or want name");

            UniValue *top = stack.back();
            if (top->getType() != VOBJ)
                throw runtime_error("json parse: : parent not object");

            expectColon = false;
            break;
            }

        case TOK_COMMA: {
            if (!stack.size() || expectName || expectColon)
                throw runtime_error("json parse: , stack empty or want name");

            UniValue *top = stack.back();
            if (top->getType() == VOBJ)
                expectName = true;
            break;
            }

        case TOK_KW_NULL:
        case TOK_KW_TRUE:
        case TOK_KW_FALSE: {
            if (!stack.size() || expectName || expectColon)
                throw runtime_error("json parse: ntf stack empty or want name");

            VType utyp;
            switch (tok) {
            case TOK_KW_NULL:  utyp = VNULL; break;
            case TOK_KW_TRUE:  utyp = VTRUE; break;
            case TOK_KW_FALSE: utyp = VFALSE; break;
            default: assert(0); break;
            }

            UniValue tmpVal(utyp);
            UniValue *top = stack.back();
            top->values.push_back(tmpVal);

            break;
            }

        case TOK_NUMBER: {
            if (!stack.size() || expectName || expectColon)
                throw runtime_error("json parse digits: stack empty or want name");

            UniValue tmpVal(VNUM, tokenVal);
            UniValue *top = stack.back();
            top->values.push_back(tmpVal);

            break;
            }

        case TOK_STRING: {
            if (!stack.size())
                throw runtime_error("json parse string: stack empty");

            UniValue *top = stack.back();

            if (expectName) {
                top->keys.push_back(tokenVal);
                expectName = false;
                expectColon = true;
            } else {
                UniValue tmpVal(VSTR, tokenVal);
                top->values.push_back(tmpVal);
            }

            break;
            }

        default:
            throw runtime_error("json parse string: illegal expression");
        }
    }

    if (stack.size() != 0) {
        char msg[64];
        sprintf(msg, "json parse: too many toplevel obj, %lu", stack.size());
        throw runtime_error(msg);
    }
}

<|endoftext|>


<commit_before>/** \brief A tool for installing IxTheo and KrimDok from scratch on Ubuntu and Centos systems.
 *  \author Dr. Johannes Ruscheinski (johannes.ruscheinski@uni-tuebingen.de)
 *
 *  \copyright 2016 Universitätsbiblothek Tübingen.  All rights reserved.
 *
 *  This program is free software: you can redistribute it and/or modify
 *  it under the terms of the GNU Affero General Public License as
 *  published by the Free Software Foundation, either version 3 of the
 *  License, or (at your option) any later version.
 *
 *  This program is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *  GNU Affero General Public License for more details.
 *
 *  You should have received a copy of the GNU Affero General Public License
 *  along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
#include <iostream>
#include <unistd.h>
#include "Compiler.h"
#include "ExecUtil.h"
#include "FileUtil.h"
#include "StringUtil.h"
#include "util.h"

void Usage() {
    std::cerr << "Usage: " << ::progname << " vufind_system_type\n";
    std::cerr << "       where \"vufind_system_type\" must be either \"krimdok\" or \"ixtheo\".\n\n";
    std::exit(EXIT_FAILURE);
}

// Print a log message to the terminal with a bright green background.
void Echo(const std::string &log_message) {
    std::cout << "\x1B" << "[42m--- " << log_message << "\x1B" << "[0m\n";
}

enum VuFindSystemType { KRIMDOK, IXTHEO };
enum OSSystemType { UBUNTU, CENTOS };

OSSystemType DetermineOSSystemType() {
    std::string file_contents;
    if (FileUtil::ReadString("/etc/issue", &file_contents)
        and StringUtil::FindCaseInsensitive(file_contents, "ubuntu") != std::string::npos)
        return UBUNTU;
    if (FileUtil::ReadString("/etc/redhat_release", &file_contents)
        and StringUtil::FindCaseInsensitive(file_contents, "centos") != std::string::npos)
        return CENTOS;
    Error("you're probably not on an Ubunto nor on a CentOS system!");
}

// Returns true if a line starting with "line_prefix" was found in "filename", o/w returns false.
bool FileContainsLineStartingWith(const std::string &filename, const std::string &line_prefix) {
    std::string file_contents;
    if (unlikely(not FileUtil::ReadString(filename, &file_contents)))
        Error("in FileContainsLineStartingWith: could not read the contents of \"" + filename + "\"!");

    std::vector<std::string> lines;
    StringUtil::Split(file_contents, '\n', &lines);
    for (const auto &line : lines) {
        if (StringUtil::StartsWith(line, line_prefix))
            return true;
    }

    return false;
}

void AppendLineToFileOrDie(const std::string &filename, std::string line) {
    line += '\n';
    std::unique_ptr<File> file(FileUtil::OpenForAppeningOrDie(filename));
    if (file->size() == 0) {
        if (unlikely(file->write(line.data(), line.size()) != line.size()))
            Error("in AppendLineToFileOrDie: failed to append a line to \"" + filename + "\"!");
        return;
    }
    if (unlikely(not file->seek(-1, SEEK_END)))
        Error("in AppendLineToFileOrDie: failed to seek to the last byte in \"" + filename + "\"!");

    // Do we need to append a newline before appending our new line?
    const int last_char(file->get());
    if (last_char != '\n')
        line = '\n' + line; // Apprently we do.

    if (unlikely(file->write(line.data(), line.size()) != line.size()))
        Error("in AppendLineToFileOrDie: failed to write a line at the end of \"" + filename + "\"!");
}

void InsertFsTabLineOrDie(const std::string &line) {
    const std::string::size_type first_space_pos(line.find(' '));
    if (unlikely(first_space_pos == std::string::npos))
        Error("InsertFsTabLineOrDie: could not find a space in \"" + line + "\"!");
    if (unlikely(first_space_pos == 0))
        Error("InsertFsTabLineOrDie: first field of \"" + line + "\" must not be empty!");
    const std::string first_field_plus_space(line.substr(0, first_space_pos + 1));
    if (FileContainsLineStartingWith("/etc/fstab", first_field_plus_space))
        return; // Nothing to do.
    AppendLineToFileOrDie("/etc/fstab", line);
}

void ExecOrDie(const std::string &command, const std::vector<std::string> &arguments) {
    int exit_code;
    if ((exit_code = ExecUtil::Exec(command, arguments)) != 0)
        Error("Failed to execute \"" + command + "\"! (exit code was " + std::to_string(exit_code) + ")");
}

void MountDepartmentDriveOrDie() {
    InsertFsTabLineOrDie("//sn00.zdv.uni-tuebingen.de/ZE020150 /mnt/ZE020150 cifs "
                         "credentials=/root/.smbcredentials,workgroup=uni-tuebingen.de,uid=root,gid=root,auto 0 0");
    ExecOrDie("/bin/mount", { "--all" });
    Echo("mounted department drive");
}

void ExecuteCommandSequence(
    const std::vector<std::pair<std::string, std::vector<std::string>>> &commands_and_arguments)
{
    for (const auto &command_and_arguments : commands_and_arguments)
        ExecOrDie(command_and_arguments.first, command_and_arguments.second);
}

inline std::pair<std::string, std::vector<std::string>> CmdAndArgs(const std::string &command,
                                                                   const std::vector<std::string> &arguments)
{
    return std::pair<std::string, std::vector<std::string>>(command, arguments);
}

void InstallUbuntuSoftwarePackages() {
    const std::vector<std::pair<std::string, std::vector<std::string>>> &commands_and_arguments {
        CmdAndArgs("/usr/bin/add-apt-repository", { "ppa:ubuntu-lxc/lxd-stable" }),
        CmdAndArgs("/usr/bin/apt", { "update" }),
        CmdAndArgs(
            "/usr/bin/apt",
            { "install", "-y", "clang", "golang", "wget", "curl", "git", "apache2", "libapache2-mod-gnutls",
                    "mysql-server", "php7.0", "php7.0-dev", "php-pear", "php7.0-json", "php7.0-ldap", "php7.0-mcrypt",
              "php7.0-mysql", "php7.0-xsl", "php7.0-intl", "php7.0-gd", "libapache2-mod-php7.0", "composer",
              "openjdk-8-jdk", "libmagic-dev", "libpcre3-dev", "libssl-dev", "libkyotocabinet-dev", "mutt",
              "libxml2-dev", "libmysqlclient-dev", "libcurl4-openssl-dev", "ant", "libtokyocabinet-dev",
              "liblz4-tool", "libarchive-dev", "libboost-all-dev", "clang-3.8", "clang++-3.8", "clang", "golang" }),
        CmdAndArgs("/usr/sbin/a2enmod", { "rewrite" }),
        CmdAndArgs("/usr/sbin/phpenmod", { "mcrypt" }),
        CmdAndArgs("/etc/init.d/apache2", { "restart" }),
//        CmdAndArgs("mysql_secure_installation", {  }),
    };
    ExecuteCommandSequence(commands_and_arguments);
    Echo("installed software packages");
}

void InstallCentOSSoftwarePackages() {
    std::vector<std::string> rpm_package_install_args;
    FileUtil::GetFileNameList("*.\\\\.rpm", &rpm_package_install_args,
                              "/mnt/ZE020150/IT-Abteilung/02_Projekte/11_KrimDok_neu/05_Pakete/");
    rpm_package_install_args.insert(rpm_package_install_args.begin(), "-y");
    rpm_package_install_args.insert(rpm_package_install_args.begin(), "install");

    const std::vector<std::pair<std::string, std::vector<std::string>>> &commands_and_arguments {
        CmdAndArgs("/bin/yum", { "update" }),
        CmdAndArgs("/bin/yum", { "-y", "install", "epel-release" }),
        CmdAndArgs(
            "/bin/yum",
            { "-y", "install", "mawk", "git", "mariadb", "mariadb-server", "httpd", "php", "php-devel", "php-mcrypt",
              "php-intl", "php-ldap", "php-mysql", "php-xsl", "php-gd", "php-mbstring", "php-mcrypt",
              "java-*-openjdk-devel", "mawk", "mod_ssl", "epel-release", "wget", "policycoreutils-python" }),
        CmdAndArgs("systemctl", { "start", "mariadb.service" }),
        CmdAndArgs("mysql_secure_installation", {  }),
        CmdAndArgs(
            "/bin/wget",
            { "http://download.opensuse.org/repositories/security:shibboleth/CentOS_7/security:shibboleth.repo",
              "--directory-prefix=/etc/yum.repos.d/" }),
        CmdAndArgs(
            "/bin/yum",
            { "-y", "install", "curl-openssl", "mutt", "golang", "lsof", "clang", "gcc-c++.x86_64", "file-devel",
              "pcre-devel", "openssl-devel", "kyotocabinet-devel", "tokyocabinet-devel", "poppler-utils", "libwebp",
              "mariadb-devel.x86_64", "libxml2-devel.x86_64", "libcurl-openssl-devel.x86_64", "ant", "lz4", "unzip",
              "libarchive-devel", "boost-devel" }),
        CmdAndArgs("/bin/yum", rpm_package_install_args),
        CmdAndArgs(
            "/bin/ln",
            { "-s", "/usr/share/tessdata/deu.traineddata", "/usr/share/tesseract/tessdata/deu.traineddata" }),
    };
    ExecuteCommandSequence(commands_and_arguments);
}

void InstallSoftwarePackages(const OSSystemType os_system_type) {
    if (os_system_type == UBUNTU)
        InstallUbuntuSoftwarePackages();
    else
        InstallCentOSSoftwarePackages();
}

int main(int argc, char **argv) {
    ::progname = argv[0];

    if (argc != 2)
        Usage();

    VuFindSystemType vufind_system_type;
    if (::strcasecmp(argv[1], "krimdok") == 0)
        vufind_system_type = KRIMDOK;
    else if (::strcasecmp(argv[1], "ixtheo") == 0)
        vufind_system_type = IXTHEO;
    else
        Error("system type msut be either \"krimdok\" or \"ixtheo\"!");
    (void)vufind_system_type;

    const OSSystemType os_system_type(DetermineOSSystemType());

    if (::geteuid() != 0)
        Error("you must execute this program as root!");

    try {
        MountDepartmentDriveOrDie();
        InstallSoftwarePackages(os_system_type);
    } catch (const std::exception &x) {
        Error("caught exception: " + std::string(x.what()));
    }
}
<commit_msg>Fixed an incorrect pathname.<commit_after>/** \brief A tool for installing IxTheo and KrimDok from scratch on Ubuntu and Centos systems.
 *  \author Dr. Johannes Ruscheinski (johannes.ruscheinski@uni-tuebingen.de)
 *
 *  \copyright 2016 Universitätsbiblothek Tübingen.  All rights reserved.
 *
 *  This program is free software: you can redistribute it and/or modify
 *  it under the terms of the GNU Affero General Public License as
 *  published by the Free Software Foundation, either version 3 of the
 *  License, or (at your option) any later version.
 *
 *  This program is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *  GNU Affero General Public License for more details.
 *
 *  You should have received a copy of the GNU Affero General Public License
 *  along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
#include <iostream>
#include <unistd.h>
#include "Compiler.h"
#include "ExecUtil.h"
#include "FileUtil.h"
#include "StringUtil.h"
#include "util.h"

void Usage() {
    std::cerr << "Usage: " << ::progname << " vufind_system_type\n";
    std::cerr << "       where \"vufind_system_type\" must be either \"krimdok\" or \"ixtheo\".\n\n";
    std::exit(EXIT_FAILURE);
}

// Print a log message to the terminal with a bright green background.
void Echo(const std::string &log_message) {
    std::cout << "\x1B" << "[42m--- " << log_message << "\x1B" << "[0m\n";
}

enum VuFindSystemType { KRIMDOK, IXTHEO };
enum OSSystemType { UBUNTU, CENTOS };
static std::string master_password;

OSSystemType DetermineOSSystemType() {
    std::string file_contents;
    if (FileUtil::ReadString("/etc/issue", &file_contents)
        and StringUtil::FindCaseInsensitive(file_contents, "ubuntu") != std::string::npos)
        return UBUNTU;
    if (FileUtil::ReadString("/etc/redhat-release", &file_contents)
        and StringUtil::FindCaseInsensitive(file_contents, "centos") != std::string::npos)
        return CENTOS;
    Error("you're probably not on an Ubuntu nor on a CentOS system!");
}

void ExecOrDie(const std::string &command, const std::vector<std::string> &arguments) {
    int exit_code;
    if ((exit_code = ExecUtil::Exec(command, arguments)) != 0)
        Error("Failed to execute \"" + command + "\"! (exit code was " + std::to_string(exit_code) + ")");
}

void MountLUKSContainerOrDie() {
    FileUtil::AutoTempFile key_file_temp_file;
    if (not FileUtil::WriteString(key_file_temp_file.getFilePath(), master_password))
        Error("failed to write the master password into the temporary LUKS key file!");

    const std::string LUKS_IMAGE_FILE("/usr/local/ub_tools/configs.ext4.luks");
    const std::string LUKS_MOUNT_POINT("/usr/local/ub_tools/configs");
    ExecOrDie(ExecUtil::Which("cryptsetup"),
              { "luksOpen", "--batch-mode", "--key-file", key_file_temp_file.getFilePath(), LUKS_IMAGE_FILE,
                "configs" });
    ExecOrDie("/bin/mount", { "/dev/mapper/configs", LUKS_MOUNT_POINT });
}

void ExecuteCommandSequence(
    const std::vector<std::pair<std::string, std::vector<std::string>>> &commands_and_arguments)
{
    for (const auto &command_and_arguments : commands_and_arguments)
        ExecOrDie(command_and_arguments.first, command_and_arguments.second);
}

inline std::pair<std::string, std::vector<std::string>> CmdAndArgs(const std::string &command,
                                                                   const std::vector<std::string> &arguments)
{
    return std::pair<std::string, std::vector<std::string>>(command, arguments);
}

void InstallUbuntuSoftwarePackages(const VuFindSystemType vufind_system_type) {
    const std::vector<std::pair<std::string, std::vector<std::string>>> &commands_and_arguments {
        CmdAndArgs("/usr/bin/add-apt-repository", { "--yes", "ppa:ubuntu-lxc/lxd-stable" }),
        CmdAndArgs("/usr/bin/apt", { "update" }),
        CmdAndArgs(
            "/usr/bin/apt",
            { "install", "--yes", "clang", "golang", "wget", "curl", "git", "apache2", "libapache2-mod-gnutls",
                    "mysql-server", "php7.0", "php7.0-dev", "php-pear", "php7.0-json", "php7.0-ldap", "php7.0-mcrypt",
              "php7.0-mysql", "php7.0-xsl", "php7.0-intl", "php7.0-gd", "libapache2-mod-php7.0", "composer",
              "openjdk-8-jdk", "libmagic-dev", "libpcre3-dev", "libssl-dev", "libkyotocabinet-dev", "mutt",
              "libxml2-dev", "libmysqlclient-dev", "libcurl4-openssl-dev", "ant", "libtokyocabinet-dev",
              "liblz4-tool", "libarchive-dev", "libboost-all-dev", "clang-3.8", "clang++-3.8", "clang", "golang" }),
        CmdAndArgs("/usr/sbin/a2enmod", { "rewrite" }),
        CmdAndArgs("/usr/sbin/phpenmod", { "mcrypt" }),
        CmdAndArgs("/etc/init.d/apache2", { "restart" }),
//        CmdAndArgs("mysql_secure_installation", {  }),
    };
    ExecuteCommandSequence(commands_and_arguments);

    // If installing a KrimDok system we need tesseract:
    if (vufind_system_type == KRIMDOK)
        ExecOrDie("/usr/bin/apt", { "install", "--yes", "tesseract-ocr-all" });
}

void InstallCentOSSoftwarePackages(const VuFindSystemType vufind_system_type) {
    const std::vector<std::pair<std::string, std::vector<std::string>>> &commands_and_arguments {
        CmdAndArgs("/bin/yum", { "update" }),
        CmdAndArgs("/bin/yum", { "-y", "install", "epel-release" }),
        CmdAndArgs(
            "/bin/yum",
            { "-y", "install", "mawk", "git", "mariadb", "mariadb-server", "httpd", "php", "php-devel", "php-mcrypt",
              "php-intl", "php-ldap", "php-mysql", "php-xsl", "php-gd", "php-mbstring", "php-mcrypt",
              "java-*-openjdk-devel", "mawk", "mod_ssl", "epel-release", "wget", "policycoreutils-python" }),
        CmdAndArgs("systemctl", { "start", "mariadb.service" }),
        CmdAndArgs("mysql_secure_installation", {  }),
        CmdAndArgs(
            "/bin/wget",
            { "http://download.opensuse.org/repositories/security:shibboleth/CentOS_7/security:shibboleth.repo",
              "--directory-prefix=/etc/yum.repos.d/" }),
        CmdAndArgs(
            "/bin/yum",
            { "-y", "install", "curl-openssl", "mutt", "golang", "lsof", "clang", "gcc-c++.x86_64", "file-devel",
              "pcre-devel", "openssl-devel", "kyotocabinet-devel", "tokyocabinet-devel", "poppler-utils", "libwebp",
              "mariadb-devel.x86_64", "libxml2-devel.x86_64", "libcurl-openssl-devel.x86_64", "ant", "lz4", "unzip",
              "libarchive-devel", "boost-devel" }),
        CmdAndArgs(
            "/bin/ln",
            { "-s", "/usr/share/tessdata/deu.traineddata", "/usr/share/tesseract/tessdata/deu.traineddata" }),
    };
    ExecuteCommandSequence(commands_and_arguments);

    if (vufind_system_type == KRIMDOK) {
        std::vector<std::string> rpm_package_install_args;
        FileUtil::GetFileNameList("*.\\\\.rpm", &rpm_package_install_args,
                                  "/mnt/ZE020150/IT-Abteilung/02_Projekte/11_KrimDok_neu/05_Pakete/");
        rpm_package_install_args.insert(rpm_package_install_args.begin(), "-y");
        rpm_package_install_args.insert(rpm_package_install_args.begin(), "install");
        ExecOrDie("/bin/yum", rpm_package_install_args);
    }
}

void InstallSoftwarePackages(const OSSystemType os_system_type,  const VuFindSystemType vufind_system_type) {
    if (os_system_type == UBUNTU)
        InstallUbuntuSoftwarePackages(vufind_system_type);
    else
        InstallCentOSSoftwarePackages(vufind_system_type);
    Echo("installed software packages");
}

void GetMasterPassword() {
    errno = 0;
    master_password = ::getpass("Master password >");
    if (errno != 0)
        Error("failed to read the password from the terminal!");
}

int main(int argc, char **argv) {
    ::progname = argv[0];

    if (argc != 2)
        Usage();

    if (::geteuid() != 0)
        Error("you must execute this program as root!");

    GetMasterPassword();

    VuFindSystemType vufind_system_type;
    if (::strcasecmp(argv[1], "krimdok") == 0)
        vufind_system_type = KRIMDOK;
    else if (::strcasecmp(argv[1], "ixtheo") == 0)
        vufind_system_type = IXTHEO;
    else
        Error("system type must be either \"krimdok\" or \"ixtheo\"!");

    const OSSystemType os_system_type(DetermineOSSystemType());

    try {
        MountLUKSContainerOrDie();
        InstallSoftwarePackages(os_system_type, vufind_system_type);
    } catch (const std::exception &x) {
        Error("caught exception: " + std::string(x.what()));
    }
}
<|endoftext|>


<commit_before><commit_msg>making a behavior-preserving change to see if my commit messages fix works<commit_after><|endoftext|>