<commit_before>#include <itkImageRegionConstIterator.h>

#include "rtkSheppLoganPhantomFilter.h"
#include "rtkDrawSheppLoganFilter.h"
#include "rtkFDKConeBeamReconstructionFilter.h"
#include "rtkConstantImageSource.h"
#include "rtkDisplacedDetectorImageFilter.h"

template<class TImage>
void CheckImageQuality(typename TImage::Pointer recon, typename TImage::Pointer ref)
{
  typedef itk::ImageRegionConstIterator<TImage> ImageIteratorType;
  ImageIteratorType itTest( recon, recon->GetBufferedRegion() );
  ImageIteratorType itRef( ref, ref->GetBufferedRegion() );

  typedef double ErrorType;
  ErrorType TestError = 0.;
  ErrorType EnerError = 0.;

  itTest.GoToBegin();
  itRef.GoToBegin();

  while( !itRef.IsAtEnd() )
    {
    typename TImage::PixelType TestVal = itTest.Get();
    typename TImage::PixelType RefVal = itRef.Get();
    TestError += vcl_abs(RefVal - TestVal);
    EnerError += vcl_pow(ErrorType(RefVal - TestVal), 2.);
    ++itTest;
    ++itRef;
    }
  // Error per Pixel
  ErrorType ErrorPerPixel = TestError/ref->GetBufferedRegion().GetNumberOfPixels();
  std::cout << "\nError per Pixel = " << ErrorPerPixel << std::endl;
  // MSE
  ErrorType MSE = EnerError/ref->GetBufferedRegion().GetNumberOfPixels();
  std::cout << "MSE = " << MSE << std::endl;
  // PSNR
  ErrorType PSNR = 20*log10(2.0) - 10*log10(MSE);
  std::cout << "PSNR = " << PSNR << "dB" << std::endl;
  // QI
  ErrorType QI = (2.0-ErrorPerPixel)/2.0;
  std::cout << "QI = " << QI << std::endl;

  // Checking results
  if (ErrorPerPixel > 0.06)
  {
    std::cerr << "Test Failed, Error per pixel not valid! "
              << ErrorPerPixel << " instead of 0.06." << std::endl;
    exit( EXIT_FAILURE);
  }
  if (PSNR < 24.)
  {
    std::cerr << "Test Failed, PSNR not valid! "
              << PSNR << " instead of 24." << std::endl;
    exit( EXIT_FAILURE);
  }
}

int main(int argc, char* argv[])
{
  const unsigned int Dimension = 3;
  typedef float                                    OutputPixelType;
  typedef itk::Image< OutputPixelType, Dimension > OutputImageType;
  const unsigned int NumberOfProjectionImages = 180;

  // Constant image sources
  typedef rtk::ConstantImageSource< OutputImageType > ConstantImageSourceType;
  ConstantImageSourceType::PointType origin;
  ConstantImageSourceType::SizeType size;
  ConstantImageSourceType::SpacingType spacing;

  ConstantImageSourceType::Pointer tomographySource  = ConstantImageSourceType::New();
  origin[0] = -127.;
  origin[1] = -127.;
  origin[2] = -127.;
  size[0] = 128;
  size[1] = 128;
  size[2] = 128;
  spacing[0] = 2.;
  spacing[1] = 2.;
  spacing[2] = 2.;
  tomographySource->SetOrigin( origin );
  tomographySource->SetSpacing( spacing );
  tomographySource->SetSize( size );
  tomographySource->SetConstant( 0. );

  ConstantImageSourceType::Pointer projectionsSource = ConstantImageSourceType::New();
  origin[0] = -254.;
  origin[1] = -254.;
  origin[2] = -254.;
  size[0] = 128;
  size[1] = 128;
  size[2] = NumberOfProjectionImages;
  spacing[0] = 4.;
  spacing[1] = 4.;
  spacing[2] = 4.;
  projectionsSource->SetOrigin( origin );
  projectionsSource->SetSpacing( spacing );
  projectionsSource->SetSize( size );
  projectionsSource->SetConstant( 0. );

  // Shepp Logan projections filter
  typedef rtk::SheppLoganPhantomFilter<OutputImageType, OutputImageType> SLPType;
  SLPType::Pointer slp=SLPType::New();
  slp->SetInput( projectionsSource->GetOutput() );

  // Displaced detector weighting
  typedef rtk::DisplacedDetectorImageFilter< OutputImageType > DDFType;
  DDFType::Pointer ddf = DDFType::New();
  ddf->SetInput( slp->GetOutput() );

  // Create a reference object (in this case a 3D phantom reference).
  typedef rtk::DrawSheppLoganFilter<OutputImageType, OutputImageType> DSLType;
  DSLType::Pointer dsl = DSLType::New();
  dsl->SetInput( tomographySource->GetOutput() );
  TRY_AND_EXIT_ON_ITK_EXCEPTION( dsl->Update() );

  // FDK reconstruction filtering
  typedef rtk::FDKConeBeamReconstructionFilter< OutputImageType > FDKCPUType;
  FDKCPUType::Pointer feldkamp = FDKCPUType::New();
  feldkamp->SetInput( 0, tomographySource->GetOutput() );
  feldkamp->SetInput( 1, ddf->GetOutput() );

  std::cout << "\n\n****** Case 1: positive offset in geometry ******" << std::endl;
  typedef rtk::ThreeDCircularProjectionGeometry GeometryType;
  GeometryType::Pointer geometry = GeometryType::New();
  slp->SetGeometry(geometry);
  ddf->SetGeometry( geometry );
  feldkamp->SetGeometry( geometry );
  for(unsigned int noProj=0; noProj<NumberOfProjectionImages; noProj++)
    geometry->AddProjection(600., 1200., noProj*360./NumberOfProjectionImages, 120., 0.);
  slp->Update();
  TRY_AND_EXIT_ON_ITK_EXCEPTION( feldkamp->Update() );
  CheckImageQuality<OutputImageType>(feldkamp->GetOutput(), dsl->GetOutput());

  std::cout << "\n\n****** Case 2: negative offset in geometry ******" << std::endl;
  geometry = GeometryType::New();
  slp->SetGeometry(geometry);
  ddf->SetGeometry( geometry );
  feldkamp->SetGeometry( geometry );
  for(unsigned int noProj=0; noProj<NumberOfProjectionImages; noProj++)
    geometry->AddProjection(600., 1200., noProj*360./NumberOfProjectionImages, -120., 0.);
  TRY_AND_EXIT_ON_ITK_EXCEPTION( feldkamp->Update() );
  CheckImageQuality<OutputImageType>(feldkamp->GetOutput(), dsl->GetOutput());

  std::cout << "\n\n****** Case 3: no displacement ******" << std::endl;
  geometry = GeometryType::New();
  slp->SetGeometry(geometry);
  ddf->SetGeometry( geometry );
  feldkamp->SetGeometry( geometry );
  for(unsigned int noProj=0; noProj<NumberOfProjectionImages; noProj++)
    geometry->AddProjection(600., 1200., noProj*360./NumberOfProjectionImages);
  projectionsSource->SetOrigin(origin);
  TRY_AND_EXIT_ON_ITK_EXCEPTION( feldkamp->Update() );
  CheckImageQuality<OutputImageType>(feldkamp->GetOutput(), dsl->GetOutput());

  std::cout << "\n\n****** Case 4: negative offset in origin ******" << std::endl;
  geometry = GeometryType::New();
  slp->SetGeometry(geometry);
  ddf->SetGeometry( geometry );
  feldkamp->SetGeometry( geometry );
  for(unsigned int noProj=0; noProj<NumberOfProjectionImages; noProj++)
    geometry->AddProjection(600., 1200., noProj*360./NumberOfProjectionImages);
  origin[0] = -400;
  projectionsSource->SetOrigin(origin);
  TRY_AND_EXIT_ON_ITK_EXCEPTION( feldkamp->Update() );
  CheckImageQuality<OutputImageType>(feldkamp->GetOutput(), dsl->GetOutput());

  std::cout << "\n\n****** Case 5: positive offset in origin ******" << std::endl;
  origin[0] = -100;
  projectionsSource->SetOrigin(origin);
  TRY_AND_EXIT_ON_ITK_EXCEPTION( feldkamp->Update() );
  CheckImageQuality<OutputImageType>(feldkamp->GetOutput(), dsl->GetOutput());

  std::cout << "\n\nTest PASSED! " << std::endl;
  exit(EXIT_SUCCESS);
}
<commit_msg>Threshold ErrorPerPixel increased to 0.061<commit_after>#include <itkImageRegionConstIterator.h>

#include "rtkSheppLoganPhantomFilter.h"
#include "rtkDrawSheppLoganFilter.h"
#include "rtkFDKConeBeamReconstructionFilter.h"
#include "rtkConstantImageSource.h"
#include "rtkDisplacedDetectorImageFilter.h"

template<class TImage>
void CheckImageQuality(typename TImage::Pointer recon, typename TImage::Pointer ref)
{
  typedef itk::ImageRegionConstIterator<TImage> ImageIteratorType;
  ImageIteratorType itTest( recon, recon->GetBufferedRegion() );
  ImageIteratorType itRef( ref, ref->GetBufferedRegion() );

  typedef double ErrorType;
  ErrorType TestError = 0.;
  ErrorType EnerError = 0.;

  itTest.GoToBegin();
  itRef.GoToBegin();

  while( !itRef.IsAtEnd() )
    {
    typename TImage::PixelType TestVal = itTest.Get();
    typename TImage::PixelType RefVal = itRef.Get();
    TestError += vcl_abs(RefVal - TestVal);
    EnerError += vcl_pow(ErrorType(RefVal - TestVal), 2.);
    ++itTest;
    ++itRef;
    }
  // Error per Pixel
  ErrorType ErrorPerPixel = TestError/ref->GetBufferedRegion().GetNumberOfPixels();
  std::cout << "\nError per Pixel = " << ErrorPerPixel << std::endl;
  // MSE
  ErrorType MSE = EnerError/ref->GetBufferedRegion().GetNumberOfPixels();
  std::cout << "MSE = " << MSE << std::endl;
  // PSNR
  ErrorType PSNR = 20*log10(2.0) - 10*log10(MSE);
  std::cout << "PSNR = " << PSNR << "dB" << std::endl;
  // QI
  ErrorType QI = (2.0-ErrorPerPixel)/2.0;
  std::cout << "QI = " << QI << std::endl;

  // Checking results
  if (ErrorPerPixel > 0.061)
  {
    std::cerr << "Test Failed, Error per pixel not valid! "
              << ErrorPerPixel << " instead of 0.06." << std::endl;
    exit( EXIT_FAILURE);
  }
  if (PSNR < 24.)
  {
    std::cerr << "Test Failed, PSNR not valid! "
              << PSNR << " instead of 24." << std::endl;
    exit( EXIT_FAILURE);
  }
}

int main(int argc, char* argv[])
{
  const unsigned int Dimension = 3;
  typedef float                                    OutputPixelType;
  typedef itk::Image< OutputPixelType, Dimension > OutputImageType;
  const unsigned int NumberOfProjectionImages = 180;

  // Constant image sources
  typedef rtk::ConstantImageSource< OutputImageType > ConstantImageSourceType;
  ConstantImageSourceType::PointType origin;
  ConstantImageSourceType::SizeType size;
  ConstantImageSourceType::SpacingType spacing;

  ConstantImageSourceType::Pointer tomographySource  = ConstantImageSourceType::New();
  origin[0] = -127.;
  origin[1] = -127.;
  origin[2] = -127.;
  size[0] = 128;
  size[1] = 128;
  size[2] = 128;
  spacing[0] = 2.;
  spacing[1] = 2.;
  spacing[2] = 2.;
  tomographySource->SetOrigin( origin );
  tomographySource->SetSpacing( spacing );
  tomographySource->SetSize( size );
  tomographySource->SetConstant( 0. );

  ConstantImageSourceType::Pointer projectionsSource = ConstantImageSourceType::New();
  origin[0] = -254.;
  origin[1] = -254.;
  origin[2] = -254.;
  size[0] = 128;
  size[1] = 128;
  size[2] = NumberOfProjectionImages;
  spacing[0] = 4.;
  spacing[1] = 4.;
  spacing[2] = 4.;
  projectionsSource->SetOrigin( origin );
  projectionsSource->SetSpacing( spacing );
  projectionsSource->SetSize( size );
  projectionsSource->SetConstant( 0. );

  // Shepp Logan projections filter
  typedef rtk::SheppLoganPhantomFilter<OutputImageType, OutputImageType> SLPType;
  SLPType::Pointer slp=SLPType::New();
  slp->SetInput( projectionsSource->GetOutput() );

  // Displaced detector weighting
  typedef rtk::DisplacedDetectorImageFilter< OutputImageType > DDFType;
  DDFType::Pointer ddf = DDFType::New();
  ddf->SetInput( slp->GetOutput() );

  // Create a reference object (in this case a 3D phantom reference).
  typedef rtk::DrawSheppLoganFilter<OutputImageType, OutputImageType> DSLType;
  DSLType::Pointer dsl = DSLType::New();
  dsl->SetInput( tomographySource->GetOutput() );
  TRY_AND_EXIT_ON_ITK_EXCEPTION( dsl->Update() );

  // FDK reconstruction filtering
  typedef rtk::FDKConeBeamReconstructionFilter< OutputImageType > FDKCPUType;
  FDKCPUType::Pointer feldkamp = FDKCPUType::New();
  feldkamp->SetInput( 0, tomographySource->GetOutput() );
  feldkamp->SetInput( 1, ddf->GetOutput() );

  std::cout << "\n\n****** Case 1: positive offset in geometry ******" << std::endl;
  typedef rtk::ThreeDCircularProjectionGeometry GeometryType;
  GeometryType::Pointer geometry = GeometryType::New();
  slp->SetGeometry(geometry);
  ddf->SetGeometry( geometry );
  feldkamp->SetGeometry( geometry );
  for(unsigned int noProj=0; noProj<NumberOfProjectionImages; noProj++)
    geometry->AddProjection(600., 1200., noProj*360./NumberOfProjectionImages, 120., 0.);
  slp->Update();
  TRY_AND_EXIT_ON_ITK_EXCEPTION( feldkamp->Update() );
  CheckImageQuality<OutputImageType>(feldkamp->GetOutput(), dsl->GetOutput());

  std::cout << "\n\n****** Case 2: negative offset in geometry ******" << std::endl;
  geometry = GeometryType::New();
  slp->SetGeometry(geometry);
  ddf->SetGeometry( geometry );
  feldkamp->SetGeometry( geometry );
  for(unsigned int noProj=0; noProj<NumberOfProjectionImages; noProj++)
    geometry->AddProjection(600., 1200., noProj*360./NumberOfProjectionImages, -120., 0.);
  TRY_AND_EXIT_ON_ITK_EXCEPTION( feldkamp->Update() );
  CheckImageQuality<OutputImageType>(feldkamp->GetOutput(), dsl->GetOutput());

  std::cout << "\n\n****** Case 3: no displacement ******" << std::endl;
  geometry = GeometryType::New();
  slp->SetGeometry(geometry);
  ddf->SetGeometry( geometry );
  feldkamp->SetGeometry( geometry );
  for(unsigned int noProj=0; noProj<NumberOfProjectionImages; noProj++)
    geometry->AddProjection(600., 1200., noProj*360./NumberOfProjectionImages);
  projectionsSource->SetOrigin(origin);
  TRY_AND_EXIT_ON_ITK_EXCEPTION( feldkamp->Update() );
  CheckImageQuality<OutputImageType>(feldkamp->GetOutput(), dsl->GetOutput());

  std::cout << "\n\n****** Case 4: negative offset in origin ******" << std::endl;
  geometry = GeometryType::New();
  slp->SetGeometry(geometry);
  ddf->SetGeometry( geometry );
  feldkamp->SetGeometry( geometry );
  for(unsigned int noProj=0; noProj<NumberOfProjectionImages; noProj++)
    geometry->AddProjection(600., 1200., noProj*360./NumberOfProjectionImages);
  origin[0] = -400;
  projectionsSource->SetOrigin(origin);
  TRY_AND_EXIT_ON_ITK_EXCEPTION( feldkamp->Update() );
  CheckImageQuality<OutputImageType>(feldkamp->GetOutput(), dsl->GetOutput());

  std::cout << "\n\n****** Case 5: positive offset in origin ******" << std::endl;
  origin[0] = -100;
  projectionsSource->SetOrigin(origin);
  TRY_AND_EXIT_ON_ITK_EXCEPTION( feldkamp->Update() );
  CheckImageQuality<OutputImageType>(feldkamp->GetOutput(), dsl->GetOutput());

  std::cout << "\n\nTest PASSED! " << std::endl;
  exit(EXIT_SUCCESS);
}
<|endoftext|>


<commit_before>// packetfield.cpp

#include "packetfield.h"
#include <gtest/gtest.h>

using namespace boostcraft::network;

TEST(PacketFieldTests, ShortField)
{
    boost::asio::streambuf b;
    std::ostream os(&b);
    os.put(0x10);
    os.put(0x80);

    uint16_t val = 0;

    ShortField field(val);

    field.readFrom(b);

    ASSERT_EQ(0x1080, val);
}

TEST(PacketFieldTests, DoubleField)
{
    boost::asio::streambuf b;
    std::ostream os(&b);

    // Double value 1337.1337 in network byte order:
    os.put(0x40);
    os.put(0x94);
    os.put(0xE4);
    os.put(0x88);
    os.put(0xE8);
    os.put(0xA7);
    os.put(0x1D);
    os.put(0xE7);

    double val = 0.0d;

    DoubleField field(val);
    field.readFrom(b);

    // This number should be exactly representable
    ASSERT_EQ(1337.1337, val);
}

TEST(PacketFieldTests, DoubleFieldContinuable)
{
    boost::asio::streambuf b;
    std::ostream os(&b);

    double val = 0.0d;

    DoubleField field(val);

    // All 8 bytes should still be required at first
    ASSERT_EQ(8, field.readFrom(b));

    // Double value 4.6855785559756050e-277
    // Put only the first three bytes in the buffer
    os.put(0x06);
    os.put(0x90);
    os.put(0x9c);

    // Should need five more bytes
    ASSERT_EQ(5, field.readFrom(b));

    os.put(0xa5);

    ASSERT_EQ(4, field.readFrom(b));

    os.put(0x45);
    os.put(0x8c);
    os.put(0xa9);
    os.put(0xc6);

    ASSERT_EQ(0, field.readFrom(b));

    // This value should be exactly represented so direct equality is OK
    ASSERT_EQ(4.6855785559756050e-277, val);
}

<commit_msg>More unit tests for packetfield<commit_after>// packetfield.cpp

#include "packetfield.h"
#include <gtest/gtest.h>

using namespace boostcraft::network;

TEST(PacketFieldTests, ByteField)
{
    boost::asio::streambuf b;
    std::ostream os(&b);
    os.put(0x42);

    uint8_t val = 0;
    ByteField field(val);
    field.readFrom(b);

    ASSERT_EQ(0x42, val);
}

TEST(PacketFieldTests, ShortField)
{
    boost::asio::streambuf b;
    std::ostream os(&b);
    os.put(0x10);
    os.put(0x80);

    uint16_t val = 0;
    ShortField field(val);
    field.readFrom(b);

    ASSERT_EQ(0x1080, val);
}

TEST(PacketFieldTests, LongField)
{
    boost::asio::streambuf b;
    std::ostream os(&b);
    os.put(0x10);
    os.put(0x20);
    os.put(0x30);
    os.put(0x40);
    os.put(0x50);
    os.put(0x60);
    os.put(0x70);
    os.put(0x80);

    uint64_t val = 0;
    LongField field(val);
    field.readFrom(b);

    ASSERT_EQ(0x1020304050607080LLU, val);
}

TEST(PacketFieldTests, DoubleField)
{
    boost::asio::streambuf b;
    std::ostream os(&b);

    // Double value 1337.1337 in network byte order:
    os.put(0x40);
    os.put(0x94);
    os.put(0xE4);
    os.put(0x88);
    os.put(0xE8);
    os.put(0xA7);
    os.put(0x1D);
    os.put(0xE7);

    double val = 0.0d;

    DoubleField field(val);
    field.readFrom(b);

    // This number should be exactly representable
    ASSERT_EQ(1337.1337, val);
}

TEST(PacketFieldTests, DoubleFieldContinuable)
{
    boost::asio::streambuf b;
    std::ostream os(&b);

    double val = 0.0d;

    DoubleField field(val);

    // All 8 bytes should still be required at first
    ASSERT_EQ(8, field.readFrom(b));

    // Double value 4.6855785559756050e-277
    // Put only the first three bytes in the buffer
    os.put(0x06);
    os.put(0x90);
    os.put(0x9c);

    // Should need five more bytes
    ASSERT_EQ(5, field.readFrom(b));

    os.put(0xa5);

    ASSERT_EQ(4, field.readFrom(b));

    os.put(0x45);
    os.put(0x8c);
    os.put(0xa9);
    os.put(0xc6);

    ASSERT_EQ(0, field.readFrom(b));

    // This value should be exactly represented so direct equality is OK
    ASSERT_EQ(4.6855785559756050e-277, val);
}

/*
 * Verify that UTF-8 strings are read correctly across multiple buffers.
 */
TEST(PacketFieldTests, UtfStringContinuable)
{
    boost::asio::streambuf b;
    std::ostream os(&b);

    /*
     * For this test, one quatrain of the Rubaiyat of Omar Khayyam, in Farsi
     * and Fitzgerald's English, plus one quatrain of the Rubaiyat of a Persian
     * kitten, in plain and simple everyday unoriental speech.
     */
    std::string teststr(
        u8"Why, all the Saints and Sages who discuss'd\n"
        "Of the Two Worlds so wisely -- they are thrust\n"
        "Like foolish Prophets forth; their Words to Scorn\n"
        "Are scatter'd, and their Mouths are stopt with Dust.\n"
        "\n"
        "آنانكه ز پيش رفته‌اند اى ساقى\n"
        "درخاك غرور خفته‌اند اى ساقى\n"
        "رو باده خور و حقيقت از من بشنو\n"
        "باد است هرآنچه گفته‌اند اى ساقى\n"
        "\n"
        "They say the Lion and the Lizard keep\n"
        "The Courts where Jamshyd gloried and drank deep.\n"
        "The Lion is my cousin; I don't know\n"
        "Who Jamshyd is--nor shall it break my sleep.\n");

    os.put(teststr.length() >> 8);
    os.put(teststr.length() & 0xFF);

    std::string val;
    String8Field field(val, 0);

    // Only write the first 42 bytes at first; the first attempt to read should
    // then come up len-42 bytes short and consume all 42 available bytes.
    os << teststr.substr(0, 42);
    size_t bytesleft = field.readFrom(b);
    ASSERT_EQ(teststr.length() - 42, bytesleft);
    ASSERT_EQ(0, b.size());

    // Write the next 112 bytes and try reading from the buffer again
    os << teststr.substr(42, 112);
    bytesleft = field.readFrom(b);
    ASSERT_EQ(teststr.length() - 42 - 112, bytesleft);
    ASSERT_EQ(0, b.size());

    // Finally, write the rest of the string
    os << teststr.substr(42+112);
    bytesleft = field.readFrom(b);
    ASSERT_EQ(0, bytesleft);
    ASSERT_EQ(0, b.size());

    // Check that the read string is the same as the original
    ASSERT_EQ(teststr, val);
}

/*
 * Verify that multiple fields can be read successively from a single buffer
 */
TEST(PacketFieldTests, MultipleFields)
{
    boost::asio::streambuf b;
    std::ostream os(&b);

    // Fields:
    //  (string8) "Avogadro's number (not Avocado) is:"
    //  (float)   0x66FF0C30 = 6.0221414e+23
    //  (byte)    0x2A
    //  (byte)    0xFE
    //  (long)    0xDEADBEEF00C0FFEE
    std::string avocado("Avogadro's number (not Avocado) is:");
    os.put(avocado.length() >> 8);
    os.put(avocado.length() & 0xFF);
    os << avocado;
    std::vector<uint8_t> bytes = { 0x66, 0xFF, 0x0C, 0x30,
        0x2A, 0xFE, 0xDE, 0xAD, 0xBE, 0xEF, 0x00, 0xC0, 0xFF, 0xEE };
    os.write((char*)&bytes[0], bytes.size());

    size_t bufsize = b.size();

    std::string msg;
    float avogadro = 0.0f;
    uint8_t foo = 0;
    uint8_t bar = 0;
    uint64_t baz = 0;

    String8Field msgField(msg, 0);
    FloatField avoField(avogadro);
    ByteField fooField(foo);
    ByteField barField(bar);
    LongField bazField(baz);

    ASSERT_EQ(0, msgField.readFrom(b));
    ASSERT_EQ(avocado, msg);
    ASSERT_EQ(bufsize - 2 - avocado.length(), b.size());
    bufsize -= 2 + avocado.length();

    ASSERT_EQ(0, avoField.readFrom(b));
    ASSERT_FLOAT_EQ(6.0221414e+23, avogadro);
    ASSERT_EQ(bufsize - 4, b.size());
    bufsize -= 4;

    ASSERT_EQ(0, fooField.readFrom(b));
    ASSERT_EQ(0x2A, foo);
    ASSERT_EQ(bufsize - 1, b.size());
    bufsize -= 1;

    ASSERT_EQ(0, barField.readFrom(b));
    ASSERT_EQ(0xFE, bar);
    ASSERT_EQ(bufsize - 1, b.size());
    bufsize -= 1;

    ASSERT_EQ(0, bazField.readFrom(b));
    ASSERT_EQ(0xDEADBEEF00C0FFEE, baz);
    ASSERT_EQ(0, b.size());
}

<|endoftext|>


<commit_before>/*
 * Copyright (c) 2011 - 2017 Stephen F. Booth <me@sbooth.org>
 * See https://github.com/sbooth/SFBAudioEngine/blob/master/LICENSE.txt for license information
 */

#include <AudioToolbox/AudioFormat.h>

#include "MonkeysAudioDecoder.h"
#include "CFErrorUtilities.h"
#include "Logger.h"

#define PLATFORM_APPLE

#include <mac/All.h>
#include <mac/MACLib.h>

#include <mac/IO.h>

namespace {

	void RegisterMonkeysAudioDecoder() __attribute__ ((constructor));
	void RegisterMonkeysAudioDecoder()
	{
		SFB::Audio::Decoder::RegisterSubclass<SFB::Audio::MonkeysAudioDecoder>();
	}

}

#pragma mark IO Interface

// The I/O interface for MAC
class SFB::Audio::MonkeysAudioDecoder::APEIOInterface : public APE::CIO
{
public:
	explicit APEIOInterface(SFB::InputSource& inputSource)
		: mInputSource(inputSource)
	{}

	inline virtual int Open(const wchar_t * pName, bool bOpenReadOnly)
	{
#pragma unused(pName)
#pragma unused(bOpenReadOnly)

		return ERROR_INVALID_INPUT_FILE;
	}

	inline virtual int Close()
	{
		return ERROR_SUCCESS;
	}

	virtual int Read(void * pBuffer, unsigned int nBytesToRead, unsigned int * pBytesRead)
	{
		SInt64 bytesRead = mInputSource.Read(pBuffer, nBytesToRead);
		if(-1 == bytesRead)
			return ERROR_IO_READ;

		*pBytesRead = (unsigned int)bytesRead;

		return ERROR_SUCCESS;
	}

	inline virtual int Write(const void * pBuffer, unsigned int nBytesToWrite, unsigned int * pBytesWritten)
	{
#pragma unused(pBuffer)
#pragma unused(nBytesToWrite)
#pragma unused(pBytesWritten)

		return ERROR_IO_WRITE;
	}

	virtual int Seek(int64_t nDistance, unsigned int nMoveMode)
	{
		if(!mInputSource.SupportsSeeking())
			return ERROR_IO_READ;

		SInt64 offset = nDistance;
		switch(nMoveMode) {
			case SEEK_SET:
				// offset remains unchanged
				break;
			case SEEK_CUR:
				offset += mInputSource.GetOffset();
				break;
			case SEEK_END:
				offset += mInputSource.GetLength();
				break;
		}

		return (!mInputSource.SeekToOffset(offset));
	}

	inline virtual int Create(const wchar_t * pName)
	{
#pragma unused(pName)
		return ERROR_IO_WRITE;
	}

	inline virtual int Delete()
	{
		return ERROR_IO_WRITE;
	}

	inline virtual int SetEOF()
	{
		return ERROR_IO_WRITE;
	}

	inline virtual int GetPosition()
	{
		return (int)mInputSource.GetOffset();
	}

	inline virtual unsigned int GetSize()
	{
		return (unsigned int)mInputSource.GetLength();
	}

	inline virtual int GetName(wchar_t * pBuffer)
	{
#pragma unused(pBuffer)
		return ERROR_SUCCESS;
	}

private:

	SFB::InputSource& mInputSource;
};

#pragma mark Static Methods

CFArrayRef SFB::Audio::MonkeysAudioDecoder::CreateSupportedFileExtensions()
{
	CFStringRef supportedExtensions [] = { CFSTR("ape") };
	return CFArrayCreate(kCFAllocatorDefault, (const void **)supportedExtensions, 1, &kCFTypeArrayCallBacks);
}

CFArrayRef SFB::Audio::MonkeysAudioDecoder::CreateSupportedMIMETypes()
{
	CFStringRef supportedMIMETypes [] = { CFSTR("audio/monkeys-audio"), CFSTR("audio/x-monkeys-audio") };
	return CFArrayCreate(kCFAllocatorDefault, (const void **)supportedMIMETypes, 2, &kCFTypeArrayCallBacks);
}

bool SFB::Audio::MonkeysAudioDecoder::HandlesFilesWithExtension(CFStringRef extension)
{
	if(nullptr == extension)
		return false;

	if(kCFCompareEqualTo == CFStringCompare(extension, CFSTR("ape"), kCFCompareCaseInsensitive))
		return true;

	return false;
}

bool SFB::Audio::MonkeysAudioDecoder::HandlesMIMEType(CFStringRef mimeType)
{
	if(nullptr == mimeType)
		return false;

	if(kCFCompareEqualTo == CFStringCompare(mimeType, CFSTR("audio/monkeys-audio"), kCFCompareCaseInsensitive))
		return true;

	if(kCFCompareEqualTo == CFStringCompare(mimeType, CFSTR("audio/x-monkeys-audio"), kCFCompareCaseInsensitive))
		return true;

	return false;
}

SFB::Audio::Decoder::unique_ptr SFB::Audio::MonkeysAudioDecoder::CreateDecoder(InputSource::unique_ptr inputSource)
{
	return unique_ptr(new MonkeysAudioDecoder(std::move(inputSource)));
}

#pragma mark Creation and Destruction

SFB::Audio::MonkeysAudioDecoder::MonkeysAudioDecoder(InputSource::unique_ptr inputSource)
	: Decoder(std::move(inputSource))
{}

#pragma mark Functionality

bool SFB::Audio::MonkeysAudioDecoder::_Open(CFErrorRef *error)
{
	auto ioInterface = 	std::unique_ptr<APEIOInterface>(new APEIOInterface(GetInputSource()));

	auto decompressor = std::unique_ptr<APE::IAPEDecompress>(CreateIAPEDecompressEx(ioInterface.get(), nullptr));
	if(!decompressor) {
		if(error) {
			SFB::CFString description(CFCopyLocalizedString(CFSTR("The file “%@” is not a valid Monkey's Audio file."), ""));
			SFB::CFString failureReason(CFCopyLocalizedString(CFSTR("Not a Monkey's Audio file"), ""));
			SFB::CFString recoverySuggestion(CFCopyLocalizedString(CFSTR("The file's extension may not match the file's type."), ""));

			*error = CreateErrorForURL(Decoder::ErrorDomain, Decoder::InputOutputError, description, mInputSource->GetURL(), failureReason, recoverySuggestion);
		}

		return false;
	}

	mDecompressor = std::move(decompressor);
	mIOInterface = std::move(ioInterface);

	// The file format
	mFormat.mFormatID			= kAudioFormatLinearPCM;
	mFormat.mFormatFlags		= kAudioFormatFlagIsSignedInteger | kAudioFormatFlagsNativeEndian | kAudioFormatFlagIsPacked;

	mFormat.mBitsPerChannel		= (UInt32)mDecompressor->GetInfo(APE::APE_INFO_BITS_PER_SAMPLE);
	mFormat.mSampleRate			= mDecompressor->GetInfo(APE::APE_INFO_SAMPLE_RATE);
	mFormat.mChannelsPerFrame	= (UInt32)mDecompressor->GetInfo(APE::APE_INFO_CHANNELS);

	mFormat.mBytesPerPacket		= (mFormat.mBitsPerChannel / 8) * mFormat.mChannelsPerFrame;
	mFormat.mFramesPerPacket	= 1;
	mFormat.mBytesPerFrame		= mFormat.mBytesPerPacket * mFormat.mFramesPerPacket;

	mFormat.mReserved			= 0;

	// Set up the source format
	mSourceFormat.mFormatID				= kAudioFormatMonkeysAudio;

	mSourceFormat.mSampleRate			= mFormat.mSampleRate;
	mSourceFormat.mChannelsPerFrame		= mFormat.mChannelsPerFrame;
	mSourceFormat.mBitsPerChannel		= mFormat.mBitsPerChannel;

	switch(mFormat.mChannelsPerFrame) {
		case 1:		mChannelLayout = ChannelLayout::ChannelLayoutWithTag(kAudioChannelLayoutTag_Mono);			break;
		case 2:		mChannelLayout = ChannelLayout::ChannelLayoutWithTag(kAudioChannelLayoutTag_Stereo);		break;
		case 4:		mChannelLayout = ChannelLayout::ChannelLayoutWithTag(kAudioChannelLayoutTag_Quadraphonic);	break;
	}

	return true;
}

bool SFB::Audio::MonkeysAudioDecoder::_Close(CFErrorRef */*error*/)
{
	mIOInterface.reset();
	mDecompressor.reset();

	return true;
}

SFB::CFString SFB::Audio::MonkeysAudioDecoder::_GetSourceFormatDescription() const
{
	return CFString(nullptr,
					CFSTR("Monkey's Audio, %u channels, %u Hz"),
					mSourceFormat.mChannelsPerFrame,
					(unsigned int)mSourceFormat.mSampleRate);
}

UInt32 SFB::Audio::MonkeysAudioDecoder::_ReadAudio(AudioBufferList *bufferList, UInt32 frameCount)
{
	int64_t blocksRead = 0;
	if(ERROR_SUCCESS != mDecompressor->GetData((char *)bufferList->mBuffers[0].mData, (int64_t)frameCount, &blocksRead)) {
		LOGGER_ERR("org.sbooth.AudioEngine.Decoder.MonkeysAudio", "Monkey's Audio invalid checksum");
		return 0;
	}

	bufferList->mBuffers[0].mDataByteSize = (UInt32)blocksRead * mFormat.mBytesPerFrame;
	bufferList->mBuffers[0].mNumberChannels = mFormat.mChannelsPerFrame;

	return (UInt32)blocksRead;
}

SInt64 SFB::Audio::MonkeysAudioDecoder::_GetTotalFrames() const
{
	return mDecompressor->GetInfo(APE::APE_DECOMPRESS_TOTAL_BLOCKS);
}

SInt64 SFB::Audio::MonkeysAudioDecoder::_GetCurrentFrame() const
{
	return mDecompressor->GetInfo(APE::APE_DECOMPRESS_CURRENT_BLOCK);
}

SInt64 SFB::Audio::MonkeysAudioDecoder::_SeekToFrame(SInt64 frame)
{
	if(ERROR_SUCCESS != mDecompressor->Seek((int)frame)) {
		LOGGER_ERR("org.sbooth.AudioEngine.Decoder.MonkeysAudio", "mDecompressor->Seek() failed");
		return -1;
	}

	return this->GetCurrentFrame();
}
<commit_msg>Updated to MAC SDK 519<commit_after>/*
 * Copyright (c) 2011 - 2020 Stephen F. Booth <me@sbooth.org>
 * See https://github.com/sbooth/SFBAudioEngine/blob/master/LICENSE.txt for license information
 */

#include <AudioToolbox/AudioFormat.h>

#include "MonkeysAudioDecoder.h"
#include "CFErrorUtilities.h"
#include "Logger.h"

#define PLATFORM_APPLE

#include <MAC/All.h>
#include <MAC/MACLib.h>

#include <MAC/IO.h>

namespace {

	void RegisterMonkeysAudioDecoder() __attribute__ ((constructor));
	void RegisterMonkeysAudioDecoder()
	{
		SFB::Audio::Decoder::RegisterSubclass<SFB::Audio::MonkeysAudioDecoder>();
	}

}

#pragma mark IO Interface

// The I/O interface for MAC
class SFB::Audio::MonkeysAudioDecoder::APEIOInterface : public APE::CIO
{
public:
	explicit APEIOInterface(SFB::InputSource& inputSource)
		: mInputSource(inputSource)
	{}

	inline virtual int Open(const wchar_t * pName, bool bOpenReadOnly)
	{
#pragma unused(pName)
#pragma unused(bOpenReadOnly)

		return ERROR_INVALID_INPUT_FILE;
	}

	inline virtual int Close()
	{
		return ERROR_SUCCESS;
	}

	virtual int Read(void * pBuffer, unsigned int nBytesToRead, unsigned int * pBytesRead)
	{
		SInt64 bytesRead = mInputSource.Read(pBuffer, nBytesToRead);
		if(-1 == bytesRead)
			return ERROR_IO_READ;

		*pBytesRead = (unsigned int)bytesRead;

		return ERROR_SUCCESS;
	}

	inline virtual int Write(const void * pBuffer, unsigned int nBytesToWrite, unsigned int * pBytesWritten)
	{
#pragma unused(pBuffer)
#pragma unused(nBytesToWrite)
#pragma unused(pBytesWritten)

		return ERROR_IO_WRITE;
	}

	virtual APE::int64 PerformSeek()
	{
		if(!mInputSource.SupportsSeeking())
			return ERROR_IO_READ;

		SInt64 offset = m_nSeekPosition;
		switch(m_nSeekMethod) {
			case SEEK_SET:
				// offset remains unchanged
				break;
			case SEEK_CUR:
				offset += mInputSource.GetOffset();
				break;
			case SEEK_END:
				offset += mInputSource.GetLength();
				break;
		}

		return (!mInputSource.SeekToOffset(offset));
	}

	inline virtual int Create(const wchar_t * pName)
	{
#pragma unused(pName)
		return ERROR_IO_WRITE;
	}

	inline virtual int Delete()
	{
		return ERROR_IO_WRITE;
	}

	inline virtual int SetEOF()
	{
		return ERROR_IO_WRITE;
	}

	inline virtual APE::int64 GetPosition()
	{
		return mInputSource.GetOffset();
	}

	inline virtual APE::int64 GetSize()
	{
		return mInputSource.GetLength();
	}

	inline virtual int GetName(wchar_t * pBuffer)
	{
#pragma unused(pBuffer)
		return ERROR_SUCCESS;
	}

private:

	SFB::InputSource& mInputSource;
};

#pragma mark Static Methods

CFArrayRef SFB::Audio::MonkeysAudioDecoder::CreateSupportedFileExtensions()
{
	CFStringRef supportedExtensions [] = { CFSTR("ape") };
	return CFArrayCreate(kCFAllocatorDefault, (const void **)supportedExtensions, 1, &kCFTypeArrayCallBacks);
}

CFArrayRef SFB::Audio::MonkeysAudioDecoder::CreateSupportedMIMETypes()
{
	CFStringRef supportedMIMETypes [] = { CFSTR("audio/monkeys-audio"), CFSTR("audio/x-monkeys-audio") };
	return CFArrayCreate(kCFAllocatorDefault, (const void **)supportedMIMETypes, 2, &kCFTypeArrayCallBacks);
}

bool SFB::Audio::MonkeysAudioDecoder::HandlesFilesWithExtension(CFStringRef extension)
{
	if(nullptr == extension)
		return false;

	if(kCFCompareEqualTo == CFStringCompare(extension, CFSTR("ape"), kCFCompareCaseInsensitive))
		return true;

	return false;
}

bool SFB::Audio::MonkeysAudioDecoder::HandlesMIMEType(CFStringRef mimeType)
{
	if(nullptr == mimeType)
		return false;

	if(kCFCompareEqualTo == CFStringCompare(mimeType, CFSTR("audio/monkeys-audio"), kCFCompareCaseInsensitive))
		return true;

	if(kCFCompareEqualTo == CFStringCompare(mimeType, CFSTR("audio/x-monkeys-audio"), kCFCompareCaseInsensitive))
		return true;

	return false;
}

SFB::Audio::Decoder::unique_ptr SFB::Audio::MonkeysAudioDecoder::CreateDecoder(InputSource::unique_ptr inputSource)
{
	return unique_ptr(new MonkeysAudioDecoder(std::move(inputSource)));
}

#pragma mark Creation and Destruction

SFB::Audio::MonkeysAudioDecoder::MonkeysAudioDecoder(InputSource::unique_ptr inputSource)
	: Decoder(std::move(inputSource))
{}

#pragma mark Functionality

bool SFB::Audio::MonkeysAudioDecoder::_Open(CFErrorRef *error)
{
	auto ioInterface = 	std::unique_ptr<APEIOInterface>(new APEIOInterface(GetInputSource()));

	auto decompressor = std::unique_ptr<APE::IAPEDecompress>(CreateIAPEDecompressEx(ioInterface.get(), nullptr));
	if(!decompressor) {
		if(error) {
			SFB::CFString description(CFCopyLocalizedString(CFSTR("The file “%@” is not a valid Monkey's Audio file."), ""));
			SFB::CFString failureReason(CFCopyLocalizedString(CFSTR("Not a Monkey's Audio file"), ""));
			SFB::CFString recoverySuggestion(CFCopyLocalizedString(CFSTR("The file's extension may not match the file's type."), ""));

			*error = CreateErrorForURL(Decoder::ErrorDomain, Decoder::InputOutputError, description, mInputSource->GetURL(), failureReason, recoverySuggestion);
		}

		return false;
	}

	mDecompressor = std::move(decompressor);
	mIOInterface = std::move(ioInterface);

	// The file format
	mFormat.mFormatID			= kAudioFormatLinearPCM;
	mFormat.mFormatFlags		= kAudioFormatFlagIsSignedInteger | kAudioFormatFlagsNativeEndian | kAudioFormatFlagIsPacked;

	mFormat.mBitsPerChannel		= (UInt32)mDecompressor->GetInfo(APE::APE_INFO_BITS_PER_SAMPLE);
	mFormat.mSampleRate			= mDecompressor->GetInfo(APE::APE_INFO_SAMPLE_RATE);
	mFormat.mChannelsPerFrame	= (UInt32)mDecompressor->GetInfo(APE::APE_INFO_CHANNELS);

	mFormat.mBytesPerPacket		= (mFormat.mBitsPerChannel / 8) * mFormat.mChannelsPerFrame;
	mFormat.mFramesPerPacket	= 1;
	mFormat.mBytesPerFrame		= mFormat.mBytesPerPacket * mFormat.mFramesPerPacket;

	mFormat.mReserved			= 0;

	// Set up the source format
	mSourceFormat.mFormatID				= kAudioFormatMonkeysAudio;

	mSourceFormat.mSampleRate			= mFormat.mSampleRate;
	mSourceFormat.mChannelsPerFrame		= mFormat.mChannelsPerFrame;
	mSourceFormat.mBitsPerChannel		= mFormat.mBitsPerChannel;

	switch(mFormat.mChannelsPerFrame) {
		case 1:		mChannelLayout = ChannelLayout::ChannelLayoutWithTag(kAudioChannelLayoutTag_Mono);			break;
		case 2:		mChannelLayout = ChannelLayout::ChannelLayoutWithTag(kAudioChannelLayoutTag_Stereo);		break;
		case 4:		mChannelLayout = ChannelLayout::ChannelLayoutWithTag(kAudioChannelLayoutTag_Quadraphonic);	break;
	}

	return true;
}

bool SFB::Audio::MonkeysAudioDecoder::_Close(CFErrorRef */*error*/)
{
	mIOInterface.reset();
	mDecompressor.reset();

	return true;
}

SFB::CFString SFB::Audio::MonkeysAudioDecoder::_GetSourceFormatDescription() const
{
	return CFString(nullptr,
					CFSTR("Monkey's Audio, %u channels, %u Hz"),
					mSourceFormat.mChannelsPerFrame,
					(unsigned int)mSourceFormat.mSampleRate);
}

UInt32 SFB::Audio::MonkeysAudioDecoder::_ReadAudio(AudioBufferList *bufferList, UInt32 frameCount)
{
	int64_t blocksRead = 0;
	if(ERROR_SUCCESS != mDecompressor->GetData((char *)bufferList->mBuffers[0].mData, (int64_t)frameCount, &blocksRead)) {
		LOGGER_ERR("org.sbooth.AudioEngine.Decoder.MonkeysAudio", "Monkey's Audio invalid checksum");
		return 0;
	}

	bufferList->mBuffers[0].mDataByteSize = (UInt32)blocksRead * mFormat.mBytesPerFrame;
	bufferList->mBuffers[0].mNumberChannels = mFormat.mChannelsPerFrame;

	return (UInt32)blocksRead;
}

SInt64 SFB::Audio::MonkeysAudioDecoder::_GetTotalFrames() const
{
	return mDecompressor->GetInfo(APE::APE_DECOMPRESS_TOTAL_BLOCKS);
}

SInt64 SFB::Audio::MonkeysAudioDecoder::_GetCurrentFrame() const
{
	return mDecompressor->GetInfo(APE::APE_DECOMPRESS_CURRENT_BLOCK);
}

SInt64 SFB::Audio::MonkeysAudioDecoder::_SeekToFrame(SInt64 frame)
{
	if(ERROR_SUCCESS != mDecompressor->Seek(frame)) {
		LOGGER_ERR("org.sbooth.AudioEngine.Decoder.MonkeysAudio", "mDecompressor->Seek() failed");
		return -1;
	}

	return this->GetCurrentFrame();
}
<|endoftext|>


<commit_before>/******************************************************************************
 * This file is part of the Mula project
 * Copyright (c) 2011 Laszlo Papp <lpapp@kde.org>
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2.1 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with this library; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 */

#include "offsetcachefile.h"

#include "file.h"
#include "wordentry.h"

#include <QtCore/QVector>
#include <QtCore/QFile>
#include <QtCore/QtGlobal>
#include <QtCore/QDir>
#include <QtCore/QDateTime>
#include <QtCore/QDebug>
#include <QtCore/QPair>
#include <QtGui/QDesktopServices>

#include <arpa/inet.h>

using namespace MulaPluginStarDict;

class OffsetCacheFile::Private
{
    public:
        Private()
            : wordCount(0)
            , cacheMagicString("StarDict's Cache, Version: 0.1")
            , mappedData(0)
        {
        }

        ~Private()
        {
        }

        void fill(QByteArray data, int entryCount, long index);

        static const int entriesPerPage = 32;

        QVector<quint32> wordOffset;
        QFile indexFile;
        ulong wordCount;

        QByteArray wordEntryBuffer; // 256 + sizeof(quint32)*2) - The length of "word_str" should be less than 256. See src/tools/DICTFILE_FORMAT.

        QPair<int, QByteArray> first;
        QPair<int, QByteArray> last;
        QPair<int, QByteArray> middle;
        QPair<int, QByteArray> realLast;

        QByteArray pageData;

        long entryIndex;
        QList<WordEntry> entries;

        QByteArray cacheMagicString;
        QFile mapFile;
        uchar *mappedData;
};

void
OffsetCacheFile::Private::fill(QByteArray data, int entryCount, long index)
{
    entryIndex = index;
    ulong position = 0;
    for (int i = 0; i < entryCount; ++i)
    {
        entries[i].setData(data.mid(position));
        position = qstrlen(data.mid(position)) + 1;
        entries[i].setDataOffset(ntohl(*reinterpret_cast<quint32 *>(data.mid(position).data())));
        position += sizeof(quint32);
        entries[i].setDataSize(ntohl(*reinterpret_cast<quint32 *>(data.mid(position).data())));
        position += sizeof(quint32);
    }
}

OffsetCacheFile::OffsetCacheFile()
    : d(new Private)
{
}

OffsetCacheFile::~OffsetCacheFile()
{
}

QByteArray
OffsetCacheFile::readFirstOnPageKey(long pageIndex)
{
    d->indexFile.seek(d->wordOffset.at(pageIndex));
    int pageSize = d->wordOffset.at(pageIndex + 1) - d->wordOffset.at(pageIndex);
    d->wordEntryBuffer = d->indexFile.read(qMin(d->wordEntryBuffer.size(), pageSize)); //TODO: check returned values, deal with word entry that strlen>255.
    return d->wordEntryBuffer;
}

QByteArray
OffsetCacheFile::firstOnPageKey(long pageIndex)
{
    if (pageIndex < d->middle.first)
    {
        if (pageIndex == d->first.first)
            return d->first.second;

        return readFirstOnPageKey(pageIndex);
    }
    else if (pageIndex > d->middle.first)
    {
        if (pageIndex == d->last.first)
            return d->last.second;

        return readFirstOnPageKey(pageIndex);
    }
    else
    {
        return d->middle.second;
    }
}

QStringList
OffsetCacheFile::cacheLocations(const QString& url)
{
    QStringList result;
    result.append(url + ".oft");

    QString cacheLocation = QDesktopServices::storageLocation(QDesktopServices::CacheLocation);
    QFileInfo cacheLocationFileInfo(cacheLocation);
    QDir cacheLocationDir;

    if (!cacheLocationFileInfo.exists() && cacheLocationDir.mkdir(cacheLocation) == false)
        return result;

    if (!cacheLocationFileInfo.isDir())
        return result;

    result.append(cacheLocation + QDir::separator() + "sdcv" + QDir::separator() + QFileInfo(url).fileName() + ".oft");
    return result;
}

bool
OffsetCacheFile::loadCache(const QString& url)
{
    foreach (const QString& cacheLocation, cacheLocations(url))
    {
        QFileInfo fileInfoIndex(url);
        QFileInfo fileInfoCache(cacheLocation);

        if (fileInfoCache.lastModified() < fileInfoIndex.lastModified())
            continue;

        d->mapFile.unmap(d->mappedData);
        d->mapFile.setFileName(cacheLocation);
        if( !d->mapFile.open( QIODevice::ReadOnly ) )
        {
            qDebug() << "Failed to open file:" << cacheLocation;
            return -1;
        }

        d->mappedData = d->mapFile.map(0, d->mapFile.size());
        if (d->mappedData == NULL)
        {
            qDebug() << Q_FUNC_INFO << QString("Mapping the file %1 failed!").arg(cacheLocation);
            return false;
        }

        if (d->cacheMagicString != d->mappedData)
            continue;

        memcpy(d->wordOffset.data(), d->mappedData + d->cacheMagicString.size(), d->wordOffset.size()*sizeof(d->wordOffset.at(0)));
        return true;
    }

    return false;
}

bool
OffsetCacheFile::saveCache(const QString& url)
{
    foreach (const QString& cacheLocation, cacheLocations(url))
    {
        QFile file(cacheLocation);
        if( !file.open( QIODevice::WriteOnly ) )
        {
            qDebug() << "Failed to open file for writing:" << cacheLocation;
            return -1;
        }

        d->mapFile.unmap(d->mappedData);
        d->mapFile.setFileName(cacheLocation);
        if( !d->mapFile.open( QIODevice::ReadOnly ) )
        {
            qDebug() << "Failed to open file:" << cacheLocation;
            return -1;
        }

        d->mappedData = d->mapFile.map(0, d->mapFile.size());
        if (d->mappedData == NULL)
        {
            qDebug() << Q_FUNC_INFO << QString("Mapping the file %1 failed!").arg(cacheLocation);
            return false;
        }

        if (file.write(d->cacheMagicString) != d->cacheMagicString.size())
            continue;

        if (file.write(reinterpret_cast<const char*>(d->wordOffset.data()), sizeof(d->wordOffset.at(0))*d->wordOffset.size())
                != d->wordOffset.at(0)*d->wordOffset.size())
            continue;

        file.close();

        qDebug() << "Save to cache" << url;

        return true;
    }

    return false;
}

bool
OffsetCacheFile::load(const QString& url, int wc, qulonglong fileSize)
{
    Q_UNUSED(fileSize);

    d->wordCount = wc;
    qulonglong npages = (wc - 1) / d->entriesPerPage + 2;
    d->wordOffset.resize(npages);

    if (!loadCache(url))
    { //map file will close after finish of block
        d->mapFile.setFileName(url);
        if (!d->mapFile.open(QIODevice::ReadOnly))
        {
            qDebug() << "Failed to open file:" << url;
            return -1;
        }

        d->mappedData = d->mapFile.map(0, d->mapFile.size());
        if (d->mappedData == NULL)
        {
            qDebug() << Q_FUNC_INFO << QString("Mapping the file %1 failed!").arg(url);
            return false;
        }

        QByteArray byteArray = QByteArray::fromRawData(reinterpret_cast<const char*>(d->mappedData), d->mapFile.size());

        int position = 0;
        int j = 0;
        for (int i = 0; i < wc; i++)
        {
            if (i % d->entriesPerPage == 0)
            {
                d->wordOffset[j] = position;
                ++j;
            }

            position += qstrlen(byteArray.mid(position)) + 1 + 2 * sizeof(quint32);
        }

        d->wordOffset[j] = position;

        if (!saveCache(url))
            qDebug() << "Cache update failed";
    }

    d->indexFile.setFileName(url);
    if (!d->indexFile.open(QIODevice::ReadOnly))
    {
        qDebug() << "Failed to open file:" << url;
        d->wordOffset.resize(0);
        return false;
    }

    d->first = qMakePair(0, readFirstOnPageKey(0));
    d->last = qMakePair(d->wordOffset.size() - 2, readFirstOnPageKey(d->wordOffset.size() - 2));
    d->middle = qMakePair((d->wordOffset.size() - 2) / 2, readFirstOnPageKey((d->wordOffset.size() - 2) / 2));
    d->realLast = qMakePair(wc - 1, key(wc - 1));

    return true;
}

ulong
OffsetCacheFile::loadPage(long pageIndex)
{
    ulong entryCount = d->entriesPerPage;
    if (pageIndex == ulong(d->wordOffset.size() - 2) && (entryCount = d->wordCount % d->entriesPerPage) == 0)
    {
        entryCount = d->entriesPerPage;
    }

    if (pageIndex != d->entryIndex)
    {
        d->indexFile.seek(d->wordOffset.at(pageIndex));

        d->pageData = d->indexFile.read(d->wordOffset[pageIndex + 1] - d->wordOffset[pageIndex]);
        d->fill(d->pageData, entryCount, pageIndex);
    }

    return entryCount;
}

QByteArray
OffsetCacheFile::key(long index)
{
    loadPage(index / d->entriesPerPage);
    ulong indexInPage = index % d->entriesPerPage;
    setWordEntryOffset(d->entries.at(indexInPage).dataOffset());
    setWordEntrySize(d->entries.at(indexInPage).dataSize());

    return d->entries.at(indexInPage).data();
}

bool
OffsetCacheFile::lookup(const QByteArray& word, long &index)
{
    bool found = false;
    long indexFrom;
    long indexTo = d->wordOffset.size() - 2;
    int cmpint;
    long indexThisIndex;
    if (stardictStringCompare(word, d->first.second) < 0)
    {
        index = 0;
        return false;
    }
    else if (stardictStringCompare(word, d->realLast.second) > 0)
    {
        index = invalidIndex;
        return false;
    }
    else
    {
        indexFrom = 0;
        indexThisIndex = 0;
        while (indexFrom <= indexTo)
        {
            indexThisIndex = (indexFrom + indexTo) / 2;
            cmpint = stardictStringCompare(word, firstOnPageKey(indexThisIndex));
            if (cmpint > 0)
                indexFrom = indexThisIndex + 1;
            else if (cmpint < 0)
                indexTo = indexThisIndex - 1;
            else
            {
                found = true;
                break;
            }
        }
        if (!found)
            index = indexTo;    //prev
        else
            index = indexThisIndex;
    }

    if (!found)
    {
        ulong netr = loadPage(index);
        indexFrom = 1; // Needn't search the first word anymore.
        indexTo = netr - 1;
        indexThisIndex = 0;
        while (indexFrom <= indexTo)
        {
            indexThisIndex = (indexFrom + indexTo) / 2;
            cmpint = stardictStringCompare(word, d->entries.at(indexThisIndex).data());
            if (cmpint > 0)
                indexFrom = indexThisIndex + 1;
            else if (cmpint < 0)
                indexTo = indexThisIndex - 1;
            else
            {
                found = true;
                break;
            }
        }

        index *= d->entriesPerPage;
        if (!found)
            index += indexFrom;    //next
        else
            index += indexThisIndex;
    }
    else
    {
        index *= d->entriesPerPage;
    }

    return found;
}

quint32
OffsetCacheFile::wordEntryOffset() const
{
    return AbstractIndexFile::wordEntryOffset();
}

void
OffsetCacheFile::setWordEntryOffset(quint32 wordEntryOffset)
{
    AbstractIndexFile::setWordEntryOffset(wordEntryOffset);
}

quint32
OffsetCacheFile::wordEntrySize() const
{
    return AbstractIndexFile::wordEntrySize();
}

void
OffsetCacheFile::setWordEntrySize(quint32 wordEntrySize)
{
    AbstractIndexFile::setWordEntrySize(wordEntrySize);
}
<commit_msg>Initialize the entryIndex to an invalid index the private class ctor<commit_after>/******************************************************************************
 * This file is part of the Mula project
 * Copyright (c) 2011 Laszlo Papp <lpapp@kde.org>
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2.1 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with this library; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 */

#include "offsetcachefile.h"

#include "file.h"
#include "wordentry.h"

#include <QtCore/QVector>
#include <QtCore/QFile>
#include <QtCore/QtGlobal>
#include <QtCore/QDir>
#include <QtCore/QDateTime>
#include <QtCore/QDebug>
#include <QtCore/QPair>
#include <QtGui/QDesktopServices>

#include <arpa/inet.h>

using namespace MulaPluginStarDict;

class OffsetCacheFile::Private
{
    public:
        Private()
            : wordCount(0)
            , entryIndex(-1)
            , cacheMagicString("StarDict's Cache, Version: 0.1")
            , mappedData(0)
        {
        }

        ~Private()
        {
        }

        void fill(QByteArray data, int entryCount, long index);

        static const int entriesPerPage = 32;

        QVector<quint32> wordOffset;
        QFile indexFile;
        ulong wordCount;

        QByteArray wordEntryBuffer; // 256 + sizeof(quint32)*2) - The length of "word_str" should be less than 256. See src/tools/DICTFILE_FORMAT.

        QPair<int, QByteArray> first;
        QPair<int, QByteArray> last;
        QPair<int, QByteArray> middle;
        QPair<int, QByteArray> realLast;

        QByteArray pageData;

        long entryIndex;
        QList<WordEntry> entries;

        QByteArray cacheMagicString;
        QFile mapFile;
        uchar *mappedData;
};

void
OffsetCacheFile::Private::fill(QByteArray data, int entryCount, long index)
{
    entryIndex = index;
    ulong position = 0;
    for (int i = 0; i < entryCount; ++i)
    {
        entries[i].setData(data.mid(position));
        position = qstrlen(data.mid(position)) + 1;
        entries[i].setDataOffset(ntohl(*reinterpret_cast<quint32 *>(data.mid(position).data())));
        position += sizeof(quint32);
        entries[i].setDataSize(ntohl(*reinterpret_cast<quint32 *>(data.mid(position).data())));
        position += sizeof(quint32);
    }
}

OffsetCacheFile::OffsetCacheFile()
    : d(new Private)
{
}

OffsetCacheFile::~OffsetCacheFile()
{
}

QByteArray
OffsetCacheFile::readFirstOnPageKey(long pageIndex)
{
    d->indexFile.seek(d->wordOffset.at(pageIndex));
    int pageSize = d->wordOffset.at(pageIndex + 1) - d->wordOffset.at(pageIndex);
    d->wordEntryBuffer = d->indexFile.read(qMin(d->wordEntryBuffer.size(), pageSize)); //TODO: check returned values, deal with word entry that strlen>255.
    return d->wordEntryBuffer;
}

QByteArray
OffsetCacheFile::firstOnPageKey(long pageIndex)
{
    if (pageIndex < d->middle.first)
    {
        if (pageIndex == d->first.first)
            return d->first.second;

        return readFirstOnPageKey(pageIndex);
    }
    else if (pageIndex > d->middle.first)
    {
        if (pageIndex == d->last.first)
            return d->last.second;

        return readFirstOnPageKey(pageIndex);
    }
    else
    {
        return d->middle.second;
    }
}

QStringList
OffsetCacheFile::cacheLocations(const QString& url)
{
    QStringList result;
    result.append(url + ".oft");

    QString cacheLocation = QDesktopServices::storageLocation(QDesktopServices::CacheLocation);
    QFileInfo cacheLocationFileInfo(cacheLocation);
    QDir cacheLocationDir;

    if (!cacheLocationFileInfo.exists() && cacheLocationDir.mkdir(cacheLocation) == false)
        return result;

    if (!cacheLocationFileInfo.isDir())
        return result;

    result.append(cacheLocation + QDir::separator() + "sdcv" + QDir::separator() + QFileInfo(url).fileName() + ".oft");
    return result;
}

bool
OffsetCacheFile::loadCache(const QString& url)
{
    foreach (const QString& cacheLocation, cacheLocations(url))
    {
        QFileInfo fileInfoIndex(url);
        QFileInfo fileInfoCache(cacheLocation);

        if (fileInfoCache.lastModified() < fileInfoIndex.lastModified())
            continue;

        d->mapFile.unmap(d->mappedData);
        d->mapFile.setFileName(cacheLocation);
        if( !d->mapFile.open( QIODevice::ReadOnly ) )
        {
            qDebug() << "Failed to open file:" << cacheLocation;
            return -1;
        }

        d->mappedData = d->mapFile.map(0, d->mapFile.size());
        if (d->mappedData == NULL)
        {
            qDebug() << Q_FUNC_INFO << QString("Mapping the file %1 failed!").arg(cacheLocation);
            return false;
        }

        if (d->cacheMagicString != d->mappedData)
            continue;

        memcpy(d->wordOffset.data(), d->mappedData + d->cacheMagicString.size(), d->wordOffset.size()*sizeof(d->wordOffset.at(0)));
        return true;
    }

    return false;
}

bool
OffsetCacheFile::saveCache(const QString& url)
{
    foreach (const QString& cacheLocation, cacheLocations(url))
    {
        QFile file(cacheLocation);
        if( !file.open( QIODevice::WriteOnly ) )
        {
            qDebug() << "Failed to open file for writing:" << cacheLocation;
            return -1;
        }

        d->mapFile.unmap(d->mappedData);
        d->mapFile.setFileName(cacheLocation);
        if( !d->mapFile.open( QIODevice::ReadOnly ) )
        {
            qDebug() << "Failed to open file:" << cacheLocation;
            return -1;
        }

        d->mappedData = d->mapFile.map(0, d->mapFile.size());
        if (d->mappedData == NULL)
        {
            qDebug() << Q_FUNC_INFO << QString("Mapping the file %1 failed!").arg(cacheLocation);
            return false;
        }

        if (file.write(d->cacheMagicString) != d->cacheMagicString.size())
            continue;

        if (file.write(reinterpret_cast<const char*>(d->wordOffset.data()), sizeof(d->wordOffset.at(0))*d->wordOffset.size())
                != d->wordOffset.at(0)*d->wordOffset.size())
            continue;

        file.close();

        qDebug() << "Save to cache" << url;

        return true;
    }

    return false;
}

bool
OffsetCacheFile::load(const QString& url, int wc, qulonglong fileSize)
{
    Q_UNUSED(fileSize);

    d->wordCount = wc;
    qulonglong npages = (wc - 1) / d->entriesPerPage + 2;
    d->wordOffset.resize(npages);

    if (!loadCache(url))
    { //map file will close after finish of block
        d->mapFile.setFileName(url);
        if (!d->mapFile.open(QIODevice::ReadOnly))
        {
            qDebug() << "Failed to open file:" << url;
            return -1;
        }

        d->mappedData = d->mapFile.map(0, d->mapFile.size());
        if (d->mappedData == NULL)
        {
            qDebug() << Q_FUNC_INFO << QString("Mapping the file %1 failed!").arg(url);
            return false;
        }

        QByteArray byteArray = QByteArray::fromRawData(reinterpret_cast<const char*>(d->mappedData), d->mapFile.size());

        int position = 0;
        int j = 0;
        for (int i = 0; i < wc; i++)
        {
            if (i % d->entriesPerPage == 0)
            {
                d->wordOffset[j] = position;
                ++j;
            }

            position += qstrlen(byteArray.mid(position)) + 1 + 2 * sizeof(quint32);
        }

        d->wordOffset[j] = position;

        if (!saveCache(url))
            qDebug() << "Cache update failed";
    }

    d->indexFile.setFileName(url);
    if (!d->indexFile.open(QIODevice::ReadOnly))
    {
        qDebug() << "Failed to open file:" << url;
        d->wordOffset.resize(0);
        return false;
    }

    d->first = qMakePair(0, readFirstOnPageKey(0));
    d->last = qMakePair(d->wordOffset.size() - 2, readFirstOnPageKey(d->wordOffset.size() - 2));
    d->middle = qMakePair((d->wordOffset.size() - 2) / 2, readFirstOnPageKey((d->wordOffset.size() - 2) / 2));
    d->realLast = qMakePair(wc - 1, key(wc - 1));

    return true;
}

ulong
OffsetCacheFile::loadPage(long pageIndex)
{
    ulong entryCount = d->entriesPerPage;
    if (pageIndex == ulong(d->wordOffset.size() - 2) && (entryCount = d->wordCount % d->entriesPerPage) == 0)
    {
        entryCount = d->entriesPerPage;
    }

    if (pageIndex != d->entryIndex)
    {
        d->indexFile.seek(d->wordOffset.at(pageIndex));

        d->pageData = d->indexFile.read(d->wordOffset[pageIndex + 1] - d->wordOffset[pageIndex]);
        d->fill(d->pageData, entryCount, pageIndex);
    }

    return entryCount;
}

QByteArray
OffsetCacheFile::key(long index)
{
    loadPage(index / d->entriesPerPage);
    ulong indexInPage = index % d->entriesPerPage;
    setWordEntryOffset(d->entries.at(indexInPage).dataOffset());
    setWordEntrySize(d->entries.at(indexInPage).dataSize());

    return d->entries.at(indexInPage).data();
}

bool
OffsetCacheFile::lookup(const QByteArray& word, long &index)
{
    bool found = false;
    long indexFrom;
    long indexTo = d->wordOffset.size() - 2;
    int cmpint;
    long indexThisIndex;
    if (stardictStringCompare(word, d->first.second) < 0)
    {
        index = 0;
        return false;
    }
    else if (stardictStringCompare(word, d->realLast.second) > 0)
    {
        index = invalidIndex;
        return false;
    }
    else
    {
        indexFrom = 0;
        indexThisIndex = 0;
        while (indexFrom <= indexTo)
        {
            indexThisIndex = (indexFrom + indexTo) / 2;
            cmpint = stardictStringCompare(word, firstOnPageKey(indexThisIndex));
            if (cmpint > 0)
                indexFrom = indexThisIndex + 1;
            else if (cmpint < 0)
                indexTo = indexThisIndex - 1;
            else
            {
                found = true;
                break;
            }
        }
        if (!found)
            index = indexTo;    //prev
        else
            index = indexThisIndex;
    }

    if (!found)
    {
        ulong netr = loadPage(index);
        indexFrom = 1; // Needn't search the first word anymore.
        indexTo = netr - 1;
        indexThisIndex = 0;
        while (indexFrom <= indexTo)
        {
            indexThisIndex = (indexFrom + indexTo) / 2;
            cmpint = stardictStringCompare(word, d->entries.at(indexThisIndex).data());
            if (cmpint > 0)
                indexFrom = indexThisIndex + 1;
            else if (cmpint < 0)
                indexTo = indexThisIndex - 1;
            else
            {
                found = true;
                break;
            }
        }

        index *= d->entriesPerPage;
        if (!found)
            index += indexFrom;    //next
        else
            index += indexThisIndex;
    }
    else
    {
        index *= d->entriesPerPage;
    }

    return found;
}

quint32
OffsetCacheFile::wordEntryOffset() const
{
    return AbstractIndexFile::wordEntryOffset();
}

void
OffsetCacheFile::setWordEntryOffset(quint32 wordEntryOffset)
{
    AbstractIndexFile::setWordEntryOffset(wordEntryOffset);
}

quint32
OffsetCacheFile::wordEntrySize() const
{
    return AbstractIndexFile::wordEntrySize();
}

void
OffsetCacheFile::setWordEntrySize(quint32 wordEntrySize)
{
    AbstractIndexFile::setWordEntrySize(wordEntrySize);
}
<|endoftext|>


<commit_before>#include "caffe2/operators/load_save_op.h"

namespace caffe2 {

template <>
void LoadOp<CPUContext>::SetCurrentDevice(BlobProto* proto) {
  if (proto->has_tensor()) {
    proto->mutable_tensor()->mutable_device_detail()->set_device_type(CPU);
  }
}

namespace {
REGISTER_CPU_OPERATOR(DBExists, DBExistsOp<CPUContext>);
REGISTER_CPU_OPERATOR(Load, LoadOp<CPUContext>);
REGISTER_CPU_OPERATOR(Save, SaveOp<CPUContext>);
REGISTER_CPU_OPERATOR(Checkpoint, CheckpointOp<CPUContext>);
// CPU Operator old name: do NOT use, we may deprecate this later.
REGISTER_CPU_OPERATOR(Snapshot, CheckpointOp<CPUContext>);

OPERATOR_SCHEMA(DBExists)
    .NumInputs(0)
    .NumOutputs(1)
    .SetDoc(R"DOC(
Checks if the DB exists.
)DOC")
    .Output(0, "exists", "A scalar bool Tensor.")
    .Arg(
        "absolute_path",
        "(int, default 0) if set, use the db path directly and do not prepend "
        "the current root folder of the workspace.")
    .Arg("db_name", "(string) the path to the db to load.")
    .Arg("db_type", "(string) the type of the db.");

OPERATOR_SCHEMA(Load)
    .NumInputs(0, 1)
    .NumOutputs(0, INT_MAX)
    .SetDoc(R"DOC(
The Load operator loads a set of serialized blobs from a db. It takes no
input and [0, infinity) number of outputs, using the db keys to match the db
entries with the outputs.

If an input is passed, then it is assumed that that input blob is a
DBReader to load from, and we ignore the db and db_type arguments.

)DOC")
    .Arg(
        "absolute_path",
        "(int, default 0) if set, use the db path directly and do not prepend "
        "the current root folder of the workspace.")
    .Arg(
        "strip_prefix",
        "(string, default=\"\") characters in the provided blob "
        " names that match strip_prefix will be removed prior to saving."
        " Also, characters that precede strip_prefix will be removed. Useful "
        " for removing device scope from blob names.")
    .Arg("db", "(string) the path to the db to load.")
    .Arg("db_type", "(string) the type of the db.")
    .Arg(
        "keep_device",
        "(int, default 0) if nonzero, the blobs are loaded into the device that "
        "is specified in the serialized BlobProto. Otherwise, the device will be "
        "set as the one that the Load operator is being run under.")
    .Arg(
        "load_all",
        "(int, default 0) if nonzero, will load all blobs pointed to by the db "
        "to the workspace overwriting/creating blobs as needed.")
    .Arg(
        "allow_incomplete",
        "(bool, default false) if true, will allow not loading all the output "
        "blobs specified in the outputs")
    .Arg(
        "source_blob_names",
        "(list of strings) if set, used instead of output "
        "blob names, to specify which blobs in the db shall be loaded. Must be "
        "the same length as number of output blobs.");

OPERATOR_SCHEMA(Save)
    .NumInputs(1, INT_MAX)
    .NumOutputs(0)
    .SetDoc(R"DOC(
The Save operator saves a set of blobs to a db. It takes [1, infinity) number
of inputs and has no output. The contents of the inputs are written into the
db specified by the arguments.
)DOC")
    .Arg(
        "absolute_path",
        "(int, default 0) if set, use the db path directly and do not prepend "
        "the current root folder of the workspace.")
     .Arg(
         "strip_prefix",
         "(string, default=\"\") characters in the provided blob "
         " names that match strip_prefix will be removed prior to saving."
         " Also, characters that precede strip_prefix will be removed. Useful "
         " for removing device scope from blob names.")
    .Arg(
        "blob_name_overrides",
        "(list of strings) if set, used instead of original "
        "blob names. Must be the same length as number of blobs.")
    .Arg("db", "(string) the path to the db to load.")
    .Arg("db_type", "(string) the type of the db.");

OPERATOR_SCHEMA(Checkpoint)
    .NumInputs(1, INT_MAX)
    .NumOutputs(0)
    .SetDoc(R"DOC(
The Checkpoint operator is similar to the Save operator, but allows one to save
to db every few iterations, with a db name that is appended with the iteration
count. It takes [1, infinity) number of inputs and has no output. The first
input has to be a TensorCPU of type int and has size 1 (i.e. the iteration
counter). This is determined whether we need to do checkpointing.
)DOC")
    .Arg(
        "absolute_path",
        "(int, default 0) if set, use the db path directly and do not prepend "
        "the current root folder of the workspace.")
    .Arg(
        "db",
        "(string) a template string that one can combine with the "
        "iteration to create the final db name. For example, "
        "\"/home/lonestarr/checkpoint_%08d.db\"")
    .Arg("db_type", "(string) the type of the db.")
    .Arg(
        "every",
        "(int, default 1) the checkpointing is carried out when "
        "(iter mod every) is zero.");

NO_GRADIENT(Load);
SHOULD_NOT_DO_GRADIENT(DBExists);
SHOULD_NOT_DO_GRADIENT(Save);
SHOULD_NOT_DO_GRADIENT(Checkpoint);
SHOULD_NOT_DO_GRADIENT(Snapshot);
}  // namespace
}  // namespace caffe2
<commit_msg>Fix Caffe2 LoadOp docs<commit_after>#include "caffe2/operators/load_save_op.h"

namespace caffe2 {

template <>
void LoadOp<CPUContext>::SetCurrentDevice(BlobProto* proto) {
  if (proto->has_tensor()) {
    proto->mutable_tensor()->mutable_device_detail()->set_device_type(CPU);
  }
}

namespace {
REGISTER_CPU_OPERATOR(DBExists, DBExistsOp<CPUContext>);
REGISTER_CPU_OPERATOR(Load, LoadOp<CPUContext>);
REGISTER_CPU_OPERATOR(Save, SaveOp<CPUContext>);
REGISTER_CPU_OPERATOR(Checkpoint, CheckpointOp<CPUContext>);
// CPU Operator old name: do NOT use, we may deprecate this later.
REGISTER_CPU_OPERATOR(Snapshot, CheckpointOp<CPUContext>);

OPERATOR_SCHEMA(DBExists)
    .NumInputs(0)
    .NumOutputs(1)
    .SetDoc(R"DOC(
Checks if the DB exists.
)DOC")
    .Output(0, "exists", "A scalar bool Tensor.")
    .Arg(
        "absolute_path",
        "(int, default 0) if set, use the db path directly and do not prepend "
        "the current root folder of the workspace.")
    .Arg("db_name", "(string) the path to the db to load.")
    .Arg("db_type", "(string) the type of the db.");

OPERATOR_SCHEMA(Load)
    .NumInputs(0, 1)
    .NumOutputs(0, INT_MAX)
    .SetDoc(R"DOC(
The Load operator loads a set of serialized blobs from a db. It takes no
input and [0, infinity) number of outputs, using the db keys to match the db
entries with the outputs.

If an input is passed, then it is assumed that that input blob is a
DBReader to load from, and we ignore the db and db_type arguments.

)DOC")
    .Arg(
        "absolute_path",
        "(int, default 0) if set, use the db path directly and do not prepend "
        "the current root folder of the workspace.")
    .Arg(
        "add_prefix",
        "(string, default=\"\") blobs will be prefixed with this when loading."
        "Useful for avoiding collisions with blobs existing in the workspace."
        "The output blob names specified to this op should include this prefix.")
    .Arg(
        "strip_prefix",
        "(string, default=\"\") characters in the provided blob "
        " names that match strip_prefix will be removed prior to loading."
        " Also, characters that precede strip_prefix will be removed. Useful "
        " for removing device scope from blob names.")
    .Arg("db", "(string) the path to the db to load.")
    .Arg("db_type", "(string) the type of the db.")
    .Arg(
        "keep_device",
        "(int, default 0) if nonzero, the blobs are loaded into the device that "
        "is specified in the serialized BlobProto. Otherwise, the device will be "
        "set as the one that the Load operator is being run under.")
    .Arg(
        "load_all",
        "(int, default 0) if nonzero, will load all blobs pointed to by the db "
        "to the workspace overwriting/creating blobs as needed.")
    .Arg(
        "allow_incomplete",
        "(bool, default false) if true, will allow not loading all the output "
        "blobs specified in the outputs")
    .Arg(
        "source_blob_names",
        "(list of strings) if set, used instead of output "
        "blob names, to specify which blobs in the db shall be loaded. Must be "
        "the same length as number of output blobs.");

OPERATOR_SCHEMA(Save)
    .NumInputs(1, INT_MAX)
    .NumOutputs(0)
    .SetDoc(R"DOC(
The Save operator saves a set of blobs to a db. It takes [1, infinity) number
of inputs and has no output. The contents of the inputs are written into the
db specified by the arguments.
)DOC")
    .Arg(
        "absolute_path",
        "(int, default 0) if set, use the db path directly and do not prepend "
        "the current root folder of the workspace.")
     .Arg(
         "strip_prefix",
         "(string, default=\"\") characters in the provided blob "
         " names that match strip_prefix will be removed prior to saving."
         " Also, characters that precede strip_prefix will be removed. Useful "
         " for removing device scope from blob names.")
    .Arg(
        "blob_name_overrides",
        "(list of strings) if set, used instead of original "
        "blob names. Must be the same length as number of blobs.")
    .Arg("db", "(string) the path to the db to load.")
    .Arg("db_type", "(string) the type of the db.");

OPERATOR_SCHEMA(Checkpoint)
    .NumInputs(1, INT_MAX)
    .NumOutputs(0)
    .SetDoc(R"DOC(
The Checkpoint operator is similar to the Save operator, but allows one to save
to db every few iterations, with a db name that is appended with the iteration
count. It takes [1, infinity) number of inputs and has no output. The first
input has to be a TensorCPU of type int and has size 1 (i.e. the iteration
counter). This is determined whether we need to do checkpointing.
)DOC")
    .Arg(
        "absolute_path",
        "(int, default 0) if set, use the db path directly and do not prepend "
        "the current root folder of the workspace.")
    .Arg(
        "db",
        "(string) a template string that one can combine with the "
        "iteration to create the final db name. For example, "
        "\"/home/lonestarr/checkpoint_%08d.db\"")
    .Arg("db_type", "(string) the type of the db.")
    .Arg(
        "every",
        "(int, default 1) the checkpointing is carried out when "
        "(iter mod every) is zero.");

NO_GRADIENT(Load);
SHOULD_NOT_DO_GRADIENT(DBExists);
SHOULD_NOT_DO_GRADIENT(Save);
SHOULD_NOT_DO_GRADIENT(Checkpoint);
SHOULD_NOT_DO_GRADIENT(Snapshot);
}  // namespace
}  // namespace caffe2
<|endoftext|>


<commit_before>/******************************************************************************
 * This file is part of the Mula project
 * Copyright (c) 2011 Laszlo Papp <lpapp@kde.org>
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2.1 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with this library; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 */

#include "offsetcachefile.h"

#include "file.h"
#include "wordentry.h"

#include <QtCore/QVector>
#include <QtCore/QFile>
#include <QtCore/QtGlobal>
#include <QtCore/QDir>
#include <QtCore/QDateTime>
#include <QtCore/QDebug>
#include <QtCore/QPair>
#include <QtGui/QDesktopServices>

#include <arpa/inet.h>

using namespace MulaPluginStarDict;

class OffsetCacheFile::Private
{
    public:
        Private()
            : wordCount(0)
            , pageIndex(-1)
            , cacheMagicString("StarDict's Cache, Version: 0.1")
            , mappedData(0)
        {
        }

        ~Private()
        {
        }

        // The length of "word_str" should be less than 256, and then offset, size
        static const int wordEntrySize = 256 + sizeof(quint32)*2;

        static const int pageEntryNumber = 32;

        QVector<quint32> pageOffsetList;
        QFile indexFile;
        ulong wordCount;

        // index/date based key and value pair
        QPair<int, QByteArray> first;
        QPair<int, QByteArray> last;
        QPair<int, QByteArray> middle;
        QPair<int, QByteArray> realLast;

        long pageIndex;
        QList<WordEntry> entries;

        QByteArray cacheMagicString;
        QFile mapFile;
        uchar *mappedData;
};

OffsetCacheFile::OffsetCacheFile()
    : d(new Private)
{
}

OffsetCacheFile::~OffsetCacheFile()
{
}

QByteArray
OffsetCacheFile::readFirstWordDataOnPage(long pageIndex)
{
    int pageSize = d->pageOffsetList.at(pageIndex + 1) - d->pageOffsetList.at(pageIndex);
    int wordEntrySize = d->wordEntrySize;

    d->indexFile.seek(d->pageOffsetList.at(pageIndex));
    return d->indexFile.read(qMin(wordEntrySize, pageSize)); //TODO: check returned values, deal with word entry that strlen>255.
}

QByteArray
OffsetCacheFile::firstWordDataOnPage(long pageIndex)
{
    if (pageIndex < d->middle.first)
    {
        if (pageIndex == d->first.first)
            return d->first.second;

        return readFirstWordDataOnPage(pageIndex);
    }
    else if (pageIndex > d->middle.first)
    {
        if (pageIndex == d->last.first)
            return d->last.second;

        return readFirstWordDataOnPage(pageIndex);
    }
    else
    {
        return d->middle.second;
    }
}

QStringList
OffsetCacheFile::cacheLocations(const QString& completeFilePath)
{
    QStringList result;
    result.append(completeFilePath + ".oft");

    QString cacheLocation = QDesktopServices::storageLocation(QDesktopServices::CacheLocation);
    QFileInfo cacheLocationFileInfo(cacheLocation);
    QDir cacheLocationDir;

    if (!cacheLocationFileInfo.exists() && cacheLocationDir.mkdir(cacheLocation) == false)
        return result;

    if (!cacheLocationFileInfo.isDir())
        return result;

    result.append(cacheLocation + QDir::separator() + "sdcv" + QDir::separator() + QFileInfo(completeFilePath).fileName() + ".oft");
    return result;
}

bool
OffsetCacheFile::loadCache(const QString& completeFilePath)
{
    foreach (const QString& cacheLocation, cacheLocations(completeFilePath))
    {
        QFileInfo fileInfoIndex(completeFilePath);
        QFileInfo fileInfoCache(cacheLocation);

        if (fileInfoCache.lastModified() < fileInfoIndex.lastModified())
            continue;

        d->mapFile.unmap(d->mappedData);
        d->mapFile.setFileName(cacheLocation);
        if( !d->mapFile.open( QIODevice::ReadOnly ) )
        {
            qDebug() << "Failed to open file:" << cacheLocation;
            return false;
        }

        d->mappedData = d->mapFile.map(0, d->mapFile.size());
        if (d->mappedData == NULL)
        {
            qDebug() << Q_FUNC_INFO << QString("Mapping the file %1 failed!").arg(cacheLocation);
            return false;
        }

        if (d->cacheMagicString != d->mappedData)
            continue;

        memcpy(d->pageOffsetList.data(), d->mappedData + d->cacheMagicString.size(), d->pageOffsetList.size()*sizeof(d->pageOffsetList.at(0)));
        return true;
    }

    return false;
}

bool
OffsetCacheFile::saveCache(const QString& completeFilePath)
{
    foreach (const QString& cacheLocation, cacheLocations(completeFilePath))
    {
        QFile file(cacheLocation);
        if( !file.open( QIODevice::WriteOnly ) )
        {
            qDebug() << "Failed to open file for writing:" << cacheLocation;
            return false;
        }

        d->mapFile.unmap(d->mappedData);
        d->mapFile.setFileName(cacheLocation);
        if( !d->mapFile.open( QIODevice::ReadOnly ) )
        {
            qDebug() << "Failed to open file:" << cacheLocation;
            return false;
        }

        d->mappedData = d->mapFile.map(0, d->mapFile.size());
        if (d->mappedData == NULL)
        {
            qDebug() << Q_FUNC_INFO << QString("Mapping the file %1 failed!").arg(cacheLocation);
            return false;
        }

        if (file.write(d->cacheMagicString) != d->cacheMagicString.size())
            continue;

        if (file.write(reinterpret_cast<const char*>(d->pageOffsetList.data()), sizeof(d->pageOffsetList.at(0))*d->pageOffsetList.size())
                != d->pageOffsetList.at(0)*d->pageOffsetList.size())
            continue;

        file.close();

        qDebug() << "Save to cache" << completeFilePath;

        return true;
    }

    return false;
}

int
OffsetCacheFile::loadPage(int pageIndex)
{
    int wordEntryCountOnPage;

    if (pageIndex == (d->pageOffsetList.size() - 2) && (d->wordCount % d->pageEntryNumber) != 0)
        wordEntryCountOnPage = d->wordCount % d->pageEntryNumber;
    else
        wordEntryCountOnPage = d->pageEntryNumber;

    if (pageIndex != d->pageIndex)
    {
        d->pageIndex = pageIndex;
        d->indexFile.seek(d->pageOffsetList.at(pageIndex));
        QByteArray pageData = d->indexFile.read(d->pageOffsetList.at(pageIndex + 1) - d->pageOffsetList.at(pageIndex));

        ulong position = 0;
        d->entries.reserve(wordEntryCountOnPage);
        for (int i = 0; i < wordEntryCountOnPage; ++i)
        {
            d->entries[i].setData(pageData.mid(position));
            position = qstrlen(pageData.mid(position)) + 1;
            d->entries[i].setDataOffset(ntohl(*reinterpret_cast<quint32 *>(pageData.mid(position).data())));
            position += sizeof(quint32);
            d->entries[i].setDataSize(ntohl(*reinterpret_cast<quint32 *>(pageData.mid(position).data())));
            position += sizeof(quint32);
        }
    }

    return wordEntryCountOnPage;
}

QByteArray
OffsetCacheFile::key(long index)
{
    loadPage(index / d->pageEntryNumber);
    ulong indexInPage = index % d->pageEntryNumber;
    setWordEntryOffset(d->entries.at(indexInPage).dataOffset());
    setWordEntrySize(d->entries.at(indexInPage).dataSize());

    return d->entries.at(indexInPage).data();
}

bool
OffsetCacheFile::load(const QString& completeFilePath, int wordCount, qulonglong fileSize)
{
    Q_UNUSED(fileSize);

    d->wordCount = wordCount;

    if (!loadCache(completeFilePath))
    { //map file will close after finish of block
        d->mapFile.unmap(d->mappedData);
        d->mapFile.setFileName(completeFilePath);
        if (!d->mapFile.open(QIODevice::ReadOnly))
        {
            qDebug() << "Failed to open file:" << completeFilePath;
            return -1;
        }

        d->mappedData = d->mapFile.map(0, d->mapFile.size());
        if (d->mappedData == NULL)
        {
            qDebug() << Q_FUNC_INFO << QString("Mapping the file %1 failed!").arg(completeFilePath);
            return false;
        }

        QByteArray byteArray = QByteArray::fromRawData(reinterpret_cast<const char*>(d->mappedData), d->mapFile.size());

        int position = 0;
        d->pageOffsetList.clear();
        int wordTerminatorOffsetSizeLength = 1 + 2 * sizeof(quint32);
        for (int i = 0; i < wordCount; ++i)
        {
            if (i % d->pageEntryNumber == 0)
                d->pageOffsetList.append(position);

            position += qstrlen(byteArray.mid(position)) + wordTerminatorOffsetSizeLength;
        }

        d->pageOffsetList.append(position);

        if (!saveCache(completeFilePath))
            qDebug() << "Cache update failed";
    }

    d->indexFile.setFileName(completeFilePath);
    if (!d->indexFile.open(QIODevice::ReadOnly))
    {
        qDebug() << "Failed to open file:" << completeFilePath;
        d->pageOffsetList.clear();
        return false;
    }

    d->first = qMakePair(0, readFirstWordDataOnPage(0));
    d->last = qMakePair(d->pageOffsetList.size() - 2, readFirstWordDataOnPage(d->pageOffsetList.size() - 2));
    d->middle = qMakePair((d->pageOffsetList.size() - 2) / 2, readFirstWordDataOnPage((d->pageOffsetList.size() - 2) / 2));
    d->realLast = qMakePair(wordCount - 1, key(wordCount - 1));

    return true;
}

bool
OffsetCacheFile::lookup(const QByteArray& word, long &index)
{
    bool found = false;
    long indexFrom;
    long indexTo = d->pageOffsetList.size() - 2;
    int cmpint;
    long indexThisIndex;
    if (stardictStringCompare(word, d->first.second) < 0)
    {
        index = 0;
        return false;
    }
    else if (stardictStringCompare(word, d->realLast.second) > 0)
    {
        index = invalidIndex;
        return false;
    }
    else
    {
        indexFrom = 0;
        indexThisIndex = 0;
        while (indexFrom <= indexTo)
        {
            indexThisIndex = (indexFrom + indexTo) / 2;
            cmpint = stardictStringCompare(word, firstWordDataOnPage(indexThisIndex));
            if (cmpint > 0)
                indexFrom = indexThisIndex + 1;
            else if (cmpint < 0)
                indexTo = indexThisIndex - 1;
            else
            {
                found = true;
                break;
            }
        }
        if (!found)
            index = indexTo;    //prev
        else
            index = indexThisIndex;
    }

    if (!found)
    {
        ulong netr = loadPage(index);
        indexFrom = 1; // Needn't search the first word anymore.
        indexTo = netr - 1;
        indexThisIndex = 0;
        while (indexFrom <= indexTo)
        {
            indexThisIndex = (indexFrom + indexTo) / 2;
            cmpint = stardictStringCompare(word, d->entries.at(indexThisIndex).data());
            if (cmpint > 0)
                indexFrom = indexThisIndex + 1;
            else if (cmpint < 0)
                indexTo = indexThisIndex - 1;
            else
            {
                found = true;
                break;
            }
        }

        index *= d->pageEntryNumber;
        if (!found)
            index += indexFrom;    //next
        else
            index += indexThisIndex;
    }
    else
    {
        index *= d->pageEntryNumber;
    }

    return found;
}

quint32
OffsetCacheFile::wordEntryOffset() const
{
    return AbstractIndexFile::wordEntryOffset();
}

void
OffsetCacheFile::setWordEntryOffset(quint32 wordEntryOffset)
{
    AbstractIndexFile::setWordEntryOffset(wordEntryOffset);
}

quint32
OffsetCacheFile::wordEntrySize() const
{
    return AbstractIndexFile::wordEntrySize();
}

void
OffsetCacheFile::setWordEntrySize(quint32 wordEntrySize)
{
    AbstractIndexFile::setWordEntrySize(wordEntrySize);
}
<commit_msg>Refactor the for loop by using append instead of direct indexing<commit_after>/******************************************************************************
 * This file is part of the Mula project
 * Copyright (c) 2011 Laszlo Papp <lpapp@kde.org>
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2.1 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with this library; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 */

#include "offsetcachefile.h"

#include "file.h"
#include "wordentry.h"

#include <QtCore/QVector>
#include <QtCore/QFile>
#include <QtCore/QtGlobal>
#include <QtCore/QDir>
#include <QtCore/QDateTime>
#include <QtCore/QDebug>
#include <QtCore/QPair>
#include <QtGui/QDesktopServices>

#include <arpa/inet.h>

using namespace MulaPluginStarDict;

class OffsetCacheFile::Private
{
    public:
        Private()
            : wordCount(0)
            , pageIndex(-1)
            , cacheMagicString("StarDict's Cache, Version: 0.1")
            , mappedData(0)
        {
        }

        ~Private()
        {
        }

        // The length of "word_str" should be less than 256, and then offset, size
        static const int wordEntrySize = 256 + sizeof(quint32)*2;

        static const int pageEntryNumber = 32;

        QVector<quint32> pageOffsetList;
        QFile indexFile;
        ulong wordCount;

        // index/date based key and value pair
        QPair<int, QByteArray> first;
        QPair<int, QByteArray> last;
        QPair<int, QByteArray> middle;
        QPair<int, QByteArray> realLast;

        long pageIndex;
        QList<WordEntry> entries;

        QByteArray cacheMagicString;
        QFile mapFile;
        uchar *mappedData;
};

OffsetCacheFile::OffsetCacheFile()
    : d(new Private)
{
}

OffsetCacheFile::~OffsetCacheFile()
{
}

QByteArray
OffsetCacheFile::readFirstWordDataOnPage(long pageIndex)
{
    int pageSize = d->pageOffsetList.at(pageIndex + 1) - d->pageOffsetList.at(pageIndex);
    int wordEntrySize = d->wordEntrySize;

    d->indexFile.seek(d->pageOffsetList.at(pageIndex));
    return d->indexFile.read(qMin(wordEntrySize, pageSize)); //TODO: check returned values, deal with word entry that strlen>255.
}

QByteArray
OffsetCacheFile::firstWordDataOnPage(long pageIndex)
{
    if (pageIndex < d->middle.first)
    {
        if (pageIndex == d->first.first)
            return d->first.second;

        return readFirstWordDataOnPage(pageIndex);
    }
    else if (pageIndex > d->middle.first)
    {
        if (pageIndex == d->last.first)
            return d->last.second;

        return readFirstWordDataOnPage(pageIndex);
    }
    else
    {
        return d->middle.second;
    }
}

QStringList
OffsetCacheFile::cacheLocations(const QString& completeFilePath)
{
    QStringList result;
    result.append(completeFilePath + ".oft");

    QString cacheLocation = QDesktopServices::storageLocation(QDesktopServices::CacheLocation);
    QFileInfo cacheLocationFileInfo(cacheLocation);
    QDir cacheLocationDir;

    if (!cacheLocationFileInfo.exists() && cacheLocationDir.mkdir(cacheLocation) == false)
        return result;

    if (!cacheLocationFileInfo.isDir())
        return result;

    result.append(cacheLocation + QDir::separator() + "sdcv" + QDir::separator() + QFileInfo(completeFilePath).fileName() + ".oft");
    return result;
}

bool
OffsetCacheFile::loadCache(const QString& completeFilePath)
{
    foreach (const QString& cacheLocation, cacheLocations(completeFilePath))
    {
        QFileInfo fileInfoIndex(completeFilePath);
        QFileInfo fileInfoCache(cacheLocation);

        if (fileInfoCache.lastModified() < fileInfoIndex.lastModified())
            continue;

        d->mapFile.unmap(d->mappedData);
        d->mapFile.setFileName(cacheLocation);
        if( !d->mapFile.open( QIODevice::ReadOnly ) )
        {
            qDebug() << "Failed to open file:" << cacheLocation;
            return false;
        }

        d->mappedData = d->mapFile.map(0, d->mapFile.size());
        if (d->mappedData == NULL)
        {
            qDebug() << Q_FUNC_INFO << QString("Mapping the file %1 failed!").arg(cacheLocation);
            return false;
        }

        if (d->cacheMagicString != d->mappedData)
            continue;

        memcpy(d->pageOffsetList.data(), d->mappedData + d->cacheMagicString.size(), d->pageOffsetList.size()*sizeof(d->pageOffsetList.at(0)));
        return true;
    }

    return false;
}

bool
OffsetCacheFile::saveCache(const QString& completeFilePath)
{
    foreach (const QString& cacheLocation, cacheLocations(completeFilePath))
    {
        QFile file(cacheLocation);
        if( !file.open( QIODevice::WriteOnly ) )
        {
            qDebug() << "Failed to open file for writing:" << cacheLocation;
            return false;
        }

        d->mapFile.unmap(d->mappedData);
        d->mapFile.setFileName(cacheLocation);
        if( !d->mapFile.open( QIODevice::ReadOnly ) )
        {
            qDebug() << "Failed to open file:" << cacheLocation;
            return false;
        }

        d->mappedData = d->mapFile.map(0, d->mapFile.size());
        if (d->mappedData == NULL)
        {
            qDebug() << Q_FUNC_INFO << QString("Mapping the file %1 failed!").arg(cacheLocation);
            return false;
        }

        if (file.write(d->cacheMagicString) != d->cacheMagicString.size())
            continue;

        if (file.write(reinterpret_cast<const char*>(d->pageOffsetList.data()), sizeof(d->pageOffsetList.at(0))*d->pageOffsetList.size())
                != d->pageOffsetList.at(0)*d->pageOffsetList.size())
            continue;

        file.close();

        qDebug() << "Save to cache" << completeFilePath;

        return true;
    }

    return false;
}

int
OffsetCacheFile::loadPage(int pageIndex)
{
    int wordEntryCountOnPage;

    if (pageIndex == (d->pageOffsetList.size() - 2) && (d->wordCount % d->pageEntryNumber) != 0)
        wordEntryCountOnPage = d->wordCount % d->pageEntryNumber;
    else
        wordEntryCountOnPage = d->pageEntryNumber;

    if (pageIndex != d->pageIndex)
    {
        d->pageIndex = pageIndex;
        d->indexFile.seek(d->pageOffsetList.at(pageIndex));
        QByteArray pageData = d->indexFile.read(d->pageOffsetList.at(pageIndex + 1) - d->pageOffsetList.at(pageIndex));

        ulong position = 0;
        d->entries.reserve(wordEntryCountOnPage);
        for (int i = 0; i < wordEntryCountOnPage; ++i)
        {
            WordEntry wordEntry;
            wordEntry.setData(pageData.mid(position));
            position = qstrlen(pageData.mid(position)) + 1;
            wordEntry.setDataOffset(ntohl(*reinterpret_cast<quint32 *>(pageData.mid(position).data())));
            position += sizeof(quint32);
            wordEntry.setDataSize(ntohl(*reinterpret_cast<quint32 *>(pageData.mid(position).data())));
            position += sizeof(quint32);

            d->entries.append(wordEntry);
        }
    }

    return wordEntryCountOnPage;
}

QByteArray
OffsetCacheFile::key(long index)
{
    loadPage(index / d->pageEntryNumber);
    ulong indexInPage = index % d->pageEntryNumber;
    setWordEntryOffset(d->entries.at(indexInPage).dataOffset());
    setWordEntrySize(d->entries.at(indexInPage).dataSize());

    return d->entries.at(indexInPage).data();
}

bool
OffsetCacheFile::load(const QString& completeFilePath, int wordCount, qulonglong fileSize)
{
    Q_UNUSED(fileSize);

    d->wordCount = wordCount;

    if (!loadCache(completeFilePath))
    { //map file will close after finish of block
        d->mapFile.unmap(d->mappedData);
        d->mapFile.setFileName(completeFilePath);
        if (!d->mapFile.open(QIODevice::ReadOnly))
        {
            qDebug() << "Failed to open file:" << completeFilePath;
            return -1;
        }

        d->mappedData = d->mapFile.map(0, d->mapFile.size());
        if (d->mappedData == NULL)
        {
            qDebug() << Q_FUNC_INFO << QString("Mapping the file %1 failed!").arg(completeFilePath);
            return false;
        }

        QByteArray byteArray = QByteArray::fromRawData(reinterpret_cast<const char*>(d->mappedData), d->mapFile.size());

        int position = 0;
        d->pageOffsetList.clear();
        int wordTerminatorOffsetSizeLength = 1 + 2 * sizeof(quint32);
        for (int i = 0; i < wordCount; ++i)
        {
            if (i % d->pageEntryNumber == 0)
                d->pageOffsetList.append(position);

            position += qstrlen(byteArray.mid(position)) + wordTerminatorOffsetSizeLength;
        }

        d->pageOffsetList.append(position);

        if (!saveCache(completeFilePath))
            qDebug() << "Cache update failed";
    }

    d->indexFile.setFileName(completeFilePath);
    if (!d->indexFile.open(QIODevice::ReadOnly))
    {
        qDebug() << "Failed to open file:" << completeFilePath;
        d->pageOffsetList.clear();
        return false;
    }

    d->first = qMakePair(0, readFirstWordDataOnPage(0));
    d->last = qMakePair(d->pageOffsetList.size() - 2, readFirstWordDataOnPage(d->pageOffsetList.size() - 2));
    d->middle = qMakePair((d->pageOffsetList.size() - 2) / 2, readFirstWordDataOnPage((d->pageOffsetList.size() - 2) / 2));
    d->realLast = qMakePair(wordCount - 1, key(wordCount - 1));

    return true;
}

bool
OffsetCacheFile::lookup(const QByteArray& word, long &index)
{
    bool found = false;
    long indexFrom;
    long indexTo = d->pageOffsetList.size() - 2;
    int cmpint;
    long indexThisIndex;
    if (stardictStringCompare(word, d->first.second) < 0)
    {
        index = 0;
        return false;
    }
    else if (stardictStringCompare(word, d->realLast.second) > 0)
    {
        index = invalidIndex;
        return false;
    }
    else
    {
        indexFrom = 0;
        indexThisIndex = 0;
        while (indexFrom <= indexTo)
        {
            indexThisIndex = (indexFrom + indexTo) / 2;
            cmpint = stardictStringCompare(word, firstWordDataOnPage(indexThisIndex));
            if (cmpint > 0)
                indexFrom = indexThisIndex + 1;
            else if (cmpint < 0)
                indexTo = indexThisIndex - 1;
            else
            {
                found = true;
                break;
            }
        }
        if (!found)
            index = indexTo;    //prev
        else
            index = indexThisIndex;
    }

    if (!found)
    {
        ulong netr = loadPage(index);
        indexFrom = 1; // Needn't search the first word anymore.
        indexTo = netr - 1;
        indexThisIndex = 0;
        while (indexFrom <= indexTo)
        {
            indexThisIndex = (indexFrom + indexTo) / 2;
            cmpint = stardictStringCompare(word, d->entries.at(indexThisIndex).data());
            if (cmpint > 0)
                indexFrom = indexThisIndex + 1;
            else if (cmpint < 0)
                indexTo = indexThisIndex - 1;
            else
            {
                found = true;
                break;
            }
        }

        index *= d->pageEntryNumber;
        if (!found)
            index += indexFrom;    //next
        else
            index += indexThisIndex;
    }
    else
    {
        index *= d->pageEntryNumber;
    }

    return found;
}

quint32
OffsetCacheFile::wordEntryOffset() const
{
    return AbstractIndexFile::wordEntryOffset();
}

void
OffsetCacheFile::setWordEntryOffset(quint32 wordEntryOffset)
{
    AbstractIndexFile::setWordEntryOffset(wordEntryOffset);
}

quint32
OffsetCacheFile::wordEntrySize() const
{
    return AbstractIndexFile::wordEntrySize();
}

void
OffsetCacheFile::setWordEntrySize(quint32 wordEntrySize)
{
    AbstractIndexFile::setWordEntrySize(wordEntrySize);
}
<|endoftext|>