<commit_before>// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements.  See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership.  The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License.  You may obtain a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied.  See the License for the
// specific language governing permissions and limitations
// under the License.

use std::fs::File;
use std::io::{self, Read, BufReader, Seek, SeekFrom};
use std::rc::Rc;
use std::cell::RefCell;

use basic::{Compression, Encoding};
use errors::{Result, ParquetError};
use file::metadata::{RowGroupMetaData, FileMetaData, ParquetMetaData};
use byteorder::{LittleEndian, ByteOrder};
use thrift::transport::{TTransport, TBufferTransport};
use thrift::protocol::TCompactInputProtocol;
use parquet_thrift::parquet::FileMetaData as TFileMetaData;
use parquet_thrift::parquet::{PageType, PageHeader};
use schema::types;
use column::page::{Page, PageReader};
use compression::{Codec, create_codec};
use util::memory::{Buffer, MutableBuffer, ByteBuffer};

// ----------------------------------------------------------------------
// APIs for file & row group readers

/// Parquet file reader API. With this, user can get metadata information
/// about the Parquet file, and can get reader for each row group.
pub trait FileReader {
  /// Get metadata information about this file
  fn metadata(&self) -> &ParquetMetaData;

  /// Get the total number of row groups for this file
  fn num_row_groups(&self) -> usize;

  /// Get the `i`th row group reader. Note this doesn't do bound check.
  /// N.B.: Box<..> has 'static lifetime in default, but here we need
  /// the lifetime to be the same as this. Otherwise, the row group metadata
  /// stored in the row group reader may outlive the file reader.
  fn get_row_group<'a>(&'a self, i: usize) -> Result<Box<RowGroupReader + 'a>>;
}

/// Parquet row group reader API. With this, user can get metadata
/// information about the row group, as well as readers for each individual
/// column chunk
/// The lifetime 'a is for the metadata inherited from the parent file reader
pub trait RowGroupReader<'a> {
  /// Get metadata information about this row group.
  /// The result metadata is owned by the parent file reader.
  fn metadata(&self) -> &'a RowGroupMetaData;

  /// Get the total number of column chunks in this row group
  fn num_columns(&self) -> usize;

  /// Get page reader for the `i`th column chunk
  fn get_page_reader<'b>(&'b self, i: usize) -> Result<Box<PageReader + 'b>>;
}

/// A thin wrapper on `BufReader` to be used by Thrift transport
pub struct TMemoryBuffer<'a, T> where T: 'a + Read {
  buf_reader: &'a mut BufReader<T>,
  offset: usize
}

impl<'a, T: 'a + Read> TMemoryBuffer<'a, T> {
  pub fn new(buf_reader: &'a mut BufReader<T>) -> Self {
    Self { buf_reader: buf_reader, offset: 0 }
  }
}

impl<'a, T: 'a + Read> Read for TMemoryBuffer<'a, T> {
  fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> {
    let bytes_read = self.buf_reader.read(buf)?;
    self.offset += bytes_read;
    Ok(bytes_read)

  }
}

impl<'a, T: 'a + Read> io::Write for TMemoryBuffer<'a, T> {
  fn write(&mut self, _: &[u8]) -> io::Result<usize> {
    panic!("Not implemented")
  }

  fn flush(&mut self) -> io::Result<()> {
    panic!("Not implemented")
  }
}

// ----------------------------------------------------------------------
// Serialized impl for file & row group readers

const FOOTER_SIZE: usize = 8;
const PARQUET_MAGIC: [u8; 4] = [b'P', b'A', b'R', b'1'];

pub struct SerializedFileReader {
  buf: BufReader<File>,
  metadata: ParquetMetaData
}

impl SerializedFileReader {
  pub fn new(file: File) -> Result<Self> {
    let mut buf = BufReader::new(file);
    let metadata = Self::parse_metadata(&mut buf)?;
    Ok(Self { buf: buf, metadata: metadata })
  }

  fn parse_metadata(buf: &mut BufReader<File>) -> Result<ParquetMetaData> {
    let file_metadata = buf.get_ref().metadata()?;
    let file_size = file_metadata.len();
    if file_size < (FOOTER_SIZE as u64) {
      return Err(parse_err!("Corrputed file, smaller than file footer"));
    }
    let mut footer_buffer: [u8; FOOTER_SIZE] = [0; FOOTER_SIZE];
    buf.seek(SeekFrom::End(-(FOOTER_SIZE as i64)))?;
    buf.read_exact(&mut footer_buffer)?;
    if footer_buffer[4..] != PARQUET_MAGIC {
      return Err(parse_err!("Invalid parquet file. Corrupt footer."));
    }
    let metadata_len = LittleEndian::read_i32(&footer_buffer[0..4]);
    if metadata_len < 0 {
      return Err(parse_err!(
        "Invalid parquet file. Metadata length is less than zero ({})",
        metadata_len));
    }
    let mut metadata_buffer = vec![0; metadata_len as usize];
    let metadata_start: i64 = file_size as i64 - FOOTER_SIZE as i64 - metadata_len as i64;
    if metadata_start < 0 {
      return Err(parse_err!(
        "Invalid parquet file. Metadata start is less than zero ({})",
        metadata_start))
    }
    buf.seek(SeekFrom::Start(metadata_start as u64))?;
    buf.read_exact(metadata_buffer.as_mut_slice())
      .map_err(|e| io_err!(e, "Failed to read metadata"))?;

    // TODO: do row group filtering
    let mut transport = TBufferTransport::with_capacity(metadata_len as usize, 0);
    transport.set_readable_bytes(metadata_buffer.as_mut_slice());
    let transport = Rc::new(RefCell::new(Box::new(transport) as Box<TTransport>));
    let mut prot = TCompactInputProtocol::new(transport);
    let mut t_file_metadata: TFileMetaData = TFileMetaData::read_from_in_protocol(&mut prot)
      .map_err(|e| thrift_err!(e, "Could not parse metadata"))?;
    let schema: Box<types::Type> = types::from_thrift(&mut t_file_metadata.schema)?;
    let mut row_groups = Vec::new();
    for rg in t_file_metadata.row_groups {
      row_groups.push(RowGroupMetaData::from_thrift(rg)?);
    }

    let file_metadata = FileMetaData::new(
      t_file_metadata.version,
      t_file_metadata.num_rows,
      t_file_metadata.created_by,
      schema);
    Ok(ParquetMetaData::new(file_metadata, row_groups))
  }
}

impl FileReader for SerializedFileReader {
  fn metadata(&self) -> &ParquetMetaData {
    &self.metadata
  }

  fn num_row_groups(&self) -> usize {
    self.metadata.num_row_groups()
  }

  fn get_row_group<'a>(&'a self, i: usize) -> Result<Box<RowGroupReader + 'a>> {
    let row_group_metadata = self.metadata.row_group(i);
    let f = self.buf.get_ref().try_clone()?;
    Ok(Box::new(SerializedRowGroupReader::new(f, row_group_metadata)))
  }
}

/// A serialized impl for row group reader
/// Here 'a is the lifetime for the row group metadata, which is owned
/// by the parent Parquet file reader
pub struct SerializedRowGroupReader<'a> {
  buf: BufReader<File>,
  metadata: &'a RowGroupMetaData
}

impl<'a> SerializedRowGroupReader<'a> {
  pub fn new(file: File, metadata: &'a RowGroupMetaData) -> Self {
    let buf = BufReader::new(file);
    Self { buf: buf, metadata: metadata }
  }
}

impl<'a> RowGroupReader<'a> for SerializedRowGroupReader<'a> {
  fn metadata(&self) -> &'a RowGroupMetaData {
    self.metadata
  }

  fn num_columns(&self) -> usize {
    self.metadata.num_columns()
  }

  // TODO: fix PARQUET-816
  fn get_page_reader<'b>(&'b self, i: usize) -> Result<Box<PageReader + 'b>> {
    let col = self.metadata.column(i);
    let mut col_start = col.data_page_offset();
    if col.has_dictionary_page() {
      col_start = col.dictionary_page_offset().unwrap();
    }
    let col_length = col.compressed_size() as u64;
    let f = self.buf.get_ref().try_clone()?;
    let mut buf = BufReader::new(f);
    let _ = buf.seek(SeekFrom::Start(col_start as u64));
    let page_reader = SerializedPageReader::new(
      buf.take(col_length).into_inner(), col.num_values(), col.compression())?;
    Ok(Box::new(page_reader))
  }
}

/// A serialized impl for Parquet page reader
pub struct SerializedPageReader {
  /// The buffer which contains exactly the bytes for the column trunk
  /// to be read by this page reader
  buf: BufReader<File>,

  /// The compression codec for this column chunk. Only set for
  /// non-PLAIN codec.
  decompressor: Option<Box<Codec>>,

  /// The number of values we have seen so far
  seen_num_values: i64,

  /// The number of total values in this column chunk
  total_num_values: i64,
}

impl SerializedPageReader {
  pub fn new(buf: BufReader<File>, total_num_values: i64,
             compression: Compression) -> Result<Self> {
    let decompressor = create_codec(compression)?;
    let result =
      Self { buf: buf, total_num_values: total_num_values, seen_num_values: 0,
             decompressor: decompressor };
    Ok(result)
  }

  fn read_page_header(&mut self) -> Result<PageHeader> {
    let transport = Rc::new(RefCell::new(
      Box::new(TMemoryBuffer::new(&mut self.buf)) as Box<TTransport>));
    let mut prot = TCompactInputProtocol::new(transport);
    let page_header = PageHeader::read_from_in_protocol(&mut prot)?;
    Ok(page_header)
  }
}

impl PageReader for SerializedPageReader {
  fn get_next_page(&mut self) -> Result<Option<Page>> {
    while self.seen_num_values < self.total_num_values {
      let page_header = self.read_page_header()?;
      let compressed_len = page_header.compressed_page_size as usize;
      let uncompressed_len = page_header.uncompressed_page_size as usize;
      let mut buffer = ByteBuffer::new(compressed_len);
      self.buf.read_exact(buffer.mut_data())?;

      // TODO: page header could be huge because of statistics. We should
      // set a maximum page header size and abort if that is exceeded.
      if let Some(decompressor) = self.decompressor.as_mut() {
        let mut decompressed_buffer = vec!();
        let decompressed_size = decompressor.decompress(buffer.data(), &mut decompressed_buffer)?;
        if decompressed_size != uncompressed_len {
          return Err(decode_err!("Actual decompressed size doesn't \
            match the expected one ({} vs {})", decompressed_size, uncompressed_len));
        }
        buffer.set_data(decompressed_buffer);
      }

      // TODO: process statistics
      let result = match page_header.type_ {
        PageType::DICTIONARY_PAGE => {
          assert!(page_header.dictionary_page_header.is_some());
          let dict_header = page_header.dictionary_page_header.as_ref().unwrap();
          let is_sorted = match dict_header.is_sorted {
            None => false,
            Some(b) => b
          };
          self.seen_num_values += dict_header.num_values as i64;
          Page::DictionaryPage {
            buf: Box::new(buffer), num_values: dict_header.num_values as u32,
            encoding: Encoding::from(dict_header.encoding), is_sorted: is_sorted
          }
        },
        PageType::DATA_PAGE => {
          assert!(page_header.data_page_header.is_some());
          let header = page_header.data_page_header.as_ref().unwrap();
          self.seen_num_values += header.num_values as i64;
          Page::DataPage {
            buf: Box::new(buffer), num_values: header.num_values as u32,
            encoding: Encoding::from(header.encoding),
            def_level_encoding: Encoding::from(header.definition_level_encoding),
            rep_level_encoding: Encoding::from(header.repetition_level_encoding)
          }
        },
        PageType::DATA_PAGE_V2 => {
          assert!(page_header.data_page_header_v2.is_some());
          let header = page_header.data_page_header_v2.as_ref().unwrap();
          let is_compressed = match header.is_compressed {
            Some(b) => b,
            None => true
          };
          self.seen_num_values += header.num_values as i64;
          Page::DataPageV2 {
            buf: Box::new(buffer), num_values: header.num_values as u32,
            encoding: Encoding::from(header.encoding),
            num_nulls: header.num_nulls as u32, num_rows: header.num_rows as u32,
            def_levels_byte_len: header.definition_levels_byte_length as u32,
            rep_levels_byte_len: header.repetition_levels_byte_length as u32,
            is_compressed: is_compressed
          }
        },
        _ => {
          // For unknown page type (e.g., INDEX_PAGE), skip and read next.
          continue;
        }
      };
      return Ok(Some(result))
    }

    // We are at the end of this column chunk and no more page left. Return None.
    Ok(None)
  }
}

<commit_msg>Add basic tests for file reader<commit_after>// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements.  See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership.  The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License.  You may obtain a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied.  See the License for the
// specific language governing permissions and limitations
// under the License.

use std::fs::File;
use std::io::{self, Read, BufReader, Seek, SeekFrom};
use std::rc::Rc;
use std::cell::RefCell;

use basic::{Compression, Encoding};
use errors::{Result, ParquetError};
use file::metadata::{RowGroupMetaData, FileMetaData, ParquetMetaData};
use byteorder::{LittleEndian, ByteOrder};
use thrift::transport::{TTransport, TBufferTransport};
use thrift::protocol::TCompactInputProtocol;
use parquet_thrift::parquet::FileMetaData as TFileMetaData;
use parquet_thrift::parquet::{PageType, PageHeader};
use schema::types;
use column::page::{Page, PageReader};
use compression::{Codec, create_codec};
use util::memory::{Buffer, MutableBuffer, ByteBuffer};

// ----------------------------------------------------------------------
// APIs for file & row group readers

/// Parquet file reader API. With this, user can get metadata information
/// about the Parquet file, and can get reader for each row group.
pub trait FileReader {
  /// Get metadata information about this file
  fn metadata(&self) -> &ParquetMetaData;

  /// Get the total number of row groups for this file
  fn num_row_groups(&self) -> usize;

  /// Get the `i`th row group reader. Note this doesn't do bound check.
  /// N.B.: Box<..> has 'static lifetime in default, but here we need
  /// the lifetime to be the same as this. Otherwise, the row group metadata
  /// stored in the row group reader may outlive the file reader.
  fn get_row_group<'a>(&'a self, i: usize) -> Result<Box<RowGroupReader + 'a>>;
}

/// Parquet row group reader API. With this, user can get metadata
/// information about the row group, as well as readers for each individual
/// column chunk
/// The lifetime 'a is for the metadata inherited from the parent file reader
pub trait RowGroupReader<'a> {
  /// Get metadata information about this row group.
  /// The result metadata is owned by the parent file reader.
  fn metadata(&self) -> &'a RowGroupMetaData;

  /// Get the total number of column chunks in this row group
  fn num_columns(&self) -> usize;

  /// Get page reader for the `i`th column chunk
  fn get_page_reader<'b>(&'b self, i: usize) -> Result<Box<PageReader + 'b>>;
}

/// A thin wrapper on `BufReader` to be used by Thrift transport
pub struct TMemoryBuffer<'a, T> where T: 'a + Read {
  buf_reader: &'a mut BufReader<T>,
  offset: usize
}

impl<'a, T: 'a + Read> TMemoryBuffer<'a, T> {
  pub fn new(buf_reader: &'a mut BufReader<T>) -> Self {
    Self { buf_reader: buf_reader, offset: 0 }
  }
}

impl<'a, T: 'a + Read> Read for TMemoryBuffer<'a, T> {
  fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> {
    let bytes_read = self.buf_reader.read(buf)?;
    self.offset += bytes_read;
    Ok(bytes_read)

  }
}

impl<'a, T: 'a + Read> io::Write for TMemoryBuffer<'a, T> {
  fn write(&mut self, _: &[u8]) -> io::Result<usize> {
    panic!("Not implemented")
  }

  fn flush(&mut self) -> io::Result<()> {
    panic!("Not implemented")
  }
}

// ----------------------------------------------------------------------
// Serialized impl for file & row group readers

const FOOTER_SIZE: usize = 8;
const PARQUET_MAGIC: [u8; 4] = [b'P', b'A', b'R', b'1'];

pub struct SerializedFileReader {
  buf: BufReader<File>,
  metadata: ParquetMetaData
}

impl SerializedFileReader {
  pub fn new(file: File) -> Result<Self> {
    let mut buf = BufReader::new(file);
    let metadata = Self::parse_metadata(&mut buf)?;
    Ok(Self { buf: buf, metadata: metadata })
  }

  fn parse_metadata(buf: &mut BufReader<File>) -> Result<ParquetMetaData> {
    let file_metadata = buf.get_ref().metadata()?;
    let file_size = file_metadata.len();
    if file_size < (FOOTER_SIZE as u64) {
      return Err(parse_err!("Corrputed file, smaller than file footer"));
    }
    let mut footer_buffer: [u8; FOOTER_SIZE] = [0; FOOTER_SIZE];
    buf.seek(SeekFrom::End(-(FOOTER_SIZE as i64)))?;
    buf.read_exact(&mut footer_buffer)?;
    if footer_buffer[4..] != PARQUET_MAGIC {
      return Err(parse_err!("Invalid parquet file. Corrupt footer."));
    }
    let metadata_len = LittleEndian::read_i32(&footer_buffer[0..4]);
    if metadata_len < 0 {
      return Err(parse_err!(
        "Invalid parquet file. Metadata length is less than zero ({})",
        metadata_len));
    }
    let mut metadata_buffer = vec![0; metadata_len as usize];
    let metadata_start: i64 = file_size as i64 - FOOTER_SIZE as i64 - metadata_len as i64;
    if metadata_start < 0 {
      return Err(parse_err!(
        "Invalid parquet file. Metadata start is less than zero ({})",
        metadata_start))
    }
    buf.seek(SeekFrom::Start(metadata_start as u64))?;
    buf.read_exact(metadata_buffer.as_mut_slice())
      .map_err(|e| io_err!(e, "Failed to read metadata"))?;

    // TODO: do row group filtering
    let mut transport = TBufferTransport::with_capacity(metadata_len as usize, 0);
    transport.set_readable_bytes(metadata_buffer.as_mut_slice());
    let transport = Rc::new(RefCell::new(Box::new(transport) as Box<TTransport>));
    let mut prot = TCompactInputProtocol::new(transport);
    let mut t_file_metadata: TFileMetaData = TFileMetaData::read_from_in_protocol(&mut prot)
      .map_err(|e| thrift_err!(e, "Could not parse metadata"))?;
    let schema: Box<types::Type> = types::from_thrift(&mut t_file_metadata.schema)?;
    let mut row_groups = Vec::new();
    for rg in t_file_metadata.row_groups {
      row_groups.push(RowGroupMetaData::from_thrift(rg)?);
    }

    let file_metadata = FileMetaData::new(
      t_file_metadata.version,
      t_file_metadata.num_rows,
      t_file_metadata.created_by,
      schema);
    Ok(ParquetMetaData::new(file_metadata, row_groups))
  }
}

impl FileReader for SerializedFileReader {
  fn metadata(&self) -> &ParquetMetaData {
    &self.metadata
  }

  fn num_row_groups(&self) -> usize {
    self.metadata.num_row_groups()
  }

  fn get_row_group<'a>(&'a self, i: usize) -> Result<Box<RowGroupReader + 'a>> {
    let row_group_metadata = self.metadata.row_group(i);
    let f = self.buf.get_ref().try_clone()?;
    Ok(Box::new(SerializedRowGroupReader::new(f, row_group_metadata)))
  }
}

/// A serialized impl for row group reader
/// Here 'a is the lifetime for the row group metadata, which is owned
/// by the parent Parquet file reader
pub struct SerializedRowGroupReader<'a> {
  buf: BufReader<File>,
  metadata: &'a RowGroupMetaData
}

impl<'a> SerializedRowGroupReader<'a> {
  pub fn new(file: File, metadata: &'a RowGroupMetaData) -> Self {
    let buf = BufReader::new(file);
    Self { buf: buf, metadata: metadata }
  }
}

impl<'a> RowGroupReader<'a> for SerializedRowGroupReader<'a> {
  fn metadata(&self) -> &'a RowGroupMetaData {
    self.metadata
  }

  fn num_columns(&self) -> usize {
    self.metadata.num_columns()
  }

  // TODO: fix PARQUET-816
  fn get_page_reader<'b>(&'b self, i: usize) -> Result<Box<PageReader + 'b>> {
    let col = self.metadata.column(i);
    let mut col_start = col.data_page_offset();
    if col.has_dictionary_page() {
      col_start = col.dictionary_page_offset().unwrap();
    }
    let col_length = col.compressed_size() as u64;
    let f = self.buf.get_ref().try_clone()?;
    let mut buf = BufReader::new(f);
    let _ = buf.seek(SeekFrom::Start(col_start as u64));
    let page_reader = SerializedPageReader::new(
      buf.take(col_length).into_inner(), col.num_values(), col.compression())?;
    Ok(Box::new(page_reader))
  }
}

/// A serialized impl for Parquet page reader
pub struct SerializedPageReader {
  /// The buffer which contains exactly the bytes for the column trunk
  /// to be read by this page reader
  buf: BufReader<File>,

  /// The compression codec for this column chunk. Only set for
  /// non-PLAIN codec.
  decompressor: Option<Box<Codec>>,

  /// The number of values we have seen so far
  seen_num_values: i64,

  /// The number of total values in this column chunk
  total_num_values: i64,
}

impl SerializedPageReader {
  pub fn new(buf: BufReader<File>, total_num_values: i64,
             compression: Compression) -> Result<Self> {
    let decompressor = create_codec(compression)?;
    let result =
      Self { buf: buf, total_num_values: total_num_values, seen_num_values: 0,
             decompressor: decompressor };
    Ok(result)
  }

  fn read_page_header(&mut self) -> Result<PageHeader> {
    let transport = Rc::new(RefCell::new(
      Box::new(TMemoryBuffer::new(&mut self.buf)) as Box<TTransport>));
    let mut prot = TCompactInputProtocol::new(transport);
    let page_header = PageHeader::read_from_in_protocol(&mut prot)?;
    Ok(page_header)
  }
}

impl PageReader for SerializedPageReader {
  fn get_next_page(&mut self) -> Result<Option<Page>> {
    while self.seen_num_values < self.total_num_values {
      let page_header = self.read_page_header()?;
      let compressed_len = page_header.compressed_page_size as usize;
      let uncompressed_len = page_header.uncompressed_page_size as usize;
      let mut buffer = ByteBuffer::new(compressed_len);
      self.buf.read_exact(buffer.mut_data())?;

      // TODO: page header could be huge because of statistics. We should
      // set a maximum page header size and abort if that is exceeded.
      if let Some(decompressor) = self.decompressor.as_mut() {
        let mut decompressed_buffer = vec!();
        let decompressed_size = decompressor.decompress(buffer.data(), &mut decompressed_buffer)?;
        if decompressed_size != uncompressed_len {
          return Err(decode_err!("Actual decompressed size doesn't \
            match the expected one ({} vs {})", decompressed_size, uncompressed_len));
        }
        buffer.set_data(decompressed_buffer);
      }

      // TODO: process statistics
      let result = match page_header.type_ {
        PageType::DICTIONARY_PAGE => {
          assert!(page_header.dictionary_page_header.is_some());
          let dict_header = page_header.dictionary_page_header.as_ref().unwrap();
          let is_sorted = match dict_header.is_sorted {
            None => false,
            Some(b) => b
          };
          self.seen_num_values += dict_header.num_values as i64;
          Page::DictionaryPage {
            buf: Box::new(buffer), num_values: dict_header.num_values as u32,
            encoding: Encoding::from(dict_header.encoding), is_sorted: is_sorted
          }
        },
        PageType::DATA_PAGE => {
          assert!(page_header.data_page_header.is_some());
          let header = page_header.data_page_header.as_ref().unwrap();
          self.seen_num_values += header.num_values as i64;
          Page::DataPage {
            buf: Box::new(buffer), num_values: header.num_values as u32,
            encoding: Encoding::from(header.encoding),
            def_level_encoding: Encoding::from(header.definition_level_encoding),
            rep_level_encoding: Encoding::from(header.repetition_level_encoding)
          }
        },
        PageType::DATA_PAGE_V2 => {
          assert!(page_header.data_page_header_v2.is_some());
          let header = page_header.data_page_header_v2.as_ref().unwrap();
          let is_compressed = match header.is_compressed {
            Some(b) => b,
            None => true
          };
          self.seen_num_values += header.num_values as i64;
          Page::DataPageV2 {
            buf: Box::new(buffer), num_values: header.num_values as u32,
            encoding: Encoding::from(header.encoding),
            num_nulls: header.num_nulls as u32, num_rows: header.num_rows as u32,
            def_levels_byte_len: header.definition_levels_byte_length as u32,
            rep_levels_byte_len: header.repetition_levels_byte_length as u32,
            is_compressed: is_compressed
          }
        },
        _ => {
          // For unknown page type (e.g., INDEX_PAGE), skip and read next.
          continue;
        }
      };
      return Ok(Some(result))
    }

    // We are at the end of this column chunk and no more page left. Return None.
    Ok(None)
  }
}

#[cfg(test)]
mod tests {
  use super::*;
  use std::fs;
  use std::env;

  #[test]
  fn test_file_reader() {
    let test_file = get_test_file("alltypes_plain.parquet");
    println!("test file = {:?}", test_file);
    let reader_result = SerializedFileReader::new(test_file);
    assert!(reader_result.is_ok());
    let reader = reader_result.unwrap();

    // Test contents in Parquet metadata
    let metadata: &ParquetMetaData = reader.metadata();
    assert_eq!(metadata.num_row_groups(), 1);

    // Test contents in file metadata
    let file_metadata: &FileMetaData = metadata.file_metadata();
    assert!(file_metadata.created_by().is_some());
    assert_eq!(file_metadata.created_by().as_ref().unwrap(),
      "impala version 1.3.0-INTERNAL (build 8a48ddb1eff84592b3fc06bc6f51ec120e1fffc9)");
    assert_eq!(file_metadata.num_rows(), 8);
    assert_eq!(file_metadata.version(), 1);

    // Test contents in row group metadata
    let row_group_metadata: &RowGroupMetaData = metadata.row_group(0);
    assert_eq!(row_group_metadata.num_columns(), 11);
    assert_eq!(row_group_metadata.num_rows(), 8);
    assert_eq!(row_group_metadata.total_byte_size(), 671);

    // Test row group reader
    let row_group_reader_result = reader.get_row_group(0);
    assert!(row_group_reader_result.is_ok());
    let row_group_reader: Box<RowGroupReader> = row_group_reader_result.unwrap();

    // Test page readers
    // TODO: test for every column
    let page_reader_0_result = row_group_reader.get_page_reader(0);
    assert!(page_reader_0_result.is_ok());
    let mut page_reader_0: Box<PageReader> = page_reader_0_result.unwrap();
    let mut page_count = 0;
    while let Ok(Some(page)) = page_reader_0.get_next_page() {
      let is_dict_type = match page {
        Page::DictionaryPage{ buf, num_values, encoding, is_sorted } => {
          assert_eq!(buf.size(), 32);
          assert_eq!(num_values, 8);
          assert_eq!(encoding, Encoding::PLAIN_DICTIONARY);
          assert_eq!(is_sorted, false);
          true
        },
        _ => false
      };
      assert!(is_dict_type);
      page_count += 1;
    }
    assert_eq!(page_count, 1);
  }

  fn get_test_file<'a>(file_name: &str) -> fs::File {
    let mut path_buf = env::current_dir().unwrap();
    path_buf.push("data");
    path_buf.push(file_name);
    let file = File::open(path_buf.as_path());
    assert!(file.is_ok());
    file.unwrap()
  }
}
<|endoftext|>


<commit_before>use std::marker::PhantomData;
use gl;
use gl::types::*;
use context::Context;
use renderbuffer::Renderbuffer;
use texture::{Texture, TextureType, ImageTargetType};
use types::{BufferBits, GLError};

pub struct Framebuffer {
    gl_id: GLuint
}

impl Framebuffer {
    pub unsafe fn from_gl(id: GLuint) -> Self {
        Framebuffer { gl_id: id }
    }

    pub fn gl_id(&self) -> GLuint {
        self.gl_id
    }
}

impl Drop for Framebuffer {
    fn drop(&mut self) {
        unsafe {
            gl::DeleteFramebuffers(1, &self.gl_id as *const GLuint);
        }
    }
}

impl Context {
    pub unsafe fn gen_framebuffer(&self) -> Framebuffer {
        let mut id : GLuint = 0;

        gl::GenFramebuffers(1, &mut id as *mut GLuint);
        dbg_gl_sanity_check! {
            GLError::InvalidValue => "`n` is negative",
            _ => "Unknown error"
        }

        Framebuffer {
            gl_id: id
        }
    }
}

gl_enum! {
    pub gl_enum FramebufferAttachment {
        ColorAttachment0 as COLOR_ATTACHMENT0 = gl::COLOR_ATTACHMENT0,
        DepthAttachment as DEPTH_ATTACHMENT = gl::DEPTH_ATTACHMENT,
        StencilAttachment as STENCIL_ATTACHMENT = gl::STENCIL_ATTACHMENT
    }
}

pub struct FramebufferBinding<'a> {
    phantom: PhantomData<&'a mut Framebuffer>
}

impl<'a> FramebufferBinding<'a> {
    fn target(&self) -> GLenum {
        gl::FRAMEBUFFER
    }

    pub fn renderbuffer(&mut self,
                        attachment: FramebufferAttachment,
                        renderbuffer: &mut Renderbuffer)
    {
        unsafe {
            gl::FramebufferRenderbuffer(self.target(),
                                        attachment.gl_enum(),
                                        gl::RENDERBUFFER,
                                        renderbuffer.gl_id());
            dbg_gl_sanity_check! {
                GLError::InvalidEnum => "`target` is not `GL_FRAMEBUFFER`, `attachment` is not a valid attachment point, or `renderbuffer` is not `GL_RENDERBUFFER` and `renderbuffer` is not 0",
                GLError::InvalidOperation => "Framebuffer 0 is bound, or `renderbuffer` is neither 0 nor the name of an existing renderbuffer object",
                _ => "Unknown error"
            }
        }
    }

    pub fn texture_2d<T: TextureType>(&mut self,
                                      attachment: FramebufferAttachment,
                                      tex_target: T::ImageTargetType,
                                      texture: &mut Texture<T>,
                                      level: i32)
    {
        debug_assert!(level == 0);

        unsafe {
            gl::FramebufferTexture2D(self.target(),
                                     attachment.gl_enum(),
                                     tex_target.gl_enum(),
                                     texture.gl_id(),
                                     level as GLint);
            dbg_gl_sanity_check! {
                GLError::InvalidEnum => "`target` is not `GL_FRAMEBUFFER`, `attachment` is not an accepted attachment point, or `textarget` is not an accepted texture target and texture is not 0",
                GLError::InvalidValue => "`level` is not 0 and `texture` is not 0",
                GLError::InvalidOperation => "Framebuffer object 0 is bound, `texture` is neither 0 nor the name of an existing texture object, or `textarget` is not a valid target for `texture`",
                _ => "Unknown error"
            }
        }
    }

    pub fn clear(&mut self, buffers: BufferBits) {
        unsafe {
            gl::Clear(buffers.bits());
            dbg_gl_sanity_check! {
                GLError::InvalidValue => "`mask` includes a bit other than an allowed value",
                _ => "Unkown error"
            }
        }
    }
}

pub struct FramebufferBinder;
impl FramebufferBinder {
    pub unsafe fn current_binding<'a>(&'a mut self) -> FramebufferBinding<'a> {
        FramebufferBinding { phantom: PhantomData }
    }

    pub fn bind<'a>(&'a mut self, fbo: &mut Framebuffer)
        -> FramebufferBinding<'a>
    {
        let binding = FramebufferBinding { phantom: PhantomData };
        unsafe {
            gl::BindFramebuffer(binding.target(), fbo.gl_id());
            dbg_gl_sanity_check! {
                GLError::InvalidEnum => "`target` is not `GL_FRAMEBUFFER`",
                _ => "Unknown error"
            }
        }
        binding
    }
}
<commit_msg>Use `RenderbufferTarget` variant within `FramebufferBinding.renderbuffer()`<commit_after>use std::marker::PhantomData;
use gl;
use gl::types::*;
use context::Context;
use renderbuffer::{Renderbuffer, RenderbufferTarget};
use texture::{Texture, TextureType, ImageTargetType};
use types::{BufferBits, GLError};

pub struct Framebuffer {
    gl_id: GLuint
}

impl Framebuffer {
    pub unsafe fn from_gl(id: GLuint) -> Self {
        Framebuffer { gl_id: id }
    }

    pub fn gl_id(&self) -> GLuint {
        self.gl_id
    }
}

impl Drop for Framebuffer {
    fn drop(&mut self) {
        unsafe {
            gl::DeleteFramebuffers(1, &self.gl_id as *const GLuint);
        }
    }
}

impl Context {
    pub unsafe fn gen_framebuffer(&self) -> Framebuffer {
        let mut id : GLuint = 0;

        gl::GenFramebuffers(1, &mut id as *mut GLuint);
        dbg_gl_sanity_check! {
            GLError::InvalidValue => "`n` is negative",
            _ => "Unknown error"
        }

        Framebuffer {
            gl_id: id
        }
    }
}

gl_enum! {
    pub gl_enum FramebufferAttachment {
        ColorAttachment0 as COLOR_ATTACHMENT0 = gl::COLOR_ATTACHMENT0,
        DepthAttachment as DEPTH_ATTACHMENT = gl::DEPTH_ATTACHMENT,
        StencilAttachment as STENCIL_ATTACHMENT = gl::STENCIL_ATTACHMENT
    }
}

pub struct FramebufferBinding<'a> {
    phantom: PhantomData<&'a mut Framebuffer>
}

impl<'a> FramebufferBinding<'a> {
    fn target(&self) -> GLenum {
        gl::FRAMEBUFFER
    }

    pub fn renderbuffer(&mut self,
                        attachment: FramebufferAttachment,
                        renderbuffer: &mut Renderbuffer)
    {
        let renderbuffer_target = RenderbufferTarget::Renderbuffer;
        unsafe {
            gl::FramebufferRenderbuffer(self.target(),
                                        attachment.gl_enum(),
                                        renderbuffer_target.gl_enum(),
                                        renderbuffer.gl_id());
            dbg_gl_sanity_check! {
                GLError::InvalidEnum => "`target` is not `GL_FRAMEBUFFER`, `attachment` is not a valid attachment point, or `renderbuffer` is not `GL_RENDERBUFFER` and `renderbuffer` is not 0",
                GLError::InvalidOperation => "Framebuffer 0 is bound, or `renderbuffer` is neither 0 nor the name of an existing renderbuffer object",
                _ => "Unknown error"
            }
        }
    }

    pub fn texture_2d<T: TextureType>(&mut self,
                                      attachment: FramebufferAttachment,
                                      tex_target: T::ImageTargetType,
                                      texture: &mut Texture<T>,
                                      level: i32)
    {
        debug_assert!(level == 0);

        unsafe {
            gl::FramebufferTexture2D(self.target(),
                                     attachment.gl_enum(),
                                     tex_target.gl_enum(),
                                     texture.gl_id(),
                                     level as GLint);
            dbg_gl_sanity_check! {
                GLError::InvalidEnum => "`target` is not `GL_FRAMEBUFFER`, `attachment` is not an accepted attachment point, or `textarget` is not an accepted texture target and texture is not 0",
                GLError::InvalidValue => "`level` is not 0 and `texture` is not 0",
                GLError::InvalidOperation => "Framebuffer object 0 is bound, `texture` is neither 0 nor the name of an existing texture object, or `textarget` is not a valid target for `texture`",
                _ => "Unknown error"
            }
        }
    }

    pub fn clear(&mut self, buffers: BufferBits) {
        unsafe {
            gl::Clear(buffers.bits());
            dbg_gl_sanity_check! {
                GLError::InvalidValue => "`mask` includes a bit other than an allowed value",
                _ => "Unkown error"
            }
        }
    }
}

pub struct FramebufferBinder;
impl FramebufferBinder {
    pub unsafe fn current_binding<'a>(&'a mut self) -> FramebufferBinding<'a> {
        FramebufferBinding { phantom: PhantomData }
    }

    pub fn bind<'a>(&'a mut self, fbo: &mut Framebuffer)
        -> FramebufferBinding<'a>
    {
        let binding = FramebufferBinding { phantom: PhantomData };
        unsafe {
            gl::BindFramebuffer(binding.target(), fbo.gl_id());
            dbg_gl_sanity_check! {
                GLError::InvalidEnum => "`target` is not `GL_FRAMEBUFFER`",
                _ => "Unknown error"
            }
        }
        binding
    }
}
<|endoftext|>


<commit_before>use std::ptr::null;
use std::str::raw;
use libc::{c_int, c_char, c_double};
use sync::mutex::{StaticMutex, MUTEX_INIT};

#[link(name="gdal")]
extern {
    fn OGRRegisterAll();
    fn OGROpen(pszName: *c_char, bUpdate: c_int, pahDriverList: *()) -> *();
    fn OGR_DS_GetLayerCount(hDS: *()) -> c_int;
    fn OGR_DS_Destroy(hDataSource: *());
    fn OGR_DS_GetLayer(hDS: *(), iLayer: c_int) -> *();
    fn OGR_L_GetNextFeature(hLayer: *()) -> *();
    fn OGR_F_GetFieldIndex(hFeat: *(), pszName: *c_char) -> c_int;
    fn OGR_F_GetFieldAsString(hFeat: *(), iField: c_int) -> *c_char;
    fn OGR_F_GetFieldAsDouble(hFeat: *(), iField: c_int) -> c_double;
    fn OGR_F_GetGeometryRef(hFeat: *()) -> *();
    fn OGR_F_Destroy(hFeat: *());
    fn OGR_G_ExportToWkt(hGeom: *(), ppszSrcText: **c_char) -> c_int;
    fn OGRFree(ptr: *());
}

static mut LOCK: StaticMutex = MUTEX_INIT;
static mut registered_drivers: bool = false;

fn register_drivers() {
    unsafe {
        let _g = LOCK.lock();
        if ! registered_drivers {
            OGRRegisterAll();
            registered_drivers = true;
        }
    }
}

pub struct VectorDataset {
    c_dataset: *(),
}

impl VectorDataset {
    pub fn get_layer_count(&self) -> int {
        return unsafe { OGR_DS_GetLayerCount(self.c_dataset) } as int;
    }

    pub fn get_layer<'a>(&'a self, idx: int) -> Option<Layer<'a>> {
        let c_layer = unsafe { OGR_DS_GetLayer(self.c_dataset, idx as c_int) };
        return match c_layer.is_null() {
            true  => None,
            false => Some(Layer{vector_dataset: self, c_layer: c_layer}),
        };
    }
}

impl Drop for VectorDataset {
    fn drop(&mut self) {
        unsafe { OGR_DS_Destroy(self.c_dataset); }
    }
}

pub struct Layer<'a> {
    vector_dataset: &'a VectorDataset,
    c_layer: *(),
}

impl<'a> Layer<'a> {
    pub fn features<'a>(&'a self) -> FeatureIterator<'a> {
        return FeatureIterator{layer: self};
    }
}

pub struct FeatureIterator<'a> {
    layer: &'a Layer<'a>,
}

impl<'a> Iterator<Feature<'a>> for FeatureIterator<'a> {
    #[inline]
    fn next(&mut self) -> Option<Feature<'a>> {
        let c_feature = unsafe { OGR_L_GetNextFeature(self.layer.c_layer) };
        return match c_feature.is_null() {
            true  => None,
            false => Some(Feature{layer: self.layer, c_feature: c_feature}),
        };
    }
}

pub struct Feature<'a> {
    layer: &'a Layer<'a>,
    c_feature: *(),
}

impl<'a> Feature<'a> {
    pub fn get_string_field(&self, name: String) -> String {
        return name.with_c_str(|c_name| {
            unsafe {
                let field_id = OGR_F_GetFieldIndex(self.c_feature, c_name);
                let c_value = OGR_F_GetFieldAsString(self.c_feature, field_id);
                return raw::from_c_str(c_value);
            }
        });
    }

    pub fn get_f64_field(&self, name: String) -> f64 {
        return name.with_c_str(|c_name| {
            unsafe {
                let field_id = OGR_F_GetFieldIndex(self.c_feature, c_name);
                let c_value = OGR_F_GetFieldAsDouble(self.c_feature, field_id);
                return c_value as f64;
            }
        });
    }

    pub fn get_wkt(&self) -> String {
        unsafe {
            let c_geom = OGR_F_GetGeometryRef(self.c_feature);
            let c_wkt: *c_char = null();
            OGR_G_ExportToWkt(c_geom, &c_wkt);
            let wkt = raw::from_c_str(c_wkt);
            OGRFree(c_wkt as *());
            return wkt;
        }
    }
}

#[unsafe_destructor]
impl<'a> Drop for Feature<'a> {
    fn drop(&mut self) {
        unsafe { OGR_F_Destroy(self.c_feature); }
    }
}

pub fn open(path: &Path) -> Option<VectorDataset> {
    register_drivers();
    let filename = path.as_str().unwrap();
    let c_dataset = filename.with_c_str(|c_filename| {
        return unsafe { OGROpen(c_filename, 0, null()) };
    });
    return match c_dataset.is_null() {
        true  => None,
        false => Some(VectorDataset{c_dataset: c_dataset}),
    };
}

#[cfg(test)]
mod test {
    use std::os::getenv;
    use std::path::Path;
    use super::{Feature, open};

    fn fixture_path(name: &str) -> Path {
        let envvar = "RUST_GDAL_TEST_FIXTURES";
        let fixtures = match getenv(envvar) {
            Some(p) => Path::new(p),
            None => fail!("Environment variable {} not set", envvar)
        };
        let rv = fixtures.join(name);
        return rv;
    }

    fn assert_almost_eq(a: f64, b: f64) {
        let f: f64 = a / b;
        assert!(f < 1.00001);
        assert!(f > 0.99999);
    }

    #[test]
    fn test_layer_count() {
        let ds = open(&fixture_path("roads.geojson")).unwrap();
        assert_eq!(ds.get_layer_count(), 1);
    }

    #[test]
    fn test_iterate_features() {
        let ds = open(&fixture_path("roads.geojson")).unwrap();
        let layer = ds.get_layer(0).unwrap();
        let features: Vec<Feature> = layer.features().collect();
        assert_eq!(features.len(), 21);
    }

    #[test]
    fn test_get_string_field() {
        let ds = open(&fixture_path("roads.geojson")).unwrap();
        let layer = ds.get_layer(0).unwrap();
        let feature = layer.features().next().unwrap();
        assert_eq!(feature.get_string_field("highway".to_string()),
                   "footway".to_string());
        assert_eq!(
            layer.features()
                 .count(|f|
                    f.get_string_field("highway".to_string()) ==
                    "residential".to_string()),
            2);
    }

    #[test]
    fn test_get_float_field() {
        let ds = open(&fixture_path("roads.geojson")).unwrap();
        let layer = ds.get_layer(0).unwrap();
        let feature = layer.features().next().unwrap();
        assert_almost_eq(feature.get_f64_field("sort_key".to_string()), -9.0);
    }

    #[test]
    fn test_get_wkt() {
        let ds = open(&fixture_path("roads.geojson")).unwrap();
        let layer = ds.get_layer(0).unwrap();
        let feature = layer.features().next().unwrap();
        let wkt = feature.get_wkt();
        assert_eq!(wkt, "LINESTRING (26.1019276 44.4302748,26.1019382 44.4303191,26.1020002 44.4304202)".to_string());
    }
}
<commit_msg>a single function that gets field values<commit_after>use std::ptr::null;
use std::str::raw;
use libc::{c_int, c_char, c_double};
use sync::mutex::{StaticMutex, MUTEX_INIT};

#[link(name="gdal")]
extern {
    fn OGRRegisterAll();
    fn OGROpen(pszName: *c_char, bUpdate: c_int, pahDriverList: *()) -> *();
    fn OGR_DS_GetLayerCount(hDS: *()) -> c_int;
    fn OGR_DS_Destroy(hDataSource: *());
    fn OGR_DS_GetLayer(hDS: *(), iLayer: c_int) -> *();
    fn OGR_L_GetNextFeature(hLayer: *()) -> *();
    fn OGR_F_GetFieldIndex(hFeat: *(), pszName: *c_char) -> c_int;
    fn OGR_F_GetFieldDefnRef(hFeat: *(), i: c_int) -> *();
    fn OGR_F_GetFieldAsString(hFeat: *(), iField: c_int) -> *c_char;
    fn OGR_F_GetFieldAsDouble(hFeat: *(), iField: c_int) -> c_double;
    fn OGR_F_GetGeometryRef(hFeat: *()) -> *();
    fn OGR_F_Destroy(hFeat: *());
    fn OGR_G_ExportToWkt(hGeom: *(), ppszSrcText: **c_char) -> c_int;
    fn OGR_Fld_GetType(hDefn: *()) -> c_int;
    fn OGRFree(ptr: *());
}

static OFTInteger:        c_int = 0;
static OFTIntegerList:    c_int = 1;
static OFTReal:           c_int = 2;
static OFTRealList:       c_int = 3;
static OFTString:         c_int = 4;
static OFTStringList:     c_int = 5;
static OFTWideString:     c_int = 6;
static OFTWideStringList: c_int = 7;
static OFTBinary:         c_int = 8;
static OFTDate:           c_int = 9;
static OFTTime:           c_int = 10;
static OFTDateTime:       c_int = 11;

static mut LOCK: StaticMutex = MUTEX_INIT;
static mut registered_drivers: bool = false;

pub enum FieldValue {
    StringValue(String),
    F64Value(f64),
}

fn register_drivers() {
    unsafe {
        let _g = LOCK.lock();
        if ! registered_drivers {
            OGRRegisterAll();
            registered_drivers = true;
        }
    }
}

pub struct VectorDataset {
    c_dataset: *(),
}

impl VectorDataset {
    pub fn get_layer_count(&self) -> int {
        return unsafe { OGR_DS_GetLayerCount(self.c_dataset) } as int;
    }

    pub fn get_layer<'a>(&'a self, idx: int) -> Option<Layer<'a>> {
        let c_layer = unsafe { OGR_DS_GetLayer(self.c_dataset, idx as c_int) };
        return match c_layer.is_null() {
            true  => None,
            false => Some(Layer{vector_dataset: self, c_layer: c_layer}),
        };
    }
}

impl Drop for VectorDataset {
    fn drop(&mut self) {
        unsafe { OGR_DS_Destroy(self.c_dataset); }
    }
}

pub struct Layer<'a> {
    vector_dataset: &'a VectorDataset,
    c_layer: *(),
}

impl<'a> Layer<'a> {
    pub fn features<'a>(&'a self) -> FeatureIterator<'a> {
        return FeatureIterator{layer: self};
    }
}

pub struct FeatureIterator<'a> {
    layer: &'a Layer<'a>,
}

impl<'a> Iterator<Feature<'a>> for FeatureIterator<'a> {
    #[inline]
    fn next(&mut self) -> Option<Feature<'a>> {
        let c_feature = unsafe { OGR_L_GetNextFeature(self.layer.c_layer) };
        return match c_feature.is_null() {
            true  => None,
            false => Some(Feature{layer: self.layer, c_feature: c_feature}),
        };
    }
}

pub struct Feature<'a> {
    layer: &'a Layer<'a>,
    c_feature: *(),
}

impl<'a> Feature<'a> {
    pub fn get_field(&self, name: String) -> FieldValue {
        return name.with_c_str(|c_name| unsafe {
            let field_id = OGR_F_GetFieldIndex(self.c_feature, c_name);
            let field_defn = OGR_F_GetFieldDefnRef(self.c_feature, field_id);
            let field_type = OGR_Fld_GetType(field_defn);
            return match field_type {
                OFTString => {
                    let rv = OGR_F_GetFieldAsString(self.c_feature, field_id);
                    return StringValue(raw::from_c_str(rv));
                },
                OFTReal => {
                    let rv = OGR_F_GetFieldAsDouble(self.c_feature, field_id);
                    return F64Value(rv as f64);
                },
                _ => fail!("Unknown field type {}", field_type)
            }
        });
    }

    pub fn get_string_field(&self, name: String) -> String {
        return match self.get_field(name) {
            StringValue(rv) => rv,
            _ => fail!("not a string")
        }
    }

    pub fn get_f64_field(&self, name: String) -> f64 {
        return match self.get_field(name) {
            F64Value(rv) => rv,
            _ => fail!("not an f64")
        }
    }

    pub fn get_wkt(&self) -> String {
        unsafe {
            let c_geom = OGR_F_GetGeometryRef(self.c_feature);
            let c_wkt: *c_char = null();
            OGR_G_ExportToWkt(c_geom, &c_wkt);
            let wkt = raw::from_c_str(c_wkt);
            OGRFree(c_wkt as *());
            return wkt;
        }
    }
}

#[unsafe_destructor]
impl<'a> Drop for Feature<'a> {
    fn drop(&mut self) {
        unsafe { OGR_F_Destroy(self.c_feature); }
    }
}

pub fn open(path: &Path) -> Option<VectorDataset> {
    register_drivers();
    let filename = path.as_str().unwrap();
    let c_dataset = filename.with_c_str(|c_filename| {
        return unsafe { OGROpen(c_filename, 0, null()) };
    });
    return match c_dataset.is_null() {
        true  => None,
        false => Some(VectorDataset{c_dataset: c_dataset}),
    };
}

#[cfg(test)]
mod test {
    use std::os::getenv;
    use std::path::Path;
    use super::{Feature, open};

    fn fixture_path(name: &str) -> Path {
        let envvar = "RUST_GDAL_TEST_FIXTURES";
        let fixtures = match getenv(envvar) {
            Some(p) => Path::new(p),
            None => fail!("Environment variable {} not set", envvar)
        };
        let rv = fixtures.join(name);
        return rv;
    }

    fn assert_almost_eq(a: f64, b: f64) {
        let f: f64 = a / b;
        assert!(f < 1.00001);
        assert!(f > 0.99999);
    }

    #[test]
    fn test_layer_count() {
        let ds = open(&fixture_path("roads.geojson")).unwrap();
        assert_eq!(ds.get_layer_count(), 1);
    }

    #[test]
    fn test_iterate_features() {
        let ds = open(&fixture_path("roads.geojson")).unwrap();
        let layer = ds.get_layer(0).unwrap();
        let features: Vec<Feature> = layer.features().collect();
        assert_eq!(features.len(), 21);
    }

    #[test]
    fn test_get_string_field() {
        let ds = open(&fixture_path("roads.geojson")).unwrap();
        let layer = ds.get_layer(0).unwrap();
        let feature = layer.features().next().unwrap();
        assert_eq!(feature.get_string_field("highway".to_string()),
                   "footway".to_string());
        assert_eq!(
            layer.features()
                 .count(|f|
                    f.get_string_field("highway".to_string()) ==
                    "residential".to_string()),
            2);
    }

    #[test]
    fn test_get_float_field() {
        let ds = open(&fixture_path("roads.geojson")).unwrap();
        let layer = ds.get_layer(0).unwrap();
        let feature = layer.features().next().unwrap();
        assert_almost_eq(feature.get_f64_field("sort_key".to_string()), -9.0);
    }

    #[test]
    fn test_get_wkt() {
        let ds = open(&fixture_path("roads.geojson")).unwrap();
        let layer = ds.get_layer(0).unwrap();
        let feature = layer.features().next().unwrap();
        let wkt = feature.get_wkt();
        assert_eq!(wkt, "LINESTRING (26.1019276 44.4302748,26.1019382 44.4303191,26.1020002 44.4304202)".to_string());
    }
}
<|endoftext|>


<commit_before>use std::path::Path;
use std::collections::HashMap;
use graphics::character::{ CharacterCache, Character };
use graphics::types::FontSize;
use glium::Texture2d;
use glium::backend::Facade;
use image::{ Luma, ImageBuffer };
use freetype::{ self, Face };
use ::backend::DrawTexture;

fn load_character<F: Facade>(face: &Face, facade: &F, font_size: FontSize,
                             character: char)
    -> Character<DrawTexture>
{
    face.set_pixel_sizes(0, font_size).unwrap();
    face.load_char(character as usize, freetype::face::DEFAULT).unwrap();
    let glyph = face.glyph().get_glyph().unwrap();
    let bitmap_glyph = glyph.to_bitmap(freetype::render_mode::RenderMode::Normal, None).unwrap();
    let bitmap = bitmap_glyph.bitmap();
    let texture = Texture2d::new(
        facade,
        ImageBuffer::<Luma<u8>, _>::from_raw(
            bitmap.width() as u32, bitmap.rows() as u32,
            bitmap.buffer().iter().map(|&pix| pix).collect::<Vec<_>>()
        ).expect("failed to create glyph texture")
    );
    let glyph_size = glyph.advance();
    Character {
        offset: [
            bitmap_glyph.left() as f64,
            bitmap_glyph.top() as f64
        ],
        size: [
            (glyph_size.x >> 16) as f64,
            (glyph_size.y >> 16) as f64
        ],
        texture: DrawTexture::new(texture),
    }
}

pub struct GlyphCache<F> {
    face: Face<'static>,
    data: HashMap<FontSize, HashMap<char, Character<DrawTexture>>>,
    facade: F,
}

impl<F> GlyphCache<F> {
     /// Constructor for a GlyphCache.
    pub fn new(font: &Path, facade: F) -> Result<Self, freetype::error::Error> {
        let freetype = try!(freetype::Library::init());
        let face = try!(freetype.new_face(font, 0));
        Ok(GlyphCache {
            face: face,
            data: HashMap::new(),
            facade: facade,
        })
    }
}

impl<F: Facade> CharacterCache for GlyphCache<F> {
    type Texture = DrawTexture;

    fn character<'a>(&'a mut self, font_size: FontSize, character: char)
        -> &'a Character<DrawTexture>
    {
        use std::collections::hash_map::Entry::{Vacant, Occupied};
        let size_cache: &'a mut HashMap<char, _> = match self.data.entry(font_size) {
            Vacant(entry) => entry.insert(HashMap::new()),
            Occupied(entry) => entry.into_mut(),
        };
        match size_cache.entry(character) {
            Vacant(entry) => entry.insert(load_character(&self.face, &self.facade, font_size, character)),
            Occupied(entry) => entry.into_mut(),
        }
    }
}
<commit_msg>Fix texture creation<commit_after>use std::path::Path;
use std::collections::HashMap;
use graphics::character::{ CharacterCache, Character };
use graphics::types::FontSize;
use glium::Texture2d;
use glium::backend::Facade;
use image::{ Rgba, ImageBuffer };
use freetype::{ self, Face };
use ::back_end::DrawTexture;

fn load_character<F: Facade>(face: &Face, facade: &F, font_size: FontSize,
                             character: char)
    -> Character<DrawTexture>
{
    face.set_pixel_sizes(0, font_size).unwrap();
    face.load_char(character as usize, freetype::face::DEFAULT).unwrap();
    let glyph = face.glyph().get_glyph().unwrap();
    let bitmap_glyph = glyph.to_bitmap(freetype::render_mode::RenderMode::Normal, None).unwrap();
    let bitmap = bitmap_glyph.bitmap();
    let texture = Texture2d::new(
        facade,
        ImageBuffer::<Rgba<u8>, _>::from_raw(
            bitmap.width() as u32, bitmap.rows() as u32,
            bitmap.buffer().iter()
                .flat_map(|&pix| vec![255, 255, 255, pix].into_iter())
                .collect::<Vec<_>>()
        ).expect("failed to create glyph texture")
    );
    let glyph_size = glyph.advance();
    Character {
        offset: [
            bitmap_glyph.left() as f64,
            bitmap_glyph.top() as f64
        ],
        size: [
            (glyph_size.x >> 16) as f64,
            (glyph_size.y >> 16) as f64
        ],
        texture: DrawTexture::new(texture),
    }
}

pub struct GlyphCache<F> {
    face: Face<'static>,
    data: HashMap<FontSize, HashMap<char, Character<DrawTexture>>>,
    facade: F,
}

impl<F> GlyphCache<F> {
     /// Constructor for a GlyphCache.
    pub fn new(font: &Path, facade: F) -> Result<Self, freetype::error::Error> {
        let freetype = try!(freetype::Library::init());
        let face = try!(freetype.new_face(font, 0));
        Ok(GlyphCache {
            face: face,
            data: HashMap::new(),
            facade: facade,
        })
    }
}

impl<F: Facade> CharacterCache for GlyphCache<F> {
    type Texture = DrawTexture;

    fn character<'a>(&'a mut self, font_size: FontSize, character: char)
        -> &'a Character<DrawTexture>
    {
        use std::collections::hash_map::Entry::{Vacant, Occupied};
        let size_cache: &'a mut HashMap<char, _> = match self.data.entry(font_size) {
            Vacant(entry) => entry.insert(HashMap::new()),
            Occupied(entry) => entry.into_mut(),
        };
        match size_cache.entry(character) {
            Vacant(entry) => entry.insert(load_character(&self.face, &self.facade, font_size, character)),
            Occupied(entry) => entry.into_mut(),
        }
    }
}
<|endoftext|>


<commit_before>// Copyright 2014 Google Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

//! Standardized isolation of logic as a `process` wrapper around a `thread`.
//!
//! Basic wrapper around threads to provide some standard API. All long-living Hat threads run
//! as a `process`. The main difference between a normal Rust `thread` and a Hat `process` is
//! currently that the `process` has a bounded input-channel and a standard implementation of a
//! synchronous `send_reply()`.

/// A long-living `thread` is promoted to a standard `process`.
///
/// To create a new `process`, simply define a `Msg` type, a `Reply` type and a state struct
/// that implements the `MsgHandler` trait. You can then create a new process with:
///
/// ```rust,ignore
/// enum Msg {Ping, Foo}
/// enum Reply {Pong, Bar}
///
/// struct MyHandler {
///   fn handle(&mut self, msg: Msg, reply: |Reply|) {
///     match Msg {
///       Ping => return reply(Pong),
///       Foo => return reply(Bar),
///     }
///   }
/// }
///
/// fn main() {
///   let p = Process::new(MyHandler::new());
///   let pong = p.send_reply(Ping);
/// }
/// ```
///
/// The handler must always call `reply()` to release any call to `send_reply()`. It is allowed to
/// call `reply()` "early" - before all work has been done - but calling `reply()` is usually the
/// last thing a handler does. The idiom `return reply(...)` is a simple way of guaranteeing this.
///
/// A handler is allowed to call `reply()` **exactly** once. Not calling it or calling it multiple
/// times will likely cause runtime panicure when using `send_reply()`.

use std::fmt;
use std::thread;
use std::sync::{mpsc, Arc, Mutex};

pub struct Process<Msg, Reply, E> {
    sender: mpsc::SyncSender<(Msg, mpsc::Sender<Result<Reply, E>>)>,
    poison_after: Option<i64>,
    poisoned: Arc<Mutex<bool>>,
}

/// When cloning a `process` we clone the input-channel, allowing multiple threads to share the same
/// `process`.
impl<Msg: Send, Reply: Send, E> Clone for Process<Msg, Reply, E> {
    fn clone(&self) -> Process<Msg, Reply, E> {
        Process {
            sender: self.sender.clone(),
            poison_after: None,
            poisoned: self.poisoned.clone(),
        }
    }
}

pub trait MsgHandler<Msg, Reply> {
    type Err;

    fn handle(&mut self,
              msg: Msg,
              callback: Box<Fn(Result<Reply, Self::Err>)>)
              -> Result<(), Self::Err>;
}

impl<Msg: 'static + Send, Reply: 'static + Send, E> Process<Msg, Reply, E> {
    /// Create and start a new process using `handler`.
    pub fn new<H, F>(handler_proc: F) -> Result<Process<Msg, Reply, E>, E>
        where H: 'static + MsgHandler<Msg, Reply, Err = E> + Send,
              E: 'static + From<mpsc::RecvError> + fmt::Debug + Send,
              F: FnOnce() -> Result<H, E>
    {
        Process::new_with_poison(handler_proc, None)
    }

    pub fn new_with_poison<H, F>(handler_proc: F,
                                 poison_after: Option<i64>)
                                 -> Result<Process<Msg, Reply, E>, E>
        where H: 'static + MsgHandler<Msg, Reply, Err = E> + Send,
              E: 'static + From<mpsc::RecvError> + fmt::Debug + Send,
              F: FnOnce() -> Result<H, E>
    {
        let (sender, receiver) = mpsc::sync_channel(10);
        let p = Process {
            sender: sender,
            poison_after: poison_after,
            poisoned: Arc::new(Mutex::new(false)),
        };

        try!(p.start(receiver, handler_proc));

        Ok(p)
    }

    fn start<H, F>(&self,
                   receiver: mpsc::Receiver<(Msg, mpsc::Sender<Result<Reply, E>>)>,
                   handler_proc: F)
                   -> Result<(), E>
        where H: 'static + MsgHandler<Msg, Reply, Err = E> + Send,
              E: 'static + From<mpsc::RecvError> + fmt::Debug + Send,
              F: FnOnce() -> Result<H, E>
    {
        let poison_after = self.poison_after;
        let mut my_handler = try!(handler_proc());

        let poisoned = self.poisoned.clone();

        thread::spawn(move || {
            // fork handler
            let mut handle_msg = |msg, rep: mpsc::Sender<Result<Reply, E>>| {
                let did_reply = Arc::new(Mutex::new(false));
                let local_did_reply = did_reply.clone();
                let local_rep = rep.clone();
                let ret = my_handler.handle(msg,
                                            Box::new(move |r| {
                                                *local_did_reply.lock().unwrap() = true;
                                                local_rep.send(r)
                                                    .expect("Message reply ignored");
                                            }));
                match ret {
                    Ok(()) => assert_eq!(*did_reply.lock().unwrap(), true),
                    Err(e) => {
                        if *did_reply.lock().expect("Message reply ignored") {
                            return Err(format!("Encountered unrecoverable error: {:?}", e));
                        } else {
                            rep.send(Err(e)).expect("Message reply ignored");
                        }
                    }
                }

                Ok(())
            };
            let mut do_loop = || {
                match poison_after {
                    Some(msg_count) => {
                        for _ in 0..msg_count {
                            if let Ok((msg, rep)) = receiver.recv() {
                                try!(handle_msg(msg, rep))
                            } else {
                                return Ok(());  // Clean shutdown.
                            }
                        }
                        return Err("Process has reached message limit".to_string());
                    }
                    None => {
                        while let Ok((msg, rep)) = receiver.recv() {
                            try!(handle_msg(msg, rep))
                        }
                    }
                };
                Ok(())
            };
            while let Err(e) = do_loop() {
                *poisoned.lock().unwrap() = true;  // This process is in a polluted state.
                warn!("Poisoned process: {:?}", e);
            }
        });

        Ok(())
    }

    /// Synchronous send.
    ///
    /// Will always wait for a reply from the receiving `process`.
    pub fn send_reply(&self, msg: Msg) -> Result<Reply, E>
        where E: From<String>
    {
        let (sender, receiver) = mpsc::channel();

        if *self.poisoned.lock().unwrap() {
            return Err(From::from("Refusing to forward message to poisoned process".to_string()));
        }

        if let Err(e) = self.sender.send((msg, sender)) {
            return Err(From::from("Could not send message: process is dead".to_string()));
        }

        match receiver.recv() {
            Err(e) => Err(From::from("Could not read reply".to_string())),
            Ok(reply) => reply,
        }
    }
}
<commit_msg>Include channel error in message.<commit_after>// Copyright 2014 Google Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

//! Standardized isolation of logic as a `process` wrapper around a `thread`.
//!
//! Basic wrapper around threads to provide some standard API. All long-living Hat threads run
//! as a `process`. The main difference between a normal Rust `thread` and a Hat `process` is
//! currently that the `process` has a bounded input-channel and a standard implementation of a
//! synchronous `send_reply()`.

/// A long-living `thread` is promoted to a standard `process`.
///
/// To create a new `process`, simply define a `Msg` type, a `Reply` type and a state struct
/// that implements the `MsgHandler` trait. You can then create a new process with:
///
/// ```rust,ignore
/// enum Msg {Ping, Foo}
/// enum Reply {Pong, Bar}
///
/// struct MyHandler {
///   fn handle(&mut self, msg: Msg, reply: |Reply|) {
///     match Msg {
///       Ping => return reply(Pong),
///       Foo => return reply(Bar),
///     }
///   }
/// }
///
/// fn main() {
///   let p = Process::new(MyHandler::new());
///   let pong = p.send_reply(Ping);
/// }
/// ```
///
/// The handler must always call `reply()` to release any call to `send_reply()`. It is allowed to
/// call `reply()` "early" - before all work has been done - but calling `reply()` is usually the
/// last thing a handler does. The idiom `return reply(...)` is a simple way of guaranteeing this.
///
/// A handler is allowed to call `reply()` **exactly** once. Not calling it or calling it multiple
/// times will likely cause runtime panicure when using `send_reply()`.

use std::fmt;
use std::thread;
use std::sync::{mpsc, Arc, Mutex};

pub struct Process<Msg, Reply, E> {
    sender: mpsc::SyncSender<(Msg, mpsc::Sender<Result<Reply, E>>)>,
    poison_after: Option<i64>,
    poisoned: Arc<Mutex<bool>>,
}

/// When cloning a `process` we clone the input-channel, allowing multiple threads to share the same
/// `process`.
impl<Msg: Send, Reply: Send, E> Clone for Process<Msg, Reply, E> {
    fn clone(&self) -> Process<Msg, Reply, E> {
        Process {
            sender: self.sender.clone(),
            poison_after: None,
            poisoned: self.poisoned.clone(),
        }
    }
}

pub trait MsgHandler<Msg, Reply> {
    type Err;

    fn handle(&mut self,
              msg: Msg,
              callback: Box<Fn(Result<Reply, Self::Err>)>)
              -> Result<(), Self::Err>;
}

impl<Msg: 'static + Send, Reply: 'static + Send, E> Process<Msg, Reply, E> {
    /// Create and start a new process using `handler`.
    pub fn new<H, F>(handler_proc: F) -> Result<Process<Msg, Reply, E>, E>
        where H: 'static + MsgHandler<Msg, Reply, Err = E> + Send,
              E: 'static + From<mpsc::RecvError> + fmt::Debug + Send,
              F: FnOnce() -> Result<H, E>
    {
        Process::new_with_poison(handler_proc, None)
    }

    pub fn new_with_poison<H, F>(handler_proc: F,
                                 poison_after: Option<i64>)
                                 -> Result<Process<Msg, Reply, E>, E>
        where H: 'static + MsgHandler<Msg, Reply, Err = E> + Send,
              E: 'static + From<mpsc::RecvError> + fmt::Debug + Send,
              F: FnOnce() -> Result<H, E>
    {
        let (sender, receiver) = mpsc::sync_channel(10);
        let p = Process {
            sender: sender,
            poison_after: poison_after,
            poisoned: Arc::new(Mutex::new(false)),
        };

        try!(p.start(receiver, handler_proc));

        Ok(p)
    }

    fn start<H, F>(&self,
                   receiver: mpsc::Receiver<(Msg, mpsc::Sender<Result<Reply, E>>)>,
                   handler_proc: F)
                   -> Result<(), E>
        where H: 'static + MsgHandler<Msg, Reply, Err = E> + Send,
              E: 'static + From<mpsc::RecvError> + fmt::Debug + Send,
              F: FnOnce() -> Result<H, E>
    {
        let poison_after = self.poison_after;
        let mut my_handler = try!(handler_proc());

        let poisoned = self.poisoned.clone();

        thread::spawn(move || {
            // fork handler
            let mut handle_msg = |msg, rep: mpsc::Sender<Result<Reply, E>>| {
                let did_reply = Arc::new(Mutex::new(false));
                let local_did_reply = did_reply.clone();
                let local_rep = rep.clone();
                let ret = my_handler.handle(msg,
                                            Box::new(move |r| {
                                                *local_did_reply.lock().unwrap() = true;
                                                local_rep.send(r)
                                                    .expect("Message reply ignored");
                                            }));
                match ret {
                    Ok(()) => assert_eq!(*did_reply.lock().unwrap(), true),
                    Err(e) => {
                        if *did_reply.lock().expect("Message reply ignored") {
                            return Err(format!("Encountered unrecoverable error: {:?}", e));
                        } else {
                            rep.send(Err(e)).expect("Message reply ignored");
                        }
                    }
                }

                Ok(())
            };
            let mut do_loop = || {
                match poison_after {
                    Some(msg_count) => {
                        for _ in 0..msg_count {
                            if let Ok((msg, rep)) = receiver.recv() {
                                try!(handle_msg(msg, rep))
                            } else {
                                return Ok(());  // Clean shutdown.
                            }
                        }
                        return Err("Process has reached message limit".to_string());
                    }
                    None => {
                        while let Ok((msg, rep)) = receiver.recv() {
                            try!(handle_msg(msg, rep))
                        }
                    }
                };
                Ok(())
            };
            while let Err(e) = do_loop() {
                *poisoned.lock().unwrap() = true;  // This process is in a polluted state.
                warn!("Poisoned process: {:?}", e);
            }
        });

        Ok(())
    }

    /// Synchronous send.
    ///
    /// Will always wait for a reply from the receiving `process`.
    pub fn send_reply(&self, msg: Msg) -> Result<Reply, E>
        where E: From<String>
    {
        let (sender, receiver) = mpsc::channel();

        if *self.poisoned.lock().unwrap() {
            return Err(From::from("Refusing to forward message to poisoned process".to_string()));
        }

        if let Err(e) = self.sender.send((msg, sender)) {
            return Err(From::from(format!("Could not send message: process is dead: {}", e)));
        }

        match receiver.recv() {
            Err(e) => Err(From::from(format!("Could not read reply: {}", e))),
            Ok(reply) => reply,
        }
    }
}
<|endoftext|>


<commit_before>use std::collections::{BTreeMap, HashSet};
use std::hash::Hasher;
use std::iter;
use std::str::CharIndices;
use std::ops;

use itertools::{GroupBy, Itertools};

use util::*;
use super::*;
use Query::*;

/// A basic implementation of an `Index`, the inverted index is a data structure that maps
/// from words to postings.
#[derive(Debug)]
pub struct InvertedIndex {
    // Maps terms to their postings
    index: BTreeMap<String, PostingsMap>,
    // Maps doc ids to their docs
    docs: BTreeMap<String, Document>,
}

impl InvertedIndex {
    pub fn new() -> InvertedIndex {
        InvertedIndex { index: BTreeMap::new(), docs: BTreeMap::new() }
    }

    /// A basic implementation of index, splits the document's content into whitespace-separated
    /// words, and inserts each word-document pair into the map.
    pub fn index(&mut self, doc: Document) {
        let analyzed = analyze_doc(doc.content());
        let previous_version = self.docs.insert(doc.id.clone(), doc.clone());
        if let Some(previous_version) = previous_version {
            let previous_analyzed = analyze_doc(previous_version.content());
            for (ngram, _) in previous_analyzed {
                let is_empty = {
                    let docs_for_ngram = self.index.get_mut(&ngram).unwrap();
                    docs_for_ngram.remove(&doc.id);
                    docs_for_ngram.is_empty()
                };
                if is_empty {
                    self.index.remove(&ngram);
                }
            }
        }

        for (ngram, highlighted) in analyzed {
            let mut highlights = self.index
                                     .entry(ngram)
                                     .or_insert_with(BTreeMap::new)
                                     .entry(doc.id.clone())
                                     .or_insert_with(Vec::new);
            let coalesce_idx = highlights.binary_search(&highlighted).err().unwrap();
            highlights.coalesce(coalesce_idx, highlighted);
        }
    }

    fn postings(&self, query: &str) -> PostingsMap {
        let unique_terms: HashSet<_> = query.split_whitespace().map(str::to_lowercase).collect();
        unique_terms.into_iter().flat_map(|word| self.index.get(&word)).merge_postings()
    }

    pub fn query(&self, query: &Query) -> Vec<SearchResult> {
        let postings = self.query_rec(query);
        self.compute_results(postings)
    }

    fn query_rec(&self, query: &Query) -> PostingsMap {
        match *query {
            Match(query) => self.postings(query),
            And(queries) => {
                let postings: Vec<_> = queries.iter().map(|q| self.query_rec(q)).collect();
                postings.intersect_postings()
            }
            Or(queries) => queries.into_iter().map(|q| self.query_rec(q)).merge_postings(),
        }
    }

    /// A basic search implementation that splits the query's content into whitespace-separated
    /// words, looks up the set of Documents for each word, and then concatenates the sets.
    pub fn search(&self, query: &str) -> Vec<SearchResult> {
        let postings = self.postings(query);
        self.compute_results(postings)
    }

    fn compute_results(&self, postings: PostingsMap) -> Vec<SearchResult> {
        let mut results: Vec<_> = postings.into_iter()
                                          .map(|(doc_id, index_map)| {
                                              SearchResult::new(&self.docs[&doc_id],
                                                                index_map)
                                          })
                                          .collect();
        results.sort_by(|result1, result2| result2.score().partial_cmp(&result1.score()).unwrap());
        results
    }
}

#[allow(unused_attributes)]
#[rustfmt_skip]
fn analyze_doc(doc: &str) ->
    iter::FlatMap<
        iter::Filter<
            GroupBy<bool, CharIndices, fn(&(usize, char)) -> bool>,
            fn(&(bool, Vec<(usize, char)>)) -> bool>,
        iter::Map<ops::Range<usize>, Ngrams>,
        fn((bool, Vec<(usize, char)>)) -> iter::Map<ops::Range<usize>, Ngrams>>
{
    doc.char_indices()
        .group_by(is_whitespace as fn(&(usize, char)) -> bool)
        .filter(not_whitespace as fn(&(bool, Vec<(usize, char)>)) -> bool)
        .flat_map(ngrams)
}

fn ngrams((_, chars): (bool, Vec<(usize, char)>)) -> iter::Map<ops::Range<usize>, Ngrams> {
    (1..chars.len() + 1).map(Ngrams::new(chars))
}

fn not_whitespace(&(is_whitespace, _): &(bool, Vec<(usize, char)>)) -> bool {
    !is_whitespace
}

fn is_whitespace(&(_, c): &(usize, char)) -> bool {
    c.is_whitespace()
}

struct Ngrams {
    chars: Vec<(usize, char)>,
}

impl Ngrams {
    fn new(chars: Vec<(usize, char)>) -> Ngrams {
        Ngrams { chars: chars }
    }
}

impl Fn<(usize,)> for Ngrams {
    extern "rust-call" fn call(&self, (to,): (usize,)) -> (String, (usize, usize)) {
        let word = self.chars[..to].iter().flat_map(|&(_, c)| c.to_lowercase()).collect();
        let start = self.chars[0].0;
        let (last_idx, last_char) = self.chars[to - 1];
        let finish = last_idx + last_char.len_utf8();
        (word, (start, finish))
    }
}

impl FnMut<(usize,)> for Ngrams {
    extern "rust-call" fn call_mut(&mut self, to: (usize,)) -> (String, (usize, usize)) {
        self.call(to)
    }
}

impl FnOnce<(usize,)> for Ngrams {
    type Output = (String, (usize, usize));
    extern "rust-call" fn call_once(self, to: (usize,)) -> (String, (usize, usize)) {
        self.call(to)
    }
}

#[cfg(test)]
mod test {
    use super::super::*;
    use Query::{And, Match, Or};
    use std::collections::BTreeMap;

    #[test]
    fn test_ngrams() {
        let mut index = InvertedIndex::new();
        let doc1 = Document::new("1", "learn to program in rust today");
        let doc2 = Document::new("2", "what did you today do");
        index.index(doc1.clone());
        index.index(doc2.clone());
        let search_results = index.search("to");
        let expected: BTreeMap<_, _> = [(doc1.clone(), vec![(6, 8), (25, 27)]),
                                        (doc2, vec![(13, 15)])]
                                           .iter()
                                           .cloned()
                                           .collect();
        assert_eq!(search_results.len(), expected.len());
        for search_result in &search_results {
            assert_eq!(&search_result.highlights, &expected[search_result.doc])
        }
        assert_eq!("learn to program in rust today",
                   search_results.iter()
                                 .find(|search_result| search_result.doc == &doc1)
                                 .unwrap()
                                 .highlight("", ""));

    }

    #[test]
    fn test_highlight() {
        let mut index = InvertedIndex::new();
        let doc1 = Document::new("2", "Won\u{2019}t this split the ecosystem? Will everyone use?");
        index.index(doc1.clone());
        let expected = "Won\u{2019}t this split the *e*cosystem? Will *e*veryone use?";
        let search_results = index.search("e");
        assert_eq!(1, search_results.len());
        assert_eq!(search_results[0].highlight("*", "*"), expected);
    }

    #[test]
    fn test_unicode() {
        let mut index = InvertedIndex::new();
        let doc = Document::new("0", "嗨, 您好");
        index.index(doc.clone());
        let to_search = "您";
        let search_results = index.search(to_search);
        let &SearchResult { ref doc, ref highlights, .. } = search_results.iter().next().unwrap();
        let (begin, end) = highlights[0];
        assert_eq!(&doc.content()[begin..end], to_search);
    }

    #[test]
    fn test_update_doc() {
        let mut index = InvertedIndex::new();
        let doc = Document::new("0", "abc åäö");
        index.index(doc);
        let doc = Document::new("0", "different");
        index.index(doc);
        let search_results = index.search("å");
        assert!(search_results.is_empty());
        assert_eq!(index.docs.len(), 1);
    }

    #[test]
    fn test_ranking() {
        let mut index = InvertedIndex::new();
        let doc = Document::new("0", "beat");
        index.index(doc.clone());
        let doc2 = Document::new("1", "beast");
        index.index(doc2);
        let search_results = index.search("be");
        assert_eq!(index.docs.len(), 2);
        // "beat" should be first, since it's a closer match
        assert_eq!(search_results[0].doc, &doc);
    }

    #[test]
    fn test_duplicate_term() {
        let mut index = InvertedIndex::new();
        let doc = Document::new("0", "beat");
        index.index(doc.clone());
        let search_results = index.search("be be");
        assert_eq!(search_results.len(), 1);
    }

    #[test]
    fn test_duplicate_term2() {
        let mut index = InvertedIndex::new();
        let doc = Document::new("0", "beat");
        index.index(doc.clone());
        let search_results = index.search("be b");
        assert_eq!(search_results.len(), 1);
        assert_eq!(search_results[0].highlights, vec![(0, 2)]);
    }

    #[test]
    fn test_lowercase_search() {
        let mut index = InvertedIndex::new();
        let doc = Document::new("0", "BeAt");
        index.index(doc.clone());
        let search_results = index.search("bE");
        assert_eq!(search_results.len(), 1);
        assert_eq!(search_results[0].highlights, vec![(0, 2)]);
    }

    #[test]
    fn test_lowercase_index() {
        let mut index = InvertedIndex::new();
        let doc = Document::new("0", "BeAt");
        index.index(doc.clone());
        let search_results = index.search("be");
        assert_eq!(search_results.len(), 1);
        assert_eq!(search_results[0].highlights, vec![(0, 2)]);
    }

    #[test]
    fn test_and() {
        let mut index = InvertedIndex::new();
        let doc1 = Document::new("1", "learn to program in rust today");
        let doc2 = Document::new("2", "what did you today do");
        let doc3 = Document::new("3", "what did you do yesterday");
        index.index(doc1.clone());
        index.index(doc2.clone());
        index.index(doc3.clone());
        let search_results = index.query(&And(&[Match("today"), Match("you")]));
        let expected: BTreeMap<_, _> = [(doc2, vec![(9, 12), (13, 18)])]
                                           .iter()
                                           .cloned()
                                           .collect();
        assert_eq!(search_results.len(), expected.len());
        for search_result in &search_results {
            assert_eq!(&search_result.highlights, &expected[search_result.doc])
        }
    }

    #[test]
    fn test_and_or() {
        let mut index = InvertedIndex::new();
        let doc1 = Document::new("1", "learn to program in rust today");
        let doc2 = Document::new("2", "what did you today do");
        let doc3 = Document::new("3", "what did you do yesterday");
        index.index(doc1.clone());
        index.index(doc2.clone());
        index.index(doc3.clone());
        let search_results = index.query(&Or(&[Match("you"), And(&[Match("today"), Match("you")])]));
        let expected: BTreeMap<_, _> = [(doc2, vec![(9, 12), (13, 18)]), (doc3, vec![(9, 12)])]
                                           .iter()
                                           .cloned()
                                           .collect();
        assert_eq!(search_results.len(), expected.len());
        for search_result in &search_results {
            assert_eq!(&search_result.highlights, &expected[search_result.doc])
        }
    }
}
<commit_msg>Refactor<commit_after>use std::collections::{BTreeMap, HashSet};
use std::hash::Hasher;
use std::iter;
use std::str::CharIndices;
use std::ops;

use itertools::{GroupBy, Itertools};

use util::*;
use super::*;
use Query::*;

/// A basic implementation of an `Index`, the inverted index is a data structure that maps
/// from words to postings.
#[derive(Debug)]
pub struct InvertedIndex {
    // Maps terms to their postings
    index: BTreeMap<String, PostingsMap>,
    // Maps doc ids to their docs
    docs: BTreeMap<String, Document>,
}

impl InvertedIndex {
    pub fn new() -> InvertedIndex {
        InvertedIndex { index: BTreeMap::new(), docs: BTreeMap::new() }
    }

    /// A basic implementation of index, splits the document's content into whitespace-separated
    /// words, and inserts each word-document pair into the map.
    pub fn index(&mut self, doc: Document) {
        let analyzed = analyze_doc(doc.content());
        let previous_version = self.docs.insert(doc.id.clone(), doc.clone());
        if let Some(previous_version) = previous_version {
            let previous_analyzed = analyze_doc(previous_version.content());
            for (ngram, _) in previous_analyzed {
                let is_empty = {
                    let docs_for_ngram = self.index.get_mut(&ngram).unwrap();
                    docs_for_ngram.remove(&doc.id);
                    docs_for_ngram.is_empty()
                };
                if is_empty {
                    self.index.remove(&ngram);
                }
            }
        }

        for (ngram, highlighted) in analyzed {
            let mut highlights = self.index
                                     .entry(ngram)
                                     .or_insert_with(BTreeMap::new)
                                     .entry(doc.id.clone())
                                     .or_insert_with(Vec::new);
            let coalesce_idx = highlights.binary_search(&highlighted).err().unwrap();
            highlights.coalesce(coalesce_idx, highlighted);
        }
    }

    fn postings(&self, query: &str) -> PostingsMap {
        let unique_terms: HashSet<_> = query.split_whitespace().map(str::to_lowercase).collect();
        unique_terms.into_iter().flat_map(|word| self.index.get(&word)).merge_postings()
    }

    pub fn query(&self, query: &Query) -> Vec<SearchResult> {
        let postings = self.query_rec(query);
        self.compute_results(postings)
    }

    fn query_rec(&self, query: &Query) -> PostingsMap {
        match *query {
            Match(query) => self.postings(query),
            And(queries) => {
                let postings: Vec<_> = queries.iter().map(|q| self.query_rec(q)).collect();
                postings.intersect_postings()
            }
            Or(queries) => queries.into_iter().map(|q| self.query_rec(q)).merge_postings(),
        }
    }

    /// A basic search implementation that splits the query's content into whitespace-separated
    /// words, looks up the set of Documents for each word, and then concatenates the sets.
    pub fn search(&self, query: &str) -> Vec<SearchResult> {
        let postings = self.postings(query);
        self.compute_results(postings)
    }

    fn compute_results(&self, postings: PostingsMap) -> Vec<SearchResult> {
        let mut results: Vec<_> = postings.into_iter()
                                          .map(|(doc_id, index_map)| {
                                              SearchResult::new(&self.docs[&doc_id],
                                                                index_map)
                                          })
                                          .collect();
        results.sort_by(|result1, result2| result2.score.partial_cmp(&result1.score).unwrap());
        results
    }
}

#[allow(unused_attributes)]
#[rustfmt_skip]
fn analyze_doc(doc: &str) ->
    iter::FlatMap<
        iter::Filter<
            GroupBy<bool, CharIndices, fn(&(usize, char)) -> bool>,
            fn(&(bool, Vec<(usize, char)>)) -> bool>,
        iter::Map<ops::Range<usize>, Ngrams>,
        fn((bool, Vec<(usize, char)>)) -> iter::Map<ops::Range<usize>, Ngrams>>
{
    doc.char_indices()
        .group_by(is_whitespace as fn(&(usize, char)) -> bool)
        .filter(not_whitespace as fn(&(bool, Vec<(usize, char)>)) -> bool)
        .flat_map(ngrams)
}

fn ngrams((_, chars): (bool, Vec<(usize, char)>)) -> iter::Map<ops::Range<usize>, Ngrams> {
    (1..chars.len() + 1).map(Ngrams::new(chars))
}

fn not_whitespace(&(is_whitespace, _): &(bool, Vec<(usize, char)>)) -> bool {
    !is_whitespace
}

fn is_whitespace(&(_, c): &(usize, char)) -> bool {
    c.is_whitespace()
}

struct Ngrams {
    chars: Vec<(usize, char)>,
}

impl Ngrams {
    fn new(chars: Vec<(usize, char)>) -> Ngrams {
        Ngrams { chars: chars }
    }
}

impl Fn<(usize,)> for Ngrams {
    extern "rust-call" fn call(&self, (to,): (usize,)) -> (String, (usize, usize)) {
        let word = self.chars[..to].iter().flat_map(|&(_, c)| c.to_lowercase()).collect();
        let start = self.chars[0].0;
        let (last_idx, last_char) = self.chars[to - 1];
        let finish = last_idx + last_char.len_utf8();
        (word, (start, finish))
    }
}

impl FnMut<(usize,)> for Ngrams {
    extern "rust-call" fn call_mut(&mut self, to: (usize,)) -> (String, (usize, usize)) {
        self.call(to)
    }
}

impl FnOnce<(usize,)> for Ngrams {
    type Output = (String, (usize, usize));
    extern "rust-call" fn call_once(self, to: (usize,)) -> (String, (usize, usize)) {
        self.call(to)
    }
}

#[cfg(test)]
mod test {
    use super::super::*;
    use Query::{And, Match, Or};
    use std::collections::BTreeMap;

    #[test]
    fn test_ngrams() {
        let mut index = InvertedIndex::new();
        let doc1 = Document::new("1", "learn to program in rust today");
        let doc2 = Document::new("2", "what did you today do");
        index.index(doc1.clone());
        index.index(doc2.clone());
        let search_results = index.search("to");
        let expected: BTreeMap<_, _> = [(doc1.clone(), vec![(6, 8), (25, 27)]),
                                        (doc2, vec![(13, 15)])]
                                           .iter()
                                           .cloned()
                                           .collect();
        assert_eq!(search_results.len(), expected.len());
        for search_result in &search_results {
            assert_eq!(&search_result.highlights, &expected[search_result.doc])
        }
        assert_eq!("learn to program in rust today",
                   search_results.iter()
                                 .find(|search_result| search_result.doc == &doc1)
                                 .unwrap()
                                 .highlight("", ""));

    }

    #[test]
    fn test_highlight() {
        let mut index = InvertedIndex::new();
        let doc1 = Document::new("2", "Won\u{2019}t this split the ecosystem? Will everyone use?");
        index.index(doc1.clone());
        let expected = "Won\u{2019}t this split the *e*cosystem? Will *e*veryone use?";
        let search_results = index.search("e");
        assert_eq!(1, search_results.len());
        assert_eq!(search_results[0].highlight("*", "*"), expected);
    }

    #[test]
    fn test_unicode() {
        let mut index = InvertedIndex::new();
        let doc = Document::new("0", "嗨, 您好");
        index.index(doc.clone());
        let to_search = "您";
        let search_results = index.search(to_search);
        let &SearchResult { ref doc, ref highlights, .. } = search_results.iter().next().unwrap();
        let (begin, end) = highlights[0];
        assert_eq!(&doc.content()[begin..end], to_search);
    }

    #[test]
    fn test_update_doc() {
        let mut index = InvertedIndex::new();
        let doc = Document::new("0", "abc åäö");
        index.index(doc);
        let doc = Document::new("0", "different");
        index.index(doc);
        let search_results = index.search("å");
        assert!(search_results.is_empty());
        assert_eq!(index.docs.len(), 1);
    }

    #[test]
    fn test_ranking() {
        let mut index = InvertedIndex::new();
        let doc = Document::new("0", "beat");
        index.index(doc.clone());
        let doc2 = Document::new("1", "beast");
        index.index(doc2);
        let search_results = index.search("be");
        assert_eq!(index.docs.len(), 2);
        // "beat" should be first, since it's a closer match
        assert_eq!(search_results[0].doc, &doc);
    }

    #[test]
    fn test_duplicate_term() {
        let mut index = InvertedIndex::new();
        let doc = Document::new("0", "beat");
        index.index(doc.clone());
        let search_results = index.search("be be");
        assert_eq!(search_results.len(), 1);
    }

    #[test]
    fn test_duplicate_term2() {
        let mut index = InvertedIndex::new();
        let doc = Document::new("0", "beat");
        index.index(doc.clone());
        let search_results = index.search("be b");
        assert_eq!(search_results.len(), 1);
        assert_eq!(search_results[0].highlights, vec![(0, 2)]);
    }

    #[test]
    fn test_lowercase_search() {
        let mut index = InvertedIndex::new();
        let doc = Document::new("0", "BeAt");
        index.index(doc.clone());
        let search_results = index.search("bE");
        assert_eq!(search_results.len(), 1);
        assert_eq!(search_results[0].highlights, vec![(0, 2)]);
    }

    #[test]
    fn test_lowercase_index() {
        let mut index = InvertedIndex::new();
        let doc = Document::new("0", "BeAt");
        index.index(doc.clone());
        let search_results = index.search("be");
        assert_eq!(search_results.len(), 1);
        assert_eq!(search_results[0].highlights, vec![(0, 2)]);
    }

    #[test]
    fn test_and() {
        let mut index = InvertedIndex::new();
        let doc1 = Document::new("1", "learn to program in rust today");
        let doc2 = Document::new("2", "what did you today do");
        let doc3 = Document::new("3", "what did you do yesterday");
        index.index(doc1.clone());
        index.index(doc2.clone());
        index.index(doc3.clone());
        let search_results = index.query(&And(&[Match("today"), Match("you")]));
        let expected: BTreeMap<_, _> = [(doc2, vec![(9, 12), (13, 18)])]
                                           .iter()
                                           .cloned()
                                           .collect();
        assert_eq!(search_results.len(), expected.len());
        for search_result in &search_results {
            assert_eq!(&search_result.highlights, &expected[search_result.doc])
        }
    }

    #[test]
    fn test_and_or() {
        let mut index = InvertedIndex::new();
        let doc1 = Document::new("1", "learn to program in rust today");
        let doc2 = Document::new("2", "what did you today do");
        let doc3 = Document::new("3", "what did you do yesterday");
        index.index(doc1.clone());
        index.index(doc2.clone());
        index.index(doc3.clone());
        let search_results = index.query(&Or(&[Match("you"), And(&[Match("today"), Match("you")])]));
        let expected: BTreeMap<_, _> = [(doc2, vec![(9, 12), (13, 18)]), (doc3, vec![(9, 12)])]
                                           .iter()
                                           .cloned()
                                           .collect();
        assert_eq!(search_results.len(), expected.len());
        for search_result in &search_results {
            assert_eq!(&search_result.highlights, &expected[search_result.doc])
        }
    }
}
<|endoftext|>


<commit_before>use getopts::{getopts,OptGroup,Matches};
use help;
use creator;
use editor;

fn run_command(command: &str, value: &str) {
    match command {
        "new"  => creator::new(value),
        "open" => editor::new(value)
    }
}

#[test]
fn run_command_prints_value() {
  assert_eq!(run_command("value"), false);
}

pub fn init(args: Vec<String>) {
    let program = args[0].clone();
    let opts = help::opts();

    let matches = match getopts(args.tail(), opts) {
        Ok(m) => { m }
        Err(f) => { fail!(f.to_string()) }
    };

    if matches.opt_present("h") {
        help::print_usage(program.as_slice(), opts);
        return;
    }

    let input = if !matches.free.is_empty() && !matches.free.len() == 2 {
fn validate_command(matches: Matches) -> bool {
    let command = if matches.free.len() == 2 {
        matches.free[0].clone()
    } else {
        help::print_usage(program.as_slice(), opts);
        return;
    };

    if !verify_command(input.as_slice()) {
        help::print_usage(program.as_slice(), opts);
        return;
    return match command {
        "new"  => true,
        "open" => true,
        _      => false
    }
}

    run_command(input.as_slice(), matches.free[1].clone().as_slice());
#[test]
fn validate_command_new_returns_true() {
    let matches = get_matches([String::from_str("new muxed")], help::opts());
    assert_eq!(validate_command(matches), true);
}

#[test]
fn validate_command_open_returns_true() {
    let matches = get_matches([String::from_str("open muxed")], help::opts());
    assert_eq!(validate_command(matches), true);
}

#[test]
fn validate_command_value_returns_false() {
    let matches = get_matches([String::from_str("value")], help::opts());
    assert_eq!(validate_command(matches), false);
}

#[test]
#[should_fail]
fn get_matches_returns_failure_with_bad_opts() {
    get_matches([String::from_str("-m")], help::opts());
}

<commit_msg>Move init method to the top of the file.<commit_after>use getopts::{getopts,OptGroup,Matches};
use help;
use creator;
use editor;

pub fn init(args: Vec<String>) {
    let program = args[0].clone();
    let opts = help::opts();
    let matches = get_matches(args.tail(), opts.clone());

fn run_command(command: &str, value: &str) {
    match command {
        "new"  => creator::new(value),
        "open" => editor::new(value)
    if matches.opt_present("h") || !validate_command(matches) {
        help::print_usage(program.as_slice(), opts);
        return;
    }
}

#[test]
fn run_command_prints_value() {
  assert_eq!(run_command("value"), false);
}

        Ok(m) => { m }
        Err(f) => { fail!(f.to_string()) }
    }

fn validate_command(matches: Matches) -> bool {
    let command = if matches.free.len() == 2 {
        matches.free[0].clone()
    };

    return match command {
        "new"  => true,
        "open" => true,
        _      => false
    }
}

#[test]
fn validate_command_new_returns_true() {
    let matches = get_matches([String::from_str("new muxed")], help::opts());
    assert_eq!(validate_command(matches), true);
}

#[test]
fn validate_command_open_returns_true() {
    let matches = get_matches([String::from_str("open muxed")], help::opts());
    assert_eq!(validate_command(matches), true);
}

#[test]
fn validate_command_value_returns_false() {
    let matches = get_matches([String::from_str("value")], help::opts());
    assert_eq!(validate_command(matches), false);
}

#[test]
#[should_fail]
fn get_matches_returns_failure_with_bad_opts() {
    get_matches([String::from_str("-m")], help::opts());
}

<|endoftext|>