<commit_before>use rutorch::*;
use std::ops::{Index, IndexMut};
use std::convert::From;
use std::cmp::max;
use std::hash::{Hash, Hasher};

use storage::*;
use rand;
use {Ixs, RcMut};

#[derive(Hash)]
pub enum TensorKind {
    FloatTensor(Tensor<f32>),
    LongTensor(Tensor<i64>),
}

pub type TensorList<T> = Vec<Tensor<T>>;
pub type TensorKindList = Vec<TensorKind>;
pub type RefTensorList<'a, T> = Vec<&'a mut Tensor<T>>;
pub type RefTensorKindList<'a> = Vec<&'a TensorKind>;
pub type TensorId = i32;

impl PartialEq for TensorKind {
    fn eq(&self, other: &TensorKind) -> bool {
        use self::TensorKind::{FloatTensor, LongTensor};
        match (self, other) {
            (&FloatTensor(ref t1), &FloatTensor(ref t2)) => t1.id == t2.id,
            (&LongTensor(ref t1), &LongTensor(ref t2)) => t1.id == t2.id,
            _ => false,
        }
    }
}
impl Eq for TensorKind {}
impl Clone for TensorKind {
    fn clone(&self) -> Self {
        use self::TensorKind::{FloatTensor, LongTensor};
        match *self {
            FloatTensor(ref t) => FloatTensor(t.clone()),
            LongTensor(ref t) => LongTensor(t.clone()),
        }
    }
}

impl<T> From<Tensor<T>> for TensorKind {
    default fn from(input: Tensor<T>) -> Self {
        unreachable!()
    }
}

impl From<Tensor<f32>> for TensorKind {
    fn from(input: Tensor<f32>) -> TensorKind {
        TensorKind::FloatTensor(input)
    }
}

impl From<Tensor<i64>> for TensorKind {
    fn from(input: Tensor<i64>) -> TensorKind {
        TensorKind::LongTensor(input)
    }
}

impl<T> From<TensorKind> for Tensor<T> {
    default fn from(input: TensorKind) -> Tensor<T> {
        panic!("bad cast")
    }
}
impl From<TensorKind> for Tensor<f32> {
    fn from(input: TensorKind) -> Self {
        match input {
            TensorKind::FloatTensor(v) => v,
            _ => unimplemented!(),
        }
    }
}

impl From<TensorKind> for Tensor<i64> {
    fn from(input: TensorKind) -> Self {
        match input {
            TensorKind::LongTensor(v) => v,
            _ => unimplemented!(),
        }
    }
}

pub struct Tensor<T> {
    pub id: i32,
    value: RcMut<TensorImpl<T, Output = T>>,
}
impl<T> Hash for Tensor<T> {
    fn hash<H: Hasher>(&self, state: &mut H) {
        self.id.hash(state)
    }
}

impl<T: Copy> Index<usize> for Tensor<T> {
    type Output = T;
    fn index(&self, idx: usize) -> &Self::Output {

        //        self.value.borrow_mut().index(idx as isize)
        unimplemented!()
    }
}

impl<T: Copy> Index<i32> for Tensor<T> {
    type Output = T;
    fn index(&self, idx: i32) -> &Self::Output {

        //        self.value.borrow_mut().index(idx as isize)
        unimplemented!()
    }
}

pub trait New<D, T> {
    fn new(args: D) -> T;
    fn new_(&self, args: D) -> T;
}

impl<T> New<usize, Tensor<T>> for Tensor<T> {
    fn new(args: usize) -> Self {
        unimplemented!()
    }
    fn new_(&self, args: usize) -> Self {
        unimplemented!()
    }
}
impl<T> New<Vec<usize>, Tensor<T>> for Tensor<T> {
    fn new(args: Vec<usize>) -> Self {
        unimplemented!()
    }
    fn new_(&self, args: Vec<usize>) -> Self {
        unimplemented!()
    }
}

impl<T> Tensor<T> {
    pub fn len(&self) -> usize {
        unimplemented!()
    }
    pub fn size(&self) -> Vec<usize> {
        unimplemented!()
    }
    pub fn zero_(&mut self) -> Self {
        unimplemented!()
    }
    pub fn add_(&mut self, rhs: &Tensor<T>) {
        unimplemented!()
    }
    pub fn cuda(&self) -> Self {
        unimplemented!()
    }
    pub fn cpu(&self) -> Self {
        unimplemented!()
    }
    pub fn reduce_max(&self, axis: usize) -> (Self, Tensor<i64>) {
        unimplemented!()
    }
    pub fn tensor_eq(&self, rhs: &Self) -> Tensor<i64> {
        unimplemented!()
    }
    pub fn sum(&self) -> u32 {
        unimplemented!()
    }
    pub fn s(&self, dim: usize) -> Self {
        unimplemented!()
    }
}

impl<T> Default for Tensor<T> {
    fn default() -> Self {
        unimplemented!()
    }
}

impl<T> Clone for Tensor<T> {
    fn clone(&self) -> Self {
        Tensor {
            id: self.id,
            value: self.value.clone(),
        }
    }
}

impl<T: Copy> Index<isize> for Tensor<T> {
    type Output = T;

    fn index(&self, idx: isize) -> &Self::Output {
        unimplemented!()
    }
}
pub trait TensorImpl<T>: Index<Ixs, Output = T> {
    //fn view<'a>(&self, dims: &[i32]) -> Tensor<'a>;
}

pub struct FloatTensor {
    t: *mut THFloatTensor,
    storage: FloatStorage,
    dims: Vec<isize>,
}

impl Default for FloatTensor {
    fn default() -> Self {
        FloatTensor::new()
    }
}

impl FloatTensor {
    pub fn new() -> Self {
        unsafe {
            FloatTensor {
                t: THFloatTensor_new(),
                storage: FloatStorage::new(),
                dims: Vec::new(),
            }
        }
    }
    pub fn with_capacity(dims: &[isize]) -> Self {
        let size = dims.iter().product();
        let storage = FloatStorage::with_capacity(size);
        let strides = vec![1; dims.len()];
        let mut t = THFloatTensor {
            size: dims.clone().as_ptr() as *mut ::std::os::raw::c_long,
            stride: strides.as_ptr() as *mut ::std::os::raw::c_long,
            nDimension: dims.len() as i32,
            storage: storage.t,
            storageOffset: 0,
            refcount: 1,
            flag: TH_TENSOR_REFCOUNTED as i8,
        };
        FloatTensor {
            t: &mut t,
            storage: storage,
            dims: Vec::from(dims),
        }
    }
    pub fn randn(dims: &[isize]) -> Self {
        /* XXX */
        let mut t = FloatTensor::with_capacity(dims);
        for x in t.storage.iter_mut() {
            *x = rand::random::<f32>()
        }
        t
    }
}

impl<'a> Index<&'a [isize]> for FloatTensor {
    type Output = f32;

    fn index(&self, idx: &'a [isize]) -> &Self::Output {
        let mut index: isize = 0;
        let lastidx = max(0, idx.len() as isize - 1) as usize;
        if idx.len() != self.dims.len() {
            panic!("bad dimlen")
        }
        for i in 0..lastidx {
            if idx[i] >= self.dims[i] {
                panic!("bad dimlen")
            }
            index += idx[i] * self.dims[i]
        }
        if idx[lastidx] >= self.dims[lastidx] {
            panic!("bad dimlen")
        }
        index += idx[lastidx];
        &self.storage[index]
    }
}

impl<'a> IndexMut<&'a [isize]> for FloatTensor {
    fn index_mut(&mut self, idx: &'a [isize]) -> &mut Self::Output {
        let mut index: isize = 0;
        let lastidx = max(0, idx.len() as isize - 1) as usize;
        if idx.len() != self.dims.len() {
            panic!("bad dimlen")
        }
        for i in 0..lastidx {
            if idx[i] >= self.dims[i] {
                panic!("bad dimlen")
            }
            index += idx[i] * self.dims[i]
        }
        if idx[lastidx] >= self.dims[lastidx] {
            panic!("bad dimlen")
        }
        index += idx[lastidx];
        &mut self.storage[index]
    }
}
<commit_msg>silence unused_variables warning for default functions, implement From for borrowed TensorKind<commit_after>use rutorch::*;
use std::ops::{Index, IndexMut};
use std::convert::From;
use std::cmp::max;
use std::hash::{Hash, Hasher};

use storage::*;
use rand;
use {Ixs, RcMut};

#[derive(Hash)]
pub enum TensorKind {
    FloatTensor(Tensor<f32>),
    LongTensor(Tensor<i64>),
}

pub type TensorList<T> = Vec<Tensor<T>>;
pub type TensorKindList = Vec<TensorKind>;
pub type RefTensorList<'a, T> = Vec<&'a mut Tensor<T>>;
pub type RefTensorKindList<'a> = Vec<&'a TensorKind>;
pub type TensorId = i32;

impl PartialEq for TensorKind {
    fn eq(&self, other: &TensorKind) -> bool {
        use self::TensorKind::{FloatTensor, LongTensor};
        match (self, other) {
            (&FloatTensor(ref t1), &FloatTensor(ref t2)) => t1.id == t2.id,
            (&LongTensor(ref t1), &LongTensor(ref t2)) => t1.id == t2.id,
            _ => false,
        }
    }
}
impl Eq for TensorKind {}
impl Clone for TensorKind {
    fn clone(&self) -> Self {
        use self::TensorKind::{FloatTensor, LongTensor};
        match *self {
            FloatTensor(ref t) => FloatTensor(t.clone()),
            LongTensor(ref t) => LongTensor(t.clone()),
        }
    }
}

impl<T> From<Tensor<T>> for TensorKind {
    #[allow(unused_variables)]
    default fn from(input: Tensor<T>) -> Self {
        unreachable!()
    }
}

impl From<Tensor<f32>> for TensorKind {
    fn from(input: Tensor<f32>) -> TensorKind {
        TensorKind::FloatTensor(input)
    }
}

impl From<Tensor<i64>> for TensorKind {
    fn from(input: Tensor<i64>) -> TensorKind {
        TensorKind::LongTensor(input)
    }
}

impl<T> From<TensorKind> for Tensor<T> {
    #[allow(unused_variables)]
    default fn from(input: TensorKind) -> Tensor<T> {
        panic!("bad cast")
    }
}

impl<'a, T> From<&'a TensorKind> for &'a Tensor<T> {
    #[allow(unused_variables)]
    default fn from(input: &'a TensorKind) -> &'a Tensor<T> {
        panic!("bad cast")
    }
}

impl<'a> From<&'a TensorKind> for &'a Tensor<f32> {
    fn from(input: &'a TensorKind) -> Self {
        match *input {
            TensorKind::FloatTensor(ref t) => t,
            _ => unreachable!(),
        }
    }
}

impl<'a> From<&'a TensorKind> for &'a Tensor<i64> {
    fn from(input: &'a TensorKind) -> Self {
        match *input {
            TensorKind::LongTensor(ref t) => t,
            _ => unreachable!(),
        }
    }
}

impl From<TensorKind> for Tensor<f32> {
    fn from(input: TensorKind) -> Self {
        match input {
            TensorKind::FloatTensor(v) => v,
            _ => unimplemented!(),
        }
    }
}

impl From<TensorKind> for Tensor<i64> {
    fn from(input: TensorKind) -> Self {
        match input {
            TensorKind::LongTensor(v) => v,
            _ => unimplemented!(),
        }
    }
}
pub struct Tensor<T> {
    pub id: i32,
    value: RcMut<TensorImpl<T, Output = T>>,
}
impl<T> Hash for Tensor<T> {
    fn hash<H: Hasher>(&self, state: &mut H) {
        self.id.hash(state)
    }
}

impl<T: Copy> Index<usize> for Tensor<T> {
    type Output = T;
    fn index(&self, idx: usize) -> &Self::Output {

        //        self.value.borrow_mut().index(idx as isize)
        unimplemented!()
    }
}

impl<T: Copy> Index<i32> for Tensor<T> {
    type Output = T;
    fn index(&self, idx: i32) -> &Self::Output {

        //        self.value.borrow_mut().index(idx as isize)
        unimplemented!()
    }
}

pub trait New<D, T> {
    fn new(args: D) -> T;
    fn new_(&self, args: D) -> T;
}

impl<T> New<usize, Tensor<T>> for Tensor<T> {
    fn new(args: usize) -> Self {
        unimplemented!()
    }
    fn new_(&self, args: usize) -> Self {
        unimplemented!()
    }
}
impl<T> New<Vec<usize>, Tensor<T>> for Tensor<T> {
    fn new(args: Vec<usize>) -> Self {
        unimplemented!()
    }
    fn new_(&self, args: Vec<usize>) -> Self {
        unimplemented!()
    }
}

impl<T> Tensor<T> {
    pub fn len(&self) -> usize {
        unimplemented!()
    }
    pub fn size(&self) -> Vec<usize> {
        unimplemented!()
    }
    pub fn zero_(&mut self) -> Self {
        unimplemented!()
    }
    pub fn add_(&mut self, rhs: &Tensor<T>) {
        unimplemented!()
    }
    pub fn cuda(&self) -> Self {
        unimplemented!()
    }
    pub fn cpu(&self) -> Self {
        unimplemented!()
    }
    pub fn reduce_max(&self, axis: usize) -> (Self, Tensor<i64>) {
        unimplemented!()
    }
    pub fn tensor_eq(&self, rhs: &Self) -> Tensor<i64> {
        unimplemented!()
    }
    pub fn sum(&self) -> u32 {
        unimplemented!()
    }
    pub fn s(&self, dim: usize) -> Self {
        unimplemented!()
    }
}

impl<T> Default for Tensor<T> {
    fn default() -> Self {
        unimplemented!()
    }
}

impl<T> Clone for Tensor<T> {
    fn clone(&self) -> Self {
        Tensor {
            id: self.id,
            value: self.value.clone(),
        }
    }
}

impl<T: Copy> Index<isize> for Tensor<T> {
    type Output = T;

    fn index(&self, idx: isize) -> &Self::Output {
        unimplemented!()
    }
}
pub trait TensorImpl<T>: Index<Ixs, Output = T> {
    //fn view<'a>(&self, dims: &[i32]) -> Tensor<'a>;
}

pub struct FloatTensor {
    t: *mut THFloatTensor,
    storage: FloatStorage,
    dims: Vec<isize>,
}

impl Default for FloatTensor {
    fn default() -> Self {
        FloatTensor::new()
    }
}

impl FloatTensor {
    pub fn new() -> Self {
        unsafe {
            FloatTensor {
                t: THFloatTensor_new(),
                storage: FloatStorage::new(),
                dims: Vec::new(),
            }
        }
    }
    pub fn with_capacity(dims: &[isize]) -> Self {
        let size = dims.iter().product();
        let storage = FloatStorage::with_capacity(size);
        let strides = vec![1; dims.len()];
        let mut t = THFloatTensor {
            size: dims.clone().as_ptr() as *mut ::std::os::raw::c_long,
            stride: strides.as_ptr() as *mut ::std::os::raw::c_long,
            nDimension: dims.len() as i32,
            storage: storage.t,
            storageOffset: 0,
            refcount: 1,
            flag: TH_TENSOR_REFCOUNTED as i8,
        };
        FloatTensor {
            t: &mut t,
            storage: storage,
            dims: Vec::from(dims),
        }
    }
    pub fn randn(dims: &[isize]) -> Self {
        /* XXX */
        let mut t = FloatTensor::with_capacity(dims);
        for x in t.storage.iter_mut() {
            *x = rand::random::<f32>()
        }
        t
    }
}

impl<'a> Index<&'a [isize]> for FloatTensor {
    type Output = f32;

    fn index(&self, idx: &'a [isize]) -> &Self::Output {
        let mut index: isize = 0;
        let lastidx = max(0, idx.len() as isize - 1) as usize;
        if idx.len() != self.dims.len() {
            panic!("bad dimlen")
        }
        for i in 0..lastidx {
            if idx[i] >= self.dims[i] {
                panic!("bad dimlen")
            }
            index += idx[i] * self.dims[i]
        }
        if idx[lastidx] >= self.dims[lastidx] {
            panic!("bad dimlen")
        }
        index += idx[lastidx];
        &self.storage[index]
    }
}

impl<'a> IndexMut<&'a [isize]> for FloatTensor {
    fn index_mut(&mut self, idx: &'a [isize]) -> &mut Self::Output {
        let mut index: isize = 0;
        let lastidx = max(0, idx.len() as isize - 1) as usize;
        if idx.len() != self.dims.len() {
            panic!("bad dimlen")
        }
        for i in 0..lastidx {
            if idx[i] >= self.dims[i] {
                panic!("bad dimlen")
            }
            index += idx[i] * self.dims[i]
        }
        if idx[lastidx] >= self.dims[lastidx] {
            panic!("bad dimlen")
        }
        index += idx[lastidx];
        &mut self.storage[index]
    }
}
<|endoftext|>


<commit_before>use super::Collector;
use collector::top_collector::TopCollector;
use collector::top_collector::TopSegmentCollector;
use collector::SegmentCollector;
use fastfield::FastFieldReader;
use fastfield::FastValue;
use schema::Field;
use DocAddress;
use Result;
use SegmentLocalId;
use SegmentReader;

/// The Top Field Collector keeps track of the K documents
/// sorted by a fast field in the index
///
/// The implementation is based on a `BinaryHeap`.
/// The theorical complexity for collecting the top `K` out of `n` documents
/// is `O(n log K)`.
///
/// ```rust
/// #[macro_use]
/// extern crate tantivy;
/// # use tantivy::schema::{Schema, Field, FAST, TEXT};
/// # use tantivy::{Index, Result, DocAddress};
/// # use tantivy::query::{Query, QueryParser};
/// use tantivy::collector::TopDocs;
///
/// # fn main() {
/// #   let mut schema_builder = Schema::builder();
/// #   let title = schema_builder.add_text_field("title", TEXT);
/// #   let rating = schema_builder.add_u64_field("rating", FAST);
/// #   let schema = schema_builder.build();
/// #   let index = Index::create_in_ram(schema);
/// #   let mut index_writer = index.writer_with_num_threads(1, 3_000_000).unwrap();
/// #   index_writer.add_document(doc!(
/// #       title => "The Name of the Wind",
/// #       rating => 92u64,
/// #   ));
/// #   index_writer.add_document(doc!(title => "The Diary of Muadib", rating => 97u64));
/// #   index_writer.add_document(doc!(title => "A Dairy Cow", rating => 63u64));
/// #   index_writer.add_document(doc!(title => "The Diary of a Young Girl", rating => 80u64));
/// #   index_writer.commit().unwrap();
/// #   index.load_searchers().unwrap();
///	#   let query = QueryParser::for_index(&index, vec![title]).parse_query("diary").unwrap();
/// #   let top_docs = docs_sorted_by_rating(&index, &query, rating).unwrap();
/// #   assert_eq!(top_docs,
/// #            vec![(97u64, DocAddress(0u32, 1)),
/// #                 (80u64, DocAddress(0u32, 3))]);
/// # }
/// #
/// /// Searches the document matching the given query, and
/// /// collects the top 10 documents, order by the `field`
/// /// given in argument.
/// ///
/// /// `field` is required to be a FAST field.
/// fn docs_sorted_by_rating(index: &Index, query: &Query, sort_by_field: Field)
///     -> Result<Vec<(u64, DocAddress)>> {
///
///     // This is where we build our collector!
///     let top_docs_by_rating = TopDocs::with_limit(2).order_by_field(sort_by_field);
///
///     // ... and here is our documents. Not this is a simple vec.
///     // The `u64` in the pair is the value of our fast field for each documents.
///     index.searcher()
///          .search(query, &top_docs_by_rating)
/// }
/// ```
pub struct TopDocsByField<T> {
    collector: TopCollector<T>,
    field: Field,
}

impl<T: FastValue + PartialOrd + Clone> TopDocsByField<T> {
    /// Creates a top field collector, with a number of documents equal to "limit".
    ///
    /// The given field name must be a fast field, otherwise the collector have an error while
    /// collecting results.
    ///
    /// # Panics
    /// The method panics if limit is 0
    pub(crate) fn new(field: Field, limit: usize) -> TopDocsByField<T> {
        TopDocsByField {
            collector: TopCollector::with_limit(limit),
            field,
        }
    }
}

impl<T: FastValue + PartialOrd + Send + Sync + 'static> Collector for TopDocsByField<T> {
    type Fruit = Vec<(T, DocAddress)>;

    type Child = TopFieldSegmentCollector<T>;

    fn for_segment(
        &self,
        segment_local_id: SegmentLocalId,
        reader: &SegmentReader,
    ) -> Result<TopFieldSegmentCollector<T>> {
        let collector = self.collector.for_segment(segment_local_id, reader)?;
        let reader = reader.fast_field_reader(self.field)?;
        Ok(TopFieldSegmentCollector { collector, reader })
    }

    fn requires_scoring(&self) -> bool {
        false
    }

    fn merge_fruits(
        &self,
        segment_fruits: Vec<Vec<(T, DocAddress)>>,
    ) -> Result<Vec<(T, DocAddress)>> {
        self.collector.merge_fruits(segment_fruits)
    }
}

pub struct TopFieldSegmentCollector<T: FastValue + PartialOrd> {
    collector: TopSegmentCollector<T>,
    reader: FastFieldReader<T>,
}

impl<T: FastValue + PartialOrd + Send + Sync + 'static> SegmentCollector
    for TopFieldSegmentCollector<T>
{
    type Fruit = Vec<(T, DocAddress)>;

    fn collect(&mut self, doc: u32, _score: f32) {
        let field_value = self.reader.get(doc);
        self.collector.collect(doc, field_value);
    }

    fn harvest(self) -> Vec<(T, DocAddress)> {
        self.collector.harvest()
    }
}

#[cfg(test)]
mod tests {
    use super::TopDocsByField;
    use collector::Collector;
    use collector::TopDocs;
    use query::Query;
    use query::QueryParser;
    use schema::Field;
    use schema::IntOptions;
    use schema::{Schema, FAST, TEXT};
    use DocAddress;
    use Index;
    use IndexWriter;
    use TantivyError;

    const TITLE: &str = "title";
    const SIZE: &str = "size";

    #[test]
    fn test_top_collector_not_at_capacity() {
        let mut schema_builder = Schema::builder();
        let title = schema_builder.add_text_field(TITLE, TEXT);
        let size = schema_builder.add_u64_field(SIZE, FAST);
        let schema = schema_builder.build();
        let (index, query) = index("beer", title, schema, |index_writer| {
            index_writer.add_document(doc!(
                title => "bottle of beer",
                size => 12u64,
            ));
            index_writer.add_document(doc!(
                title => "growler of beer",
                size => 64u64,
            ));
            index_writer.add_document(doc!(
                title => "pint of beer",
                size => 16u64,
            ));
        });
        let searcher = index.searcher();

        let top_collector = TopDocs::with_limit(4).order_by_field(size);
        let top_docs: Vec<(u64, DocAddress)> = searcher.search(&query, &top_collector).unwrap();
        assert_eq!(
            top_docs,
            vec![
                (64, DocAddress(0, 1)),
                (16, DocAddress(0, 2)),
                (12, DocAddress(0, 0))
            ]
        );
    }

    #[test]
    #[should_panic]
    fn test_field_does_not_exist() {
        let mut schema_builder = Schema::builder();
        let title = schema_builder.add_text_field(TITLE, TEXT);
        let size = schema_builder.add_u64_field(SIZE, FAST);
        let schema = schema_builder.build();
        let (index, _) = index("beer", title, schema, |index_writer| {
            index_writer.add_document(doc!(
                title => "bottle of beer",
                size => 12u64,
            ));
        });
        let searcher = index.searcher();
        let top_collector: TopDocsByField<u64> = TopDocs::with_limit(4).order_by_field(Field(2));
        let segment_reader = searcher.segment_reader(0u32);
        top_collector
            .for_segment(0, segment_reader)
            .expect("should panic");
    }

    #[test]
    fn test_field_not_fast_field() {
        let mut schema_builder = Schema::builder();
        let title = schema_builder.add_text_field(TITLE, TEXT);
        let size = schema_builder.add_u64_field(SIZE, IntOptions::default());
        let schema = schema_builder.build();
        let (index, _) = index("beer", title, schema, |index_writer| {
            index_writer.add_document(doc!(
                title => "bottle of beer",
                size => 12u64,
            ));
        });
        let searcher = index.searcher();
        let segment = searcher.segment_reader(0);
        let top_collector: TopDocsByField<u64> = TopDocs::with_limit(4).order_by_field(size);
        assert_matches!(
            top_collector
                .for_segment(0, segment)
                .map(|_| ())
                .unwrap_err(),
            TantivyError::FastFieldError(_)
        );
    }

    fn index(
        query: &str,
        query_field: Field,
        schema: Schema,
        mut doc_adder: impl FnMut(&mut IndexWriter) -> (),
    ) -> (Index, Box<Query>) {
        let index = Index::create_in_ram(schema);

        let mut index_writer = index.writer_with_num_threads(1, 3_000_000).unwrap();
        doc_adder(&mut index_writer);
        index_writer.commit().unwrap();
        index.load_searchers().unwrap();

        let query_parser = QueryParser::for_index(&index, vec![query_field]);
        let query = query_parser.parse_query(query).unwrap();
        (index, query)
    }
}
<commit_msg>Added comment on the constructor fo TopDocSByField<commit_after>use super::Collector;
use collector::top_collector::TopCollector;
use collector::top_collector::TopSegmentCollector;
use collector::SegmentCollector;
use fastfield::FastFieldReader;
use fastfield::FastValue;
use schema::Field;
use DocAddress;
use Result;
use SegmentLocalId;
use SegmentReader;

/// The Top Field Collector keeps track of the K documents
/// sorted by a fast field in the index
///
/// The implementation is based on a `BinaryHeap`.
/// The theorical complexity for collecting the top `K` out of `n` documents
/// is `O(n log K)`.
///
/// ```rust
/// #[macro_use]
/// extern crate tantivy;
/// # use tantivy::schema::{Schema, Field, FAST, TEXT};
/// # use tantivy::{Index, Result, DocAddress};
/// # use tantivy::query::{Query, QueryParser};
/// use tantivy::collector::TopDocs;
///
/// # fn main() {
/// #   let mut schema_builder = Schema::builder();
/// #   let title = schema_builder.add_text_field("title", TEXT);
/// #   let rating = schema_builder.add_u64_field("rating", FAST);
/// #   let schema = schema_builder.build();
/// #   let index = Index::create_in_ram(schema);
/// #   let mut index_writer = index.writer_with_num_threads(1, 3_000_000).unwrap();
/// #   index_writer.add_document(doc!(
/// #       title => "The Name of the Wind",
/// #       rating => 92u64,
/// #   ));
/// #   index_writer.add_document(doc!(title => "The Diary of Muadib", rating => 97u64));
/// #   index_writer.add_document(doc!(title => "A Dairy Cow", rating => 63u64));
/// #   index_writer.add_document(doc!(title => "The Diary of a Young Girl", rating => 80u64));
/// #   index_writer.commit().unwrap();
/// #   index.load_searchers().unwrap();
///	#   let query = QueryParser::for_index(&index, vec![title]).parse_query("diary").unwrap();
/// #   let top_docs = docs_sorted_by_rating(&index, &query, rating).unwrap();
/// #   assert_eq!(top_docs,
/// #            vec![(97u64, DocAddress(0u32, 1)),
/// #                 (80u64, DocAddress(0u32, 3))]);
/// # }
/// #
/// /// Searches the document matching the given query, and
/// /// collects the top 10 documents, order by the `field`
/// /// given in argument.
/// ///
/// /// `field` is required to be a FAST field.
/// fn docs_sorted_by_rating(index: &Index, query: &Query, sort_by_field: Field)
///     -> Result<Vec<(u64, DocAddress)>> {
///
///     // This is where we build our collector!
///     let top_docs_by_rating = TopDocs::with_limit(2).order_by_field(sort_by_field);
///
///     // ... and here is our documents. Not this is a simple vec.
///     // The `u64` in the pair is the value of our fast field for each documents.
///     index.searcher()
///          .search(query, &top_docs_by_rating)
/// }
/// ```
pub struct TopDocsByField<T> {
    collector: TopCollector<T>,
    field: Field,
}

impl<T: FastValue + PartialOrd + Clone> TopDocsByField<T> {
    /// Creates a top field collector, with a number of documents equal to "limit".
    ///
    /// The given field name must be a fast field, otherwise the collector have an error while
    /// collecting results.
    ///
    /// This constructor is crate-private. Client are supposed to call
    /// build `TopDocsByField`  object using the `TopDocs` API.
    ///
    /// e.g.: `TopDocs::with_limit(2).order_by_field(sort_by_field)`
    ///
    /// # Panics
    /// The method panics if limit is 0
    pub(crate) fn new(field: Field, limit: usize) -> TopDocsByField<T> {
        TopDocsByField {
            collector: TopCollector::with_limit(limit),
            field,
        }
    }
}

impl<T: FastValue + PartialOrd + Send + Sync + 'static> Collector for TopDocsByField<T> {
    type Fruit = Vec<(T, DocAddress)>;

    type Child = TopFieldSegmentCollector<T>;

    fn for_segment(
        &self,
        segment_local_id: SegmentLocalId,
        reader: &SegmentReader,
    ) -> Result<TopFieldSegmentCollector<T>> {
        let collector = self.collector.for_segment(segment_local_id, reader)?;
        let reader = reader.fast_field_reader(self.field)?;
        Ok(TopFieldSegmentCollector { collector, reader })
    }

    fn requires_scoring(&self) -> bool {
        false
    }

    fn merge_fruits(
        &self,
        segment_fruits: Vec<Vec<(T, DocAddress)>>,
    ) -> Result<Vec<(T, DocAddress)>> {
        self.collector.merge_fruits(segment_fruits)
    }
}

pub struct TopFieldSegmentCollector<T: FastValue + PartialOrd> {
    collector: TopSegmentCollector<T>,
    reader: FastFieldReader<T>,
}

impl<T: FastValue + PartialOrd + Send + Sync + 'static> SegmentCollector
    for TopFieldSegmentCollector<T>
{
    type Fruit = Vec<(T, DocAddress)>;

    fn collect(&mut self, doc: u32, _score: f32) {
        let field_value = self.reader.get(doc);
        self.collector.collect(doc, field_value);
    }

    fn harvest(self) -> Vec<(T, DocAddress)> {
        self.collector.harvest()
    }
}

#[cfg(test)]
mod tests {
    use super::TopDocsByField;
    use collector::Collector;
    use collector::TopDocs;
    use query::Query;
    use query::QueryParser;
    use schema::Field;
    use schema::IntOptions;
    use schema::{Schema, FAST, TEXT};
    use DocAddress;
    use Index;
    use IndexWriter;
    use TantivyError;

    const TITLE: &str = "title";
    const SIZE: &str = "size";

    #[test]
    fn test_top_collector_not_at_capacity() {
        let mut schema_builder = Schema::builder();
        let title = schema_builder.add_text_field(TITLE, TEXT);
        let size = schema_builder.add_u64_field(SIZE, FAST);
        let schema = schema_builder.build();
        let (index, query) = index("beer", title, schema, |index_writer| {
            index_writer.add_document(doc!(
                title => "bottle of beer",
                size => 12u64,
            ));
            index_writer.add_document(doc!(
                title => "growler of beer",
                size => 64u64,
            ));
            index_writer.add_document(doc!(
                title => "pint of beer",
                size => 16u64,
            ));
        });
        let searcher = index.searcher();

        let top_collector = TopDocs::with_limit(4).order_by_field(size);
        let top_docs: Vec<(u64, DocAddress)> = searcher.search(&query, &top_collector).unwrap();
        assert_eq!(
            top_docs,
            vec![
                (64, DocAddress(0, 1)),
                (16, DocAddress(0, 2)),
                (12, DocAddress(0, 0))
            ]
        );
    }

    #[test]
    #[should_panic]
    fn test_field_does_not_exist() {
        let mut schema_builder = Schema::builder();
        let title = schema_builder.add_text_field(TITLE, TEXT);
        let size = schema_builder.add_u64_field(SIZE, FAST);
        let schema = schema_builder.build();
        let (index, _) = index("beer", title, schema, |index_writer| {
            index_writer.add_document(doc!(
                title => "bottle of beer",
                size => 12u64,
            ));
        });
        let searcher = index.searcher();
        let top_collector: TopDocsByField<u64> = TopDocs::with_limit(4).order_by_field(Field(2));
        let segment_reader = searcher.segment_reader(0u32);
        top_collector
            .for_segment(0, segment_reader)
            .expect("should panic");
    }

    #[test]
    fn test_field_not_fast_field() {
        let mut schema_builder = Schema::builder();
        let title = schema_builder.add_text_field(TITLE, TEXT);
        let size = schema_builder.add_u64_field(SIZE, IntOptions::default());
        let schema = schema_builder.build();
        let (index, _) = index("beer", title, schema, |index_writer| {
            index_writer.add_document(doc!(
                title => "bottle of beer",
                size => 12u64,
            ));
        });
        let searcher = index.searcher();
        let segment = searcher.segment_reader(0);
        let top_collector: TopDocsByField<u64> = TopDocs::with_limit(4).order_by_field(size);
        assert_matches!(
            top_collector
                .for_segment(0, segment)
                .map(|_| ())
                .unwrap_err(),
            TantivyError::FastFieldError(_)
        );
    }

    fn index(
        query: &str,
        query_field: Field,
        schema: Schema,
        mut doc_adder: impl FnMut(&mut IndexWriter) -> (),
    ) -> (Index, Box<Query>) {
        let index = Index::create_in_ram(schema);

        let mut index_writer = index.writer_with_num_threads(1, 3_000_000).unwrap();
        doc_adder(&mut index_writer);
        index_writer.commit().unwrap();
        index.load_searchers().unwrap();

        let query_parser = QueryParser::for_index(&index, vec![query_field]);
        let query = query_parser.parse_query(query).unwrap();
        (index, query)
    }
}
<|endoftext|>


<commit_before>use std::collections::BTreeMap;
use std::path::{PathBuf, Path};
use config::Config;

#[derive(Debug, Clone, Default, Hash, PartialEq, Eq, PartialOrd)]
pub struct Trace {
    pub line: u64,
    pub address: Option<u64>,
    pub hits: u64,
}

pub struct TraceMap<'a> {
    traces: BTreeMap<PathBuf, Vec<Trace>>,
    config: &'a Config
}

impl<'a> TraceMap<'a> {

    pub fn new(config: &'a Config) -> TraceMap<'a> {
        TraceMap {
            config: config,
            traces: BTreeMap::new(),
        }
    }

    pub fn get_child_traces(&self, root: &Path) -> Vec<&Trace> {
        self.traces.iter()
                   .filter(|&(ref k, _)| k.starts_with(root))
                   .flat_map(|(_, ref v)| v.iter())
                   .collect()
    }
}
<commit_msg>Added function to get all traces<commit_after>use std::collections::BTreeMap;
use std::path::{PathBuf, Path};
use config::Config;

#[derive(Debug, Clone, Default, Hash, PartialEq, Eq, PartialOrd)]
pub struct Trace {
    pub line: u64,
    pub address: Option<u64>,
    pub hits: u64,
}

/// Stores all the program traces mapped to files and provides an interface to
/// add, query and change traces.
pub struct TraceMap<'a> {
    traces: BTreeMap<PathBuf, Vec<Trace>>,
    config: &'a Config
}

impl<'a> TraceMap<'a> {

    pub fn new(config: &'a Config) -> TraceMap<'a> {
        TraceMap {
            config: config,
            traces: BTreeMap::new(),
        }
    }

    pub fn add_trace(&mut self, file: &Path, trace: Trace) {
        if self.traces.contains_key(file) {
            if let Some(trace_vec) = self.traces.get_mut(file) {
                trace_vec.push(trace);
            }
        } else {
            self.traces.insert(file.to_path_buf(), vec![trace]);
        }
    }

    pub fn get_trace(&self, address: u64) -> Option<&Trace> {
        self.all_traces()
            .iter()
            .find(|x| x.address == Some(address))
            .map(|x| *x)
    }

    pub fn get_trace_mut(&mut self, address: u64) -> Option<&mut Trace> {
        None
    }

    pub fn get_child_traces(&self, root: &Path) -> Vec<&Trace> {
        self.traces.iter()
                   .filter(|&(ref k, _)| k.starts_with(root))
                   .flat_map(|(_, ref v)| v.iter())
                   .collect()
    }

    fn all_traces(&self) -> Vec<&Trace> {
        self.traces.values().flat_map(|ref x| x.iter()).collect()
    }

}
<|endoftext|>


<commit_before>#[cfg(feature = "regex1")]
use regex::Regex;
use serde_json::{Value, Map, Number};
use url::percent_encoding::percent_decode;

use merge::merge;
use helpers::{create_array, push_item_to_array};

#[cfg(feature = "regex1")]
lazy_static! {
    static ref PARENT_REGEX: Regex = Regex::new(r"^([^\]\[]+)").unwrap();
    static ref CHILD_REGEX: Regex = Regex::new(r"(\[[^\]\[]*\])").unwrap();
}

type Object = Map<String, Value>;

#[derive(Debug)]
#[allow(missing_copy_implementations)]
pub enum ParseErrorKind {
    DecodingError,
    Other
}

#[derive(Debug)]
pub struct ParseError {
    pub kind: ParseErrorKind,
    pub message: String
}

pub type ParseResult<T> = Result<T,ParseError>;

pub fn decode_component(source: &str) -> Result<String,String> {
    let result = percent_decode(source.as_bytes()).decode_utf8_lossy().to_string();
    return Ok(result);
}

fn parse_pairs(body: &str) -> Vec<(&str, Option<&str>)> {

    let mut pairs = vec![];

    for part in body.split("&") {
        let separator = part.find("]=")
                            .and_then(|pos| Some(pos+1))
                            .or_else(|| part.find("="));

        match separator {
            None => pairs.push((part, None)),
            Some(pos) => {
                let key = &part[..pos];
                let val = &part[(pos + 1)..];
                pairs.push((key, Some(val)));
            }
        }
    }

    return pairs
}

#[cfg(feature = "regex1")]
fn parse_key(key: &str) -> ParseResult<Vec<String>> {
    let mut keys: Vec<String> = vec![];

    match PARENT_REGEX.captures(key) {
        Some(captures) => {
            match decode_component(captures.get(1).unwrap().as_str()) {
                Ok(decoded_key) => keys.push(decoded_key),
                Err(err_msg) => return Err(ParseError{ kind: ParseErrorKind::DecodingError, message: err_msg })
            }
        }
        None => ()
    };

    for captures in CHILD_REGEX.captures_iter(key) {
        match decode_component(captures.get(1).unwrap().as_str()) {
            Ok(decoded_key) => keys.push(decoded_key),
            Err(err_msg) => return Err(ParseError{ kind: ParseErrorKind::DecodingError, message: err_msg })
        }
    }

    Ok(keys)
}

#[cfg(not(feature = "regex1"))]
fn parse_key(key: &str) -> ParseResult<Vec<String>> {
    let mut keys: Vec<String> = vec![];

    match key.split(|c| c=='[' || c==']').next() {
        Some(parent) if !parent.is_empty() =>  {
            match decode_component(parent) {
                Ok(decoded_key) => keys.push(decoded_key),
                Err(err_msg) => return Err(ParseError{ kind: ParseErrorKind::DecodingError, message: err_msg })
            }
        }
        _ => ()
    }

    let mut prev_bracket = None;
    for (idx, ch) in key.char_indices() {
        match ch {
            '[' => prev_bracket = Some(idx),
            ']' => {
                if let Some(prev_idx) = prev_bracket {
                    prev_bracket = None;
                    let child = &key[prev_idx..=idx];
                    match decode_component(child) {
                        Ok(decoded_key) => keys.push(decoded_key),
                        Err(err_msg) => return Err(ParseError{ kind: ParseErrorKind::DecodingError, message: err_msg })
                    }
                }
            }
            _ => (),
        }
    }

    Ok(keys)
}

fn cleanup_key(key: &str) -> &str {
    if key.starts_with("[") && key.ends_with("]") {
        &key[1..(key.len()-1)]
    } else {
        key
    }
}

fn create_idx_merger(idx: u64, obj: Value) -> Value {
    let mut tree = Object::new();
    tree.insert("__idx".to_string(), Value::Number(Number::from(idx)));
    tree.insert("__object".to_string(), obj);
    return Value::Object(tree)
}

fn create_object_with_key(key: String, obj: Value) -> Value {
    let mut tree = Object::new();
    tree.insert(key, obj);
    return Value::Object(tree)
}

fn apply_object(keys: &[String], val: Value) -> Value {

    if keys.len() > 0 {
        let key = keys.get(0).unwrap();
        if key == "[]" {
            let mut new_array = create_array();
            let item = apply_object(&keys[1..], val);
            push_item_to_array(&mut new_array, item);
            return new_array;
        } else {
            let key = cleanup_key(key);
            let array_index = key.parse();

            match array_index {
                Ok(idx) => {
                    let result = apply_object(&keys[1..], val);
                    let item = create_idx_merger(idx, result);
                    return item;
                },
                Err(_) => {
                    return create_object_with_key(key.to_string(), apply_object(&keys[1..], val));
                }
            }
        }

    } else {
        return val;
    }
}

pub fn parse(params: &str) -> ParseResult<Value> {
    let tree = Object::new();
    let mut obj = Value::Object(tree);
    let decoded_params = match decode_component(params) {
        Ok(val) => val,
        Err(err) => return Err(ParseError{ kind: ParseErrorKind::DecodingError, message: err })
    };
    let pairs = parse_pairs(&decoded_params);
    for &(key, value) in pairs.iter() {
        let parse_key_res = try!(parse_key(key));
        let key_chain = &parse_key_res[0..];
        let decoded_value = match decode_component(value.unwrap_or("")) {
            Ok(val) => val,
            Err(err) => return Err(ParseError{ kind: ParseErrorKind::DecodingError, message: err })
        };
        let partial = apply_object(key_chain, Value::String(decoded_value));
        merge(&mut obj, &partial);
    }

    Ok(obj)
}

#[cfg(test)]
mod tests {
    use parse;
    use serde_json::{Value, to_string};

    fn eq_str(value: Value, string: &str) {
        assert_eq!(&to_string(&value).unwrap(), string)
    }

    #[test]
    fn it_parses_simple_string() {
        eq_str(parse("0=foo").unwrap(), r#"{"0":"foo"}"#);
        eq_str(parse("a[<=>]==23").unwrap(), r#"{"a":{"<=>":"=23"}}"#);
        eq_str(parse(" foo = bar = baz ").unwrap(), r#"{" foo ":" bar = baz "}"#);
    }

    #[test]
    fn it_parses_nested_string() {
        eq_str(parse("a[b][c][d][e][f][g][h]=i").unwrap(),
            r#"{"a":{"b":{"c":{"d":{"e":{"f":{"g":{"h":"i"}}}}}}}}"#);
    }

    #[test]
    fn it_parses_simple_array() {
        eq_str(parse("a=b&a=c&a=d&a=e").unwrap(),
            r#"{"a":["b","c","d","e"]}"#);
    }

    #[test]
    fn it_parses_explicit_array() {
        eq_str(parse("a[]=b&a[]=c&a[]=d").unwrap(),
            r#"{"a":["b","c","d"]}"#);
    }

    #[test]
    fn it_parses_nested_array() {
        eq_str(parse("a[b][]=c&a[b][]=d").unwrap(),
            r#"{"a":{"b":["c","d"]}}"#);
    }

    #[test]
    fn it_allows_to_specify_array_indexes() {
        eq_str(parse("a[0][]=c&a[1][]=d").unwrap(),
            r#"{"a":[["c"],["d"]]}"#);
    }

    #[test]
    fn it_transforms_arrays_to_object() {
        eq_str(parse("foo[0]=bar&foo[bad]=baz").unwrap(),
            r#"{"foo":{"0":"bar","bad":"baz"}}"#);

        eq_str(parse("foo[0][a]=a&foo[0][b]=b&foo[1][a]=aa&foo[1][b]=bb").unwrap(),
            r#"{"foo":[{"a":"a","b":"b"},{"a":"aa","b":"bb"}]}"#);
    }

    #[test]
    fn it_doesnt_produce_empty_keys() {
        eq_str(parse("_r=1&").unwrap(),
            r#"{"_r":"1"}"#);
    }

    #[test]
    fn it_supports_encoded_strings() {
        eq_str(parse("a[b%20c]=c%20d").unwrap(),
            r#"{"a":{"b c":"c d"}}"#);
    }

    #[test]
    fn it_parses_explicit_encoded_array() {
        eq_str(parse("a%5B%5D=b&a%5B%5D=c&a%5B%5D=d").unwrap(),
            r#"{"a":["b","c","d"]}"#);
    }
}
<commit_msg>refactor(parser): Cleanup 'try' linter warning (see rust-lang/rust#61000)<commit_after>#[cfg(feature = "regex1")]
use regex::Regex;
use serde_json::{Value, Map, Number};
use url::percent_encoding::percent_decode;

use merge::merge;
use helpers::{create_array, push_item_to_array};

#[cfg(feature = "regex1")]
lazy_static! {
    static ref PARENT_REGEX: Regex = Regex::new(r"^([^\]\[]+)").unwrap();
    static ref CHILD_REGEX: Regex = Regex::new(r"(\[[^\]\[]*\])").unwrap();
}

type Object = Map<String, Value>;

#[derive(Debug)]
#[allow(missing_copy_implementations)]
pub enum ParseErrorKind {
    DecodingError,
    Other
}

#[derive(Debug)]
pub struct ParseError {
    pub kind: ParseErrorKind,
    pub message: String
}

pub type ParseResult<T> = Result<T,ParseError>;

pub fn decode_component(source: &str) -> Result<String,String> {
    let result = percent_decode(source.as_bytes()).decode_utf8_lossy().to_string();
    return Ok(result);
}

fn parse_pairs(body: &str) -> Vec<(&str, Option<&str>)> {

    let mut pairs = vec![];

    for part in body.split("&") {
        let separator = part.find("]=")
                            .and_then(|pos| Some(pos+1))
                            .or_else(|| part.find("="));

        match separator {
            None => pairs.push((part, None)),
            Some(pos) => {
                let key = &part[..pos];
                let val = &part[(pos + 1)..];
                pairs.push((key, Some(val)));
            }
        }
    }

    return pairs
}

#[cfg(feature = "regex1")]
fn parse_key(key: &str) -> ParseResult<Vec<String>> {
    let mut keys: Vec<String> = vec![];

    match PARENT_REGEX.captures(key) {
        Some(captures) => {
            match decode_component(captures.get(1).unwrap().as_str()) {
                Ok(decoded_key) => keys.push(decoded_key),
                Err(err_msg) => return Err(ParseError{ kind: ParseErrorKind::DecodingError, message: err_msg })
            }
        }
        None => ()
    };

    for captures in CHILD_REGEX.captures_iter(key) {
        match decode_component(captures.get(1).unwrap().as_str()) {
            Ok(decoded_key) => keys.push(decoded_key),
            Err(err_msg) => return Err(ParseError{ kind: ParseErrorKind::DecodingError, message: err_msg })
        }
    }

    Ok(keys)
}

#[cfg(not(feature = "regex1"))]
fn parse_key(key: &str) -> ParseResult<Vec<String>> {
    let mut keys: Vec<String> = vec![];

    match key.split(|c| c=='[' || c==']').next() {
        Some(parent) if !parent.is_empty() =>  {
            match decode_component(parent) {
                Ok(decoded_key) => keys.push(decoded_key),
                Err(err_msg) => return Err(ParseError{ kind: ParseErrorKind::DecodingError, message: err_msg })
            }
        }
        _ => ()
    }

    let mut prev_bracket = None;
    for (idx, ch) in key.char_indices() {
        match ch {
            '[' => prev_bracket = Some(idx),
            ']' => {
                if let Some(prev_idx) = prev_bracket {
                    prev_bracket = None;
                    let child = &key[prev_idx..=idx];
                    match decode_component(child) {
                        Ok(decoded_key) => keys.push(decoded_key),
                        Err(err_msg) => return Err(ParseError{ kind: ParseErrorKind::DecodingError, message: err_msg })
                    }
                }
            }
            _ => (),
        }
    }

    Ok(keys)
}

fn cleanup_key(key: &str) -> &str {
    if key.starts_with("[") && key.ends_with("]") {
        &key[1..(key.len()-1)]
    } else {
        key
    }
}

fn create_idx_merger(idx: u64, obj: Value) -> Value {
    let mut tree = Object::new();
    tree.insert("__idx".to_string(), Value::Number(Number::from(idx)));
    tree.insert("__object".to_string(), obj);
    return Value::Object(tree)
}

fn create_object_with_key(key: String, obj: Value) -> Value {
    let mut tree = Object::new();
    tree.insert(key, obj);
    return Value::Object(tree)
}

fn apply_object(keys: &[String], val: Value) -> Value {

    if keys.len() > 0 {
        let key = keys.get(0).unwrap();
        if key == "[]" {
            let mut new_array = create_array();
            let item = apply_object(&keys[1..], val);
            push_item_to_array(&mut new_array, item);
            return new_array;
        } else {
            let key = cleanup_key(key);
            let array_index = key.parse();

            match array_index {
                Ok(idx) => {
                    let result = apply_object(&keys[1..], val);
                    let item = create_idx_merger(idx, result);
                    return item;
                },
                Err(_) => {
                    return create_object_with_key(key.to_string(), apply_object(&keys[1..], val));
                }
            }
        }

    } else {
        return val;
    }
}

pub fn parse(params: &str) -> ParseResult<Value> {
    let tree = Object::new();
    let mut obj = Value::Object(tree);
    let decoded_params = match decode_component(params) {
        Ok(val) => val,
        Err(err) => return Err(ParseError{ kind: ParseErrorKind::DecodingError, message: err })
    };
    let pairs = parse_pairs(&decoded_params);
    for &(key, value) in pairs.iter() {
        let parse_key_res = parse_key(key)?;
        let key_chain = &parse_key_res[0..];
        let decoded_value = match decode_component(value.unwrap_or("")) {
            Ok(val) => val,
            Err(err) => return Err(ParseError{ kind: ParseErrorKind::DecodingError, message: err })
        };
        let partial = apply_object(key_chain, Value::String(decoded_value));
        merge(&mut obj, &partial);
    }

    Ok(obj)
}

#[cfg(test)]
mod tests {
    use parse;
    use serde_json::{Value, to_string};

    fn eq_str(value: Value, string: &str) {
        assert_eq!(&to_string(&value).unwrap(), string)
    }

    #[test]
    fn it_parses_simple_string() {
        eq_str(parse("0=foo").unwrap(), r#"{"0":"foo"}"#);
        eq_str(parse("a[<=>]==23").unwrap(), r#"{"a":{"<=>":"=23"}}"#);
        eq_str(parse(" foo = bar = baz ").unwrap(), r#"{" foo ":" bar = baz "}"#);
    }

    #[test]
    fn it_parses_nested_string() {
        eq_str(parse("a[b][c][d][e][f][g][h]=i").unwrap(),
            r#"{"a":{"b":{"c":{"d":{"e":{"f":{"g":{"h":"i"}}}}}}}}"#);
    }

    #[test]
    fn it_parses_simple_array() {
        eq_str(parse("a=b&a=c&a=d&a=e").unwrap(),
            r#"{"a":["b","c","d","e"]}"#);
    }

    #[test]
    fn it_parses_explicit_array() {
        eq_str(parse("a[]=b&a[]=c&a[]=d").unwrap(),
            r#"{"a":["b","c","d"]}"#);
    }

    #[test]
    fn it_parses_nested_array() {
        eq_str(parse("a[b][]=c&a[b][]=d").unwrap(),
            r#"{"a":{"b":["c","d"]}}"#);
    }

    #[test]
    fn it_allows_to_specify_array_indexes() {
        eq_str(parse("a[0][]=c&a[1][]=d").unwrap(),
            r#"{"a":[["c"],["d"]]}"#);
    }

    #[test]
    fn it_transforms_arrays_to_object() {
        eq_str(parse("foo[0]=bar&foo[bad]=baz").unwrap(),
            r#"{"foo":{"0":"bar","bad":"baz"}}"#);

        eq_str(parse("foo[0][a]=a&foo[0][b]=b&foo[1][a]=aa&foo[1][b]=bb").unwrap(),
            r#"{"foo":[{"a":"a","b":"b"},{"a":"aa","b":"bb"}]}"#);
    }

    #[test]
    fn it_doesnt_produce_empty_keys() {
        eq_str(parse("_r=1&").unwrap(),
            r#"{"_r":"1"}"#);
    }

    #[test]
    fn it_supports_encoded_strings() {
        eq_str(parse("a[b%20c]=c%20d").unwrap(),
            r#"{"a":{"b c":"c d"}}"#);
    }

    #[test]
    fn it_parses_explicit_encoded_array() {
        eq_str(parse("a%5B%5D=b&a%5B%5D=c&a%5B%5D=d").unwrap(),
            r#"{"a":["b","c","d"]}"#);
    }
}
<|endoftext|>


<commit_before>use pom::parser::*;
use std::char::{decode_utf16, REPLACEMENT_CHARACTER};
use std::collections::HashMap;
use std::iter::FromIterator;
use std::str::{self, FromStr};
use utils::*;

mod utils;

#[derive(Debug, PartialEq)]
pub struct Equation {
    pub left: Operand,
    pub right: Operand,
}

#[derive(Debug, PartialEq)]
pub struct Function {
    pub name: String,
    pub params: Vec<Operand>,
}

#[derive(Debug, PartialEq)]
pub enum Operand {
    Column(Column),
    Function(Function),
    Value(Value),
}

#[derive(Debug, PartialEq)]
pub struct Column {
    name: String,
}

#[derive(Debug, PartialEq)]
pub enum Value {
    Null,
    String(String),
    Number(f64),
    Bool(bool),
}

#[derive(Debug, PartialEq)]
pub enum Connector {
    And,
    Or,
}
#[derive(Debug, PartialEq)]
pub enum Direction {
    Asc,
    Desc,
}

#[derive(Debug, PartialEq)]
pub enum NullsWhere {
    First,
    Last,
}

#[derive(Debug, PartialEq)]
pub struct Order {
    pub operand: Operand,
    pub direction: Option<Direction>,
    pub nulls_where: Option<NullsWhere>,
}

#[derive(Debug, PartialEq)]
pub enum Equality {
    Eq,     // = ,  eq
    Neq,    // != , neq
    Lt,     // <,  lt
    Lte,    // <=, lte
    Gt,     // >, gt
    Gte,    // >=, gte
    In,     // IN, in
    NotIn,  // NOT IN, not_in
    Is,     // IS, is
    IsNot,  // IS NOT, is_not
    Like,   // LIKE, like
    Ilike,  // ILIKE case insensitive like, postgresql specific
    Starts, // Starts with, which will become ILIKE 'value%'
}

#[derive(Debug, PartialEq)]
pub struct Condition {
    pub left: Operand,
    pub equality: Equality,
    pub right: Operand,
}

#[derive(Debug, PartialEq)]
pub struct Filter {
    pub connector: Option<Connector>,
    pub condition: Condition,
    pub sub_filters: Vec<Filter>,
}

#[derive(Debug, PartialEq, Default)]
pub struct Select {
    pub from: Vec<Operand>,
    pub join: Vec<Join>,
    pub filters: Vec<Filter>,
    pub group_by: Vec<Operand>,
    pub having: Vec<Filter>,
    pub order_by: Vec<Order>,
    pub range: Option<Range>,
    pub equations: Vec<Equation>,
}

#[derive(Debug, PartialEq, Default)]
pub struct Page {
    pub page: i64,
    pub page_size: i64,
}

#[derive(Debug, PartialEq, Default)]
pub struct Limit {
    pub limit: i64,
    pub offset: Option<i64>,
}

#[derive(Debug, PartialEq)]
pub enum Range {
    Page(Page),
    Limit(Limit),
}

#[derive(Debug, PartialEq)]
pub enum JoinType {
    Cross,
    Inner,
    Outer,
    Natural,
}
#[derive(Debug, PartialEq)]
pub enum Modifier {
    Left,
    Right,
    Full,
}

#[derive(Debug, PartialEq)]
pub struct Join {
    pub modifier: Option<Modifier>,
    pub join_type: Option<JoinType>,
    pub table: Operand,
    pub column1: Vec<String>,
    pub column2: Vec<String>,
}

#[derive(Debug, PartialEq)]
pub enum Param {
    Condition(Condition),
    Equation(Equation),
}

fn space<'a>() -> Parser<'a, char, ()> {
    one_of(" \t\r\n").repeat(0..).discard()
}

fn ident<'a>() -> Parser<'a, char, String> {
    (is_a(alpha_or_underscore) + is_a(alphanum_or_underscore).repeat(0..))
        .map(|(ch1, rest_ch)| format!("{}{}", ch1, String::from_iter(rest_ch)))
}

/// table.column_name
fn table_column_name<'a>() -> Parser<'a, char, String> {
    (ident() - sym('.') + ident()).map(|(table, column)| format!("{}.{}", table, column)) | ident()
}

fn number<'a>() -> Parser<'a, char, f64> {
    let integer = one_of("123456789") - one_of("0123456789").repeat(0..) | sym('0');
    let frac = sym('.') + one_of("0123456789").repeat(1..);
    let exp = one_of("eE") + one_of("+-").opt() + one_of("0123456789").repeat(1..);
    let number = sym('-').opt() + integer + frac.opt() + exp.opt();
    number
        .collect()
        .map(String::from_iter)
        .convert(|s| f64::from_str(&s))
}

fn quoted_string<'a>() -> Parser<'a, char, String> {
    let special_char = sym('\\')
        | sym('/')
        | sym('"')
        | sym('b').map(|_| '\x08')
        | sym('f').map(|_| '\x0C')
        | sym('n').map(|_| '\n')
        | sym('r').map(|_| '\r')
        | sym('t').map(|_| '\t');
    let escape_sequence = sym('\\') * special_char;
    let char_string = (none_of("\\\"") | escape_sequence)
        .repeat(1..)
        .map(String::from_iter);
    let utf16_char = tag("\\u")
        * is_a(|c: char| c.is_digit(16))
            .repeat(4)
            .map(String::from_iter)
            .convert(|digits| u16::from_str_radix(&digits, 16));
    let utf16_string = utf16_char.repeat(1..).map(|chars| {
        decode_utf16(chars)
            .map(|r| r.unwrap_or(REPLACEMENT_CHARACTER))
            .collect::<String>()
    });
    let string = sym('"') * (char_string | utf16_string).repeat(0..) - sym('"');
    string.map(|strings| strings.concat())
}

fn string<'a>() -> Parser<'a, char, String> {
    let char_string = none_of("=&()").repeat(1..).map(String::from_iter);
    let string = char_string.repeat(0..);
    string.map(|strings| strings.concat())
}

fn column<'a>() -> Parser<'a, char, Column> {
    table_column_name().map(|name| Column { name })
}

fn bool<'a>() -> Parser<'a, char, bool> {
    tag("true").map(|_| true) | tag("false").map(|_| false)
}

fn value<'a>() -> Parser<'a, char, Value> {
    tag("null").map(|_| Value::Null)
        | bool().map(|v| Value::Bool(v))
        | number().map(|n| Value::Number(n))
        | string().map(|s| Value::String(s))
}

fn connector<'a>() -> Parser<'a, char, Connector> {
    sym('|').map(|_| Connector::Or) | sym('&').map(|_| Connector::And)
}

fn equality<'a>() -> Parser<'a, char, Equality> {
    tag("eq").map(|_| Equality::Eq)
        | tag("neq").map(|_| Equality::Neq)
        | tag("lte").map(|_| Equality::Lte)
        | tag("lt").map(|_| Equality::Lt)
        | tag("gte").map(|_| Equality::Gte)
        | tag("gt").map(|_| Equality::Gt)
        | tag("in").map(|_| Equality::In)
        | tag("not_in").map(|_| Equality::NotIn)
        | tag("is_not").map(|_| Equality::IsNot)
        | tag("like").map(|_| Equality::Like)
        | tag("ilike").map(|_| Equality::Ilike)
        | tag("starts").map(|_| Equality::Starts)
}

fn operand<'a>() -> Parser<'a, char, Operand> {
    column().map(Operand::Column) | function().map(Operand::Function) | value().map(Operand::Value)
}

fn operands<'a>() -> Parser<'a, char, Vec<Operand>> {
    list(call(operand), space() - sym(',') - space())
}

fn function<'a>() -> Parser<'a, char, Function> {
    (ident() - sym('(') + operands() - sym(')')).map(|(name, params)| Function { name, params })
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn test_function() {
        let input = to_chars("max(seq_no)");
        let ret = function().parse(&input).expect("must be parsed");
        assert_eq!(
            ret,
            Function {
                name: "max".into(),
                params: vec![Operand::Column(Column {
                    name: "seq_no".into()
                })]
            }
        );
    }

    #[test]
    fn test_column() {
        let input = to_chars("product_id");
        let ret = column().parse(&input).expect("must be parsed");
        assert_eq!(
            ret,
            Column {
                name: "product_id".into()
            }
        );
    }

    #[test]
    fn test_value_bool() {
        let input = to_chars("true");
        let ret = value().parse(&input).expect("must be parsed");
        assert_eq!(ret, Value::Bool(true));
    }
    #[test]
    fn test_value_bool2() {
        let input = to_chars("false");
        let ret = value().parse(&input).expect("must be parsed");
        assert_eq!(ret, Value::Bool(false));
    }
    #[test]
    fn test_value_number() {
        let input = to_chars("0.1312312");
        let ret = value().parse(&input).expect("must be parsed");
        assert_eq!(ret, Value::Number(0.1312312));
    }
    #[test]
    fn test_value_number2() {
        let input = to_chars("3.14159");
        let ret = value().parse(&input).expect("must be parsed");
        assert_eq!(ret, Value::Number(3.14159));
    }

    #[test]
    fn test_string() {
        let input = to_chars("product_id");
        let ret = string().parse(&input).expect("must be parsed");
        assert_eq!(ret, "product_id");
    }

    #[test]
    fn test_iregular_string() {
        let input = to_chars("a string value\"pr'oduct_id");
        let ret = string().parse(&input).expect("must be parsed");
        assert_eq!(ret, "a string value\"pr\'oduct_id");
    }
}
<commit_msg>Implement parser for operand with priority mitigation of values, columns, functions and other values<commit_after>use pom::parser::*;
use std::char::{decode_utf16, REPLACEMENT_CHARACTER};
use std::collections::HashMap;
use std::iter::FromIterator;
use std::str::{self, FromStr};
use utils::*;

mod utils;

#[derive(Debug, PartialEq)]
pub struct Equation {
    pub left: Operand,
    pub right: Operand,
}

#[derive(Debug, PartialEq)]
pub struct Function {
    pub name: String,
    pub params: Vec<Operand>,
}

#[derive(Debug, PartialEq)]
pub enum Operand {
    Column(Column),
    Function(Function),
    Value(Value),
}

#[derive(Debug, PartialEq)]
pub struct Column {
    name: String,
}

#[derive(Debug, PartialEq)]
pub enum Value {
    Null,
    String(String),
    Number(f64),
    Bool(bool),
}

#[derive(Debug, PartialEq)]
pub enum Connector {
    And,
    Or,
}
#[derive(Debug, PartialEq)]
pub enum Direction {
    Asc,
    Desc,
}

#[derive(Debug, PartialEq)]
pub enum NullsWhere {
    First,
    Last,
}

#[derive(Debug, PartialEq)]
pub struct Order {
    pub operand: Operand,
    pub direction: Option<Direction>,
    pub nulls_where: Option<NullsWhere>,
}

#[derive(Debug, PartialEq)]
pub enum Equality {
    Eq,     // = ,  eq
    Neq,    // != , neq
    Lt,     // <,  lt
    Lte,    // <=, lte
    Gt,     // >, gt
    Gte,    // >=, gte
    In,     // IN, in
    NotIn,  // NOT IN, not_in
    Is,     // IS, is
    IsNot,  // IS NOT, is_not
    Like,   // LIKE, like
    Ilike,  // ILIKE case insensitive like, postgresql specific
    Starts, // Starts with, which will become ILIKE 'value%'
}

#[derive(Debug, PartialEq)]
pub struct Condition {
    pub left: Operand,
    pub equality: Equality,
    pub right: Operand,
}

#[derive(Debug, PartialEq)]
pub struct Filter {
    pub connector: Option<Connector>,
    pub condition: Condition,
    pub sub_filters: Vec<Filter>,
}

#[derive(Debug, PartialEq, Default)]
pub struct Select {
    pub from: Vec<Operand>,
    pub join: Vec<Join>,
    pub filters: Vec<Filter>,
    pub group_by: Vec<Operand>,
    pub having: Vec<Filter>,
    pub order_by: Vec<Order>,
    pub range: Option<Range>,
    pub equations: Vec<Equation>,
}

#[derive(Debug, PartialEq, Default)]
pub struct Page {
    pub page: i64,
    pub page_size: i64,
}

#[derive(Debug, PartialEq, Default)]
pub struct Limit {
    pub limit: i64,
    pub offset: Option<i64>,
}

#[derive(Debug, PartialEq)]
pub enum Range {
    Page(Page),
    Limit(Limit),
}

#[derive(Debug, PartialEq)]
pub enum JoinType {
    Cross,
    Inner,
    Outer,
    Natural,
}
#[derive(Debug, PartialEq)]
pub enum Modifier {
    Left,
    Right,
    Full,
}

#[derive(Debug, PartialEq)]
pub struct Join {
    pub modifier: Option<Modifier>,
    pub join_type: Option<JoinType>,
    pub table: Operand,
    pub column1: Vec<String>,
    pub column2: Vec<String>,
}

#[derive(Debug, PartialEq)]
pub enum Param {
    Condition(Condition),
    Equation(Equation),
}

fn space<'a>() -> Parser<'a, char, ()> {
    one_of(" \t\r\n").repeat(0..).discard()
}

fn ident<'a>() -> Parser<'a, char, String> {
    (is_a(alpha_or_underscore) + is_a(alphanum_or_underscore).repeat(0..))
        .map(|(ch1, rest_ch)| format!("{}{}", ch1, String::from_iter(rest_ch)))
}

/// table.column_name
fn table_column_name<'a>() -> Parser<'a, char, String> {
    (ident() - sym('.') + ident()).map(|(table, column)| format!("{}.{}", table, column)) | ident()
}

fn number<'a>() -> Parser<'a, char, f64> {
    let integer = one_of("123456789") - one_of("0123456789").repeat(0..) | sym('0');
    let frac = sym('.') + one_of("0123456789").repeat(1..);
    let exp = one_of("eE") + one_of("+-").opt() + one_of("0123456789").repeat(1..);
    let number = sym('-').opt() + integer + frac.opt() + exp.opt();
    number
        .collect()
        .map(String::from_iter)
        .convert(|s| f64::from_str(&s))
}

fn quoted_string<'a>() -> Parser<'a, char, String> {
    let special_char = sym('\\')
        | sym('/')
        | sym('"')
        | sym('b').map(|_| '\x08')
        | sym('f').map(|_| '\x0C')
        | sym('n').map(|_| '\n')
        | sym('r').map(|_| '\r')
        | sym('t').map(|_| '\t');
    let escape_sequence = sym('\\') * special_char;
    let char_string = (none_of("\\\"") | escape_sequence)
        .repeat(1..)
        .map(String::from_iter);
    let utf16_char = tag("\\u")
        * is_a(|c: char| c.is_digit(16))
            .repeat(4)
            .map(String::from_iter)
            .convert(|digits| u16::from_str_radix(&digits, 16));
    let utf16_string = utf16_char.repeat(1..).map(|chars| {
        decode_utf16(chars)
            .map(|r| r.unwrap_or(REPLACEMENT_CHARACTER))
            .collect::<String>()
    });
    let string = sym('"') * (char_string | utf16_string).repeat(0..) - sym('"');
    string.map(|strings| strings.concat())
}

fn string<'a>() -> Parser<'a, char, String> {
    let char_string = none_of("=&()").repeat(1..).map(String::from_iter);
    let string = char_string.repeat(0..);
    string.map(|strings| strings.concat())
}

fn column<'a>() -> Parser<'a, char, Column> {
    table_column_name().map(|name| Column { name })
}

fn bool<'a>() -> Parser<'a, char, bool> {
    tag("true").map(|_| true) | tag("false").map(|_| false)
}

fn null<'a>() -> Parser<'a, char, Value> {
    tag("null").map(|_| Value::Null)
}

fn value<'a>() -> Parser<'a, char, Value> {
    null()
        | bool().map(|v| Value::Bool(v))
        | number().map(|n| Value::Number(n))
        | string().map(|s| Value::String(s))
}

fn connector<'a>() -> Parser<'a, char, Connector> {
    sym('|').map(|_| Connector::Or) | sym('&').map(|_| Connector::And)
}

fn equality<'a>() -> Parser<'a, char, Equality> {
    tag("eq").map(|_| Equality::Eq)
        | tag("neq").map(|_| Equality::Neq)
        | tag("lte").map(|_| Equality::Lte)
        | tag("lt").map(|_| Equality::Lt)
        | tag("gte").map(|_| Equality::Gte)
        | tag("gt").map(|_| Equality::Gt)
        | tag("in").map(|_| Equality::In)
        | tag("not_in").map(|_| Equality::NotIn)
        | tag("is_not").map(|_| Equality::IsNot)
        | tag("like").map(|_| Equality::Like)
        | tag("ilike").map(|_| Equality::Ilike)
        | tag("starts").map(|_| Equality::Starts)
}

fn operand<'a>() -> Parser<'a, char, Operand> {
    null().map(Operand::Value)
        | bool().map(|v| Operand::Value(Value::Bool(v)))
        | number().map(|v| Operand::Value(Value::Number(v)))
        | column().map(Operand::Column)
        | function().map(Operand::Function)
        | value().map(Operand::Value)
}

fn operands<'a>() -> Parser<'a, char, Vec<Operand>> {
    list(call(operand), space() - sym(',') - space())
}

fn function<'a>() -> Parser<'a, char, Function> {
    (ident() - sym('(') + operands() - sym(')')).map(|(name, params)| Function { name, params })
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn test_function() {
        let input = to_chars("max(seq_no)");
        let ret = function().parse(&input).expect("must be parsed");
        assert_eq!(
            ret,
            Function {
                name: "max".into(),
                params: vec![Operand::Column(Column {
                    name: "seq_no".into()
                })]
            }
        );
    }

    #[test]
    fn test_column() {
        let input = to_chars("product_id");
        let ret = column().parse(&input).expect("must be parsed");
        assert_eq!(
            ret,
            Column {
                name: "product_id".into()
            }
        );
    }

    #[test]
    fn test_value_bool() {
        let input = to_chars("true");
        let ret = value().parse(&input).expect("must be parsed");
        assert_eq!(ret, Value::Bool(true));
    }
    #[test]
    fn test_value_bool2() {
        let input = to_chars("false");
        let ret = value().parse(&input).expect("must be parsed");
        assert_eq!(ret, Value::Bool(false));
    }
    #[test]
    fn test_value_number() {
        let input = to_chars("0.1312312");
        let ret = value().parse(&input).expect("must be parsed");
        assert_eq!(ret, Value::Number(0.1312312));
    }
    #[test]
    fn test_value_number2() {
        let input = to_chars("3.14159");
        let ret = value().parse(&input).expect("must be parsed");
        assert_eq!(ret, Value::Number(3.14159));
    }

    #[test]
    fn test_string() {
        let input = to_chars("product_id");
        let ret = string().parse(&input).expect("must be parsed");
        assert_eq!(ret, "product_id");
    }

    #[test]
    fn test_iregular_string() {
        let input = to_chars("a string value\"pr'oduct_id");
        let ret = string().parse(&input).expect("must be parsed");
        assert_eq!(ret, "a string value\"pr\'oduct_id");
    }

    #[test]
    fn test_bool_operand() {
        let input = to_chars("true");
        let ret = operand().parse(&input).expect("must be parsed");
        assert_eq!(ret, Operand::Value(Value::Bool(true)));
    }
    #[test]
    fn test_null_operand() {
        let input = to_chars("null");
        let ret = operand().parse(&input).expect("must be parsed");
        assert_eq!(ret, Operand::Value(Value::Null));
    }
}
<|endoftext|>


<commit_before>#![allow(dead_code)]

use term::*;
use term::Term::*;
use self::Token::*;
use self::Error::*;

#[derive(Debug, PartialEq)]
enum Error {
    InvalidCharacter((usize, char))
}

#[derive(Debug, PartialEq, Clone, Copy)]
enum Token {
    Lambda,
    Lparen,
    Rparen,
    Index(usize)
}

#[derive(Debug)]
struct Lexer {
    tokens: Vec<Token>,
    position: usize
}

impl Lexer {
    fn new() -> Lexer {
        Lexer { tokens: Vec::new(), position: 0 }
    }

    fn tokenize(&mut self, input: &str) -> Result<(), Error> {
        let mut chars = input.chars();

        while let Some(c) = chars.next() {
            match c {
                'λ' => { self.tokens.push(Lambda) },
                '(' => { self.tokens.push(Lparen) },
                ')' => { self.tokens.push(Rparen) },
                 x  => {
                    if let Some(i) = x.to_digit(16) {
                        self.tokens.push(Index(i as usize))
                    } else if x.is_whitespace() {
                        ()
                    } else {
                        return Err(InvalidCharacter((self.position, x)))
                    }
                }
            }
            self.position += if c == 'λ' { 2 } else { 1 };
        }

        Ok(())
    }
}

#[derive(Debug)]
struct Parser {
    output: Vec<Term>,
    stack: Vec<Token>,
    temp: Vec<Term>
}

impl Parser {
    fn new() -> Parser {
        Parser { output: Vec::new(), stack: Vec::new(), temp: Vec::new() }
    }

    fn process(&mut self, tokens: &[Token]) -> Result<Term, Error> {
        let mut iter = tokens.iter();

        while let Some(&token) = iter.next() {
            println!("stack: {:?}; output: {:?}; temp: {:?}", self.stack, self.output, self.temp);
            match token {
                Lambda => {
                    self.stack.push(token)
                },
                Lparen => {
                    while !self.temp.is_empty() {
                        self.output.push(self.temp.pop().unwrap())
                    }
                    self.stack.push(token)
                },
                Rparen => {
                    if !self.temp.is_empty() {

                        let mut ret = fold_terms(self.temp.clone());
                        while self.stack.last() == Some(&Lambda) {
                            ret = abs(ret);
                        }
                        self.output.push(ret);

//                        self.output.push(fold_terms(self.temp.clone())); // TODO: some swap or other smartery
                        self.temp.clear();
                    }
                    self.stack.pop(); // drop Lparen; TODO: check if Lparen
                },
                Index(i) => {
                    if self.stack.contains(&Lparen) {
                        self.temp.push(Var(i))
                    } else {
                        self.output.push(Var(i))
                    }
                }
            }
        }

        let mut ret = fold_terms(self.output.clone());

        while let Some(Lambda) = self.stack.pop() {
            ret = abs(ret);
        }

        println!("{:?}", self);

        Ok(ret)
    }

    fn parse(&mut self, string: &str) -> Result<Term, Error> {
        let mut lexer = Lexer::new();
        try!(lexer.tokenize(string));
        let mut parser = Parser::new();
        parser.process(&lexer.tokens)
    }
}

fn fold_terms(mut terms: Vec<Term>) -> Term {
//    println!("folding terms {:?}", terms);
    if terms.len() > 1 {
        terms.reverse();
        let fst = terms.pop().unwrap();
        terms.reverse();
        terms.into_iter().fold(fst, |acc, t| app(acc, t))
    } else {
        terms.pop().unwrap()
    }
}
/*
pub fn split_parenthesized(input: &str) -> (&str, &str) {
    let mut pos = 1;
    let mut depth = 1;
    let mut chars = input.chars().skip(1);

    while let Some(c) = chars.next() {
        if depth == 0 {
            break;
        } else if c == '(' {
            depth += 1;
        } else if c == ')' {
            depth -= 1;
        }
        if c == 'λ' { pos += 2 } else { pos += 1 }
    }
//    println!("input has a parens \"{}\" and rest \"{}\"", &input[1..pos-1], &input[pos..]);
    (&input[1..pos-1], &input[pos..])
}
*/
#[cfg(test)]
mod test {
    use super::*;

    #[test]
    fn tokenization_error() {
        let fail = "λλx2";
        let mut lexer = Lexer::new();

        assert_eq!(lexer.tokenize(&fail), Err(InvalidCharacter((4, 'x'))))
    }

    #[test]
    fn tokenization_success() {
        let quine = "λ 1 ( (λ 1 1) (λ λ λ λ λ 1 4 (3 (5 5) 2) ) ) 1";
        let mut lexer = Lexer::new();

        assert!(lexer.tokenize(&quine).is_ok());
        assert_eq!(lexer.tokens, vec![Lambda, Index(1), Lparen, Lparen, Lambda, Index(1), Index(1), Rparen, Lparen, Lambda,
                                      Lambda, Lambda, Lambda, Lambda, Index(1), Index(4), Lparen, Index(3), Lparen, Index(5),
                                      Index(5), Rparen, Index(2), Rparen, Rparen, Rparen, Index(1)]
        )
    }
/*
    #[test]
    fn parse_succ() {
        let succ = "λλλ2(321)";
        let mut parser = Parser::new();

        assert_eq!(&*format!("{}", parser.parse(&succ).unwrap()), "λλλ2(321)");
    }
*/
    #[test]
    fn parse_y() {
        let y = "λ(λ2(11))(λ2(11))";
        let mut parser = Parser::new();

        assert_eq!(&*format!("{}", parser.parse(&y).unwrap()), "λ(λ2(11))(λ2(11))");
    }
}
<commit_msg>Succ and y are parsed successfully<commit_after>#![allow(dead_code)]

use term::*;
use term::Term::*;
use self::Token::*;
use self::Error::*;

#[derive(Debug, PartialEq)]
enum Error {
    InvalidCharacter((usize, char)),
    UnmatchedParentheses
}

#[derive(Debug, PartialEq, Clone, Copy)]
enum Token {
    Lambda,
    Lparen,
    Rparen,
    Index(usize)
}

#[derive(Debug)]
struct Lexer {
    tokens: Vec<Token>,
    position: usize
}

impl Lexer {
    fn new() -> Lexer {
        Lexer { tokens: Vec::new(), position: 0 }
    }

    fn tokenize(&mut self, input: &str) -> Result<(), Error> {
        let mut chars = input.chars();

        while let Some(c) = chars.next() {
            match c {
                'λ' => { self.tokens.push(Lambda) },
                '(' => { self.tokens.push(Lparen) },
                ')' => { self.tokens.push(Rparen) },
                 x  => {
                    if let Some(i) = x.to_digit(16) {
                        self.tokens.push(Index(i as usize))
                    } else if x.is_whitespace() {
                        ()
                    } else {
                        return Err(InvalidCharacter((self.position, x)))
                    }
                }
            }
            self.position += if c == 'λ' { 2 } else { 1 };
        }

        Ok(())
    }
}

#[derive(Debug)]
struct Parser {
    output: Vec<Term>,
    stack: Vec<Token>,
    temp: Vec<Term>
}

impl Parser {
    fn new() -> Parser {
        Parser { output: Vec::new(), stack: Vec::new(), temp: Vec::new() }
    }

    fn process(&mut self, tokens: &[Token]) -> Result<Term, Error> {
        let mut iter = tokens.iter();

        while let Some(&token) = iter.next() {
            match token {
                Lambda => {
                    self.stack.push(token)
                },
                Lparen => {
                    while !self.temp.is_empty() {
                        self.output.push(self.temp.pop().unwrap())
                    }
                    self.stack.push(token)
                },
                Rparen => {
                    if !self.temp.is_empty() {
                        let ret = if !self.output.is_empty() {
                            fold_terms(vec![self.output.pop().unwrap(), fold_terms(self.temp.clone())])
                        } else {
                            fold_terms(self.temp.clone())
                        };
                        self.output.push(ret); // TODO: some swap or other smartery
                        self.temp.clear();
                    }

                    if self.stack.last() == Some(&Lambda) {
                        let mut ret = self.output.pop().unwrap();
                        while self.stack.last() == Some(&Lambda) {
                            ret = abs(ret);
                            self.stack.pop();
                        }
                        self.output.push(ret);
                    }

                    if self.stack.last() == Some(&Lparen) {
                        self.stack.pop(); // drop matching Lparen
                    } else {
                        return Err(UnmatchedParentheses)
                    }
                },
                Index(i) => {
                    if self.stack.contains(&Lparen) {
                        self.temp.push(Var(i))
                    } else {
                        self.output.push(Var(i))
                    }
                }
            }
            println!("s: {:?}; o: {:?}; t: {:?}", self.stack, self.output, self.temp);
        }

        let mut ret = fold_terms(self.output.clone());

        while let Some(Lambda) = self.stack.pop() {
            ret = abs(ret);
        }

//        println!("{:?}", self);

        Ok(ret)
    }

    fn parse(&mut self, string: &str) -> Result<Term, Error> {
        let mut lexer = Lexer::new();
        try!(lexer.tokenize(string));
        let mut parser = Parser::new();
        parser.process(&lexer.tokens)
    }
}

fn fold_terms(mut terms: Vec<Term>) -> Term {
//    println!("folding terms {:?}", terms);
    if terms.len() > 1 {
        terms.reverse();
        let fst = terms.pop().unwrap();
        terms.reverse();
        terms.into_iter().fold(fst, |acc, t| app(acc, t))
    } else {
        terms.pop().unwrap()
    }
}

#[cfg(test)]
mod test {
    use super::*;

    #[test]
    fn tokenization_error() {
        let fail = "λλx2";
        let mut lexer = Lexer::new();

        assert_eq!(lexer.tokenize(&fail), Err(InvalidCharacter((4, 'x'))))
    }

    #[test]
    fn tokenization_success() {
        let quine = "λ 1 ( (λ 1 1) (λ λ λ λ λ 1 4 (3 (5 5) 2) ) ) 1";
        let mut lexer = Lexer::new();

        assert!(lexer.tokenize(&quine).is_ok());
        assert_eq!(lexer.tokens, vec![Lambda, Index(1), Lparen, Lparen, Lambda, Index(1), Index(1), Rparen, Lparen, Lambda,
                                      Lambda, Lambda, Lambda, Lambda, Index(1), Index(4), Lparen, Index(3), Lparen, Index(5),
                                      Index(5), Rparen, Index(2), Rparen, Rparen, Rparen, Index(1)]
        )
    }

    #[test]
    fn parse_succ() {
        let succ = "λλλ2(321)";
        let mut parser = Parser::new();

        assert_eq!(&*format!("{}", parser.parse(&succ).unwrap()), succ);
    }

    #[test]
    fn parse_y() {
        let y = "λ(λ2(11))(λ2(11))";
        let mut parser = Parser::new();

        assert_eq!(&*format!("{}", parser.parse(&y).unwrap()), y);
    }

    #[test]
    fn parse_quine() {
        let quine = "λ1((λ11)(λλλλλ14(3(55)2)))1";
        let mut parser = Parser::new();

        assert_eq!(&*format!("{}", parser.parse(&quine).unwrap()), quine);
    }
/*
    #[test]
    fn parse_blc() {
        let blc = "(λ11)(λλλ1(λλλλ3(λ5(3(λ2(3(λλ3(λ123)))(4(λ4(λ31(21))))))(1(2(λ12))(λ4(λ4(λ2(14)))5))))(33)2)(λ1((λ11)(λ11)))";
        let mut parser = Parser::new();

        assert_eq!(&*format!("{}", parser.parse(&blc).unwrap()), blc);
    }
    */
}
<|endoftext|>