<commit_before>/// instead of writing functions which resemble
/// ```
/// pub fn query<'a> (&'a mut self,query: String) -> &'a mut Self{
///     self.query = Some(query);
///            self
/// }
/// ```
/// and repeating it for all the attributes; it is extracted out as a macro so that code
/// is more concise see
/// @https://doc.rust-lang.org/book/method-syntax.html
///
///
///
macro_rules! builder_opt_field {
    ($field:ident, $field_type:ty) => {
        pub fn $field(mut self,
                          $field: $field_type) -> Self {
            self.$field = Some($field);
            self
        }
    };
}

macro_rules! list_as_rust {
    ($($into_type:tt)*) => (
        impl AsRustType<Vec<$($into_type)*>> for List {
            fn as_rust_type(&self) -> Result<Vec<$($into_type)*>> {
                match self.metadata.value {
                    Some(ColTypeOptionValue::CList(ref type_option)) |
                    Some(ColTypeOptionValue::CSet(ref type_option)) => {
                        let type_option_ref = type_option.as_ref();
                        let convert = self
                            .map(|bytes| as_rust_type!(type_option_ref, bytes, $($into_type)*).unwrap());

                        Ok(convert)
                    },
                    _ => Err(Error::General(format!("Invalid conversion. \
                            Cannot convert {:?} into List (valid types: List, Set).",
                            self.metadata.value)))
                }
            }
        }
    );
}

macro_rules! map_as_rust {
    ($(K $key_type:tt)*, $(V $val_type:tt)*) => (
        impl AsRustType<HashMap<$($key_type)*, $($val_type)*>> for Map {
            /// Converts `Map` into `HashMap` for blob values.
            fn as_rust_type(&self) -> Result<HashMap<$($key_type)*, $($val_type)*>> {
                match self.metadata.value {
                    Some(ColTypeOptionValue::CMap((ref key_type_option, ref val_type_option))) => {
                        let mut map = HashMap::with_capacity(self.data.len());

                        for &(ref key, ref val) in self.data.iter() {
                            let key_type_option = key_type_option.as_ref();
                            let val_type_option = val_type_option.as_ref();
                            let key = as_rust_type!(key_type_option, key, $($key_type)*)?;
                            let val = as_rust_type!(val_type_option, val, $($val_type)*)?;
                            map.insert(key, val);
                        }

                        Ok(map)
                    }
                    _ => unreachable!()
                }
            }
        }
    );
}

macro_rules! into_rust_by_name {
    (Row, $($into_type:tt)*) => (
        impl IntoRustByName<$($into_type)*> for Row {
            fn get_by_name(&self, name: &str) -> Option<Result<$($into_type)*>> {
                self.get_col_spec_by_name(name)
                    .map(|(col_spec, cbytes)| {
                        if cbytes.is_empty() {
                            return Err(column_is_empty_err());
                        }

                        let ref col_type = col_spec.col_type;
                        as_rust_type!(col_type, cbytes, $($into_type)*)
                    })
            }
        }
    );

    (UDT, $($into_type:tt)*) => (
        impl IntoRustByName<$($into_type)*> for UDT {
            fn get_by_name(&self, name: &str) -> Option<Result<$($into_type)*>> {
                self.data.get(name).map(|v| {
                    let &(ref col_type, ref bytes) = v;

                    if bytes.as_plain().is_empty() {
                        return Err(column_is_empty_err());
                    }

                    let converted = as_rust_type!(col_type, bytes, $($into_type)*);
                    converted.map_err(|err| err.into())
                })
            }
        }
    );
}

/// Decodes any Cassandra data type into the corresponding Rust type,
/// given the column type as `ColTypeOption` and the value as `CBytes`
/// plus the matching Rust type.
macro_rules! as_rust_type {
    ($data_type_option:ident, $data_value:ident, Vec<u8>) => (
        match $data_type_option.id {
            ColType::Blob => {
                decode_blob($data_value.as_plain())
                    .map_err(Into::into)
            }
            _ => Err(Error::General(format!("Invalid conversion. \
                    Cannot convert {:?} into Vec<u8> (valid types: Blob).",
                    $data_type_option.id)))
        }
    );
    ($data_type_option:ident, $data_value:ident, String) => (
        match $data_type_option.id {
            ColType::Custom => {
                decode_custom($data_value.as_slice())
                    .map_err(Into::into)
            }
            ColType::Ascii => {
                decode_ascii($data_value.as_slice())
                    .map_err(Into::into)
            }
            ColType::Varchar => {
                decode_varchar($data_value.as_slice())
                    .map_err(Into::into)
            }
            // TODO: clarify when to use decode_text.
            // it's not mentioned in
            // https://github.com/apache/cassandra/blob/trunk/doc/native_protocol_v4.spec#L582
            // ColType::XXX => decode_text($data_value)?
            _ => Err(Error::General(format!("Invalid conversion. \
                    Cannot convert {:?} into String (valid types: Custom, Ascii, Varchar).",
                    $data_type_option.id)))
        }
    );
    ($data_type_option:ident, $data_value:ident, bool) => (
        match $data_type_option.id {
            ColType::Boolean => {
                decode_boolean($data_value.as_slice())
                    .map_err(Into::into)
            }
            _ => Err(Error::General(format!("Invalid conversion. \
                    Cannot convert {:?} into bool (valid types: Boolean).",
                    $data_type_option.id)))
        }
    );
    ($data_type_option:ident, $data_value:ident, i64) => (
        match $data_type_option.id {
            ColType::Bigint => {
                decode_bigint($data_value.as_slice())
                    .map_err(Into::into)
            }
            ColType::Timestamp => {
                decode_timestamp($data_value.as_slice())
                    .map_err(Into::into)
            }
            ColType::Time => {
                decode_time($data_value.as_slice())
                    .map_err(Into::into)
            }
            ColType::Varint => {
                decode_varint($data_value.as_slice())
                    .map_err(Into::into)
            }
            _ => Err(Error::General(format!("Invalid conversion. \
                    Cannot convert {:?} into i64 (valid types: Bigint, Timestamp, Time, Variant).",
                    $data_type_option.id)))
        }
    );
    ($data_type_option:ident, $data_value:ident, i32) => (
        match $data_type_option.id {
            ColType::Int => {
                decode_int($data_value.as_slice())
                    .map_err(Into::into)
            }
            ColType::Date => {
                decode_date($data_value.as_slice())
                    .map_err(Into::into)
            }
            _ => Err(Error::General(format!("Invalid conversion. \
                    Cannot convert {:?} into i32 (valid types: Int, Date).",
                    $data_type_option.id)))
        }
    );
    ($data_type_option:ident, $data_value:ident, i16) => (
        match $data_type_option.id {
            ColType::Smallint => {
                decode_smallint($data_value.as_slice())
                    .map_err(Into::into)
            }
            _ => Err(Error::General(format!("Invalid conversion. \
                    Cannot convert {:?} into i16 (valid types: Smallint).",
                    $data_type_option.id)))
        }
    );
    ($data_type_option:ident, $data_value:ident, i8) => (
        match $data_type_option.id {
            ColType::Tinyint => {
                decode_tinyint($data_value.as_slice())
                    .map_err(Into::into)
            }
            _ => Err(Error::General(format!("Invalid conversion. \
                    Cannot convert {:?} into i8 (valid types: Tinyint).",
                    $data_type_option.id)))
        }
    );
    ($data_type_option:ident, $data_value:ident, f64) => (
        match $data_type_option.id {
            ColType::Double => {
                decode_double($data_value.as_slice())
                    .map_err(Into::into)
            }
            _ => Err(Error::General(format!("Invalid conversion. \
                    Cannot convert {:?} into f64 (valid types: Double).",
                    $data_type_option.id)))
        }
    );
    ($data_type_option:ident, $data_value:ident, f32) => (
        match $data_type_option.id {
            ColType::Decimal => {
                decode_decimal($data_value.as_slice())
                    .map_err(Into::into)
            }
            ColType::Float => {
                decode_float($data_value.as_slice())
                    .map_err(Into::into)
            }
            _ => Err(Error::General(format!("Invalid conversion. \
                    Cannot convert {:?} into f32 (valid types: Decimal, Float).",
                    $data_type_option.id)))
        }
    );
    ($data_type_option:ident, $data_value:ident, IpAddr) => (
        match $data_type_option.id {
            ColType::Inet => {
                decode_inet($data_value.as_slice())
                    .map_err(Into::into)
            }
            _ => Err(Error::General(format!("Invalid conversion. \
                    Cannot convert {:?} into IpAddr (valid types: Inet).",
                    $data_type_option.id)))
        }
    );
    ($data_type_option:ident, $data_value:ident, Uuid) => (
        match $data_type_option.id {
            ColType::Uuid |
            ColType::Timeuuid => {
                decode_timeuuid($data_value.as_slice())
                    .map_err(Into::into)
            }
            _ => Err(Error::General(format!("Invalid conversion. \
                    Cannot convert {:?} into Uuid (valid types: Uuid, Timeuuid).",
                    $data_type_option.id)))
        }
    );
    ($data_type_option:ident, $data_value:ident, List) => (
        match $data_type_option.id {
            ColType::List |
            ColType::Set => {
                decode_list($data_value.as_slice())
                    .map(|data| List::new(data, $data_type_option.clone()))
                    .map_err(Into::into)
            }
            _ => Err(Error::General(format!("Invalid conversion. \
                    Cannot convert {:?} into List (valid types: List, Set).",
                    $data_type_option.id)))
        }
    );
    ($data_type_option:ident, $data_value:ident, Map) => (
        match $data_type_option.id {
            ColType::Map => {
                decode_map($data_value.as_slice())
                    .map(|data| Map::new(data, $data_type_option.clone()))
                    .map_err(Into::into)
            }
            _ => Err(Error::General(format!("Invalid conversion. \
                    Cannot convert {:?} into Map (valid types: Map).",
                    $data_type_option.id)))
        }
    );
    ($data_type_option:ident, $data_value:ident, UDT) => (
        match *$data_type_option {
            ColTypeOption {
                id: ColType::Udt,
                value: Some(ColTypeOptionValue::UdtType(ref list_type_option))
            } => {
                decode_udt($data_value.as_slice(), list_type_option.descriptions.len())
                    .map(|data| UDT::new(data, list_type_option))
                    .map_err(Into::into)
            }
            _ => Err(Error::General(format!("Invalid conversion. \
                    Cannot convert {:?} into UDT (valid types: UDT).",
                    $data_type_option.id)))
        }
    );
    ($data_type_option:ident, $data_value:ident, Timespec) => (
        match $data_type_option.id {
            ColType::Timestamp => {
                decode_timestamp($data_value.as_slice())
                    .map(|ts| Timespec::new(ts / 1_000, (ts % 1_000 * 1_000_000) as i32))
                    .map_err(Into::into)
            }
            _ => Err(Error::General(format!("Invalid conversion. \
                    Cannot convert {:?} into Timespec (valid types: Timestamp).",
                    $data_type_option.id)))
        }
    );
}
<commit_msg>Fix rustfmt<commit_after>/// instead of writing functions which resemble
/// ```
/// pub fn query<'a> (&'a mut self,query: String) -> &'a mut Self{
///     self.query = Some(query);
///            self
/// }
/// ```
/// and repeating it for all the attributes; it is extracted out as a macro so that code
/// is more concise see
/// @https://doc.rust-lang.org/book/method-syntax.html
///
///
///
macro_rules! builder_opt_field {
    ($field:ident, $field_type:ty) => {
        pub fn $field(mut self,
                          $field: $field_type) -> Self {
            self.$field = Some($field);
            self
        }
    };
}

macro_rules! list_as_rust {
    ($($into_type:tt)*) => (
        impl AsRustType<Vec<$($into_type)*>> for List {
            fn as_rust_type(&self) -> Result<Vec<$($into_type)*>> {
                match self.metadata.value {
                    Some(ColTypeOptionValue::CList(ref type_option)) |
                    Some(ColTypeOptionValue::CSet(ref type_option)) => {
                        let type_option_ref = type_option.as_ref();
                        let convert = self
                            .map(|bytes| as_rust_type!(type_option_ref, bytes, $($into_type)*)
                                .unwrap());

                        Ok(convert)
                    },
                    _ => Err(Error::General(format!("Invalid conversion. \
                            Cannot convert {:?} into List (valid types: List, Set).",
                            self.metadata.value)))
                }
            }
        }
    );
}

macro_rules! map_as_rust {
    ($(K $key_type:tt)*, $(V $val_type:tt)*) => (
        impl AsRustType<HashMap<$($key_type)*, $($val_type)*>> for Map {
            /// Converts `Map` into `HashMap` for blob values.
            fn as_rust_type(&self) -> Result<HashMap<$($key_type)*, $($val_type)*>> {
                match self.metadata.value {
                    Some(ColTypeOptionValue::CMap((ref key_type_option, ref val_type_option))) => {
                        let mut map = HashMap::with_capacity(self.data.len());

                        for &(ref key, ref val) in self.data.iter() {
                            let key_type_option = key_type_option.as_ref();
                            let val_type_option = val_type_option.as_ref();
                            let key = as_rust_type!(key_type_option, key, $($key_type)*)?;
                            let val = as_rust_type!(val_type_option, val, $($val_type)*)?;
                            map.insert(key, val);
                        }

                        Ok(map)
                    }
                    _ => unreachable!()
                }
            }
        }
    );
}

macro_rules! into_rust_by_name {
    (Row, $($into_type:tt)*) => (
        impl IntoRustByName<$($into_type)*> for Row {
            fn get_by_name(&self, name: &str) -> Option<Result<$($into_type)*>> {
                self.get_col_spec_by_name(name)
                    .map(|(col_spec, cbytes)| {
                        if cbytes.is_empty() {
                            return Err(column_is_empty_err());
                        }

                        let ref col_type = col_spec.col_type;
                        as_rust_type!(col_type, cbytes, $($into_type)*)
                    })
            }
        }
    );

    (UDT, $($into_type:tt)*) => (
        impl IntoRustByName<$($into_type)*> for UDT {
            fn get_by_name(&self, name: &str) -> Option<Result<$($into_type)*>> {
                self.data.get(name).map(|v| {
                    let &(ref col_type, ref bytes) = v;

                    if bytes.as_plain().is_empty() {
                        return Err(column_is_empty_err());
                    }

                    let converted = as_rust_type!(col_type, bytes, $($into_type)*);
                    converted.map_err(|err| err.into())
                })
            }
        }
    );
}

/// Decodes any Cassandra data type into the corresponding Rust type,
/// given the column type as `ColTypeOption` and the value as `CBytes`
/// plus the matching Rust type.
macro_rules! as_rust_type {
    ($data_type_option:ident, $data_value:ident, Vec<u8>) => (
        match $data_type_option.id {
            ColType::Blob => {
                decode_blob($data_value.as_plain())
                    .map_err(Into::into)
            }
            _ => Err(Error::General(format!("Invalid conversion. \
                    Cannot convert {:?} into Vec<u8> (valid types: Blob).",
                    $data_type_option.id)))
        }
    );
    ($data_type_option:ident, $data_value:ident, String) => (
        match $data_type_option.id {
            ColType::Custom => {
                decode_custom($data_value.as_slice())
                    .map_err(Into::into)
            }
            ColType::Ascii => {
                decode_ascii($data_value.as_slice())
                    .map_err(Into::into)
            }
            ColType::Varchar => {
                decode_varchar($data_value.as_slice())
                    .map_err(Into::into)
            }
            // TODO: clarify when to use decode_text.
            // it's not mentioned in
            // https://github.com/apache/cassandra/blob/trunk/doc/native_protocol_v4.spec#L582
            // ColType::XXX => decode_text($data_value)?
            _ => Err(Error::General(format!("Invalid conversion. \
                    Cannot convert {:?} into String (valid types: Custom, Ascii, Varchar).",
                    $data_type_option.id)))
        }
    );
    ($data_type_option:ident, $data_value:ident, bool) => (
        match $data_type_option.id {
            ColType::Boolean => {
                decode_boolean($data_value.as_slice())
                    .map_err(Into::into)
            }
            _ => Err(Error::General(format!("Invalid conversion. \
                    Cannot convert {:?} into bool (valid types: Boolean).",
                    $data_type_option.id)))
        }
    );
    ($data_type_option:ident, $data_value:ident, i64) => (
        match $data_type_option.id {
            ColType::Bigint => {
                decode_bigint($data_value.as_slice())
                    .map_err(Into::into)
            }
            ColType::Timestamp => {
                decode_timestamp($data_value.as_slice())
                    .map_err(Into::into)
            }
            ColType::Time => {
                decode_time($data_value.as_slice())
                    .map_err(Into::into)
            }
            ColType::Varint => {
                decode_varint($data_value.as_slice())
                    .map_err(Into::into)
            }
            _ => Err(Error::General(format!("Invalid conversion. \
                    Cannot convert {:?} into i64 (valid types: Bigint, Timestamp, Time, Variant).",
                    $data_type_option.id)))
        }
    );
    ($data_type_option:ident, $data_value:ident, i32) => (
        match $data_type_option.id {
            ColType::Int => {
                decode_int($data_value.as_slice())
                    .map_err(Into::into)
            }
            ColType::Date => {
                decode_date($data_value.as_slice())
                    .map_err(Into::into)
            }
            _ => Err(Error::General(format!("Invalid conversion. \
                    Cannot convert {:?} into i32 (valid types: Int, Date).",
                    $data_type_option.id)))
        }
    );
    ($data_type_option:ident, $data_value:ident, i16) => (
        match $data_type_option.id {
            ColType::Smallint => {
                decode_smallint($data_value.as_slice())
                    .map_err(Into::into)
            }
            _ => Err(Error::General(format!("Invalid conversion. \
                    Cannot convert {:?} into i16 (valid types: Smallint).",
                    $data_type_option.id)))
        }
    );
    ($data_type_option:ident, $data_value:ident, i8) => (
        match $data_type_option.id {
            ColType::Tinyint => {
                decode_tinyint($data_value.as_slice())
                    .map_err(Into::into)
            }
            _ => Err(Error::General(format!("Invalid conversion. \
                    Cannot convert {:?} into i8 (valid types: Tinyint).",
                    $data_type_option.id)))
        }
    );
    ($data_type_option:ident, $data_value:ident, f64) => (
        match $data_type_option.id {
            ColType::Double => {
                decode_double($data_value.as_slice())
                    .map_err(Into::into)
            }
            _ => Err(Error::General(format!("Invalid conversion. \
                    Cannot convert {:?} into f64 (valid types: Double).",
                    $data_type_option.id)))
        }
    );
    ($data_type_option:ident, $data_value:ident, f32) => (
        match $data_type_option.id {
            ColType::Decimal => {
                decode_decimal($data_value.as_slice())
                    .map_err(Into::into)
            }
            ColType::Float => {
                decode_float($data_value.as_slice())
                    .map_err(Into::into)
            }
            _ => Err(Error::General(format!("Invalid conversion. \
                    Cannot convert {:?} into f32 (valid types: Decimal, Float).",
                    $data_type_option.id)))
        }
    );
    ($data_type_option:ident, $data_value:ident, IpAddr) => (
        match $data_type_option.id {
            ColType::Inet => {
                decode_inet($data_value.as_slice())
                    .map_err(Into::into)
            }
            _ => Err(Error::General(format!("Invalid conversion. \
                    Cannot convert {:?} into IpAddr (valid types: Inet).",
                    $data_type_option.id)))
        }
    );
    ($data_type_option:ident, $data_value:ident, Uuid) => (
        match $data_type_option.id {
            ColType::Uuid |
            ColType::Timeuuid => {
                decode_timeuuid($data_value.as_slice())
                    .map_err(Into::into)
            }
            _ => Err(Error::General(format!("Invalid conversion. \
                    Cannot convert {:?} into Uuid (valid types: Uuid, Timeuuid).",
                    $data_type_option.id)))
        }
    );
    ($data_type_option:ident, $data_value:ident, List) => (
        match $data_type_option.id {
            ColType::List |
            ColType::Set => {
                decode_list($data_value.as_slice())
                    .map(|data| List::new(data, $data_type_option.clone()))
                    .map_err(Into::into)
            }
            _ => Err(Error::General(format!("Invalid conversion. \
                    Cannot convert {:?} into List (valid types: List, Set).",
                    $data_type_option.id)))
        }
    );
    ($data_type_option:ident, $data_value:ident, Map) => (
        match $data_type_option.id {
            ColType::Map => {
                decode_map($data_value.as_slice())
                    .map(|data| Map::new(data, $data_type_option.clone()))
                    .map_err(Into::into)
            }
            _ => Err(Error::General(format!("Invalid conversion. \
                    Cannot convert {:?} into Map (valid types: Map).",
                    $data_type_option.id)))
        }
    );
    ($data_type_option:ident, $data_value:ident, UDT) => (
        match *$data_type_option {
            ColTypeOption {
                id: ColType::Udt,
                value: Some(ColTypeOptionValue::UdtType(ref list_type_option))
            } => {
                decode_udt($data_value.as_slice(), list_type_option.descriptions.len())
                    .map(|data| UDT::new(data, list_type_option))
                    .map_err(Into::into)
            }
            _ => Err(Error::General(format!("Invalid conversion. \
                    Cannot convert {:?} into UDT (valid types: UDT).",
                    $data_type_option.id)))
        }
    );
    ($data_type_option:ident, $data_value:ident, Timespec) => (
        match $data_type_option.id {
            ColType::Timestamp => {
                decode_timestamp($data_value.as_slice())
                    .map(|ts| Timespec::new(ts / 1_000, (ts % 1_000 * 1_000_000) as i32))
                    .map_err(Into::into)
            }
            _ => Err(Error::General(format!("Invalid conversion. \
                    Cannot convert {:?} into Timespec (valid types: Timestamp).",
                    $data_type_option.id)))
        }
    );
}
<|endoftext|>


<commit_before>use std::ops::{Add, Sub};
use std::collections::HashMap;
use std::iter::FromIterator;
use std::collections::VecDeque;

struct WordVecBuilder {
    pub word: String,
    pub count: u64,
    vec: Box<[f32; 10_000]>,
}

pub struct WordVec {
    pub word: String,
    pub count: u64,
    vec: Box<[f32; 10_000]>,
}

impl WordVecBuilder {
    fn new(word: String) -> WordVecBuilder {
        WordVecBuilder {
            word: word,
            count: 0,
            vec: Box::new([0f32; 10_000]),
        }
    }

    fn normalize(mut self, word_freqs: &Vec<f32>) -> WordVec {
        let count = self.count as f32;
        for i in 0..word_freqs.len() {
            self.vec[i] = self.vec[i] / count /  word_freqs[i];
        }

        WordVec {
            word: self.word,
            count: self.count,
            vec: self.vec,
        }
    }

    fn inc(&mut self, i: usize) {
        self.vec[i] += 1.0;
    }
}

impl WordVec {

    pub fn distance(&self, other: &WordVec) -> f32 {
        self.vec.iter()
                .zip(other.vec.iter())
                .map(|(x, y)| (x*x - y*y).abs().sqrt())
                .fold(0.0, |x, y| x + y)
    }

}

impl<'a> Add for &'a WordVec {
    type Output = WordVec;

    fn add(self, other: &WordVec) -> WordVec {
        let mut newvec = WordVec {
            word: format!("{} + {}", &self.word, &other.word),
            count: 0,
            vec: Box::new([0f32; 10_000]),
        };

        for i in 0..10_000 {
            newvec.vec[i] = self.vec[i] + other.vec[i];
        }

        newvec
    }
}

impl<'a> Sub for &'a WordVec {
    type Output = WordVec;

    fn sub(self, other: &WordVec) -> WordVec {
        let mut newvec = WordVec {
            word: format!("{} + {}", &self.word, &other.word),
            count: 0,
            vec: Box::new([0f32; 10_000]),
        };

        for i in 0..10_000 {
            newvec.vec[i] = self.vec[i] - other.vec[i];
        }

        newvec
    }
}

pub struct LanguageModel {
    words: HashMap<String, usize>,
    word_vecs: Vec<WordVec>,
}

pub struct LanguageModelBuilder {
    window_width: u32,
    words: HashMap<String, usize>,
    word_vecs: Vec<WordVecBuilder>,
}

pub struct WordAcceptor<'a> {
    window: VecDeque<String>,
    focus: usize,
    builder: &'a mut LanguageModelBuilder,
}

impl<'a> LanguageModelBuilder {
    fn new(words: Vec<String>) -> LanguageModelBuilder {
        assert_eq!(words.len(), 10_000);

        let word_vecs = words.iter()
                             .map(|s| WordVecBuilder::new(s.clone()))
                             .collect();

        let words = HashMap::from_iter(
                        words.into_iter()
                             .enumerate()
                             .map(|(a, b)| (b, a)));

        LanguageModelBuilder {
            window_width: 21,
            words: words,
            word_vecs: word_vecs,
        }
    }

    fn build(self) -> LanguageModel {
        let total_words = self.word_vecs.iter().fold(0.0, |a, b| a + b.count as f32);
        let word_freqs = self.word_vecs.iter()
                                       .map(|v| v.count as f32 / total_words as f32)
                                       .collect::<Vec<_>>();
        LanguageModel {
            words: self.words,
            word_vecs: self.word_vecs.into_iter()
                                     .map(|b| b.normalize(&word_freqs))
                                     .collect(),
        }
    }

    fn new_file(&'a mut self) -> WordAcceptor<'a> {
        WordAcceptor {
            window: VecDeque::new(),
            focus: 0,
            builder: self,
        }
    }

    fn add_word(&mut self, word: &str) {
        let idx = self.words[word];
        let word_vec = &mut self.word_vecs[idx];
        word_vec.inc(idx);
    }
}

impl<'a> WordAcceptor<'a> {
    fn word(&mut self, word: String) {
        let idx = self.builder.words.get(&word).map(|i| *i);

        if let Some(idx) = idx {
            self.window.push_back(word);
            if (self.window.len() > 21) {
                self.window.pop_front();
            }
            if (self.window.len() == 21) {
                for i in (0..21).filter(|a| *a != 10) {
                    self.builder.add_word(&self.window[i]);
                }
            }
        }
    }
}<commit_msg>remove unneccesary cast<commit_after>use std::ops::{Add, Sub};
use std::collections::HashMap;
use std::iter::FromIterator;
use std::collections::VecDeque;

struct WordVecBuilder {
    pub word: String,
    pub count: u64,
    vec: Box<[f32; 10_000]>,
}

pub struct WordVec {
    pub word: String,
    pub count: u64,
    vec: Box<[f32; 10_000]>,
}

impl WordVecBuilder {
    fn new(word: String) -> WordVecBuilder {
        WordVecBuilder {
            word: word,
            count: 0,
            vec: Box::new([0f32; 10_000]),
        }
    }

    fn normalize(mut self, word_freqs: &Vec<f32>) -> WordVec {
        let count = self.count as f32;
        for i in 0..word_freqs.len() {
            self.vec[i] = self.vec[i] / count /  word_freqs[i];
        }

        WordVec {
            word: self.word,
            count: self.count,
            vec: self.vec,
        }
    }

    fn inc(&mut self, i: usize) {
        self.vec[i] += 1.0;
    }
}

impl WordVec {

    pub fn distance(&self, other: &WordVec) -> f32 {
        self.vec.iter()
                .zip(other.vec.iter())
                .map(|(x, y)| (x*x - y*y).abs().sqrt())
                .fold(0.0, |x, y| x + y)
    }

}

impl<'a> Add for &'a WordVec {
    type Output = WordVec;

    fn add(self, other: &WordVec) -> WordVec {
        let mut newvec = WordVec {
            word: format!("{} + {}", &self.word, &other.word),
            count: 0,
            vec: Box::new([0f32; 10_000]),
        };

        for i in 0..10_000 {
            newvec.vec[i] = self.vec[i] + other.vec[i];
        }

        newvec
    }
}

impl<'a> Sub for &'a WordVec {
    type Output = WordVec;

    fn sub(self, other: &WordVec) -> WordVec {
        let mut newvec = WordVec {
            word: format!("{} + {}", &self.word, &other.word),
            count: 0,
            vec: Box::new([0f32; 10_000]),
        };

        for i in 0..10_000 {
            newvec.vec[i] = self.vec[i] - other.vec[i];
        }

        newvec
    }
}

pub struct LanguageModel {
    words: HashMap<String, usize>,
    word_vecs: Vec<WordVec>,
}

pub struct LanguageModelBuilder {
    window_width: u32,
    words: HashMap<String, usize>,
    word_vecs: Vec<WordVecBuilder>,
}

pub struct WordAcceptor<'a> {
    window: VecDeque<String>,
    focus: usize,
    builder: &'a mut LanguageModelBuilder,
}

impl<'a> LanguageModelBuilder {
    fn new(words: Vec<String>) -> LanguageModelBuilder {
        assert_eq!(words.len(), 10_000);

        let word_vecs = words.iter()
                             .map(|s| WordVecBuilder::new(s.clone()))
                             .collect();

        let words = HashMap::from_iter(
                        words.into_iter()
                             .enumerate()
                             .map(|(a, b)| (b, a)));

        LanguageModelBuilder {
            window_width: 21,
            words: words,
            word_vecs: word_vecs,
        }
    }

    fn build(self) -> LanguageModel {
        let total_words = self.word_vecs.iter().fold(0.0, |a, b| a + b.count as f32);
        let word_freqs = self.word_vecs.iter()
                                       .map(|v| v.count as f32 / total_words)
                                       .collect::<Vec<_>>();
        LanguageModel {
            words: self.words,
            word_vecs: self.word_vecs.into_iter()
                                     .map(|b| b.normalize(&word_freqs))
                                     .collect(),
        }
    }

    fn new_file(&'a mut self) -> WordAcceptor<'a> {
        WordAcceptor {
            window: VecDeque::new(),
            focus: 0,
            builder: self,
        }
    }

    fn add_word(&mut self, word: &str) {
        let idx = self.words[word];
        let word_vec = &mut self.word_vecs[idx];
        word_vec.inc(idx);
    }
}

impl<'a> WordAcceptor<'a> {
    fn word(&mut self, word: String) {
        let idx = self.builder.words.get(&word).map(|i| *i);

        if let Some(idx) = idx {
            self.window.push_back(word);
            if (self.window.len() > 21) {
                self.window.pop_front();
            }
            if (self.window.len() == 21) {
                for i in (0..21).filter(|a| *a != 10) {
                    self.builder.add_word(&self.window[i]);
                }
            }
        }
    }
}<|endoftext|>


<commit_before>//! Simplifying Candidates
//!
//! *Simplifying* a match pair `place @ pattern` means breaking it down
//! into bindings or other, simpler match pairs. For example:
//!
//! - `place @ (P1, P2)` can be simplified to `[place.0 @ P1, place.1 @ P2]`
//! - `place @ x` can be simplified to `[]` by binding `x` to `place`
//!
//! The `simplify_candidate` routine just repeatedly applies these
//! sort of simplifications until there is nothing left to
//! simplify. Match pairs cannot be simplified if they require some
//! sort of test: for example, testing which variant an enum is, or
//! testing a value against a constant.

use crate::build::Builder;
use crate::build::matches::{Ascription, Binding, MatchPair, Candidate};
use crate::hair::{self, *};
use rustc::ty;
use rustc::ty::layout::{Integer, IntegerExt, Size};
use syntax::attr::{SignedInt, UnsignedInt};
use rustc::hir::RangeEnd;

use std::mem;

impl<'a, 'gcx, 'tcx> Builder<'a, 'gcx, 'tcx> {
    pub fn simplify_candidate<'pat>(&mut self,
                                    candidate: &mut Candidate<'pat, 'tcx>) {
        // repeatedly simplify match pairs until fixed point is reached
        loop {
            let match_pairs = mem::replace(&mut candidate.match_pairs, vec![]);
            let mut changed = false;
            for match_pair in match_pairs {
                match self.simplify_match_pair(match_pair, candidate) {
                    Ok(()) => {
                        changed = true;
                    }
                    Err(match_pair) => {
                        candidate.match_pairs.push(match_pair);
                    }
                }
            }
            if !changed {
                return; // if we were not able to simplify any, done.
            }
        }
    }

    /// Tries to simplify `match_pair`, returning `Ok(())` if
    /// successful. If successful, new match pairs and bindings will
    /// have been pushed into the candidate. If no simplification is
    /// possible, `Err` is returned and no changes are made to
    /// candidate.
    fn simplify_match_pair<'pat>(&mut self,
                                 match_pair: MatchPair<'pat, 'tcx>,
                                 candidate: &mut Candidate<'pat, 'tcx>)
                                 -> Result<(), MatchPair<'pat, 'tcx>> {
        let tcx = self.hir.tcx();
        match *match_pair.pattern.kind {
            PatternKind::AscribeUserType {
                ref subpattern,
                ascription: hair::pattern::Ascription {
                    variance,
                    ref user_ty,
                    user_ty_span,
                },
            } => {
                // Apply the type ascription to the value at `match_pair.place`, which is the
                // value being matched, taking the variance field into account.
                candidate.ascriptions.push(Ascription {
                    span: user_ty_span,
                    user_ty: user_ty.clone(),
                    source: match_pair.place.clone(),
                    variance,
                });

                candidate.match_pairs.push(MatchPair::new(match_pair.place, subpattern));

                Ok(())
            }

            PatternKind::Wild => {
                // nothing left to do
                Ok(())
            }

            PatternKind::Binding { name, mutability, mode, var, ty, ref subpattern } => {
                candidate.bindings.push(Binding {
                    name,
                    mutability,
                    span: match_pair.pattern.span,
                    source: match_pair.place.clone(),
                    var_id: var,
                    var_ty: ty,
                    binding_mode: mode,
                });

                if let Some(subpattern) = subpattern.as_ref() {
                    // this is the `x @ P` case; have to keep matching against `P` now
                    candidate.match_pairs.push(MatchPair::new(match_pair.place, subpattern));
                }

                Ok(())
            }

            PatternKind::Constant { .. } => {
                // FIXME normalize patterns when possible
                Err(match_pair)
            }

            PatternKind::Range(PatternRange { lo, hi, ty, end }) => {
                let (range, bias) = match ty.sty {
                    ty::Char => {
                        (Some(('\u{0000}' as u128, '\u{10FFFF}' as u128, Size::from_bits(32))), 0)
                    }
                    ty::Int(ity) => {
                        // FIXME(49937): refactor these bit manipulations into interpret.
                        let size = Integer::from_attr(&tcx, SignedInt(ity)).size();
                        let max = !0u128 >> (128 - size.bits());
                        let bias = 1u128 << (size.bits() - 1);
                        (Some((0, max, size)), bias)
                    }
                    ty::Uint(uty) => {
                        // FIXME(49937): refactor these bit manipulations into interpret.
                        let size = Integer::from_attr(&tcx, UnsignedInt(uty)).size();
                        let max = !0u128 >> (128 - size.bits());
                        (Some((0, max, size)), 0)
                    }
                    _ => (None, 0),
                };
                if let Some((min, max, sz)) = range {
                    if let (Some(lo), Some(hi)) = (lo.val.try_to_bits(sz), hi.val.try_to_bits(sz)) {
                        // We want to compare ranges numerically, but the order of the bitwise
                        // representation of signed integers does not match their numeric order.
                        // Thus, to correct the ordering, we need to shift the range of signed
                        // integers to correct the comparison. This is achieved by XORing with a
                        // bias (see pattern/_match.rs for another pertinent example of this
                        // pattern).
                        let (lo, hi) = (lo ^ bias, hi ^ bias);
                        if lo <= min && (hi > max || hi == max && end == RangeEnd::Included) {
                            // Irrefutable pattern match.
                            return Ok(());
                        }
                    }
                }
                Err(match_pair)
            }

            PatternKind::Slice { ref prefix, ref slice, ref suffix } => {
                if prefix.is_empty() && slice.is_some() && suffix.is_empty() {
                    // irrefutable
                    self.prefix_slice_suffix(&mut candidate.match_pairs,
                                             &match_pair.place,
                                             prefix,
                                             slice.as_ref(),
                                             suffix);
                    Ok(())
                } else {
                    Err(match_pair)
                }
            }

            PatternKind::Variant { adt_def, substs, variant_index, ref subpatterns } => {
                let irrefutable = adt_def.variants.iter_enumerated().all(|(i, v)| {
                    i == variant_index || {
                        self.hir.tcx().features().never_type &&
                        self.hir.tcx().features().exhaustive_patterns &&
                        self.hir.tcx().is_variant_uninhabited_from_all_modules(v, substs)
                    }
                });
                if irrefutable {
                    let place = match_pair.place.downcast(adt_def, variant_index);
                    candidate.match_pairs.extend(self.field_match_pairs(place, subpatterns));
                    Ok(())
                } else {
                    Err(match_pair)
                }
            }

            PatternKind::Array { ref prefix, ref slice, ref suffix } => {
                self.prefix_slice_suffix(&mut candidate.match_pairs,
                                         &match_pair.place,
                                         prefix,
                                         slice.as_ref(),
                                         suffix);
                Ok(())
            }

            PatternKind::Leaf { ref subpatterns } => {
                // tuple struct, match subpats (if any)
                candidate.match_pairs
                         .extend(self.field_match_pairs(match_pair.place, subpatterns));
                Ok(())
            }

            PatternKind::Deref { ref subpattern } => {
                let place = match_pair.place.deref();
                candidate.match_pairs.push(MatchPair::new(place, subpattern));
                Ok(())
            }
        }
    }
}
<commit_msg>Rollup merge of #59101 - kenta7777:reduce-code-repetition, r=oli-obk<commit_after>//! Simplifying Candidates
//!
//! *Simplifying* a match pair `place @ pattern` means breaking it down
//! into bindings or other, simpler match pairs. For example:
//!
//! - `place @ (P1, P2)` can be simplified to `[place.0 @ P1, place.1 @ P2]`
//! - `place @ x` can be simplified to `[]` by binding `x` to `place`
//!
//! The `simplify_candidate` routine just repeatedly applies these
//! sort of simplifications until there is nothing left to
//! simplify. Match pairs cannot be simplified if they require some
//! sort of test: for example, testing which variant an enum is, or
//! testing a value against a constant.

use crate::build::Builder;
use crate::build::matches::{Ascription, Binding, MatchPair, Candidate};
use crate::hair::{self, *};
use rustc::ty;
use rustc::ty::layout::{Integer, IntegerExt, Size};
use syntax::attr::{SignedInt, UnsignedInt};
use rustc::hir::RangeEnd;
use rustc::mir::interpret::truncate;

use std::mem;

impl<'a, 'gcx, 'tcx> Builder<'a, 'gcx, 'tcx> {
    pub fn simplify_candidate<'pat>(&mut self,
                                    candidate: &mut Candidate<'pat, 'tcx>) {
        // repeatedly simplify match pairs until fixed point is reached
        loop {
            let match_pairs = mem::replace(&mut candidate.match_pairs, vec![]);
            let mut changed = false;
            for match_pair in match_pairs {
                match self.simplify_match_pair(match_pair, candidate) {
                    Ok(()) => {
                        changed = true;
                    }
                    Err(match_pair) => {
                        candidate.match_pairs.push(match_pair);
                    }
                }
            }
            if !changed {
                return; // if we were not able to simplify any, done.
            }
        }
    }

    /// Tries to simplify `match_pair`, returning `Ok(())` if
    /// successful. If successful, new match pairs and bindings will
    /// have been pushed into the candidate. If no simplification is
    /// possible, `Err` is returned and no changes are made to
    /// candidate.
    fn simplify_match_pair<'pat>(&mut self,
                                 match_pair: MatchPair<'pat, 'tcx>,
                                 candidate: &mut Candidate<'pat, 'tcx>)
                                 -> Result<(), MatchPair<'pat, 'tcx>> {
        let tcx = self.hir.tcx();
        match *match_pair.pattern.kind {
            PatternKind::AscribeUserType {
                ref subpattern,
                ascription: hair::pattern::Ascription {
                    variance,
                    ref user_ty,
                    user_ty_span,
                },
            } => {
                // Apply the type ascription to the value at `match_pair.place`, which is the
                // value being matched, taking the variance field into account.
                candidate.ascriptions.push(Ascription {
                    span: user_ty_span,
                    user_ty: user_ty.clone(),
                    source: match_pair.place.clone(),
                    variance,
                });

                candidate.match_pairs.push(MatchPair::new(match_pair.place, subpattern));

                Ok(())
            }

            PatternKind::Wild => {
                // nothing left to do
                Ok(())
            }

            PatternKind::Binding { name, mutability, mode, var, ty, ref subpattern } => {
                candidate.bindings.push(Binding {
                    name,
                    mutability,
                    span: match_pair.pattern.span,
                    source: match_pair.place.clone(),
                    var_id: var,
                    var_ty: ty,
                    binding_mode: mode,
                });

                if let Some(subpattern) = subpattern.as_ref() {
                    // this is the `x @ P` case; have to keep matching against `P` now
                    candidate.match_pairs.push(MatchPair::new(match_pair.place, subpattern));
                }

                Ok(())
            }

            PatternKind::Constant { .. } => {
                // FIXME normalize patterns when possible
                Err(match_pair)
            }

            PatternKind::Range(PatternRange { lo, hi, ty, end }) => {
                let (range, bias) = match ty.sty {
                    ty::Char => {
                        (Some(('\u{0000}' as u128, '\u{10FFFF}' as u128, Size::from_bits(32))), 0)
                    }
                    ty::Int(ity) => {
                        // FIXME(49937): refactor these bit manipulations into interpret.
                        let size = Integer::from_attr(&tcx, SignedInt(ity)).size();
                        let max = truncate(u128::max_value(), size);
                        let bias = 1u128 << (size.bits() - 1);
                        (Some((0, max, size)), bias)
                    }
                    ty::Uint(uty) => {
                        // FIXME(49937): refactor these bit manipulations into interpret.
                        let size = Integer::from_attr(&tcx, UnsignedInt(uty)).size();
                        let max = truncate(u128::max_value(), size);
                        (Some((0, max, size)), 0)
                    }
                    _ => (None, 0),
                };
                if let Some((min, max, sz)) = range {
                    if let (Some(lo), Some(hi)) = (lo.val.try_to_bits(sz), hi.val.try_to_bits(sz)) {
                        // We want to compare ranges numerically, but the order of the bitwise
                        // representation of signed integers does not match their numeric order.
                        // Thus, to correct the ordering, we need to shift the range of signed
                        // integers to correct the comparison. This is achieved by XORing with a
                        // bias (see pattern/_match.rs for another pertinent example of this
                        // pattern).
                        let (lo, hi) = (lo ^ bias, hi ^ bias);
                        if lo <= min && (hi > max || hi == max && end == RangeEnd::Included) {
                            // Irrefutable pattern match.
                            return Ok(());
                        }
                    }
                }
                Err(match_pair)
            }

            PatternKind::Slice { ref prefix, ref slice, ref suffix } => {
                if prefix.is_empty() && slice.is_some() && suffix.is_empty() {
                    // irrefutable
                    self.prefix_slice_suffix(&mut candidate.match_pairs,
                                             &match_pair.place,
                                             prefix,
                                             slice.as_ref(),
                                             suffix);
                    Ok(())
                } else {
                    Err(match_pair)
                }
            }

            PatternKind::Variant { adt_def, substs, variant_index, ref subpatterns } => {
                let irrefutable = adt_def.variants.iter_enumerated().all(|(i, v)| {
                    i == variant_index || {
                        self.hir.tcx().features().never_type &&
                        self.hir.tcx().features().exhaustive_patterns &&
                        self.hir.tcx().is_variant_uninhabited_from_all_modules(v, substs)
                    }
                });
                if irrefutable {
                    let place = match_pair.place.downcast(adt_def, variant_index);
                    candidate.match_pairs.extend(self.field_match_pairs(place, subpatterns));
                    Ok(())
                } else {
                    Err(match_pair)
                }
            }

            PatternKind::Array { ref prefix, ref slice, ref suffix } => {
                self.prefix_slice_suffix(&mut candidate.match_pairs,
                                         &match_pair.place,
                                         prefix,
                                         slice.as_ref(),
                                         suffix);
                Ok(())
            }

            PatternKind::Leaf { ref subpatterns } => {
                // tuple struct, match subpats (if any)
                candidate.match_pairs
                         .extend(self.field_match_pairs(match_pair.place, subpatterns));
                Ok(())
            }

            PatternKind::Deref { ref subpattern } => {
                let place = match_pair.place.deref();
                candidate.match_pairs.push(MatchPair::new(place, subpattern));
                Ok(())
            }
        }
    }
}
<|endoftext|>


<commit_before>// Copyright (c) 2017 fd developers
// Licensed under the Apache License, Version 2.0
// <LICENSE-APACHE or http://www.apache.org/licenses/LICENSE-2.0>
// or the MIT license <LICENSE-MIT or http://opensource.org/licenses/MIT>,
// at your option. All files in the project carrying such
// notice may not be copied, modified, or distributed except
// according to those terms.

use crate::exit_codes::ExitCode;
use crate::internal::opts::FdOptions;
use lscolors::{LsColors, Style};

use std::io::{self, StdoutLock, Write};
use std::path::{Component, Path, PathBuf};
use std::process;
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::Arc;

use ansi_term;

/// Remove the `./` prefix from a path.
fn strip_current_dir(pathbuf: &PathBuf) -> &Path {
    let mut iter = pathbuf.components();
    let mut iter_next = iter.clone();
    if iter_next.next() == Some(Component::CurDir) {
        iter.next();
    }
    iter.as_path()
}

pub fn print_entry(
    stdout: &mut StdoutLock,
    entry: &PathBuf,
    config: &FdOptions,
    wants_to_quit: &Arc<AtomicBool>,
) {
    let path = if entry.is_absolute() {
        entry.as_path()
    } else {
        strip_current_dir(entry)
    };

    let r = if let Some(ref ls_colors) = config.ls_colors {
        print_entry_colorized(stdout, path, config, ls_colors, &wants_to_quit)
    } else {
        print_entry_uncolorized(stdout, path, config)
    };

    if r.is_err() {
        // Probably a broken pipe. Exit gracefully.
        process::exit(ExitCode::GeneralError.into());
    }
}

fn print_entry_colorized(
    stdout: &mut StdoutLock,
    path: &Path,
    config: &FdOptions,
    ls_colors: &LsColors,
    wants_to_quit: &Arc<AtomicBool>,
) -> io::Result<()> {
    let default_style = ansi_term::Style::default();

    // Traverse the path and colorize each component
    for (component, style) in ls_colors.style_for_path_components(path) {
        let style = style
            .map(Style::to_ansi_term_style)
            .unwrap_or(default_style);

        let path_string = component.to_string_lossy();

        match &config.path_separator {
            None => write!(stdout, "{}", style.paint(path_string))?,
            Some(sep) => {
                let path_string = path_string.replace(std::path::MAIN_SEPARATOR, &sep);
                write!(stdout, "{}", style.paint(path_string))?
            }
        }

        if wants_to_quit.load(Ordering::Relaxed) {
            writeln!(stdout)?;
            process::exit(ExitCode::KilledBySigint.into());
        }
    }

    if config.null_separator {
        write!(stdout, "\0")
    } else {
        writeln!(stdout)
    }
}

fn print_entry_uncolorized(
    stdout: &mut StdoutLock,
    path: &Path,
    config: &FdOptions,
) -> io::Result<()> {
    let separator = if config.null_separator { "\0" } else { "\n" };

    let path_str = path.to_string_lossy();
    write!(stdout, "{}{}", path_str, separator)
}
<commit_msg>Factor into a helper function; adapt to print_entry_uncolorized.<commit_after>// Copyright (c) 2017 fd developers
// Licensed under the Apache License, Version 2.0
// <LICENSE-APACHE or http://www.apache.org/licenses/LICENSE-2.0>
// or the MIT license <LICENSE-MIT or http://opensource.org/licenses/MIT>,
// at your option. All files in the project carrying such
// notice may not be copied, modified, or distributed except
// according to those terms.

use crate::exit_codes::ExitCode;
use crate::internal::opts::FdOptions;
use lscolors::{LsColors, Style};

use std::borrow::Cow;
use std::io::{self, StdoutLock, Write};
use std::path::{Component, Path, PathBuf};
use std::process;
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::Arc;

use ansi_term;

/// Remove the `./` prefix from a path.
fn strip_current_dir(pathbuf: &PathBuf) -> &Path {
    let mut iter = pathbuf.components();
    let mut iter_next = iter.clone();
    if iter_next.next() == Some(Component::CurDir) {
        iter.next();
    }
    iter.as_path()
}

pub fn print_entry(
    stdout: &mut StdoutLock,
    entry: &PathBuf,
    config: &FdOptions,
    wants_to_quit: &Arc<AtomicBool>,
) {
    let path = if entry.is_absolute() {
        entry.as_path()
    } else {
        strip_current_dir(entry)
    };

    let r = if let Some(ref ls_colors) = config.ls_colors {
        print_entry_colorized(stdout, path, config, ls_colors, &wants_to_quit)
    } else {
        print_entry_uncolorized(stdout, path, config)
    };

    if r.is_err() {
        // Probably a broken pipe. Exit gracefully.
        process::exit(ExitCode::GeneralError.into());
    }
}

fn replace_path_separator<'a>(config: &FdOptions, str: Cow<'a, str>) -> Cow<'a, str> {
    match &config.path_separator {
        None => str,
        Some(sep) => Cow::from(str.replace(std::path::MAIN_SEPARATOR, &sep)),
    }
}

fn print_entry_colorized(
    stdout: &mut StdoutLock,
    path: &Path,
    config: &FdOptions,
    ls_colors: &LsColors,
    wants_to_quit: &Arc<AtomicBool>,
) -> io::Result<()> {
    let default_style = ansi_term::Style::default();

    // Traverse the path and colorize each component
    for (component, style) in ls_colors.style_for_path_components(path) {
        let style = style
            .map(Style::to_ansi_term_style)
            .unwrap_or(default_style);

        let path_string = component.to_string_lossy();
        let path_string = replace_path_separator(&config, path_string);
        write!(stdout, "{}", style.paint(path_string))?;

        if wants_to_quit.load(Ordering::Relaxed) {
            writeln!(stdout)?;
            process::exit(ExitCode::KilledBySigint.into());
        }
    }

    if config.null_separator {
        write!(stdout, "\0")
    } else {
        writeln!(stdout)
    }
}

fn print_entry_uncolorized(
    stdout: &mut StdoutLock,
    path: &Path,
    config: &FdOptions,
) -> io::Result<()> {
    let separator = if config.null_separator { "\0" } else { "\n" };

    let path_str = path.to_string_lossy();
    let path_str = replace_path_separator(&config, path_str);
    write!(stdout, "{}{}", path_str, separator)
}
<|endoftext|>


<commit_before>use std::str;
use std::io::prelude::*;
use std::fs::File;
use std::str::from_utf8;

use nom::*;

use error::*;

named!(string_between_quotes, delimited!(char!('\"'), is_not!("\""), char!('\"')));
named!(get_cell, take_while!(is_not_cell_end));
named!(consume_useless_chars, take_while!(is_whitespace));

macro_rules! separated_list2 (
  ($i:expr, $sep:ident!( $($args:tt)* ), $submac:ident!( $($args2:tt)* )) => (
    {
      let mut res   = ::std::vec::Vec::new();
      let mut input = $i;

      // get the first element
      let first = $submac!(input, $($args2)*);

      if let IResult::Done(i,o) = first {
         if i.len() == input.len() {
            let err : IResult<&[u8], Vec<Vec<String>>, CsvError> = IResult::Error(Err::Position(ErrorKind::SeparatedList, input)); err
          } else {
            res.push(o);
            input = i;

            loop {
              // get the separator first
              if let IResult::Done(i2,_) = $sep!(input, $($args)*) {
                if i2.len() == input.len() {
                  break;
                }
                input = i2;

                // get the element next
                if let IResult::Done(i3,o3) = $submac!(input, $($args2)*) {
                  if i3.len() == input.len() {
                    break;
                  }
                  res.push(o3);
                  input = i3;
                } else {
                  break;
                }
              } else {
                break;
              }
            }
            IResult::Done(input, res)
          }
      } else if let IResult::Incomplete(i) = first {
        IResult::Incomplete(i)
      } else {
        IResult::Done(input, ::std::vec::Vec::new())
      }
    }
  );
  ($i:expr, $submac:ident!( $($args:tt)* ), $g:expr) => (
    separated_list!($i, $submac!($($args)*), call!($g));
  );
  ($i:expr, $f:expr, $submac:ident!( $($args:tt)* )) => (
    separated_list!($i, call!($f), $submac!($($args)*));
  );
  ($i:expr, $f:expr, $g:expr) => (
    separated_list!($i, call!($f), call!($g));
  );
);

fn is_whitespace(c: u8) -> bool {
    c as char == ' ' || c as char == '\t'
}

fn is_not_cell_end(c: u8) -> bool {
    c as char != ',' && c as char != '\n'
}

fn get_column_value(input: &[u8], pos: Position) -> IResult<&[u8], &[u8], CsvError> {
    let (i, cell) = try_parse!(input,
        fix_error!(CsvError,
            preceded!(
            opt!(consume_useless_chars),
                alt!(
                    string_between_quotes
                  | get_cell
                )
            )
        )
    );

    if i.len() == 0 {
        IResult::Incomplete(Needed::Unknown)
    } else if is_not_cell_end(i[0]) {
        let p = Position { line: pos.line, column: pos.column + input.offset(i) };
        IResult::Error(Err::Code(ErrorKind::Custom(
            CsvError::InvalidCharacter(CharError::new(',', i[0] as char, &p))
        )))
    } else {
        IResult::Done(i, cell)
    }
}

fn get_string_column_value(input: &[u8], pos: Position) -> IResult<&[u8], String, CsvError> {
    map_res!(input,
        map_res!(
            dbg_dmp!(
                apply!(get_column_value, Position::new(pos.line, pos.column))
            ),
            from_utf8
        ),
        |d| {
            str::FromStr::from_str(d)
        }
    )
}

fn comma_then_column<'a>(input: &'a [u8], pos: &Position) -> IResult<&'a [u8], String, CsvError> {
    preceded!(input,
        fix_error!(CsvError, char!(',')),
        apply!(get_string_column_value, Position::new(pos.line, pos.column))
    )
}

fn many_comma_then_column(input: &[u8], pos: Position) -> IResult<&[u8], Vec<String>, CsvError> {
    many0!(
        input,
        apply!(comma_then_column, &pos)
    )
}

fn get_line_values<'a>(entry: &'a[u8], ret: &mut Vec<String>, line: usize) -> IResult<&'a[u8], &'a[u8], CsvError> {
    if entry.len() == 0 {
        IResult::Done(entry, entry)
    } else {
        let (i, col) = try_parse!(entry, apply!(get_string_column_value, Position::new(line, ret.len())));
        ret.push(col);

        match fix_error!(i, CsvError, separated_list2!(
            char!('\n'),
            apply!(many_comma_then_column, Position::new(line, ret.len()))
        )) {
            IResult::Done(i, v)    => {
                for c in v {
                    for sub_c in c {
                        ret.push(sub_c);
                    }
                }
                IResult::Done(i, &entry[..entry.offset(i)])
            },
            IResult::Incomplete(i) => IResult::Incomplete(i),
            IResult::Error(e)      => IResult::Error(e)
        }
    }
}

fn get_lines_values(mut ret: Vec<Vec<String>>, entry: &[u8]) -> Result<Vec<Vec<String>>, CsvError> {
    let mut input = entry;
    let mut line  = 0;
    loop {
        let mut v: Vec<String> = Vec::new();
        match get_line_values(input, &mut v, line) {
            IResult::Error(Err::Code(ErrorKind::Custom(e))) => return Err(e),
            IResult::Error(_)                               => return Err(CsvError::GenericError),
            IResult::Incomplete(_)                          => {
                // did we reach the end of file?
                break
            }
            IResult::Done(i,_)                              => {
                input = i;
                line += 1;
                ret.push(v);
                if input.len() == 0 {
                    break;
                }
            },
        }
    }

    Ok(ret)
}

pub fn parse_csv_from_slice(entry: &[u8]) -> Result<Vec<Vec<String>>, CsvError> {
    get_lines_values(vec!(), entry)
}

pub fn parse_csv_from_file(filename: &str) -> Result<Vec<Vec<String>>, CsvError> {
    let mut f = File::open(filename).unwrap();
    let mut buffer = vec!();

    f.read_to_end(&mut buffer).unwrap();
    parse_csv_from_slice(&buffer)
}

pub fn parse_csv(entry: &str) -> Result<Vec<Vec<String>>, CsvError> {
    parse_csv_from_slice(entry.as_bytes())
}

#[test]
fn check_string_between_quotes() {
    let f = b"\"nom\",age\ncarles,30\nlaure,28\n";

    match string_between_quotes(f) {
        IResult::Done(in_, out) => {
            assert_eq!(out, b"nom");
            assert_eq!(in_, b",age\ncarles,30\nlaure,28\n");
        },
        IResult::Incomplete(x) => panic!("incomplete: {:?}", x),
        IResult::Error(e) => panic!("error: {:?}", e),
    }
}

#[test]
fn check_get_cell() {
    let f = b"age\ncarles,30\n";
    let g = b"age2,carles,30\n";

    match get_cell(f) {
        IResult::Done(_, out) => assert_eq!(out, b"age"),
        IResult::Incomplete(x) => panic!("incomplete: {:?}", x),
        IResult::Error(e) => panic!("error: {:?}", e),
    }
    match get_cell(g) {
        IResult::Done(_, out) => assert_eq!(out, b"age2"),
        IResult::Incomplete(x) => panic!("incomplete: {:?}", x),
        IResult::Error(e) => panic!("error: {:?}", e),
    }
}

#[test]
fn check_get_line_values() {
    // no terminator, this is not a line
    //let mut cells = vec!();
    //get_line_values(&mut cells, b"\"nom\",,age", 0);
    //assert_eq!(cells, vec!("nom".to_owned(), "".to_owned(), "age".to_owned()));

    let mut cells = vec!();
    let res = get_line_values(b"\"nom\",,age\n", &mut cells, 0);
    println!("res: {:?}", res);
    assert_eq!(cells, vec!("nom".to_owned(), "".to_owned(), "age".to_owned()));

    let mut cells = vec!();
    get_line_values(b"\"nom\",age,\n", &mut cells, 0);
    assert_eq!(cells, vec!("nom".to_owned(), "age".to_owned(), "".to_owned()));

    let mut cells = vec!();
    get_line_values(b"\"nom\",age,,\"hoho\",,end\n", &mut cells, 0);
    assert_eq!(cells, vec!("nom".to_owned(), "age".to_owned(), "".to_owned(), "hoho".to_owned(), "".to_owned(), "end".to_owned()));

    let mut cells = vec!();
    let e = get_line_values(b"\"nom\" ,age,\"hoho\"", &mut cells, 0);
    assert_eq!(e,
        IResult::Error(Err::Code(ErrorKind::Custom(
            CsvError::InvalidCharacter(CharError::new(',', ' ', &Position::new(0, 5)))
        )))
    );
}

#[test]
fn check_get_lines_values() {
    let f = b"\"nom\",age\ncarles,30\nlaure,28\n";

    assert_eq!(get_lines_values(vec!(), f),
               Ok(vec!(
                       vec!("nom".to_owned(), "age".to_owned()),
                       vec!("carles".to_owned(), "30".to_owned()),
                       vec!("laure".to_owned(), "28".to_owned()))));
    let f = b"\"nom\",age\ncarles,30\nlaure,28";

    assert_eq!(get_lines_values(vec!(), f),
               Ok(vec!(
                       vec!("nom".to_owned(), "age".to_owned()),
                       vec!("carles".to_owned(), "30".to_owned()),
                       vec!("laure".to_owned(), "28".to_owned()))));
}

#[test]
fn check_parse_csv() {
    let f = "\"nom\",age\ncarles,30\nlaure,28\n";

    assert_eq!(parse_csv(f),
               Ok(vec!(
                       vec!("nom".to_owned(), "age".to_owned()),
                       vec!("carles".to_owned(), "30".to_owned()),
                       vec!("laure".to_owned(), "28".to_owned()))));
}
<commit_msg>Final fix<commit_after>use std::str;
use std::io::prelude::*;
use std::fs::File;
use std::str::from_utf8;

use nom::*;

use error::*;

named!(string_between_quotes, delimited!(char!('\"'), is_not!("\""), char!('\"')));
named!(get_cell, take_while!(is_not_cell_end));
named!(consume_useless_chars, take_while!(is_whitespace));

macro_rules! separated_list2 (
  ($i:expr, $sep:ident!( $($args:tt)* ), $submac:ident!( $($args2:tt)* )) => (
    {
      let mut res   = ::std::vec::Vec::new();
      let mut input = $i;

      // get the first element
      let first = $submac!(input, $($args2)*);

      if let IResult::Done(i, o) = first {
         if i.len() == input.len() {
            let err : IResult<&[u8], Vec<Vec<String>>, CsvError> = IResult::Error(Err::Position(ErrorKind::SeparatedList, input)); err
          } else {
            res.push(o);
            input = i;

            loop {
              // get the separator first
              if let IResult::Done(i2,_) = $sep!(input, $($args)*) {
                println!("{:?}", String::from_utf8_lossy(i2));
                if i2.len() == input.len() {
                  break;
                }
                input = i2;

                // get the element next
                if let IResult::Done(i3,o3) = $submac!(input, $($args2)*) {
                  res.push(o3);
                  input = i3;
                  if i3.len() == input.len() {
                    break;
                  }
                } else {
                  break;
                }
              } else {
                break;
              }
            }
            IResult::Done(input, res)
          }
      } else if let IResult::Incomplete(i) = first {
        IResult::Incomplete(i)
      } else {
        IResult::Done(input, ::std::vec::Vec::new())
      }
    }
  );
  ($i:expr, $submac:ident!( $($args:tt)* ), $g:expr) => (
    separated_list!($i, $submac!($($args)*), call!($g));
  );
  ($i:expr, $f:expr, $submac:ident!( $($args:tt)* )) => (
    separated_list!($i, call!($f), $submac!($($args)*));
  );
  ($i:expr, $f:expr, $g:expr) => (
    separated_list!($i, call!($f), call!($g));
  );
);

fn is_whitespace(c: u8) -> bool {
    c as char == ' ' || c as char == '\t'
}

fn is_not_cell_end(c: u8) -> bool {
    c as char != ',' && c as char != '\n'
}

fn get_column_value(input: &[u8], pos: Position) -> IResult<&[u8], &[u8], CsvError> {
    let (i, cell) = try_parse!(input,
        fix_error!(CsvError,
            preceded!(
            opt!(consume_useless_chars),
                alt!(
                    string_between_quotes | get_cell
                )
            )
        )
    );

    if i.len() == 0 {
        //IResult::Incomplete(Needed::Unknown)
        IResult::Done(i, cell)
    } else if is_not_cell_end(i[0]) {
        let p = Position { line: pos.line, column: pos.column + input.offset(i) };
        IResult::Error(Err::Code(ErrorKind::Custom(
            CsvError::InvalidCharacter(CharError::new(',', i[0] as char, &p))
        )))
    } else {
        IResult::Done(i, cell)
    }
}

fn get_string_column_value(input: &[u8], pos: Position) -> IResult<&[u8], String, CsvError> {
    map_res!(input,
        map_res!(
            dbg_dmp!(
                apply!(get_column_value, Position::new(pos.line, pos.column))
            ),
            from_utf8
        ),
        |d| {
            str::FromStr::from_str(d)
        }
    )
}

fn comma_then_column<'a>(input: &'a [u8], pos: &Position) -> IResult<&'a [u8], String, CsvError> {
    preceded!(input,
        fix_error!(CsvError, char!(',')),
        apply!(get_string_column_value, Position::new(pos.line, pos.column))
    )
}

fn many_comma_then_column(input: &[u8], pos: Position) -> IResult<&[u8], Vec<String>, CsvError> {
    many0!(
        input,
        apply!(comma_then_column, &pos)
    )
}

fn get_line_values<'a>(entry: &'a[u8], ret: &mut Vec<String>, line: usize) -> IResult<&'a[u8], &'a[u8], CsvError> {
    if entry.len() == 0 {
        IResult::Done(entry, entry)
    } else {
        let (i, col) = try_parse!(entry, apply!(get_string_column_value, Position::new(line, ret.len())));
        ret.push(col);

        match fix_error!(i, CsvError, separated_list2!(
            char!('\n'),
            apply!(many_comma_then_column, Position::new(line, ret.len()))
        )) {
            IResult::Done(i, v)    => {
                let v : Vec<Vec<String>> = v;
                for c in v {
                    for sub_c in c {
                        ret.push(sub_c);
                    }
                }
                IResult::Done(i, &entry[..entry.offset(i)])
            },
            IResult::Incomplete(i) => IResult::Incomplete(i),
            IResult::Error(e)      => IResult::Error(e)
        }
    }
}

fn get_lines_values(mut ret: Vec<Vec<String>>, entry: &[u8]) -> Result<Vec<Vec<String>>, CsvError> {
    let mut input = entry;
    let mut line  = 0;
    loop {
        let mut v: Vec<String> = Vec::new();
        match get_line_values(input, &mut v, line) {
            IResult::Error(Err::Code(ErrorKind::Custom(e))) => return Err(e),
            IResult::Error(_)                               => return Err(CsvError::GenericError),
            IResult::Incomplete(_)                          => {
                // did we reach the end of file?
                break
            }
            IResult::Done(i,_)                              => {
                input = i;
                line += 1;
                ret.push(v);
                if input.len() == 0 {
                    break;
                }
            },
        }
    }

    Ok(ret)
}

pub fn parse_csv_from_slice(entry: &[u8]) -> Result<Vec<Vec<String>>, CsvError> {
    get_lines_values(vec!(), entry)
}

pub fn parse_csv_from_file(filename: &str) -> Result<Vec<Vec<String>>, CsvError> {
    let mut f = File::open(filename).unwrap();
    let mut buffer = vec!();

    f.read_to_end(&mut buffer).unwrap();
    parse_csv_from_slice(&buffer)
}

pub fn parse_csv(entry: &str) -> Result<Vec<Vec<String>>, CsvError> {
    parse_csv_from_slice(entry.as_bytes())
}

#[test]
fn check_string_between_quotes() {
    let f = b"\"nom\",age\ncarles,30\nlaure,28\n";

    match string_between_quotes(f) {
        IResult::Done(in_, out) => {
            assert_eq!(out, b"nom");
            assert_eq!(in_, b",age\ncarles,30\nlaure,28\n");
        },
        IResult::Incomplete(x) => panic!("incomplete: {:?}", x),
        IResult::Error(e) => panic!("error: {:?}", e),
    }
}

#[test]
fn check_get_cell() {
    let f = b"age\ncarles,30\n";
    let g = b"age2,carles,30\n";

    match get_cell(f) {
        IResult::Done(_, out) => assert_eq!(out, b"age"),
        IResult::Incomplete(x) => panic!("incomplete: {:?}", x),
        IResult::Error(e) => panic!("error: {:?}", e),
    }
    match get_cell(g) {
        IResult::Done(_, out) => assert_eq!(out, b"age2"),
        IResult::Incomplete(x) => panic!("incomplete: {:?}", x),
        IResult::Error(e) => panic!("error: {:?}", e),
    }
}

#[test]
fn check_get_line_values() {
    // no terminator, this is not a line
    //let mut cells = vec!();
    //get_line_values(&mut cells, b"\"nom\",,age", 0);
    //assert_eq!(cells, vec!("nom".to_owned(), "".to_owned(), "age".to_owned()));

    let mut cells = vec!();
    let res = get_line_values(b"\"nom\",,age\n", &mut cells, 0);
    println!("res: {:?}", res);
    assert_eq!(cells, vec!("nom".to_owned(), "".to_owned(), "age".to_owned()));

    let mut cells = vec!();
    get_line_values(b"\"nom\",age,\n", &mut cells, 0);
    assert_eq!(cells, vec!("nom".to_owned(), "age".to_owned(), "".to_owned()));

    let mut cells = vec!();
    get_line_values(b"\"nom\",age,,\"hoho\",,end\n", &mut cells, 0);
    assert_eq!(cells, vec!("nom".to_owned(), "age".to_owned(), "".to_owned(), "hoho".to_owned(), "".to_owned(), "end".to_owned()));

    let mut cells = vec!();
    let e = get_line_values(b"\"nom\" ,age,\"hoho\"", &mut cells, 0);
    assert_eq!(e,
        IResult::Error(Err::Code(ErrorKind::Custom(
            CsvError::InvalidCharacter(CharError::new(',', ' ', &Position::new(0, 5)))
        )))
    );
}

#[test]
fn check_get_lines_values() {
    let f = b"\"nom\",age\ncarles,30\nlaure,28\n";

    assert_eq!(get_lines_values(vec!(), f),
               Ok(vec!(
                       vec!("nom".to_owned(), "age".to_owned()),
                       vec!("carles".to_owned(), "30".to_owned()),
                       vec!("laure".to_owned(), "28".to_owned()))));
    let f = b"\"nom\",age\ncarles,30\nlaure,28";

    assert_eq!(get_lines_values(vec!(), f),
               Ok(vec!(
                       vec!("nom".to_owned(), "age".to_owned()),
                       vec!("carles".to_owned(), "30".to_owned()),
                       vec!("laure".to_owned(), "28".to_owned()))));
}

#[test]
fn check_parse_csv() {
    let f = "\"nom\",age\ncarles,30\nlaure,28\n";

    assert_eq!(parse_csv(f),
               Ok(vec!(
                       vec!("nom".to_owned(), "age".to_owned()),
                       vec!("carles".to_owned(), "30".to_owned()),
                       vec!("laure".to_owned(), "28".to_owned()))));
}
<|endoftext|>