<commit_before>use std::fmt;
use std::str::from_utf8;

use rusticata_macros::debug::{HexU16,HexSlice};

use tls::*;
use tls_alert::*;
use tls_dh::*;
use tls_ec::*;
use tls_extensions::*;
use tls_sign_hash::*;

// ------------------------- tls.rs ------------------------------
impl<'a> fmt::Debug for TlsClientHelloContents<'a> {
    fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
        fmt.debug_struct("TlsClientHelloContents")
            .field("version", &self.version)
            .field("rand_time", &self.rand_time)
            .field("rand_data", &HexSlice{d:self.rand_data})
            .field("session_id", &self.session_id.map(|o|{HexSlice{d:o}}))
            .field("ciphers", &self.ciphers)
            .field("comp", &self.comp)
            .field("ext", &self.ext.map(|o|{HexSlice{d:o}}))
            .finish()
    }
}

impl<'a> fmt::Debug for TlsServerHelloContents<'a> {
    fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
        fmt.debug_struct("TlsServerHelloContents")
            .field("version", &self.version)
            .field("rand_time", &self.rand_time)
            .field("rand_data", &HexSlice{d:self.rand_data})
            .field("session_id", &self.session_id.map(|o|{HexSlice{d:o}}))
            .field("cipher", &self.cipher)
            .field("compression", &self.compression)
            .field("ext", &self.ext.map(|o|{HexSlice{d:o}}))
            .finish()
    }
}

impl<'a> fmt::Debug for TlsServerHelloV13Draft18Contents<'a> {
    fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
        fmt.debug_struct("TlsServerHelloV13Draft18Contents")
            .field("version", &self.version)
            .field("random", &HexSlice{d:self.random})
            .field("cipher", &self.cipher)
            .field("ext", &self.ext.map(|o|{HexSlice{d:o}}))
            .finish()
    }
}

impl<'a> fmt::Debug for TlsHelloRetryRequestContents<'a> {
    fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
        fmt.debug_struct("TlsHelloRetryRequestContents")
            .field("version", &self.version)
            .field("ext", &self.ext.map(|o|{HexSlice{d:o}}))
            .finish()
    }
}

impl<'a> fmt::Debug for RawCertificate<'a> {
    fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
        fmt.debug_struct("RawCertificate")
            .field("data", &HexSlice{d:self.data})
            .finish()
    }
}

impl<'a> fmt::Debug for TlsServerKeyExchangeContents<'a> {
    fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
        fmt.debug_struct("TlsServerKeyExchangeContents")
            .field("parameters", &HexSlice{d:self.parameters})
            .finish()
    }
}

impl<'a> fmt::Debug for TlsClientKeyExchangeContents<'a> {
    fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
        match self {
            &TlsClientKeyExchangeContents::Dh(ref p)      => fmt.write_fmt(format_args!("{:?}",HexSlice{d:p})),
            &TlsClientKeyExchangeContents::Ecdh(ref p)    => fmt.write_fmt(format_args!("{:?}",p)),
            &TlsClientKeyExchangeContents::Unknown(ref p) => fmt.write_fmt(format_args!("{:?}",HexSlice{d:p})),
        }
    }
}

impl fmt::Debug for TlsRecordHeader {
    fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
        fmt.debug_struct("TlsRecordHeader")
            .field("type", &self.record_type)
            .field("version", &HexU16{d:self.version})
            .field("len", &self.len)
            .finish()
    }
}

// ------------------------- tls_alert.rs ------------------------------
impl fmt::Debug for TlsMessageAlert {
    fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
        fmt.debug_struct("TlsMessageAlert")
            .field("severity", &self.severity)
            .field("code", &self.code)
            .finish()
    }
}

// ------------------------- tls_dh.rs ------------------------------
impl<'a> fmt::Debug for ServerDHParams<'a> {
    fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
        let gs = self.dh_g.len() * 8;
        fmt.debug_struct("ServerDHParams")
            .field("group size", &gs)
            .field("dh_p", &HexSlice{d:self.dh_p})
            .field("dh_g", &HexSlice{d:self.dh_g})
            .field("dh_ys", &HexSlice{d:self.dh_ys})
            .finish()
    }
}

// ------------------------- tls_ec.rs ------------------------------
impl<'a> fmt::Debug for ECParametersContent<'a> {
    fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
        match self {
            &ECParametersContent::ExplicitPrime(ref p) => fmt.write_fmt(format_args!("ExplicitPrime({:?})",p)),
            &ECParametersContent::ExplicitChar2(ref p) => fmt.write_fmt(format_args!("ExplicitChar2({:?})",HexSlice{d:p})),
            &ECParametersContent::NamedGroup(p)    => write!(fmt, "{}", p),
        }
    }
}

impl<'a> fmt::Debug for ECParameters<'a> {
    fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
        fmt.debug_struct("ECParameters")
            .field("curve_type", &format!("{}",self.curve_type))
            .field("params_content", &self.params_content)
            .finish()
    }
}

// ------------------------- tls_extensions.rs ------------------------------
impl<'a> fmt::Debug for TlsExtension<'a> {
    fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
        match *self {
            TlsExtension::SNI(ref v) => {
                let v : Vec<_> = v.iter().map(|&(ty,n)| {
                    let s = from_utf8(n).unwrap_or("<error decoding utf8 string>");
                    format!("type={},name={}",ty, s)
                }).collect();
                write!(fmt, "TlsExtension::SNI({:?})", v)
            },
            TlsExtension::MaxFragmentLength(l) => write!(fmt, "TlsExtension::MaxFragmentLength({})", l),
            TlsExtension::StatusRequest(data) => write!(fmt, "TlsExtension::StatusRequest({:?})", data),
            TlsExtension::EllipticCurves(ref v) => {
                let v2 : Vec<_> = v.iter().map(|&curve| {
                    format!("{}", curve)
                }).collect();
                write!(fmt, "TlsExtension::EllipticCurves({:?})", v2)
            },
            TlsExtension::EcPointFormats(v) => write!(fmt, "TlsExtension::EcPointFormats({:?})", v),
            TlsExtension::SignatureAlgorithms(ref v) => {
                let v2 : Vec<_> = v.iter().map(|&alg| {
                    format!("{}", alg)
                }).collect();
                write!(fmt, "TlsExtension::SignatureAlgorithms({:?})", v2)
            },
            TlsExtension::SessionTicket(data) => write!(fmt, "TlsExtension::SessionTicket(data={:?})", data),
            TlsExtension::KeyShareOld(data) => write!(fmt, "TlsExtension::KeyShareOld(data={:?})", HexSlice{d:data}),
            TlsExtension::KeyShare(data) => write!(fmt, "TlsExtension::KeyShare(data={:?})", HexSlice{d:data}),
            TlsExtension::PreSharedKey(data) => write!(fmt, "TlsExtension::PreSharedKey(data={:?})", HexSlice{d:data}),
            TlsExtension::EarlyData(o) => write!(fmt, "TlsExtension::EarlyData({:?})",o),
            TlsExtension::SupportedVersions(ref v) => {
                let v2 : Vec<_> = v.iter().map(|c| { format!("{}",c) }).collect();
                write!(fmt, "TlsExtension::SupportedVersions(v={:?})", v2)
            },
            TlsExtension::Cookie(data) => write!(fmt, "TlsExtension::Cookie(data={:?})", data),
            TlsExtension::PskExchangeModes(ref v) => write!(fmt, "TlsExtension::PskExchangeModes({:?})", v),
            TlsExtension::Heartbeat(mode) => write!(fmt, "TlsExtension::Heartbeat(mode={})", mode),
            TlsExtension::ALPN(ref v) => {
                let v : Vec<_> = v.iter().map(|c| {
                    let s = from_utf8(c).unwrap_or("<error decoding utf8 string>");
                    format!("{}",s)
                }).collect();
                write!(fmt, "TlsExtension::ALPN({:?})", v)
            },
            TlsExtension::SignedCertificateTimestamp(data) => write!(fmt, "TlsExtension::SignedCertificateTimestamp(data={:?})", data),
            TlsExtension::Padding(data) => write!(fmt, "TlsExtension::Padding(data={:?})", data),
            TlsExtension::EncryptThenMac => write!(fmt, "TlsExtension::EncryptThenMac"),
            TlsExtension::ExtendedMasterSecret => write!(fmt, "TlsExtension::ExtendedMasterSecret"),
            TlsExtension::OidFilters(ref v) => {
                let v : Vec<_> = v.iter().map(|c| {
                    format!("{:?}",c)
                }).collect();
                write!(fmt, "TlsExtension::OidFilters({:?})", v)
            },
            TlsExtension::PostHandshakeAuth => write!(fmt, "TlsExtension::PostHandshakeAuth"),
            TlsExtension::NextProtocolNegotiation => write!(fmt, "TlsExtension::NextProtocolNegotiation"),
            TlsExtension::RenegotiationInfo(data) => write!(fmt, "TlsExtension::RenegotiationInfo(data={:?})", data),
            TlsExtension::Grease(t,data) => write!(fmt, "TlsExtension::Grease(0x{:x},data={:?})", t, HexSlice{d:data}),
            TlsExtension::Unknown(t,data) => write!(fmt, "TlsExtension::Unknown(type=0x{:x},data={:?})", t.0, data),
        }
    }
}

// ------------------------- tls_sign_hash.rs ------------------------------
impl fmt::Debug for SignatureAndHashAlgorithm {
    fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
        fmt.debug_struct("SignatureAndHashAlgorithm")
            .field("hash", &self.hash)
            .field("sign", &self.sign)
            .finish()
    }
}

impl<'a> fmt::Debug for DigitallySigned<'a> {
    fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
        fmt.debug_struct("DigitallySigned")
            .field("alg", &self.alg)
            .field("data", &HexSlice{d:self.data})
            .finish()
    }
}

<commit_msg>Improve display of SignatureAndHashAlgorithm<commit_after>use std::fmt;
use std::str::from_utf8;

use rusticata_macros::debug::{HexU16,HexSlice};

use tls::*;
use tls_alert::*;
use tls_dh::*;
use tls_ec::*;
use tls_extensions::*;
use tls_sign_hash::*;

// ------------------------- tls.rs ------------------------------
impl<'a> fmt::Debug for TlsClientHelloContents<'a> {
    fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
        fmt.debug_struct("TlsClientHelloContents")
            .field("version", &self.version)
            .field("rand_time", &self.rand_time)
            .field("rand_data", &HexSlice{d:self.rand_data})
            .field("session_id", &self.session_id.map(|o|{HexSlice{d:o}}))
            .field("ciphers", &self.ciphers)
            .field("comp", &self.comp)
            .field("ext", &self.ext.map(|o|{HexSlice{d:o}}))
            .finish()
    }
}

impl<'a> fmt::Debug for TlsServerHelloContents<'a> {
    fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
        fmt.debug_struct("TlsServerHelloContents")
            .field("version", &self.version)
            .field("rand_time", &self.rand_time)
            .field("rand_data", &HexSlice{d:self.rand_data})
            .field("session_id", &self.session_id.map(|o|{HexSlice{d:o}}))
            .field("cipher", &self.cipher)
            .field("compression", &self.compression)
            .field("ext", &self.ext.map(|o|{HexSlice{d:o}}))
            .finish()
    }
}

impl<'a> fmt::Debug for TlsServerHelloV13Draft18Contents<'a> {
    fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
        fmt.debug_struct("TlsServerHelloV13Draft18Contents")
            .field("version", &self.version)
            .field("random", &HexSlice{d:self.random})
            .field("cipher", &self.cipher)
            .field("ext", &self.ext.map(|o|{HexSlice{d:o}}))
            .finish()
    }
}

impl<'a> fmt::Debug for TlsHelloRetryRequestContents<'a> {
    fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
        fmt.debug_struct("TlsHelloRetryRequestContents")
            .field("version", &self.version)
            .field("ext", &self.ext.map(|o|{HexSlice{d:o}}))
            .finish()
    }
}

impl<'a> fmt::Debug for RawCertificate<'a> {
    fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
        fmt.debug_struct("RawCertificate")
            .field("data", &HexSlice{d:self.data})
            .finish()
    }
}

impl<'a> fmt::Debug for TlsServerKeyExchangeContents<'a> {
    fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
        fmt.debug_struct("TlsServerKeyExchangeContents")
            .field("parameters", &HexSlice{d:self.parameters})
            .finish()
    }
}

impl<'a> fmt::Debug for TlsClientKeyExchangeContents<'a> {
    fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
        match self {
            &TlsClientKeyExchangeContents::Dh(ref p)      => fmt.write_fmt(format_args!("{:?}",HexSlice{d:p})),
            &TlsClientKeyExchangeContents::Ecdh(ref p)    => fmt.write_fmt(format_args!("{:?}",p)),
            &TlsClientKeyExchangeContents::Unknown(ref p) => fmt.write_fmt(format_args!("{:?}",HexSlice{d:p})),
        }
    }
}

impl fmt::Debug for TlsRecordHeader {
    fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
        fmt.debug_struct("TlsRecordHeader")
            .field("type", &self.record_type)
            .field("version", &HexU16{d:self.version})
            .field("len", &self.len)
            .finish()
    }
}

// ------------------------- tls_alert.rs ------------------------------
impl fmt::Debug for TlsMessageAlert {
    fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
        fmt.debug_struct("TlsMessageAlert")
            .field("severity", &self.severity)
            .field("code", &self.code)
            .finish()
    }
}

// ------------------------- tls_dh.rs ------------------------------
impl<'a> fmt::Debug for ServerDHParams<'a> {
    fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
        let gs = self.dh_g.len() * 8;
        fmt.debug_struct("ServerDHParams")
            .field("group size", &gs)
            .field("dh_p", &HexSlice{d:self.dh_p})
            .field("dh_g", &HexSlice{d:self.dh_g})
            .field("dh_ys", &HexSlice{d:self.dh_ys})
            .finish()
    }
}

// ------------------------- tls_ec.rs ------------------------------
impl<'a> fmt::Debug for ECParametersContent<'a> {
    fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
        match self {
            &ECParametersContent::ExplicitPrime(ref p) => fmt.write_fmt(format_args!("ExplicitPrime({:?})",p)),
            &ECParametersContent::ExplicitChar2(ref p) => fmt.write_fmt(format_args!("ExplicitChar2({:?})",HexSlice{d:p})),
            &ECParametersContent::NamedGroup(p)    => write!(fmt, "{}", p),
        }
    }
}

impl<'a> fmt::Debug for ECParameters<'a> {
    fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
        fmt.debug_struct("ECParameters")
            .field("curve_type", &format!("{}",self.curve_type))
            .field("params_content", &self.params_content)
            .finish()
    }
}

// ------------------------- tls_extensions.rs ------------------------------
impl<'a> fmt::Debug for TlsExtension<'a> {
    fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
        match *self {
            TlsExtension::SNI(ref v) => {
                let v : Vec<_> = v.iter().map(|&(ty,n)| {
                    let s = from_utf8(n).unwrap_or("<error decoding utf8 string>");
                    format!("type={},name={}",ty, s)
                }).collect();
                write!(fmt, "TlsExtension::SNI({:?})", v)
            },
            TlsExtension::MaxFragmentLength(l) => write!(fmt, "TlsExtension::MaxFragmentLength({})", l),
            TlsExtension::StatusRequest(data) => write!(fmt, "TlsExtension::StatusRequest({:?})", data),
            TlsExtension::EllipticCurves(ref v) => {
                let v2 : Vec<_> = v.iter().map(|&curve| {
                    format!("{}", curve)
                }).collect();
                write!(fmt, "TlsExtension::EllipticCurves({:?})", v2)
            },
            TlsExtension::EcPointFormats(v) => write!(fmt, "TlsExtension::EcPointFormats({:?})", v),
            TlsExtension::SignatureAlgorithms(ref v) => {
                let v2 : Vec<_> = v.iter().map(|&alg| {
                    let s = format!("{}", SignatureScheme(alg));
                    if s.starts_with("SignatureScheme") {
                        format!("{}", SignatureAndHashAlgorithm{hash:HashAlgorithm((alg>>8) as u8), sign:SignAlgorithm((alg&0xff) as u8)})
                    } else {
                        s
                    }
                }).collect();
                write!(fmt, "TlsExtension::SignatureAlgorithms({:?})", v2)
            },
            TlsExtension::SessionTicket(data) => write!(fmt, "TlsExtension::SessionTicket(data={:?})", data),
            TlsExtension::KeyShareOld(data) => write!(fmt, "TlsExtension::KeyShareOld(data={:?})", HexSlice{d:data}),
            TlsExtension::KeyShare(data) => write!(fmt, "TlsExtension::KeyShare(data={:?})", HexSlice{d:data}),
            TlsExtension::PreSharedKey(data) => write!(fmt, "TlsExtension::PreSharedKey(data={:?})", HexSlice{d:data}),
            TlsExtension::EarlyData(o) => write!(fmt, "TlsExtension::EarlyData({:?})",o),
            TlsExtension::SupportedVersions(ref v) => {
                let v2 : Vec<_> = v.iter().map(|c| { format!("{}",c) }).collect();
                write!(fmt, "TlsExtension::SupportedVersions(v={:?})", v2)
            },
            TlsExtension::Cookie(data) => write!(fmt, "TlsExtension::Cookie(data={:?})", data),
            TlsExtension::PskExchangeModes(ref v) => write!(fmt, "TlsExtension::PskExchangeModes({:?})", v),
            TlsExtension::Heartbeat(mode) => write!(fmt, "TlsExtension::Heartbeat(mode={})", mode),
            TlsExtension::ALPN(ref v) => {
                let v : Vec<_> = v.iter().map(|c| {
                    let s = from_utf8(c).unwrap_or("<error decoding utf8 string>");
                    format!("{}",s)
                }).collect();
                write!(fmt, "TlsExtension::ALPN({:?})", v)
            },
            TlsExtension::SignedCertificateTimestamp(data) => write!(fmt, "TlsExtension::SignedCertificateTimestamp(data={:?})", data),
            TlsExtension::Padding(data) => write!(fmt, "TlsExtension::Padding(data={:?})", data),
            TlsExtension::EncryptThenMac => write!(fmt, "TlsExtension::EncryptThenMac"),
            TlsExtension::ExtendedMasterSecret => write!(fmt, "TlsExtension::ExtendedMasterSecret"),
            TlsExtension::OidFilters(ref v) => {
                let v : Vec<_> = v.iter().map(|c| {
                    format!("{:?}",c)
                }).collect();
                write!(fmt, "TlsExtension::OidFilters({:?})", v)
            },
            TlsExtension::PostHandshakeAuth => write!(fmt, "TlsExtension::PostHandshakeAuth"),
            TlsExtension::NextProtocolNegotiation => write!(fmt, "TlsExtension::NextProtocolNegotiation"),
            TlsExtension::RenegotiationInfo(data) => write!(fmt, "TlsExtension::RenegotiationInfo(data={:?})", data),
            TlsExtension::Grease(t,data) => write!(fmt, "TlsExtension::Grease(0x{:x},data={:?})", t, HexSlice{d:data}),
            TlsExtension::Unknown(t,data) => write!(fmt, "TlsExtension::Unknown(type=0x{:x},data={:?})", t.0, data),
        }
    }
}

// ------------------------- tls_sign_hash.rs ------------------------------
impl fmt::Display for SignatureAndHashAlgorithm {
    fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
        write!(fmt, "HashSign({},{})", self.hash, self.sign)
    }
}

impl fmt::Debug for SignatureAndHashAlgorithm {
    fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
        write!(fmt, "SignatureAndHashAlgorithm({},{})", self.hash, self.sign)
    }
}

impl<'a> fmt::Debug for DigitallySigned<'a> {
    fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
        fmt.debug_struct("DigitallySigned")
            .field("alg", &self.alg)
            .field("data", &HexSlice{d:self.data})
            .finish()
    }
}

<|endoftext|>


<commit_before>use event::Event;
use sass::comment::SassComment;
use sass::rule::SassRule;
use sass::selector::SassSelector;
use sass::variable::SassVariable;
use top_level_event::TopLevelEvent;

use std::borrow::Cow::Borrowed;

fn is_ascii_whitespace(c: u8) -> bool {
   is_newline(c) || is_ascii_whitespace_no_nl(c)
}

fn is_ascii_whitespace_no_nl(c: u8) -> bool {
    c == b'\t' || c == 0x0b || c == 0x0c || c == b' '
}

fn is_newline(c: u8) -> bool {
    c == b'\n' || c == b'\r'
}

fn isnt_newline(c: u8) -> bool {
    !is_newline(c)
}

// unusual among "scan" functions in that it scans from the _back_ of the string
// TODO: should also scan unicode whitespace?
fn scan_trailing_whitespace(data: &str) -> usize {
    match data.as_bytes().iter().rev().position(|&c| !is_ascii_whitespace_no_nl(c)) {
        Some(i) => i,
        None => data.len()
    }
}

#[derive(Debug)]
pub struct Tokenizer<'a> {
    sass: &'a str,
    bytes: &'a [u8],
    offset: usize,
    state: State,
    sass_rule_stack: Vec<SassRule<'a>>,
    current_sass_rule_selectors_done: bool,
}

#[derive(PartialEq, Debug, Clone)]
enum State {
    OutsideRules,
    InVariable,
    InComment,
    InRule,
    InSelectors,
    InProperties,
    EndRule,
    Eof,
}

impl<'a> Tokenizer<'a> {
    pub fn new(sass: &'a str) -> Tokenizer<'a> {
        Tokenizer {
            sass: &sass,
            bytes: &sass.as_bytes(),
            offset: 0,
            state: State::OutsideRules,
            sass_rule_stack: Vec::new(),
            current_sass_rule_selectors_done: false,
        }
    }

    fn start_something(&mut self) -> Option<TopLevelEvent<'a>> {
        let mut current_sass_rule = SassRule::new();
        self.current_sass_rule_selectors_done = false;

        self.pick_something();
        while self.state != State::OutsideRules {
            if self.state == State::Eof { return None }

            if self.state == State::InSelectors {
                let sel = self.next_selector();
                if sel.is_some() {
                    current_sass_rule.selectors.push(sel.unwrap());
                }
            } else if self.state == State::InProperties {
                let prop = self.next_property();
                if prop.is_some() {
                    current_sass_rule.children.push(prop.unwrap());
                }
            } else if self.state == State::EndRule {
                // consume '}', should probably be checking that's what we actually have
                self.offset += 1;

                match self.sass_rule_stack.pop() {
                    Some(mut rule) => {
                        rule.children.push(Event::ChildRule(current_sass_rule));
                        current_sass_rule = rule;
                        self.pick_something();
                    },
                    None => self.state = State::OutsideRules,
                }
            } else if self.state == State::InRule {
                self.sass_rule_stack.push(current_sass_rule);
                current_sass_rule = SassRule::new();
                self.current_sass_rule_selectors_done = false;
                self.pick_something();
            } else if self.state == State::InVariable {
                let var = self.next_variable();
                if var.is_some() {
                    return var
                } else {
                    // is this really what we should be doing here? reachable?
                    self.state = State::Eof;
                }
            } else if self.state == State::InComment {
                let comment = self.next_comment();
                if comment.is_some() {
                    if self.sass_rule_stack.len() == 0 &&
                       current_sass_rule.selectors.len() == 0 &&
                       current_sass_rule.children.len() == 0 {
                           return Some(TopLevelEvent::Comment(SassComment { comment: comment.unwrap() }))
                    } else {
                        current_sass_rule.children.push(comment.unwrap());
                        self.pick_something();
                    }
                } else {
                    // is this really what we should be doing here? reachable?
                    self.state = State::Eof;
                }
            } else {
                println!("i dont know what to do for {:?}", self.state);
                println!("current sass rule = {:?}", current_sass_rule);
                self.state = State::Eof;
            }
        }

        Some(TopLevelEvent::Rule(current_sass_rule))
    }

    fn pick_something(&mut self) {
        self.skip_leading_whitespace();

        if self.offset == self.sass.len() {
            self.state = State::Eof;
            return
        }

        let c = self.bytes[self.offset];

        if c == b'}' {
            self.state = State::EndRule;
            return
        }

        if c == b'$' {
            self.state = State::InVariable;
            return
        }

        if c == b'/' && (self.offset + 1) < self.sass.len() {
            let d = self.bytes[self.offset + 1];
            if d == b'*' {
                self.state = State::InComment;
                return
            }
        }

        self.state = State::InSelectors;
    }

    fn scan_while<F>(&mut self, data: &str, f: F) -> usize
            where F: Fn(u8) -> bool {
        match data.as_bytes().iter().position(|&c| !f(c)) {
            Some(i) => i,
            None => data.len()
        }
    }

    fn skip_leading_whitespace(&mut self) {
        let mut i = self.offset;
        let limit = self.sass.len();

        while i < limit {
            let c = self.bytes[i];
            if is_ascii_whitespace(c) {
                i += self.scan_while(&self.sass[i..self.sass.len()], is_ascii_whitespace);
            } else if c == b'/' && i + 1 < limit && self.bytes[i + 1] == b'/' {
                i += self.scan_while(&self.sass[i..self.sass.len()], isnt_newline);
            } else {
                self.offset = i;
                return
            }
        }
        self.offset = limit;
    }

    fn next_comment(&mut self) -> Option<Event<'a>> {
        let comment_body_beginning = self.offset;
        let mut i = comment_body_beginning + 2;
        let limit = self.sass.len();

        while i < limit {
            match self.bytes[i..limit].iter().position(|&c| c == b'*' ) {
                Some(pos) => { i += pos; },
                None => { break; },
            }

            if self.bytes[i+1] == b'/' {
                self.offset = i + 2;
                return Some(Event::Comment(Borrowed(&self.sass[comment_body_beginning..i + 2])))
            } else {
                i += 1;
            }
        }
        None
    }

    fn next_variable(&mut self) -> Option<TopLevelEvent<'a>> {
        // TODO: can parts of this be deduplicated with properties?
        let name_beginning = self.offset;
        let mut i = name_beginning;
        let limit = self.sass.len();

        while i < limit {
            match self.bytes[i..limit].iter().position(|&c| c == b':' ) {
                Some(pos) => { i += pos; },
                None => { break; },
            }

            let name_end = i;

            i += 1;
            self.offset = i;
            self.skip_leading_whitespace();

            let value_beginning = self.offset;
            i = value_beginning;

            while i < limit {
                match self.bytes[i..limit].iter().position(|&c| c == b';') {
                    Some(pos) => { i += pos; },
                    None => { i = limit; break; },
                }

                let value_end = i;
                self.offset = i + 1;

                self.skip_leading_whitespace();

                return Some(TopLevelEvent::Variable(SassVariable{
                    name: Borrowed(&self.sass[name_beginning..name_end]),
                    value: Borrowed(&self.sass[value_beginning..value_end]),
                }))
            }
        }
        self.offset = self.sass.len();
        None
    }

    fn next_property(&mut self) -> Option<Event<'a>> {
        self.skip_leading_whitespace();

        if self.offset == self.sass.len() {
            self.state = State::Eof;
            return None
        }

        let name_beginning = self.offset;
        let mut i = name_beginning;
        let limit = self.sass.len();

        let c = self.bytes[i];
        if c == b'}' {
            self.state = State::EndRule;
            return None
        }

        let d = self.bytes[i + 1];
        if c == b'/' && d == b'*' {
            self.state = State::InComment;
            return None
        }

        while i < limit {
            match self.bytes[i..limit].iter().position(|&c| c == b':' || c == b'{') {
                Some(pos) => { i += pos; },
                None => { break; },
            }

            // Inefficient since we already skipped the whitespace and we'll have to
            // do it again but oh well
            let c = self.bytes[i];
            if c == b'{' {
                self.state = State::InRule;
                return None
            }

            let name_end = i;

            i += 1;
            self.offset = i;
            self.skip_leading_whitespace();

            let value_beginning = self.offset;
            i = value_beginning;

            while i < limit {
                match self.bytes[i..limit].iter().position(|&c| c == b';') {
                    Some(pos) => { i += pos; },
                    None => { i = limit; break; },
                }

                let value_end = i;
                self.offset = i + 1;

                self.skip_leading_whitespace();

                if self.bytes[name_beginning] == b'$' {
                    return Some(Event::Variable(SassVariable {
                        name: Borrowed(&self.sass[name_beginning..name_end]),
                        value: Borrowed(&self.sass[value_beginning..value_end])
                    }))
                } else {
                    return Some(Event::Property(
                        Borrowed(&self.sass[name_beginning..name_end]),
                        Borrowed(&self.sass[value_beginning..value_end])
                    ))
                }
            }
        }
        self.offset = self.sass.len();
        None
    }

    fn next_selector(&mut self) -> Option<SassSelector<'a>> {
        self.skip_leading_whitespace();

        let beginning = self.offset;
        let mut i = beginning;
        let limit = self.sass.len();

        while i < limit {
            match self.bytes[i..limit].iter().position(|&c| c == b',' || c == b'{' || c == b':') {
                Some(pos) => { i += pos; },
                None => { i = limit; break; },
            }

            let c = self.bytes[i];

            if c == b':' {
                self.state = State::InProperties;
                return None
            }

            if c == b',' || c == b'{' {
                let n = scan_trailing_whitespace(&self.sass[beginning..i]);
                let end = i - n;
                if end > beginning {
                    if c == b'{' {
                        if self.current_sass_rule_selectors_done {
                            self.state = State::InRule;
                            return None
                        } else {
                            self.current_sass_rule_selectors_done = true;
                            self.state = State::InProperties;
                        }
                    }
                    self.offset = i + 1;
                    return Some(SassSelector { name: Borrowed(&self.sass[beginning..end]) })
                } else {
                    // only whitespace between commas
                    self.offset += 1;
                    return self.next_selector()
                }
            }

            self.offset = i;
            if i > beginning {
                return Some(SassSelector { name: Borrowed(&self.sass[beginning..i]) })
            }
            i += 1;
        }

        if i > beginning {
            self.offset = i;
            Some(SassSelector { name: Borrowed(&self.sass[beginning..i]) })
        } else {
            self.state = State::Eof;
            None
        }
    }
}

impl<'a> Iterator for Tokenizer<'a> {
    type Item = TopLevelEvent<'a>;

    fn next(&mut self) -> Option<TopLevelEvent<'a>> {
        if self.offset < self.sass.len() {
            return self.start_something()
        }
        None
    }
}
<commit_msg>If the selectors are done, we must be in properties<commit_after>use event::Event;
use sass::comment::SassComment;
use sass::rule::SassRule;
use sass::selector::SassSelector;
use sass::variable::SassVariable;
use top_level_event::TopLevelEvent;

use std::borrow::Cow::Borrowed;

fn is_ascii_whitespace(c: u8) -> bool {
   is_newline(c) || is_ascii_whitespace_no_nl(c)
}

fn is_ascii_whitespace_no_nl(c: u8) -> bool {
    c == b'\t' || c == 0x0b || c == 0x0c || c == b' '
}

fn is_newline(c: u8) -> bool {
    c == b'\n' || c == b'\r'
}

fn isnt_newline(c: u8) -> bool {
    !is_newline(c)
}

// unusual among "scan" functions in that it scans from the _back_ of the string
// TODO: should also scan unicode whitespace?
fn scan_trailing_whitespace(data: &str) -> usize {
    match data.as_bytes().iter().rev().position(|&c| !is_ascii_whitespace_no_nl(c)) {
        Some(i) => i,
        None => data.len()
    }
}

#[derive(Debug)]
pub struct Tokenizer<'a> {
    sass: &'a str,
    bytes: &'a [u8],
    offset: usize,
    state: State,
    sass_rule_stack: Vec<SassRule<'a>>,
    current_sass_rule_selectors_done: bool,
}

#[derive(PartialEq, Debug, Clone)]
enum State {
    OutsideRules,
    InVariable,
    InComment,
    InRule,
    InSelectors,
    InProperties,
    EndRule,
    Eof,
}

impl<'a> Tokenizer<'a> {
    pub fn new(sass: &'a str) -> Tokenizer<'a> {
        Tokenizer {
            sass: &sass,
            bytes: &sass.as_bytes(),
            offset: 0,
            state: State::OutsideRules,
            sass_rule_stack: Vec::new(),
            current_sass_rule_selectors_done: false,
        }
    }

    fn start_something(&mut self) -> Option<TopLevelEvent<'a>> {
        let mut current_sass_rule = SassRule::new();
        self.current_sass_rule_selectors_done = false;

        self.pick_something();
        while self.state != State::OutsideRules {
            if self.state == State::Eof { return None }

            if self.state == State::InSelectors {
                let sel = self.next_selector();
                if sel.is_some() {
                    current_sass_rule.selectors.push(sel.unwrap());
                }
            } else if self.state == State::InProperties {
                let prop = self.next_property();
                if prop.is_some() {
                    current_sass_rule.children.push(prop.unwrap());
                }
            } else if self.state == State::EndRule {
                // consume '}', should probably be checking that's what we actually have
                self.offset += 1;

                match self.sass_rule_stack.pop() {
                    Some(mut rule) => {
                        rule.children.push(Event::ChildRule(current_sass_rule));
                        current_sass_rule = rule;
                        self.pick_something();
                    },
                    None => self.state = State::OutsideRules,
                }
            } else if self.state == State::InRule {
                self.sass_rule_stack.push(current_sass_rule);
                current_sass_rule = SassRule::new();
                self.current_sass_rule_selectors_done = false;
                self.pick_something();
            } else if self.state == State::InVariable {
                let var = self.next_variable();
                if var.is_some() {
                    return var
                } else {
                    // is this really what we should be doing here? reachable?
                    self.state = State::Eof;
                }
            } else if self.state == State::InComment {
                let comment = self.next_comment();
                if comment.is_some() {
                    if self.sass_rule_stack.len() == 0 &&
                       current_sass_rule.selectors.len() == 0 &&
                       current_sass_rule.children.len() == 0 {
                           return Some(TopLevelEvent::Comment(SassComment { comment: comment.unwrap() }))
                    } else {
                        current_sass_rule.children.push(comment.unwrap());
                        self.pick_something();
                    }
                } else {
                    // is this really what we should be doing here? reachable?
                    self.state = State::Eof;
                }
            } else {
                println!("i dont know what to do for {:?}", self.state);
                println!("current sass rule = {:?}", current_sass_rule);
                self.state = State::Eof;
            }
        }

        Some(TopLevelEvent::Rule(current_sass_rule))
    }

    fn pick_something(&mut self) {
        self.skip_leading_whitespace();

        if self.offset == self.sass.len() {
            self.state = State::Eof;
            return
        }

        let c = self.bytes[self.offset];

        if c == b'}' {
            self.state = State::EndRule;
            return
        }

        if c == b'$' {
            self.state = State::InVariable;
            return
        }

        if c == b'/' && (self.offset + 1) < self.sass.len() {
            let d = self.bytes[self.offset + 1];
            if d == b'*' {
                self.state = State::InComment;
                return
            }
        }

        if self.current_sass_rule_selectors_done {
            self.state = State::InProperties;
            return
        }

        self.state = State::InSelectors;
    }

    fn scan_while<F>(&mut self, data: &str, f: F) -> usize
            where F: Fn(u8) -> bool {
        match data.as_bytes().iter().position(|&c| !f(c)) {
            Some(i) => i,
            None => data.len()
        }
    }

    fn skip_leading_whitespace(&mut self) {
        let mut i = self.offset;
        let limit = self.sass.len();

        while i < limit {
            let c = self.bytes[i];
            if is_ascii_whitespace(c) {
                i += self.scan_while(&self.sass[i..self.sass.len()], is_ascii_whitespace);
            } else if c == b'/' && i + 1 < limit && self.bytes[i + 1] == b'/' {
                i += self.scan_while(&self.sass[i..self.sass.len()], isnt_newline);
            } else {
                self.offset = i;
                return
            }
        }
        self.offset = limit;
    }

    fn next_comment(&mut self) -> Option<Event<'a>> {
        let comment_body_beginning = self.offset;
        let mut i = comment_body_beginning + 2;
        let limit = self.sass.len();

        while i < limit {
            match self.bytes[i..limit].iter().position(|&c| c == b'*' ) {
                Some(pos) => { i += pos; },
                None => { break; },
            }

            if self.bytes[i+1] == b'/' {
                self.offset = i + 2;
                return Some(Event::Comment(Borrowed(&self.sass[comment_body_beginning..i + 2])))
            } else {
                i += 1;
            }
        }
        None
    }

    fn next_variable(&mut self) -> Option<TopLevelEvent<'a>> {
        // TODO: can parts of this be deduplicated with properties?
        let name_beginning = self.offset;
        let mut i = name_beginning;
        let limit = self.sass.len();

        while i < limit {
            match self.bytes[i..limit].iter().position(|&c| c == b':' ) {
                Some(pos) => { i += pos; },
                None => { break; },
            }

            let name_end = i;

            i += 1;
            self.offset = i;
            self.skip_leading_whitespace();

            let value_beginning = self.offset;
            i = value_beginning;

            while i < limit {
                match self.bytes[i..limit].iter().position(|&c| c == b';') {
                    Some(pos) => { i += pos; },
                    None => { i = limit; break; },
                }

                let value_end = i;
                self.offset = i + 1;

                self.skip_leading_whitespace();

                return Some(TopLevelEvent::Variable(SassVariable{
                    name: Borrowed(&self.sass[name_beginning..name_end]),
                    value: Borrowed(&self.sass[value_beginning..value_end]),
                }))
            }
        }
        self.offset = self.sass.len();
        None
    }

    fn next_property(&mut self) -> Option<Event<'a>> {
        self.skip_leading_whitespace();

        if self.offset == self.sass.len() {
            self.state = State::Eof;
            return None
        }

        let name_beginning = self.offset;
        let mut i = name_beginning;
        let limit = self.sass.len();

        let c = self.bytes[i];
        if c == b'}' {
            self.state = State::EndRule;
            return None
        }

        let d = self.bytes[i + 1];
        if c == b'/' && d == b'*' {
            self.state = State::InComment;
            return None
        }

        while i < limit {
            match self.bytes[i..limit].iter().position(|&c| c == b':' || c == b'{') {
                Some(pos) => { i += pos; },
                None => { break; },
            }

            // Inefficient since we already skipped the whitespace and we'll have to
            // do it again but oh well
            let c = self.bytes[i];
            if c == b'{' {
                self.state = State::InRule;
                return None
            }

            let name_end = i;

            i += 1;
            self.offset = i;
            self.skip_leading_whitespace();

            let value_beginning = self.offset;
            i = value_beginning;

            while i < limit {
                match self.bytes[i..limit].iter().position(|&c| c == b';') {
                    Some(pos) => { i += pos; },
                    None => { i = limit; break; },
                }

                let value_end = i;
                self.offset = i + 1;

                self.skip_leading_whitespace();

                if self.bytes[name_beginning] == b'$' {
                    return Some(Event::Variable(SassVariable {
                        name: Borrowed(&self.sass[name_beginning..name_end]),
                        value: Borrowed(&self.sass[value_beginning..value_end])
                    }))
                } else {
                    return Some(Event::Property(
                        Borrowed(&self.sass[name_beginning..name_end]),
                        Borrowed(&self.sass[value_beginning..value_end])
                    ))
                }
            }
        }
        self.offset = self.sass.len();
        None
    }

    fn next_selector(&mut self) -> Option<SassSelector<'a>> {
        self.skip_leading_whitespace();

        let beginning = self.offset;
        let mut i = beginning;
        let limit = self.sass.len();

        while i < limit {
            match self.bytes[i..limit].iter().position(|&c| c == b',' || c == b'{' || c == b':') {
                Some(pos) => { i += pos; },
                None => { i = limit; break; },
            }

            let c = self.bytes[i];

            if c == b':' {
                self.state = State::InProperties;
                return None
            }

            if c == b',' || c == b'{' {
                let n = scan_trailing_whitespace(&self.sass[beginning..i]);
                let end = i - n;
                if end > beginning {
                    if c == b'{' {
                        if self.current_sass_rule_selectors_done {
                            self.state = State::InRule;
                            return None
                        } else {
                            self.current_sass_rule_selectors_done = true;
                            self.state = State::InProperties;
                        }
                    }
                    self.offset = i + 1;
                    return Some(SassSelector { name: Borrowed(&self.sass[beginning..end]) })
                } else {
                    // only whitespace between commas
                    self.offset += 1;
                    return self.next_selector()
                }
            }

            self.offset = i;
            if i > beginning {
                return Some(SassSelector { name: Borrowed(&self.sass[beginning..i]) })
            }
            i += 1;
        }

        if i > beginning {
            self.offset = i;
            Some(SassSelector { name: Borrowed(&self.sass[beginning..i]) })
        } else {
            self.state = State::Eof;
            None
        }
    }
}

impl<'a> Iterator for Tokenizer<'a> {
    type Item = TopLevelEvent<'a>;

    fn next(&mut self) -> Option<TopLevelEvent<'a>> {
        if self.offset < self.sass.len() {
            return self.start_something()
        }
        None
    }
}
<|endoftext|>


<commit_before>const R: usize = 256;

/// 245-Way trie
pub struct Node<V> {
    val: Option<V>,
    next: Vec<Option<Node<V>>>
}

impl<V> Node<V> {
    fn new() -> Node<V> {
        Node {
            val: None,
            next: {
                let mut next = Vec::with_capacity(R);
                for _ in 0 .. R { next.push(None) }
                next
            }
        }
    }

    fn put(mut x: Option<Node<V>>, key: &str, val: V, d: usize) -> Option<Node<V>> {
        if x.is_none() {
            x = Some(Node::new());
        }
        if d == key.bytes().len() {
            x.as_mut().map(|n| n.val = Some(val));
            return x
        }
        let c = key.as_bytes()[d] as usize;
        x.as_mut().map(|n| n.next[c] = Node::put(n.next[c].take(), key, val, d+1));
        x
    }

    fn get<'a>(x: Option<&'a Node<V>>, key: &str, d: usize) -> Option<&'a Node<V>> {
        if x.is_none() {
            None
        } else if d == key.bytes().len() {
            x
        } else {
            let c = key.as_bytes()[d] as usize;
            x.as_ref().map(|n| Node::get(n.next[c].as_ref(), key, d+1)).unwrap()
        }
    }

    fn delete(mut x: Option<Node<V>>, key: &str, d: usize) -> (Option<Node<V>>, Option<V>) {
        let mut deleted = None;
        if x.is_none() {
            return (None, deleted);
        }
        if d == key.bytes().len() {
            if x.as_ref().map_or(false, |n| n.val.is_some()) {
                deleted = x.as_mut().map(|n| n.val.take()).unwrap();
            }
        } else {
            let c = key.as_bytes()[d] as usize;
            x.as_mut().map(|n| {
                // FIXME: https://github.com/rust-lang/rfcs/issues/372
                let (nx, nd) = Node::delete(n.next[c].take(), key, d+1);
                n.next[c] = nx;
                deleted = nd;
            });
        }
        if x.as_ref().map_or(false, |n| n.val.is_some()) {
            return (x, deleted);
        }
        for c in 0 .. R {
            if x.as_ref().map_or(false, |n| n.next[c].is_some()) {
                return (x, deleted);
            }
        }
        return (None, deleted);
    }
}

pub struct TrieST<V> {
    root: Option<Node<V>>,
    n: usize
}

impl<V> TrieST<V> {
    pub fn new() -> TrieST<V> {
        TrieST {
            root: None,
            n: 0
        }
    }

    pub fn put(&mut self, key: &str, val: V) {
        self.root = Node::put(self.root.take(), key, val, 0);
        self.n += 1;
    }

    pub fn get(&self, key: &str) -> Option<&V> {
        Node::get(self.root.as_ref(), key, 0).map(|n| n.val.as_ref()).unwrap_or(None)
    }

    pub fn delete(&mut self, key: &str) {
        let (root, deleted) = Node::delete(self.root.take(), key, 0);
        self.root = root;
        if deleted.is_some() {
            self.n -= 1;
        }
    }

    pub fn size(&self) -> usize {
        self.n
    }

    pub fn is_empty(&self) -> bool {
        self.n == 0
    }

    pub fn contains(&self, key: &str) -> bool {
        Node::get(self.root.as_ref(), key, 0).map(|n| n.val.is_some()).unwrap_or(false)
    }
}

#[test]
fn test_tries() {
    let mut t = TrieST::new();
    assert_eq!(t.size(), 0);
    t.put("name", "Andelf");
    t.put("tel", "110");
    t.put("addr1", "Shaanxi");
    t.put("addr long", "Beijing Haidian");
    t.put("addr2", "Beijing");

    assert_eq!(t.size(), 5);
    assert_eq!(t.get("addr2"), Some(&"Beijing"));
    assert_eq!(t.get("non-exists-key"), None);
    assert!(t.contains("tel"));
    t.delete("tel");
    assert_eq!(t.size(), 4);
    assert!(!t.contains("tel"));
}
<commit_msg>tries: implement keys(), keys_with_prefix() for TrieST<commit_after>use std::char;
use adivon::queue::Queue;

const R: usize = 256;

/// 245-Way trie
pub struct Node<V> {
    val: Option<V>,
    next: Vec<Option<Node<V>>>
}

impl<V> Node<V> {
    fn new() -> Node<V> {
        Node {
            val: None,
            next: {
                let mut next = Vec::with_capacity(R);
                for _ in 0 .. R { next.push(None) }
                next
            }
        }
    }

    fn put(mut x: Option<Node<V>>, key: &str, val: V, d: usize) -> Option<Node<V>> {
        if x.is_none() {
            x = Some(Node::new());
        }
        if d == key.bytes().len() {
            x.as_mut().map(|n| n.val = Some(val));
            return x
        }
        let c = key.as_bytes()[d] as usize;
        x.as_mut().map(|n| n.next[c] = Node::put(n.next[c].take(), key, val, d+1));
        x
    }

    fn get<'a>(x: Option<&'a Node<V>>, key: &str, d: usize) -> Option<&'a Node<V>> {
        if x.is_none() {
            None
        } else if d == key.bytes().len() {
            x
        } else {
            let c = key.as_bytes()[d] as usize;
            x.as_ref().map(|n| Node::get(n.next[c].as_ref(), key, d+1)).unwrap()
        }
    }

    fn delete(mut x: Option<Node<V>>, key: &str, d: usize) -> (Option<Node<V>>, Option<V>) {
        let mut deleted = None;
        if x.is_none() {
            return (None, deleted);
        }
        if d == key.bytes().len() {
            if x.as_ref().map_or(false, |n| n.val.is_some()) {
                deleted = x.as_mut().map(|n| n.val.take()).unwrap();
            }
        } else {
            let c = key.as_bytes()[d] as usize;
            x.as_mut().map(|n| {
                // FIXME: https://github.com/rust-lang/rfcs/issues/372
                let (nx, nd) = Node::delete(n.next[c].take(), key, d+1);
                n.next[c] = nx;
                deleted = nd;
            });
        }
        if x.as_ref().map_or(false, |n| n.val.is_some()) {
            return (x, deleted);
        }
        for c in 0 .. R {
            if x.as_ref().map_or(false, |n| n.next[c].is_some()) {
                return (x, deleted);
            }
        }
        return (None, deleted);
    }

    // use '.' as pattern
    fn collect_by_pattern(x: Option<&Node<V>>, prefix: &mut String, pattern: &str, results: &mut Queue<String>) {
        if x.is_none() {
            return;
        }
        let d = prefix.bytes().len();
        if d == pattern.bytes().len() {
            if x.map_or(false, |n| n.val.is_some()) {
                results.enqueue(prefix.clone());
            }
            return;
        }
        let c = pattern.as_bytes()[d];
        if c == '.' as u8 {
            for ch in 0 .. R {
                prefix.push(char::from_u32(ch as u32).unwrap());
                Node::collect_by_pattern(x.unwrap().next[c as usize].as_ref(),
                                         prefix, pattern, results);
                prefix.pop();
            }
        } else {
            prefix.push(c as char);
            Node::collect_by_pattern(x.unwrap().next[c as usize].as_ref(),
                                     prefix, pattern, results);
            prefix.pop();
        }
    }

    fn collect(x: Option<&Node<V>>, mut prefix: String, results: &mut Queue<String>) {
        if x.is_none() {
            return;
        }
        if x.unwrap().val.is_some() {
            results.enqueue(prefix.clone());
        }
        for c in 0 .. R {
            prefix.push(char::from_u32(c as u32).unwrap());
            Node::collect(x.unwrap().next[c].as_ref(), prefix.clone(), results);
            prefix.pop();
        }
    }
}

pub struct TrieST<V> {
    root: Option<Node<V>>,
    n: usize
}

impl<V> TrieST<V> {
    pub fn new() -> TrieST<V> {
        TrieST {
            root: None,
            n: 0
        }
    }

    pub fn put(&mut self, key: &str, val: V) {
        self.root = Node::put(self.root.take(), key, val, 0);
        self.n += 1;
    }

    pub fn get(&self, key: &str) -> Option<&V> {
        Node::get(self.root.as_ref(), key, 0).map(|n| n.val.as_ref()).unwrap_or(None)
    }

    pub fn delete(&mut self, key: &str) {
        let (root, deleted) = Node::delete(self.root.take(), key, 0);
        self.root = root;
        if deleted.is_some() {
            self.n -= 1;
        }
    }

    pub fn size(&self) -> usize {
        self.n
    }

    pub fn is_empty(&self) -> bool {
        self.n == 0
    }

    pub fn contains(&self, key: &str) -> bool {
        Node::get(self.root.as_ref(), key, 0).map(|n| n.val.is_some()).unwrap_or(false)
    }

    pub fn keys_with_prefix(&self, prefix: &str) -> Vec<String> {
        let mut results = Queue::new();
        let x = Node::get(self.root.as_ref(), prefix, 0);
        Node::collect(x, prefix.into(), &mut results);
        results.into_iter().collect()
    }

    pub fn keys(&self) -> Vec<String> {
        self.keys_with_prefix("")
    }

    // TODO:
    // keys_that_match()
    // longest_prefix_of()
}

#[test]
fn test_tries() {
    let mut t = TrieST::new();
    assert_eq!(t.size(), 0);
    t.put("name", "Andelf");
    t.put("tel", "110");
    t.put("addr1", "Shaanxi");
    t.put("addr long", "Beijing Haidian");
    t.put("addr2", "Beijing");

    assert_eq!(t.size(), 5);
    assert_eq!(t.get("addr2"), Some(&"Beijing"));
    assert_eq!(t.get("non-exists-key"), None);
    assert!(t.contains("tel"));
    t.delete("tel");
    assert_eq!(t.size(), 4);
    assert!(!t.contains("tel"));
    assert_eq!(vec!["addr long", "addr1", "addr2", "name"], t.keys());
}

// TST
// pub struct TernarySearchTrie { }
<|endoftext|>


<commit_before>/// # Module variables

use self::LpExpression::*;
use std::convert::Into;
use std::rc::Rc;

trait BoundableLp : PartialEq + Clone {
    fn lower_bound(&self, lw: f32) -> Self;
    fn upper_bound(&self, up: f32) -> Self;
}

#[derive(Debug, Clone, PartialEq)]
pub struct LpBinary {
    pub name: String
}
impl LpBinary {
    pub fn new(name: &str) -> LpBinary {
        LpBinary { name: name.to_string() }
    }
}

#[derive(Debug, Clone, PartialEq)]
pub struct LpInteger {
    pub name: String,
    pub lower_bound: Option<f32>,
    pub upper_bound: Option<f32>,
}
impl LpInteger {
    pub fn new(name: &str) -> LpInteger {
        LpInteger { name: name.to_string(), lower_bound: None, upper_bound: None }
    }
}

#[derive(Debug, Clone, PartialEq)]
pub struct LpContinuous {
    pub name: String,
    pub lower_bound: Option<f32>,
    pub upper_bound: Option<f32>,
}
impl LpContinuous {
    pub fn new(name: &str) -> LpContinuous {
        LpContinuous { name: name.to_string(), lower_bound: None, upper_bound: None }
    }
}

impl BoundableLp for LpInteger {
    fn lower_bound(&self, lw: f32) -> LpInteger {
        LpInteger {
            name: self.name.clone(),
            lower_bound: Some(lw),
            upper_bound: self.upper_bound
        }
    }
    fn upper_bound(&self, up: f32) -> LpInteger {
        LpInteger {
            name: self.name.clone(),
            lower_bound: self.lower_bound,
            upper_bound: Some(up)
        }
    }
}

/// ADT for Linear Programming Expression
#[derive(Debug, Clone, PartialEq)]
pub enum LpExpression {
    ConsInt(LpInteger),
    ConsBin(LpBinary),
    ConsCont(LpContinuous),
    MulExpr(Rc<LpExpression>, Rc<LpExpression>),
    AddExpr(Rc<LpExpression>, Rc<LpExpression>),
    SubExpr(Rc<LpExpression>, Rc<LpExpression>),
    LitVal(f32),
    EmptyExpr
}

impl LpExpression {
    pub fn dfs_remove_constant(&self) -> LpExpression {
        match self {
            &MulExpr(ref rc_e1, ref rc_e2) => {
                let ref e1 = **rc_e1;
                let ref e2 = **rc_e2;
                if let &LitVal(..) = e1 {
                    if let &LitVal(..) = e2 {
                        EmptyExpr
                    }else{
                        MulExpr(rc_e1.clone(), Rc::new(e2.dfs_remove_constant()))
                    }
                }else{
                    if let &LitVal(..) = e2 {
                        // Fix: Literal must be on the left side for multiplication
                        //MulExpr(Rc::new(dfs_remove_constant(e1)), rc_e2.clone())
                        MulExpr(rc_e2.clone(), Rc::new(e1.dfs_remove_constant()))
                    }else {
                        MulExpr(Rc::new(e1.dfs_remove_constant()), Rc::new(e2.dfs_remove_constant()))
                    }
                }
            },
            &AddExpr(ref rc_e1, ref rc_e2) => {
                let ref e1 = **rc_e1;
                let ref e2 = **rc_e2;
                if let &LitVal(..) = e1 {
                    if let &LitVal(..) = e2 {
                        EmptyExpr
                    }else {
                        e2.dfs_remove_constant()
                    }
                }else{
                    if let &LitVal(..) = e2 {
                        e1.dfs_remove_constant()
                    }else {
                        AddExpr(Rc::new(e1.dfs_remove_constant()), Rc::new(e2.dfs_remove_constant()))
                    }
                }
            },
            &SubExpr(ref rc_e1, ref rc_e2) => {
                let ref e1 = **rc_e1;
                let ref e2 = **rc_e2;
                if let &LitVal(..) = e1 {
                    if let &LitVal(..) = e2 {
                        EmptyExpr
                    }else {
                        e2.dfs_remove_constant()
                    }
                }else{
                    if let &LitVal(..) = e2 {
                        e1.dfs_remove_constant()
                    }else {
                        SubExpr(Rc::new(e1.dfs_remove_constant()), Rc::new(e2.dfs_remove_constant()))
                    }
                }
            },
            _ => self.clone()
        }
    }
}

#[derive(Debug, Clone)]
pub enum Constraint {
    /* Not supported by solver format files (lp file or mps file) !
    Greater,
    Less,
    */
    GreaterOrEqual,
    LessOrEqual,
    Equal
}

#[derive(Debug, Clone)]
pub struct LpConstraint(pub LpExpression, pub Constraint, pub LpExpression);

impl LpConstraint {
    pub fn generalize(&self) -> LpConstraint {
        // TODO: Optimize tailrec
        fn dfs_constant(expr: &LpExpression, acc: f32) -> f32 {
            match expr {
                &MulExpr(ref rc_e1, ref rc_e2) => {
                    let ref e1 = **rc_e1;
                    let ref e2 = **rc_e2;
                    if let &LitVal(ref x) = e1 {
                        if let &LitVal(ref y) = e2 {
                            acc+x*y
                        }else{
                            dfs_constant(e2, acc)
                        }
                    }else{
                        if let &LitVal(ref y) = e2 {
                            dfs_constant(e1, acc+y)
                        }else {
                            dfs_constant(e2, acc) + dfs_constant(e1, 0.0)
                        }
                    }
                },
                &AddExpr(ref rc_e1, ref rc_e2) => {
                    let ref e1 = **rc_e1;
                    let ref e2 = **rc_e2;
                    if let &LitVal(ref x) = e1 {
                        if let &LitVal(ref y) = e2 {
                            acc+x+y
                        }else {
                            dfs_constant(e2, acc+x)
                        }
                    }else{
                        if let &LitVal(ref y) = e2 {
                            dfs_constant(e1, acc+y)
                        }else {
                            dfs_constant(e2, acc) + dfs_constant(e1, 0.0)
                        }
                    }
                },
                &SubExpr(ref rc_e1, ref rc_e2) => {
                    let ref e1 = **rc_e1;
                    let ref e2 = **rc_e2;
                    if let &LitVal(ref x) = e1 {
                        if let &LitVal(ref y) = e2 {
                            acc+x-y
                        }else {
                            dfs_constant(e2, acc+x)
                        }
                    }else{
                        if let &LitVal(ref y) = e2 {
                            dfs_constant(e1, acc-y)
                        }else {
                            dfs_constant(e1, acc) - dfs_constant(e2, 0.0)
                        }
                    }
                },
                _ => acc
            }
        }

        let &LpConstraint(ref lhs, ref op, ref rhs) = self;
        if let &LitVal(0.0) = rhs {
            self.clone()
        }else{
            let ref lhs_constraint = lhs - rhs;
            let constant = dfs_constant(lhs_constraint, 0.0);
            let lhs_constraint = lhs_constraint.dfs_remove_constant();
            LpConstraint(lhs_constraint, op.clone(), LitVal(-constant))
        }
    }
}

/// make a complete expression or a constraint with a vector of expressions
///
/// # Exemples
///
/// ```
/// use lp_modeler::problem::{LpObjective, LpProblem};
/// use lp_modeler::operations::LpOperations;
/// use lp_modeler::variables::{LpVariable, LpType, lp_sum};
///
/// let mut problem = LpProblem::new("My Problem", LpObjective::Maximize);
/// let ref a = LpVariable::new("a", LpType::Binary);
/// let ref b = LpVariable::new("b", LpType::Binary);
/// let ref c = LpVariable::new("c", LpType::Binary);
///
/// let ref v = vec!(a, b, c);
/// problem += lp_sum(v).equal(10.0);
/// ```
///
pub fn lp_sum<T>(expr: &Vec<T>) -> LpExpression where T : Into<LpExpression> + Clone {

    let mut expr = expr.clone();
    if let Some(e1) = expr.pop() {
        if let Some(e2) = expr.pop() {
            expr.push(e2);
            AddExpr(Rc::new(e1.into()), Rc::new(lp_sum(&expr)))
        } else {
            e1.into()
        }
    }else{
        EmptyExpr
    }
}

<commit_msg>Add macro helper implem. LpInt and LpCont<commit_after>/// # Module variables

use self::LpExpression::*;
use std::convert::Into;
use std::rc::Rc;

trait BoundableLp : PartialEq + Clone {
    fn lower_bound(&self, lw: f32) -> Self;
    fn upper_bound(&self, up: f32) -> Self;
}

#[derive(Debug, Clone, PartialEq)]
pub struct LpBinary {
    pub name: String
}
impl LpBinary {
    pub fn new(name: &str) -> LpBinary {
        LpBinary { name: name.to_string() }
    }
}

#[derive(Debug, Clone, PartialEq)]
pub struct LpInteger {
    pub name: String,
    pub lower_bound: Option<f32>,
    pub upper_bound: Option<f32>,
}
impl LpInteger {
    pub fn new(name: &str) -> LpInteger {
        LpInteger { name: name.to_string(), lower_bound: None, upper_bound: None }
    }
}

#[derive(Debug, Clone, PartialEq)]
pub struct LpContinuous {
    pub name: String,
    pub lower_bound: Option<f32>,
    pub upper_bound: Option<f32>,
}
impl LpContinuous {
    pub fn new(name: &str) -> LpContinuous {
        LpContinuous { name: name.to_string(), lower_bound: None, upper_bound: None }
    }
}

macro_rules! implement_boundable {
    ($lp_type: ident) => {
        impl BoundableLp for $lp_type {
            fn lower_bound(&self, lw: f32) -> $lp_type {
                $lp_type {
                    name: self.name.clone(),
                    lower_bound: Some(lw),
                    upper_bound: self.upper_bound
                }
            }
            fn upper_bound(&self, up: f32) -> $lp_type {
                $lp_type {
                    name: self.name.clone(),
                    lower_bound: self.lower_bound,
                    upper_bound: Some(up)
                }
            }
        }
    }
}
implement_boundable!(LpInteger);
implement_boundable!(LpContinuous);

/// ADT for Linear Programming Expression
#[derive(Debug, Clone, PartialEq)]
pub enum LpExpression {
    ConsInt(LpInteger),
    ConsBin(LpBinary),
    ConsCont(LpContinuous),
    MulExpr(Rc<LpExpression>, Rc<LpExpression>),
    AddExpr(Rc<LpExpression>, Rc<LpExpression>),
    SubExpr(Rc<LpExpression>, Rc<LpExpression>),
    LitVal(f32),
    EmptyExpr
}

impl LpExpression {
    pub fn dfs_remove_constant(&self) -> LpExpression {
        match self {
            &MulExpr(ref rc_e1, ref rc_e2) => {
                let ref e1 = **rc_e1;
                let ref e2 = **rc_e2;
                if let &LitVal(..) = e1 {
                    if let &LitVal(..) = e2 {
                        EmptyExpr
                    }else{
                        MulExpr(rc_e1.clone(), Rc::new(e2.dfs_remove_constant()))
                    }
                }else{
                    if let &LitVal(..) = e2 {
                        // Fix: Literal must be on the left side for multiplication
                        //MulExpr(Rc::new(dfs_remove_constant(e1)), rc_e2.clone())
                        MulExpr(rc_e2.clone(), Rc::new(e1.dfs_remove_constant()))
                    }else {
                        MulExpr(Rc::new(e1.dfs_remove_constant()), Rc::new(e2.dfs_remove_constant()))
                    }
                }
            },
            &AddExpr(ref rc_e1, ref rc_e2) => {
                let ref e1 = **rc_e1;
                let ref e2 = **rc_e2;
                if let &LitVal(..) = e1 {
                    if let &LitVal(..) = e2 {
                        EmptyExpr
                    }else {
                        e2.dfs_remove_constant()
                    }
                }else{
                    if let &LitVal(..) = e2 {
                        e1.dfs_remove_constant()
                    }else {
                        AddExpr(Rc::new(e1.dfs_remove_constant()), Rc::new(e2.dfs_remove_constant()))
                    }
                }
            },
            &SubExpr(ref rc_e1, ref rc_e2) => {
                let ref e1 = **rc_e1;
                let ref e2 = **rc_e2;
                if let &LitVal(..) = e1 {
                    if let &LitVal(..) = e2 {
                        EmptyExpr
                    }else {
                        e2.dfs_remove_constant()
                    }
                }else{
                    if let &LitVal(..) = e2 {
                        e1.dfs_remove_constant()
                    }else {
                        SubExpr(Rc::new(e1.dfs_remove_constant()), Rc::new(e2.dfs_remove_constant()))
                    }
                }
            },
            _ => self.clone()
        }
    }
}

#[derive(Debug, Clone)]
pub enum Constraint {
    /* Not supported by solver format files (lp file or mps file) !
    Greater,
    Less,
    */
    GreaterOrEqual,
    LessOrEqual,
    Equal
}

#[derive(Debug, Clone)]
pub struct LpConstraint(pub LpExpression, pub Constraint, pub LpExpression);

impl LpConstraint {
    pub fn generalize(&self) -> LpConstraint {
        // TODO: Optimize tailrec
        fn dfs_constant(expr: &LpExpression, acc: f32) -> f32 {
            match expr {
                &MulExpr(ref rc_e1, ref rc_e2) => {
                    let ref e1 = **rc_e1;
                    let ref e2 = **rc_e2;
                    if let &LitVal(ref x) = e1 {
                        if let &LitVal(ref y) = e2 {
                            acc+x*y
                        }else{
                            dfs_constant(e2, acc)
                        }
                    }else{
                        if let &LitVal(ref y) = e2 {
                            dfs_constant(e1, acc+y)
                        }else {
                            dfs_constant(e2, acc) + dfs_constant(e1, 0.0)
                        }
                    }
                },
                &AddExpr(ref rc_e1, ref rc_e2) => {
                    let ref e1 = **rc_e1;
                    let ref e2 = **rc_e2;
                    if let &LitVal(ref x) = e1 {
                        if let &LitVal(ref y) = e2 {
                            acc+x+y
                        }else {
                            dfs_constant(e2, acc+x)
                        }
                    }else{
                        if let &LitVal(ref y) = e2 {
                            dfs_constant(e1, acc+y)
                        }else {
                            dfs_constant(e2, acc) + dfs_constant(e1, 0.0)
                        }
                    }
                },
                &SubExpr(ref rc_e1, ref rc_e2) => {
                    let ref e1 = **rc_e1;
                    let ref e2 = **rc_e2;
                    if let &LitVal(ref x) = e1 {
                        if let &LitVal(ref y) = e2 {
                            acc+x-y
                        }else {
                            dfs_constant(e2, acc+x)
                        }
                    }else{
                        if let &LitVal(ref y) = e2 {
                            dfs_constant(e1, acc-y)
                        }else {
                            dfs_constant(e1, acc) - dfs_constant(e2, 0.0)
                        }
                    }
                },
                _ => acc
            }
        }

        let &LpConstraint(ref lhs, ref op, ref rhs) = self;
        if let &LitVal(0.0) = rhs {
            self.clone()
        }else{
            let ref lhs_constraint = lhs - rhs;
            let constant = dfs_constant(lhs_constraint, 0.0);
            let lhs_constraint = lhs_constraint.dfs_remove_constant();
            LpConstraint(lhs_constraint, op.clone(), LitVal(-constant))
        }
    }
}

/// make a complete expression or a constraint with a vector of expressions
///
/// # Exemples
///
/// ```
/// use lp_modeler::problem::{LpObjective, LpProblem};
/// use lp_modeler::operations::LpOperations;
/// use lp_modeler::variables::{LpVariable, LpType, lp_sum};
///
/// let mut problem = LpProblem::new("My Problem", LpObjective::Maximize);
/// let ref a = LpVariable::new("a", LpType::Binary);
/// let ref b = LpVariable::new("b", LpType::Binary);
/// let ref c = LpVariable::new("c", LpType::Binary);
///
/// let ref v = vec!(a, b, c);
/// problem += lp_sum(v).equal(10.0);
/// ```
///
pub fn lp_sum<T>(expr: &Vec<T>) -> LpExpression where T : Into<LpExpression> + Clone {

    let mut expr = expr.clone();
    if let Some(e1) = expr.pop() {
        if let Some(e2) = expr.pop() {
            expr.push(e2);
            AddExpr(Rc::new(e1.into()), Rc::new(lp_sum(&expr)))
        } else {
            e1.into()
        }
    }else{
        EmptyExpr
    }
}

<|endoftext|>


<commit_before>extern crate generic_array;
extern crate libc;
extern crate sha2;
extern crate xattr;

use std::ffi::{OsStr, OsString};
use std::io;
use std::os::unix::ffi::OsStrExt;
use std::os::unix::io::RawFd;
use std::path::{Component, Path, PathBuf};

use self::generic_array::GenericArray;
use self::generic_array::typenum::U64;
use self::sha2::{Sha512, Digest};
use self::xattr::FileExt;

use catfs::error;
use catfs::error::RError;
use catfs::rlibc;
use catfs::rlibc::File;

pub struct Handle {
    src_file: File,
    cache_file: File,
    dirty: bool,
}

// no-op to workaround the fact that we send the entire CatFS at start
// time, but we never send anything. Could have used Unique but that
// bounds us to rust nightly
unsafe impl Send for Handle {}

fn make_rdwr(f: &mut u32) {
    *f = (*f & !rlibc::O_ACCMODE) | rlibc::O_RDWR;
}

fn maybe_unlinkat(dir: RawFd, path: &AsRef<Path>) -> io::Result<()> {
    if let Err(e) = rlibc::unlinkat(dir, path, 0) {
        if !error::is_enoent(&e) {
            return Err(e);
        }
    }
    return Ok(());
}

pub fn mkdirat_all(dir: RawFd, path: &AsRef<Path>, mode: u32) -> io::Result<()> {
    let mut p = PathBuf::new();

    for c in path.as_ref().components() {
        if let Component::Normal(n) = c {
            p.push(n);

            if let Err(e) = rlibc::mkdirat(dir, &p, mode) {
                if e.raw_os_error().unwrap() != libc::EEXIST {
                    return Err(e);
                }
            }
        }
    }

    return Ok(());
}

impl Handle {
    pub fn create(
        src_dir: RawFd,
        cache_dir: RawFd,
        path: &AsRef<Path>,
        flags: u32,
        mode: u32,
    ) -> error::Result<Handle> {
        // need to read the cache file for writeback
        let mut cache_flags = flags.clone();
        if (cache_flags & rlibc::O_ACCMODE) == rlibc::O_WRONLY {
            make_rdwr(&mut cache_flags);
        }
        //debug!("create {:b} {:b} {:#o}", flags, cache_flags, mode);

        if let Some(parent) = path.as_ref().parent() {
            mkdirat_all(cache_dir, &parent, 0o777)?;
        }

        let src_file = File::openat(src_dir, path, flags, mode)?;
        // we are able to create the src file, then the cache file
        // shouldn't be here, but it could be because of bug/crash,
        // so unlink it first
        maybe_unlinkat(cache_dir, path)?;

        return Ok(Handle {
            src_file: src_file,
            cache_file: File::openat(cache_dir, path, cache_flags, mode)?,
            dirty: true,
        });
    }

    pub fn open(
        src_dir: RawFd,
        cache_dir: RawFd,
        path: &AsRef<Path>,
        flags: u32,
        cache_valid_if_present: bool,
    ) -> error::Result<Handle> {
        // even if file is open for write only, I still need to be
        // able to read the src for read-modify-write
        let mut flags = flags;
        if (flags & rlibc::O_ACCMODE) == rlibc::O_WRONLY {
            make_rdwr(&mut flags);
        }

        let valid = Handle::validate_cache(src_dir, cache_dir, &path, cache_valid_if_present)?;
        debug!(
            "{:?} {} a valid cache file",
            path.as_ref(),
            if valid { "is" } else { "is not" },
        );
        let mut cache_flags = flags.clone();

        if !valid {
            // mkdir the parents
            if let Some(parent) = path.as_ref().parent() {
                mkdirat_all(cache_dir, &parent, 0o777)?;
            }
            // need to cache this file so need to open it for write
            cache_flags |= rlibc::O_CREAT;
            if (cache_flags & rlibc::O_ACCMODE) == rlibc::O_RDONLY {
                make_rdwr(&mut cache_flags);
            }
        }

        let src_file: File;
        if valid && (flags & rlibc::O_ACCMODE) == rlibc::O_RDONLY {
            src_file = Default::default();
        } else {
            src_file = File::openat(src_dir, path, flags, 0o666)?;
        }

        let handle = Handle {
            src_file: src_file,
            cache_file: File::openat(cache_dir, path, cache_flags, 0o666)?,
            dirty: false,
        };

        if !valid && (flags & rlibc::O_TRUNC) == 0 {
            debug!("read ahead {:?}", path.as_ref());
            handle.copy(true)?;
        }

        return Ok(handle);
    }

    // see validate_cache.sh on how to replicate this
    pub fn src_str_to_checksum(f: &File) -> error::Result<OsString> {
        let mut s = OsString::new();
        for x in ["s3.etag"].iter() {
            if let Some(v) = f.get_xattr(&x)? {
                s.push(x);
                s.push(OsStr::new("="));
                s.push("0x");
                for b in v {
                    s.push(format!("{:x}", b));
                }
                s.push("\n");
            }
        }

        let st = f.stat()?;
        s.push(format!("{}\n", st.st_mtime));
        s.push(format!("{}\n", st.st_size));
        return Ok(s);
    }

    fn src_chksum(f: &File) -> error::Result<GenericArray<u8, U64>> {
        let s = Handle::src_str_to_checksum(f)?;
        debug!("checksum is {:?}", s);
        let mut h = Sha512::default();
        h.input(s.as_bytes());
        return Ok(h.result());
    }

    fn set_pristine(&self, pristine: bool) -> error::Result<()> {
        if pristine {
            self.cache_file.set_xattr(
                "user.catfs.src_chksum",
                Handle::src_chksum(&self.src_file)?
                    .as_slice(),
            )?;
        } else {
            self.cache_file.remove_xattr("user.catfs.src_chksum")?;
        }
        return Ok(());
    }

    fn is_pristine(src_file: &File, cache_file: &File) -> error::Result<bool> {
        if let Some(v) = cache_file.get_xattr("user.catfs.src_chksum")? {
            let expected = Handle::src_chksum(src_file)?;
            if v == expected.as_slice() {
                return Ok(true);
            } else {
                debug!("{:?} != {:?}, {} {}", v, expected, v.len(), expected.len());
                return Ok(false);
            }
        }
        debug!("user.catfs.src_chksum missing for cache_file");

        return Ok(false);
    }

    pub fn unlink(src_dir: RawFd, cache_dir: RawFd, path: &AsRef<Path>) -> io::Result<()> {
        maybe_unlinkat(cache_dir, path)?;
        return rlibc::unlinkat(src_dir, path, 0);
    }

    fn validate_cache(
        src_dir: RawFd,
        cache_dir: RawFd,
        path: &AsRef<Path>,
        cache_valid_if_present: bool,
    ) -> error::Result<bool> {
        match File::openat(src_dir, path, rlibc::O_RDONLY, 0) {
            Ok(mut src_file) => {
                match File::openat(cache_dir, path, rlibc::O_RDONLY, 0) {
                    Ok(mut cache_file) => {
                        let valid: bool;
                        if cache_valid_if_present || Handle::is_pristine(&src_file, &cache_file)? {
                            valid = true;
                        } else {
                            valid = false;
                            rlibc::unlinkat(cache_dir, path, 0)?;
                        }
                        src_file.close()?;
                        cache_file.close()?;
                        return Ok(valid);
                    }
                    Err(e) => {
                        src_file.close()?;
                        if error::try_enoent(e)? {
                            return Ok(false);
                        }
                    }
                }
            }
            Err(e) => {
                if error::try_enoent(e)? {
                    // the source file doesn't exist, the cache file shouldn't either
                    maybe_unlinkat(cache_dir, path)?;
                }
            }
        }

        return Ok(false);
    }

    pub fn read(&self, offset: u64, buf: &mut [u8]) -> error::Result<usize> {
        let nwant = buf.len();
        let mut bytes_read: usize = 0;

        while bytes_read < nwant {
            match self.cache_file.read_at(
                &mut buf[bytes_read..],
                offset + (bytes_read as u64),
            ) {
                Ok(nread) => {
                    if nread == 0 {
                        return Ok(bytes_read);
                    }
                    bytes_read += nread;
                }
                Err(e) => {
                    if bytes_read > 0 {
                        return Ok(bytes_read);
                    } else {
                        return Err(RError::from(e));
                    }
                }
            }
        }

        return Ok(bytes_read);
    }

    pub fn write(&mut self, offset: u64, buf: &[u8]) -> error::Result<usize> {
        let nwant = buf.len();
        let mut bytes_written: usize = 0;

        if !self.dirty {
            // assumes that the metadata will hit the disk before the
            // incoming data will, and not flushing
            self.set_pristine(false)?;
        }

        while bytes_written < nwant {
            match self.cache_file.write_at(
                &buf[bytes_written..],
                offset + (bytes_written as u64),
            ) {
                Ok(nwritten) => {
                    if nwritten == 0 {
                        return Ok(bytes_written);
                    }
                    bytes_written += nwritten;
                }
                Err(e) => {
                    if bytes_written > 0 {
                        return Ok(bytes_written);
                    } else {
                        return Err(RError::from(e));
                    }
                }
            }
        }

        if bytes_written != 0 {
            self.dirty = true;
        }

        return Ok(bytes_written);
    }

    pub fn flush(&mut self) -> error::Result<(bool)> {
        let mut flushed_to_src = false;
        if self.dirty {
            self.copy(false)?;
            self.dirty = false;
            self.cache_file.flush()?;
            // if file was opened for read only and cache is valid we
            // might not have opened src_file
            if self.src_file.valid() {
                self.src_file.flush()?;
                flushed_to_src = true;
            }
        }
        return Ok(flushed_to_src);
    }

    fn copy_user(rh: &File, wh: &File) -> error::Result<()> {
        let mut buf = [0u8; 32 * 1024];
        let mut offset = 0;
        loop {
            let nread = rh.read_at(&mut buf, offset)?;
            if nread == 0 {
                break;
            }
            wh.write_at(&buf[..nread], offset)?;
            offset += nread as u64;
        }
        return Ok(());
    }

    fn copy_splice(rh: &File, wh: &File) -> error::Result<()> {
        let (pin, pout) = rlibc::pipe()?;

        let mut offset = 0;
        loop {
            let nread = rlibc::splice(rh.as_raw_fd(), offset, pout, -1, 128 * 1024)?;
            if nread == 0 {
                break;
            }

            let mut written = 0;
            while written < nread {
                let nxfer = rlibc::splice(pin, -1, wh.as_raw_fd(), offset, 128 * 1024)?;

                written += nxfer;
                offset += nxfer as i64;
            }
        }

        if let Err(e) = rlibc::close(pin) {
            rlibc::close(pout)?;
            return Err(RError::from(e));
        } else {
            rlibc::close(pout)?;
        }

        return Ok(());
    }

    fn copy(&self, to_cache: bool) -> error::Result<()> {
        let rh: &File;
        let wh: &File;
        if to_cache {
            rh = &self.src_file;
            wh = &self.cache_file;
        } else {
            rh = &self.cache_file;
            wh = &self.src_file;
        }

        let size = rh.filesize()?;
        if size < wh.filesize()? {
            wh.truncate(size)?;
        }

        if let Err(e) = Handle::copy_splice(rh, wh) {
            if e.raw_os_error().unwrap() == libc::EINVAL {
                Handle::copy_user(rh, wh)?;
            }
        }

        self.set_pristine(true)?;
        return Ok(());
    }
}

impl Drop for Handle {
    fn drop(&mut self) {
        if let Err(e) = self.cache_file.close() {
            error!("!close(cache) = {}", RError::from(e));
        }
        if self.src_file.valid() {
            if let Err(e) = self.src_file.close() {
                error!("!close(src) = {}", RError::from(e));
            }
        }
    }
}
<commit_msg>log an error when a cache file is invalid<commit_after>extern crate generic_array;
extern crate libc;
extern crate sha2;
extern crate xattr;

use std::ffi::{OsStr, OsString};
use std::io;
use std::os::unix::ffi::OsStrExt;
use std::os::unix::io::RawFd;
use std::path::{Component, Path, PathBuf};

use self::generic_array::GenericArray;
use self::generic_array::typenum::U64;
use self::sha2::{Sha512, Digest};
use self::xattr::FileExt;

use catfs::error;
use catfs::error::RError;
use catfs::rlibc;
use catfs::rlibc::File;

pub struct Handle {
    src_file: File,
    cache_file: File,
    dirty: bool,
}

// no-op to workaround the fact that we send the entire CatFS at start
// time, but we never send anything. Could have used Unique but that
// bounds us to rust nightly
unsafe impl Send for Handle {}

fn make_rdwr(f: &mut u32) {
    *f = (*f & !rlibc::O_ACCMODE) | rlibc::O_RDWR;
}

fn maybe_unlinkat(dir: RawFd, path: &AsRef<Path>) -> io::Result<()> {
    if let Err(e) = rlibc::unlinkat(dir, path, 0) {
        if !error::is_enoent(&e) {
            return Err(e);
        }
    }
    return Ok(());
}

pub fn mkdirat_all(dir: RawFd, path: &AsRef<Path>, mode: u32) -> io::Result<()> {
    let mut p = PathBuf::new();

    for c in path.as_ref().components() {
        if let Component::Normal(n) = c {
            p.push(n);

            if let Err(e) = rlibc::mkdirat(dir, &p, mode) {
                if e.raw_os_error().unwrap() != libc::EEXIST {
                    return Err(e);
                }
            }
        }
    }

    return Ok(());
}

impl Handle {
    pub fn create(
        src_dir: RawFd,
        cache_dir: RawFd,
        path: &AsRef<Path>,
        flags: u32,
        mode: u32,
    ) -> error::Result<Handle> {
        // need to read the cache file for writeback
        let mut cache_flags = flags.clone();
        if (cache_flags & rlibc::O_ACCMODE) == rlibc::O_WRONLY {
            make_rdwr(&mut cache_flags);
        }
        //debug!("create {:b} {:b} {:#o}", flags, cache_flags, mode);

        if let Some(parent) = path.as_ref().parent() {
            mkdirat_all(cache_dir, &parent, 0o777)?;
        }

        let src_file = File::openat(src_dir, path, flags, mode)?;
        // we are able to create the src file, then the cache file
        // shouldn't be here, but it could be because of bug/crash,
        // so unlink it first
        maybe_unlinkat(cache_dir, path)?;

        return Ok(Handle {
            src_file: src_file,
            cache_file: File::openat(cache_dir, path, cache_flags, mode)?,
            dirty: true,
        });
    }

    pub fn open(
        src_dir: RawFd,
        cache_dir: RawFd,
        path: &AsRef<Path>,
        flags: u32,
        cache_valid_if_present: bool,
    ) -> error::Result<Handle> {
        // even if file is open for write only, I still need to be
        // able to read the src for read-modify-write
        let mut flags = flags;
        if (flags & rlibc::O_ACCMODE) == rlibc::O_WRONLY {
            make_rdwr(&mut flags);
        }

        let valid = Handle::validate_cache(src_dir, cache_dir, &path, cache_valid_if_present)?;
        debug!(
            "{:?} {} a valid cache file",
            path.as_ref(),
            if valid { "is" } else { "is not" },
        );
        let mut cache_flags = flags.clone();

        if !valid {
            // mkdir the parents
            if let Some(parent) = path.as_ref().parent() {
                mkdirat_all(cache_dir, &parent, 0o777)?;
            }
            // need to cache this file so need to open it for write
            cache_flags |= rlibc::O_CREAT;
            if (cache_flags & rlibc::O_ACCMODE) == rlibc::O_RDONLY {
                make_rdwr(&mut cache_flags);
            }
        }

        let src_file: File;
        if valid && (flags & rlibc::O_ACCMODE) == rlibc::O_RDONLY {
            src_file = Default::default();
        } else {
            src_file = File::openat(src_dir, path, flags, 0o666)?;
        }

        let handle = Handle {
            src_file: src_file,
            cache_file: File::openat(cache_dir, path, cache_flags, 0o666)?,
            dirty: false,
        };

        if !valid && (flags & rlibc::O_TRUNC) == 0 {
            debug!("read ahead {:?}", path.as_ref());
            handle.copy(true)?;
        }

        return Ok(handle);
    }

    // see validate_cache.sh on how to replicate this
    pub fn src_str_to_checksum(f: &File) -> error::Result<OsString> {
        let mut s = OsString::new();
        for x in ["s3.etag"].iter() {
            if let Some(v) = f.get_xattr(&x)? {
                s.push(x);
                s.push(OsStr::new("="));
                s.push("0x");
                for b in v {
                    s.push(format!("{:x}", b));
                }
                s.push("\n");
            }
        }

        let st = f.stat()?;
        s.push(format!("{}\n", st.st_mtime));
        s.push(format!("{}\n", st.st_size));
        return Ok(s);
    }

    fn src_chksum(f: &File) -> error::Result<GenericArray<u8, U64>> {
        let s = Handle::src_str_to_checksum(f)?;
        debug!("checksum is {:?}", s);
        let mut h = Sha512::default();
        h.input(s.as_bytes());
        return Ok(h.result());
    }

    fn set_pristine(&self, pristine: bool) -> error::Result<()> {
        if pristine {
            self.cache_file.set_xattr(
                "user.catfs.src_chksum",
                Handle::src_chksum(&self.src_file)?
                    .as_slice(),
            )?;
        } else {
            self.cache_file.remove_xattr("user.catfs.src_chksum")?;
        }
        return Ok(());
    }

    fn is_pristine(src_file: &File, cache_file: &File) -> error::Result<bool> {
        if let Some(v) = cache_file.get_xattr("user.catfs.src_chksum")? {
            let expected = Handle::src_chksum(src_file)?;
            if v == expected.as_slice() {
                return Ok(true);
            } else {
                debug!("{:?} != {:?}, {} {}", v, expected, v.len(), expected.len());
                return Ok(false);
            }
        }
        debug!("user.catfs.src_chksum missing for cache_file");

        return Ok(false);
    }

    pub fn unlink(src_dir: RawFd, cache_dir: RawFd, path: &AsRef<Path>) -> io::Result<()> {
        maybe_unlinkat(cache_dir, path)?;
        return rlibc::unlinkat(src_dir, path, 0);
    }

    fn validate_cache(
        src_dir: RawFd,
        cache_dir: RawFd,
        path: &AsRef<Path>,
        cache_valid_if_present: bool,
    ) -> error::Result<bool> {
        match File::openat(src_dir, path, rlibc::O_RDONLY, 0) {
            Ok(mut src_file) => {
                match File::openat(cache_dir, path, rlibc::O_RDONLY, 0) {
                    Ok(mut cache_file) => {
                        let valid: bool;
                        if cache_valid_if_present || Handle::is_pristine(&src_file, &cache_file)? {
                            valid = true;
                        } else {
                            error!("{:?} is not a valid cache file, deleting", path.as_ref());
                            valid = false;
                            rlibc::unlinkat(cache_dir, path, 0)?;
                        }
                        src_file.close()?;
                        cache_file.close()?;
                        return Ok(valid);
                    }
                    Err(e) => {
                        src_file.close()?;
                        if error::try_enoent(e)? {
                            return Ok(false);
                        }
                    }
                }
            }
            Err(e) => {
                if error::try_enoent(e)? {
                    // the source file doesn't exist, the cache file shouldn't either
                    maybe_unlinkat(cache_dir, path)?;
                }
            }
        }

        return Ok(false);
    }

    pub fn read(&self, offset: u64, buf: &mut [u8]) -> error::Result<usize> {
        let nwant = buf.len();
        let mut bytes_read: usize = 0;

        while bytes_read < nwant {
            match self.cache_file.read_at(
                &mut buf[bytes_read..],
                offset + (bytes_read as u64),
            ) {
                Ok(nread) => {
                    if nread == 0 {
                        return Ok(bytes_read);
                    }
                    bytes_read += nread;
                }
                Err(e) => {
                    if bytes_read > 0 {
                        return Ok(bytes_read);
                    } else {
                        return Err(RError::from(e));
                    }
                }
            }
        }

        return Ok(bytes_read);
    }

    pub fn write(&mut self, offset: u64, buf: &[u8]) -> error::Result<usize> {
        let nwant = buf.len();
        let mut bytes_written: usize = 0;

        if !self.dirty {
            // assumes that the metadata will hit the disk before the
            // incoming data will, and not flushing
            self.set_pristine(false)?;
        }

        while bytes_written < nwant {
            match self.cache_file.write_at(
                &buf[bytes_written..],
                offset + (bytes_written as u64),
            ) {
                Ok(nwritten) => {
                    if nwritten == 0 {
                        return Ok(bytes_written);
                    }
                    bytes_written += nwritten;
                }
                Err(e) => {
                    if bytes_written > 0 {
                        return Ok(bytes_written);
                    } else {
                        return Err(RError::from(e));
                    }
                }
            }
        }

        if bytes_written != 0 {
            self.dirty = true;
        }

        return Ok(bytes_written);
    }

    pub fn flush(&mut self) -> error::Result<(bool)> {
        let mut flushed_to_src = false;
        if self.dirty {
            self.copy(false)?;
            self.dirty = false;
            self.cache_file.flush()?;
            // if file was opened for read only and cache is valid we
            // might not have opened src_file
            if self.src_file.valid() {
                self.src_file.flush()?;
                flushed_to_src = true;
            }
        }
        return Ok(flushed_to_src);
    }

    fn copy_user(rh: &File, wh: &File) -> error::Result<()> {
        let mut buf = [0u8; 32 * 1024];
        let mut offset = 0;
        loop {
            let nread = rh.read_at(&mut buf, offset)?;
            if nread == 0 {
                break;
            }
            wh.write_at(&buf[..nread], offset)?;
            offset += nread as u64;
        }
        return Ok(());
    }

    fn copy_splice(rh: &File, wh: &File) -> error::Result<()> {
        let (pin, pout) = rlibc::pipe()?;

        let mut offset = 0;
        loop {
            let nread = rlibc::splice(rh.as_raw_fd(), offset, pout, -1, 128 * 1024)?;
            if nread == 0 {
                break;
            }

            let mut written = 0;
            while written < nread {
                let nxfer = rlibc::splice(pin, -1, wh.as_raw_fd(), offset, 128 * 1024)?;

                written += nxfer;
                offset += nxfer as i64;
            }
        }

        if let Err(e) = rlibc::close(pin) {
            rlibc::close(pout)?;
            return Err(RError::from(e));
        } else {
            rlibc::close(pout)?;
        }

        return Ok(());
    }

    fn copy(&self, to_cache: bool) -> error::Result<()> {
        let rh: &File;
        let wh: &File;
        if to_cache {
            rh = &self.src_file;
            wh = &self.cache_file;
        } else {
            rh = &self.cache_file;
            wh = &self.src_file;
        }

        let size = rh.filesize()?;
        if size < wh.filesize()? {
            wh.truncate(size)?;
        }

        if let Err(e) = Handle::copy_splice(rh, wh) {
            if e.raw_os_error().unwrap() == libc::EINVAL {
                Handle::copy_user(rh, wh)?;
            }
        }

        self.set_pristine(true)?;
        return Ok(());
    }
}

impl Drop for Handle {
    fn drop(&mut self) {
        if let Err(e) = self.cache_file.close() {
            error!("!close(cache) = {}", RError::from(e));
        }
        if self.src_file.valid() {
            if let Err(e) = self.src_file.close() {
                error!("!close(src) = {}", RError::from(e));
            }
        }
    }
}
<|endoftext|>


<commit_before><commit_msg>[anup] Change LogItem to be a tuple struct<commit_after><|endoftext|>