<commit_before>//
// url.rs - module for url-encoding
//

import vec::len;

enum enc_mode {
    query,
    path,
    fragment,
    userinfo
}

fn query_escape(s: str) -> str { url_escape(s, query) }

fn should_escape(c: u8, mode: enc_mode) -> bool {
    alt c {
      65u8 to 90u8  { ret false; } // A .. Z
      97u8 to 122u8 { ret false; } // a .. z
      48u8 to 57u8  { ret false; } // 0 .. 9
      _ { }
    }

    alt c {
      45u8  | 95u8 | 46u8 | 33u8 | // '-', '_', '.', '!'
      126u8 | 42u8 | 39u8 | 40u8 | // '~', '*', '\'', '('
      41u8 { false }               // ')'

      36u8 | 38u8 | 43u8 | 44u8 |  // '$', '&', '+', ','
      47u8 | 58u8 | 59u8 | 61u8 |  // '/', ':', ';', '='
      63u8 | 64u8 {                // '?', '@'
        alt mode {
          query    { true }
          fragment { false }
          path     { (c == 63u8) }
          userinfo { (c == 47u8 || c == 58u8 || c == 64u8) }
        }
      }

      _ { true }
    }
}

fn url_escape(s: str, mode: enc_mode) -> str {
    let bs = str::bytes(s);
    let src_length = len(bs);
    let mut space_count = 0u;
    let mut hex_count = 0u;
    let table =
        [48u8, 49u8, 50u8, 51u8, 52u8, 53u8, 54u8, 55u8,
         56u8, 57u8, 65u8, 66u8, 67u8, 68u8, 69u8, 70u8];

    uint::range(0u, src_length) {|i|
        let c = bs[i];
        if should_escape(bs[i], mode) {
            if c == 32u8 && mode == query {
                space_count += 1u;
            } else {
                hex_count += 1u;
            }
        }
    }

    // Nothing to do if there's no space and no escapable chars in `s`
    if space_count == 0u && hex_count == 0u {
        ret s;
    }

    let ts = vec::to_mut(vec::from_elem(src_length + 2u * hex_count, 0u8));
    let mut i = 0u;
    let mut j = 0u;
    let mut c = 0u8;

    while i < src_length {
        c = bs[i];
        if c == 32u8 && mode == query {
            ts[j] = '+' as u8;
            j += 1u;
        } else if should_escape(c, mode) {
            ts[j] = '%' as u8;
            ts[j+1u] = table[c >> 4u8];
            ts[j+2u] = table[c & 0x0f_u8];
            j += 3u;
        } else {
            ts[j] = c;
            j += 1u;
        }
        i += 1u;
    }

    str::from_bytes(vec::from_mut(ts))
}

#[cfg(test)]
mod tests {
    #[test]
    fn test_should_escape() {
        assert should_escape('A' as u8, path) == false;
        assert should_escape('0' as u8, path) == false;
        assert should_escape('?' as u8, path) == true;
        assert should_escape('A' as u8, query) == false;
        assert should_escape('0' as u8, query) == false;
        assert should_escape('@' as u8, query) == true;
        assert should_escape('/' as u8, userinfo) == true;
        assert should_escape(':' as u8, userinfo) == true;
        assert should_escape('$' as u8, userinfo) == false;
        assert should_escape('A' as u8, fragment) == false;
        assert should_escape('0' as u8, fragment) == false;
        assert should_escape(';' as u8, fragment) == false;
    }
    #[test]
    fn test_query_escape() {
        assert query_escape("a") == "a";
        assert query_escape("a z") == "a+z";
        assert query_escape("å") == "%C3%A5";
        assert query_escape("?") == "%3F";
        assert query_escape("あ") == "%E3%81%82";
    }
}
<commit_msg>url: Add url_unescape and query_unescape<commit_after>//
// url.rs - module for url-encoding
//

import vec::len;

export query_escape, query_unescape;

enum enc_mode {
    query,
    path,
    fragment,
    userinfo
}

fn query_escape(s: str) -> str { url_escape(s, query) }
fn query_unescape(s: str) -> str { url_unescape(s, query) }

#[inline(always)]
fn ishex(c: u8) -> bool {
    if 48u8 <= c && c <= 57u8 { true }
    else if 65u8 <= c && c <= 90u8  { true }
    else if 97u8 <= c && c <= 122u8 { true }
    else { false }
}

#[inline(always)]
fn unhex(c: u8) -> u8 {
    if 48u8 <= c && c <= 57u8 { c - 48u8 }
    else if 65u8 <= c && c <= 90u8  { c - 65u8 + 10u8 }
    else if 97u8 <= c && c <= 122u8 { c - 97u8 + 10u8 }
    else { fail "should be unreachable"; }
}

fn should_escape(c: u8, mode: enc_mode) -> bool {
    alt c {
      65u8 to 90u8  { ret false; } // A .. Z
      97u8 to 122u8 { ret false; } // a .. z
      48u8 to 57u8  { ret false; } // 0 .. 9
      _ { }
    }

    alt c {
      45u8  | 95u8 | 46u8 | 33u8 | // '-', '_', '.', '!'
      126u8 | 42u8 | 39u8 | 40u8 | // '~', '*', '\'', '('
      41u8 { false }               // ')'

      36u8 | 38u8 | 43u8 | 44u8 |  // '$', '&', '+', ','
      47u8 | 58u8 | 59u8 | 61u8 |  // '/', ':', ';', '='
      63u8 | 64u8 {                // '?', '@'
        alt mode {
          query    { true }
          fragment { false }
          path     { (c == 63u8) }
          userinfo { (c == 47u8 || c == 58u8 || c == 64u8) }
        }
      }

      _ { true }
    }
}

fn url_escape(s: str, mode: enc_mode) -> str {
    let bs = str::bytes(s);
    let src_length = len(bs);
    let mut space_count = 0u;
    let mut hex_count = 0u;
    let table =
        [48u8, 49u8, 50u8, 51u8, 52u8, 53u8, 54u8, 55u8,
         56u8, 57u8, 65u8, 66u8, 67u8, 68u8, 69u8, 70u8];

    uint::range(0u, src_length) {|i|
        let c = bs[i];
        if should_escape(bs[i], mode) {
            if c == 32u8 && mode == query {
                space_count += 1u;
            } else {
                hex_count += 1u;
            }
        }
    }

    // Nothing to do if there's no space and no escapable chars in `s`
    if space_count == 0u && hex_count == 0u {
        ret s;
    }

    let ts = vec::to_mut(vec::from_elem(src_length + 2u * hex_count, 0u8));
    let mut i = 0u;
    let mut j = 0u;
    let mut c = 0u8;

    while i < src_length {
        c = bs[i];
        if c == 32u8 && mode == query {
            ts[j] = '+' as u8;
            j += 1u;
        } else if should_escape(c, mode) {
            ts[j] = '%' as u8;
            ts[j+1u] = table[c >> 4u8];
            ts[j+2u] = table[c & 0x0f_u8];
            j += 3u;
        } else {
            ts[j] = c;
            j += 1u;
        }
        i += 1u;
    }

    str::from_bytes(vec::from_mut(ts))
}

fn url_unescape(s: str, mode: enc_mode) -> str {
    let bs = str::bytes(s);
    let src_length = len(bs);
    let mut n = 0u;
    let mut i = 0u;
    let mut hasplus = false;

    while i < src_length {
        let c = bs[i];
        if c == 37u8 {
            n += 1u;
            if i+2u >= src_length || !ishex(bs[i+1u]) || !ishex(bs[i+2u]) {
                fail #fmt["Invalid URL escape: '%s'", s];
            }
            i += 3u;
        } else if c == 43u8 {
            hasplus = mode == query;
            i += 1u;
        } else {
            i += 1u;
        }
    }

    if n == 0u && !hasplus {
        ret s;
    }

    let ts = vec::to_mut(vec::from_elem(src_length - 2u * n, 0u8));
    let mut i = 0u;
    let mut j = 0u;

    while i < src_length {
        let c = bs[i];
        if c == 37u8 {
            ts[j] = (unhex(bs[i+1u])<<4u) | unhex(bs[i+2u]);
            j += 1u;
            i += 3u;
        } else if c == 43u8 {
            if mode == query { ts[j] = 32u8; }
            else { ts[j] = 43u8; }
            j += 1u;
            i += 1u;
        } else {
            ts[j] = c;
            j += 1u;
            i += 1u;
        }
    }

    str::from_bytes(vec::from_mut(ts))
}

#[cfg(test)]
mod tests {
    #[test]
    fn test_should_escape() {
        assert should_escape('A' as u8, path) == false;
        assert should_escape('0' as u8, path) == false;
        assert should_escape('?' as u8, path) == true;
        assert should_escape('A' as u8, query) == false;
        assert should_escape('0' as u8, query) == false;
        assert should_escape('@' as u8, query) == true;
        assert should_escape('/' as u8, userinfo) == true;
        assert should_escape(':' as u8, userinfo) == true;
        assert should_escape('$' as u8, userinfo) == false;
        assert should_escape('A' as u8, fragment) == false;
        assert should_escape('0' as u8, fragment) == false;
        assert should_escape(';' as u8, fragment) == false;
    }
    #[test]
    fn test_query_escape() {
        assert query_escape("a") == "a";
        assert query_escape("a z") == "a+z";
        assert query_escape("å") == "%C3%A5";
        assert query_escape("?") == "%3F";
        assert query_escape("あ") == "%E3%81%82";
    }
    #[test]
    fn test_query_unescape() {
        assert query_unescape("a") == "a";
        assert query_unescape("a+z") == "a z";
        assert query_unescape("%3f") == "?";
        assert query_unescape("%C3%A5") == "å";
        assert query_unescape("%E3%81%82") == "あ";
    }
}
<|endoftext|>


<commit_before>/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements. See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership. The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License. You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied. See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */

#![crate_name="calculator"]
#![crate_type="bin"]

extern crate thrift;

use std::io::net::ip;
use thrift::protocol::{MessageType, Type};
use thrift::transport::Transport;
use thrift::protocol::Protocol;
use thrift::protocol::binary_protocol::BinaryProtocol;

// TODO: shall be generated
//#[allow(dead_code)]
struct CalculatorPingArgs;

impl CalculatorPingArgs {

    #[allow(unused_variables)]
    fn write(&self, oprot: &Protocol, transport: & mut Transport) {
        oprot.write_struct_begin(transport, "Calculator_ping_args");
        oprot.write_field_stop(transport);
        oprot.write_struct_end(transport);
    }
}

struct CalculatorPingResult;

impl CalculatorPingResult {

    fn read(&self, iprot: &Protocol, transport: & mut Transport) {
        iprot.skip(transport, Type::TStruct);
    }
}

struct CalculatorAddArgs {
    num1: i32,
    num2: i32,
}

impl CalculatorAddArgs {

    #[allow(unused_variables)]
    fn write(&self, oprot: &Protocol, transport: & mut Transport) {
        oprot.write_struct_begin(transport, "Calculator_add_args");

        oprot.write_field_begin(transport, "num1", Type::TI32, 1);
        oprot.write_i32(transport, self.num1);
        oprot.write_field_end(transport);

        oprot.write_field_begin(transport, "num2", Type::TI32, 2);
        oprot.write_i32(transport, self.num2);
        oprot.write_field_end(transport);

        oprot.write_field_stop(transport);
        oprot.write_struct_end(transport);
    }
}

struct CalculatorAddResult {
    success: Option<i32>,
}

impl CalculatorAddResult {

    #[allow(unused_variables)]
    fn read(& mut self, iprot: &Protocol, transport: & mut Transport) {
        iprot.read_struct_begin(transport);

        loop {
            let (fname, ftype, fid) = iprot.read_field_begin(transport);
            if ftype == Type::TStop {
                break;
            }
            match (fid, ftype) {
                (0, Type::TI32) => {
                    self.success = Some(iprot.read_i32(transport));
                }
                _ => {
                    iprot.skip(transport, ftype);
                }
            }
            iprot.read_field_end(transport);
        }
        iprot.read_struct_end(transport);
    }
}

struct CalculatorClient<T: Transport, P: Protocol> {
    transport: T,
    protocol: P,
}

impl <T: Transport, P: Protocol> CalculatorClient<T, P> {

    fn ping(& mut self) {
        self.send_ping();
        self.receive_ping();
    }

    #[allow(unused_variables)]
    fn send_ping(& mut self) {
        let cseqid: i32 = 0;
        self.protocol.write_message_begin(& mut self.transport, "ping", MessageType::MtCall, cseqid);

        let args = CalculatorPingArgs;
        args.write(&self.protocol, & mut self.transport);

        self.protocol.write_message_end(& mut self.transport);

        //self.transport.write_end();
        self.transport.flush().unwrap();
    }

    #[allow(unused_variables)]
    fn receive_ping(& mut self) {
      let (fname, mtype, rseqid) = self.protocol.read_message_begin(& mut self.transport);
      match mtype {
        MessageType::MtException => {
            // TODO
            //let x = ApplicationException;
            //x.read(& mut self protocol)
            //self.protocol.read_message_end();
            //transport.read_end();
            //throw x
        }
        MessageType::MtReply => {
            match fname.as_slice() {
                "ping" => {
                    let result = CalculatorPingResult;
                    result.read(&self.protocol, & mut self.transport);
                }
                _ => {
                    self.protocol.skip(& mut self.transport, Type::TStruct);
                }
            }
            self.protocol.read_message_end(& mut self.transport);
            //self.transport.read_end();
        }
        _ => {
            self.protocol.skip(& mut self.transport, Type::TStruct);
            self.protocol.read_message_end(& mut self.transport);
            //self.transport.read_end();
        }
      }
    }

    fn add(& mut self, num1: i32, num2: i32) -> Option<i32> {
        self.send_add(num1, num2);
        self.receive_add()
    }

    fn send_add(& mut self, num1: i32, num2: i32) {
        let cseqid: i32 = 0;
        self.protocol.write_message_begin(& mut self.transport, "add", MessageType::MtCall, cseqid);

        let args = CalculatorAddArgs { num1: num1, num2: num2 };
        args.write(&self.protocol, & mut self.transport);

        self.protocol.write_message_end(& mut self.transport);

        //self.transport.write_end();
        self.transport.flush().unwrap();
    }

    #[allow(unused_variables)]
    fn receive_add(& mut self) -> Option<i32> {
      let mut result = CalculatorAddResult { success: None };
      let (fname, mtype, rseqid) = self.protocol.read_message_begin(& mut self.transport);
      match mtype {
        MessageType::MtException => {
            // TODO
            //let x = ApplicationException;
            //x.read(& mut self protocol)
            //self.protocol.read_message_end();
            //transport.read_end();
            //throw x
        }
        MessageType::MtReply => {
            match fname.as_slice() {
                "add" => {
                    result.read(&self.protocol, & mut self.transport);
                }
                _ => {
                    self.protocol.skip(& mut self.transport, Type::TStruct);
                }
            }
            self.protocol.read_message_end(& mut self.transport);
            //self.transport.read_end();
        }
        _ => {
            self.protocol.skip(& mut self.transport, Type::TStruct);
            self.protocol.read_message_end(& mut self.transport);
            //self.transport.read_end();
        }
      }
      result.success
    }
}

pub fn main() {
    let server_address = "127.0.0.1:9090";
    let addr: ip::SocketAddr = std::str::FromStr::from_str(server_address)
        .expect("bad server address");
    let tcp = std::io::TcpStream::connect(addr).unwrap();

    let mut client = CalculatorClient{ protocol: BinaryProtocol, transport: tcp };

    client.ping();
    println!("ping()");

    println!("1 + 1 = {}", client.add(1, 1).unwrap());

    println!("PASS");
}
<commit_msg>Added Calculate operation and Readable trait<commit_after>/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements. See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership. The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License. You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied. See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */

#![crate_name="calculator"]
#![crate_type="bin"]

extern crate thrift;

use std::io::net::ip;
use thrift::protocol::{MessageType, Type};
use thrift::transport::Transport;
use thrift::protocol::Protocol;
use thrift::protocol::binary_protocol::BinaryProtocol;

trait Readable {
    fn read(& mut self, iprot: &Protocol, transport: & mut Transport) -> bool;
}

// TODO: shall be generated
//#[allow(dead_code)]
struct CalculatorPingArgs;

impl CalculatorPingArgs {

    #[allow(unused_variables)]
    fn write(&self, oprot: &Protocol, transport: & mut Transport) {
        oprot.write_struct_begin(transport, "Calculator_ping_args");
        oprot.write_field_stop(transport);
        oprot.write_struct_end(transport);
    }
}

struct CalculatorPingResult;

impl CalculatorPingResult {

    fn read(&self, iprot: &Protocol, transport: & mut Transport) {
        iprot.skip(transport, Type::TStruct);
    }
}

struct CalculatorAddArgs {
    num1: i32,
    num2: i32,
}

impl CalculatorAddArgs {

    #[allow(unused_variables)]
    fn write(&self, oprot: &Protocol, transport: & mut Transport) {
        oprot.write_struct_begin(transport, "Calculator_add_args");

        oprot.write_field_begin(transport, "num1", Type::TI32, 1);
        oprot.write_i32(transport, self.num1);
        oprot.write_field_end(transport);

        oprot.write_field_begin(transport, "num2", Type::TI32, 2);
        oprot.write_i32(transport, self.num2);
        oprot.write_field_end(transport);

        oprot.write_field_stop(transport);
        oprot.write_struct_end(transport);
    }
}

struct CalculatorAddResult {
    success: Option<i32>,
}

impl Readable for CalculatorAddResult {

    #[allow(unused_variables)]
    fn read(& mut self, iprot: &Protocol, transport: & mut Transport) -> bool {
        let mut have_result = false;

        iprot.read_struct_begin(transport);

        loop {
            let (fname, ftype, fid) = iprot.read_field_begin(transport);
            if ftype == Type::TStop {
                break;
            }
            match (fid, ftype) {
                (0, Type::TI32) => {
                    self.success = Some(iprot.read_i32(transport));
                    have_result = true
                }
                _ => {
                    iprot.skip(transport, ftype);
                }
            }
            iprot.read_field_end(transport);
        }
        iprot.read_struct_end(transport);
        have_result
    }
}

#[derive(Copy)]
#[allow(dead_code)]
enum Operation {
    ADD = 1,
    SUBTRACT = 2,
    MULTIPLY = 3,
    DIVIDE = 4,
}

struct Work {
    num1: i32,
    num2: i32,
    op: Operation,
    comment: Option<String>,
}

impl Work {

    #[allow(unused_variables)]
    fn write(&self, oprot: &Protocol, transport: & mut Transport) {
        oprot.write_struct_begin(transport, "Work");

        oprot.write_field_begin(transport, "num1", Type::TI32, 1);
        oprot.write_i32(transport, self.num1);
        oprot.write_field_end(transport);

        oprot.write_field_begin(transport, "num2", Type::TI32, 2);
        oprot.write_i32(transport, self.num2);
        oprot.write_field_end(transport);

        oprot.write_field_begin(transport, "op", Type::TI32, 3);
        oprot.write_i32(transport, self.op as i32);
        oprot.write_field_end(transport);

        if self.comment.is_some() {
            // FIXME
            //oprot.write_field_begin(transport, "comment", Type::TString, 4);
            //oprot.write_string(transport, self.comment.unwrap().as_slice());
            //oprot.write_field_end(transport);
        }

        oprot.write_field_stop(transport);
        oprot.write_struct_end(transport);
    }
}

struct CalculatorCalculateArgs {
    logid: i32,
    w: Work,
}

impl CalculatorCalculateArgs {

    #[allow(unused_variables)]
    fn write(&self, oprot: &Protocol, transport: & mut Transport) {
        oprot.write_struct_begin(transport, "Calculator_calculate_args");

        oprot.write_field_begin(transport, "logid", Type::TI32, 1);
        oprot.write_i32(transport, self.logid);
        oprot.write_field_end(transport);

        oprot.write_field_begin(transport, "work", Type::TStruct, 2);
        self.w.write(oprot, transport);
        oprot.write_field_end(transport);

        oprot.write_field_stop(transport);
        oprot.write_struct_end(transport);
    }
}

struct Exception;

#[allow(dead_code)]
struct CalculatorCalculateResult {
    success: Option<i32>,
    ouch: Exception
}

impl Readable for CalculatorCalculateResult {

    #[allow(unused_variables)]
    fn read(& mut self, iprot: &Protocol, transport: & mut Transport) -> bool {
        let mut have_result = false;

        iprot.read_struct_begin(transport);

        loop {
            let (fname, ftype, fid) = iprot.read_field_begin(transport);
            if ftype == Type::TStop {
                break;
            }
            match (fid, ftype) {
                (0, Type::TI32) => {
                    self.success = Some(iprot.read_i32(transport));
                    have_result = true;
                }
                (1, Type::TStruct) => {
                    // FIXME read ouch
                    iprot.skip(transport, ftype);
                }
                _ => {
                    iprot.skip(transport, ftype);
                }
            }
            iprot.read_field_end(transport);
        }
        iprot.read_struct_end(transport);
        have_result
    }
}

struct CalculatorClient<T: Transport, P: Protocol> {
    transport: T,
    protocol: P,
}

impl <T: Transport, P: Protocol> CalculatorClient<T, P> {

    fn ping(& mut self) {
        self.send_ping();
        self.receive_ping();
    }

    #[allow(unused_variables)]
    fn send_ping(& mut self) {
        let cseqid: i32 = 0;
        self.protocol.write_message_begin(& mut self.transport, "ping", MessageType::MtCall, cseqid);

        let args = CalculatorPingArgs;
        args.write(&self.protocol, & mut self.transport);

        self.protocol.write_message_end(& mut self.transport);

        //self.transport.write_end();
        self.transport.flush().unwrap();
    }

    #[allow(unused_variables)]
    fn receive_ping(& mut self) {
      let (fname, mtype, rseqid) = self.protocol.read_message_begin(& mut self.transport);
      match mtype {
        MessageType::MtException => {
            // TODO
            //let x = ApplicationException;
            //x.read(& mut self protocol)
            //self.protocol.read_message_end();
            //transport.read_end();
            //throw x
        }
        MessageType::MtReply => {
            match fname.as_slice() {
                "ping" => {
                    let result = CalculatorPingResult;
                    result.read(&self.protocol, & mut self.transport);
                }
                _ => {
                    self.protocol.skip(& mut self.transport, Type::TStruct);
                }
            }
            self.protocol.read_message_end(& mut self.transport);
            //self.transport.read_end();
        }
        _ => {
            self.protocol.skip(& mut self.transport, Type::TStruct);
            self.protocol.read_message_end(& mut self.transport);
            //self.transport.read_end();
        }
      }
    }

    fn add(& mut self, num1: i32, num2: i32) -> Option<i32> {
        self.send_add(num1, num2);
        self.receive_add()
    }

    fn send_add(& mut self, num1: i32, num2: i32) {
        let cseqid: i32 = 0;
        self.protocol.write_message_begin(& mut self.transport, "add", MessageType::MtCall, cseqid);

        let args = CalculatorAddArgs { num1: num1, num2: num2 };
        args.write(&self.protocol, & mut self.transport);

        self.protocol.write_message_end(& mut self.transport);

        //self.transport.write_end();
        self.transport.flush().unwrap();
    }

    fn receive_add(& mut self) -> Option<i32> {
        let mut result = CalculatorAddResult { success: None };
        if self.receive("add", &mut result) { result.success } else { None }
    }

    #[allow(unused_variables)]
    fn receive<R: Readable>(& mut self, op: &'static str, result: &mut R) -> bool {
        let mut have_result = false;
        let (fname, mtype, rseqid) = self.protocol.read_message_begin(& mut self.transport);
        match mtype {
            MessageType::MtException => {
                // TODO
                //let x = ApplicationException;
                //x.read(& mut self protocol)
                //self.protocol.read_message_end();
                //transport.read_end();
                //throw x
            }
            MessageType::MtReply => {
                if fname.as_slice() == op {
                    have_result = result.read(&self.protocol, & mut self.transport);
                }
                else {
                    self.protocol.skip(& mut self.transport, Type::TStruct);
                }
                self.protocol.read_message_end(& mut self.transport);
                //self.transport.read_end();
            }
            _ => {
                self.protocol.skip(& mut self.transport, Type::TStruct);
                self.protocol.read_message_end(& mut self.transport);
                //self.transport.read_end();
            }
        }
        have_result
    }

    fn calculate(& mut self, logid: i32, w: Work) -> Option<i32> {
        self.send_calculate(logid, w);
        self.receive_calculate()
    }

    fn send_calculate(& mut self, logid: i32,  w: Work) {
        let cseqid: i32 = 0;
        self.protocol.write_message_begin(& mut self.transport, "calculate", MessageType::MtCall, cseqid);

        let args = CalculatorCalculateArgs { logid: logid, w: w };
        args.write(&self.protocol, & mut self.transport);

        self.protocol.write_message_end(& mut self.transport);

        //self.transport.write_end();
        self.transport.flush().unwrap();
    }

    fn receive_calculate(& mut self) -> Option<i32> {
        let mut result = CalculatorCalculateResult { success: None, ouch: Exception };
        // FIXME: handle exception
        if self.receive("calculate", &mut result) { result.success } else { None }
    }

}

pub fn main() {
    let server_address = "127.0.0.1:9090";
    let addr: ip::SocketAddr = std::str::FromStr::from_str(server_address)
        .expect("bad server address");
    let tcp = std::io::TcpStream::connect(addr).unwrap();

    let mut client = CalculatorClient{ protocol: BinaryProtocol, transport: tcp };

    client.ping();
    println!("ping()");

    println!("1 + 1 = {}", client.add(1, 1).unwrap());

    let work = Work { op: Operation::DIVIDE, num1: 1, num2: 0, comment: None };

    match client.calculate(1, work) {
      Some(_) => {
        println!("Whoa? We can divide by zero!");
      }
      None => {
        // FIXME: use thrift exceptions
        println!("Invalid operation")
      }
    }

    let work = Work { op: Operation::SUBTRACT, num1: 15, num2: 10, comment: None };
    println!("15 - 10 = {}", client.calculate(2, work).unwrap());

    println!("PASS");
}
<|endoftext|>


<commit_before>extern crate pink_spider;

#[test]
fn it_works() {
  assert!(true);
}

#[test]
fn fetch_apple_music_track() {
    let song = pink_spider::apple_music::fetch_song("jp", "1160715431").unwrap();
    let track = pink_spider::model::Track::from_am_song(&song);
    assert_eq!(track.identifier, "1160715431");
    assert_eq!(track.title, "A Short Film");
    assert_eq!(track.owner_name.unwrap_or_default(), "LILI LIMIT");
    let artists = track.artists.unwrap();
    let artist = &artists[0];
    assert_eq!(artist.name, "LILI LIMIT");
    assert!(artist.clone().artwork_url.unwrap().len() > 0);
}

#[test]
fn fetch_apple_music_album() {
    let am_album = pink_spider::apple_music::fetch_album("jp", "1160715126").unwrap();
    let album = pink_spider::model::Album::from_am_album(&am_album);
    assert_eq!(album.identifier, "1160715126");
    assert_eq!(album.title, "a.k.a");
    assert_eq!(album.owner_name.unwrap_or_default(), "LILI LIMIT");
    let artists = album.artists.unwrap();
    let artist = &artists[0];
    assert_eq!(artist.name, "LILI LIMIT");
    assert!(artist.clone().artwork_url.unwrap().len() > 0);
}

#[test]
fn fetch_spotify_track() {
    let sp_track = pink_spider::spotify::fetch_track("3n3Ppam7vgaVa1iaRUc9Lp").unwrap();
    let track = pink_spider::model::Track::from_sp_track(&sp_track);
    assert_eq!(track.identifier, "3n3Ppam7vgaVa1iaRUc9Lp");
    assert_eq!(track.title, "Mr. Brightside");
    assert_eq!(track.owner_name.unwrap_or_default(), "The Killers");
    let artists = track.artists.unwrap();
    let artist = &artists[0];
    assert_eq!(artist.name, "The Killers");
    assert!(artist.clone().artwork_url.unwrap().len() > 0);
}

#[test]
fn fetch_spotify_album() {
    let sp_album = pink_spider::spotify::fetch_album("4OHNH3sDzIxnmUADXzv2kT").unwrap();
    let album = pink_spider::model::Album::from_sp_album(&sp_album);
    assert_eq!(album.identifier, "4OHNH3sDzIxnmUADXzv2kT");
    assert_eq!(album.title, "Hot Fuss");
    assert_eq!(album.owner_name.unwrap_or_default(), "The Killers");
    let artists = album.artists.unwrap();
    let artist = &artists[0];
    assert_eq!(artist.name, "The Killers");
    assert!(artist.clone().artwork_url.unwrap().len() > 0);
    assert!(album.tracks.len() > 0);
}

#[test]
fn fetch_soundcloud_track() {
    let sc_track = pink_spider::soundcloud::fetch_track("371851634").unwrap();
    let track = pink_spider::model::Track::from_sc_track(&sc_track);
    assert_eq!(track.identifier, "371851634");
    assert_eq!(track.title, "Down Wit That");
    assert_eq!(track.owner_name.unwrap_or_default(), "\"Chance The Rapper\"");
    let artists = track.artists.unwrap();
    let artist = &artists[0];
    assert_eq!(artist.name, "\"Chance The Rapper\"");
    assert!(artist.clone().artwork_url.unwrap().len() > 0);
}

#[test]
fn fetch_youtube_track() {
    let yt_video = pink_spider::youtube::fetch_video("Wr5f6hpYxmE").unwrap();
    let track = pink_spider::model::Track::from_yt_video(&yt_video);
    assert_eq!(track.identifier, "Wr5f6hpYxmE");
    assert_eq!(track.title, "Cornelius 『あなたがいるなら』If You're Here");
    assert_eq!(track.owner_name.unwrap_or_default(), "corneliusofficial");
    let artists = track.artists.unwrap();
    let artist = &artists[0];
    assert_eq!(artist.name, "corneliusofficial");
    assert!(artist.clone().artwork_url.unwrap().len() > 0);
}
<commit_msg>Update test<commit_after>extern crate pink_spider;

#[test]
fn it_works() {
  assert!(true);
}

#[test]
fn fetch_apple_music_track() {
    let song = pink_spider::apple_music::fetch_song("jp", "1160715431").unwrap();
    let track = pink_spider::model::Track::from_am_song(&song);
    assert_eq!(track.identifier, "1160715431");
    assert_eq!(track.title, "A Short Film");
    assert_eq!(track.owner_name.unwrap_or_default(), "LILI LIMIT");
    let artists = track.artists.unwrap();
    let artist = &artists[0];
    assert_eq!(artist.name, "LILI LIMIT");
    assert!(artist.clone().artwork_url.unwrap().len() > 0);
}

#[test]
fn fetch_apple_music_album() {
    let am_album = pink_spider::apple_music::fetch_album("jp", "1160715126").unwrap();
    let album = pink_spider::model::Album::from_am_album(&am_album);
    assert_eq!(album.identifier, "1160715126");
    assert_eq!(album.title, "a.k.a");
    assert_eq!(album.owner_name.unwrap_or_default(), "LILI LIMIT");
    let artists = album.artists.unwrap();
    let artist = &artists[0];
    assert_eq!(artist.name, "LILI LIMIT");
    assert!(artist.clone().artwork_url.unwrap().len() > 0);
}

#[test]
fn fetch_spotify_track() {
    let sp_track = pink_spider::spotify::fetch_track("3n3Ppam7vgaVa1iaRUc9Lp").unwrap();
    let track = pink_spider::model::Track::from_sp_track(&sp_track).unwrap();
    assert_eq!(track.identifier, "3n3Ppam7vgaVa1iaRUc9Lp");
    assert_eq!(track.title, "Mr. Brightside");
    assert_eq!(track.owner_name.unwrap_or_default(), "The Killers");
    let artists = track.artists.unwrap();
    let artist = &artists[0];
    assert_eq!(artist.name, "The Killers");
    assert!(artist.clone().artwork_url.unwrap().len() > 0);
}

#[test]
fn fetch_spotify_album() {
    let sp_album = pink_spider::spotify::fetch_album("4OHNH3sDzIxnmUADXzv2kT").unwrap();
    let album = pink_spider::model::Album::from_sp_album(&sp_album);
    assert_eq!(album.identifier, "4OHNH3sDzIxnmUADXzv2kT");
    assert_eq!(album.title, "Hot Fuss");
    assert_eq!(album.owner_name.unwrap_or_default(), "The Killers");
    let artists = album.artists.unwrap();
    let artist = &artists[0];
    assert_eq!(artist.name, "The Killers");
    assert!(artist.clone().artwork_url.unwrap().len() > 0);
    assert!(album.tracks.len() > 0);
}

#[test]
fn fetch_soundcloud_track() {
    let sc_track = pink_spider::soundcloud::fetch_track("371851634").unwrap();
    let track = pink_spider::model::Track::from_sc_track(&sc_track);
    assert_eq!(track.identifier, "371851634");
    assert_eq!(track.title, "Down Wit That");
    assert_eq!(track.owner_name.unwrap_or_default(), "\"Chance The Rapper\"");
    let artists = track.artists.unwrap();
    let artist = &artists[0];
    assert_eq!(artist.name, "\"Chance The Rapper\"");
    assert!(artist.clone().artwork_url.unwrap().len() > 0);
}

#[test]
fn fetch_youtube_track() {
    let yt_video = pink_spider::youtube::fetch_video("Wr5f6hpYxmE").unwrap();
    let track = pink_spider::model::Track::from_yt_video(&yt_video);
    assert_eq!(track.identifier, "Wr5f6hpYxmE");
    assert_eq!(track.title, "Cornelius 『あなたがいるなら』If You're Here");
    assert_eq!(track.owner_name.unwrap_or_default(), "corneliusofficial");
    let artists = track.artists.unwrap();
    let artist = &artists[0];
    assert_eq!(artist.name, "corneliusofficial");
    assert!(artist.clone().artwork_url.unwrap().len() > 0);
}
<|endoftext|>


<commit_before>use std::collections::VecDeque;

#[macro_use]
extern crate sliding_window;
use sliding_window::SliceRing;
use sliding_window::OptimizedSliceRing;

#[test]
fn test_slice_ring_unoptimized() {
    test_slice_ring!(VecDeque::<i32>::new());
}

#[test]
fn test_slice_ring_optimized() {
    test_slice_ring!(OptimizedSliceRing::<i32>::new());
}
// #[test]
// fn test_window_1_step_1() {
//     let mut sliding_window: SlidingWindow<i32> = SlidingWindow::new(1, 1);
//     assert!(!sliding_window.can_fill());
//     let samples: Vec<i32> = vec![];
//     sliding_window.write(&samples[..]);
//     assert!(!sliding_window.can_fill());
//     assert_eq!(sliding_window.ringbuffer.len(), 0);
//
//     let mut window: Vec<i32> = vec![0];
//     assert!(!sliding_window.read(&mut window[..]));
//     assert_eq!(window, vec![0]);
//
//     let samples: Vec<i32> = vec![1];
//     sliding_window.write(&samples[..]);
//     assert_eq!(sliding_window.ringbuffer.len(), 1);
//     assert!(sliding_window.can_fill());
//     assert!(sliding_window.read(&mut window[..]));
//     assert_eq!(window, vec![1]);
//     assert_eq!(sliding_window.ringbuffer.len(), 0);
//
//     let samples: Vec<i32> = vec![1, 2, 3];
//     sliding_window.write(&samples[..]);
//     assert_eq!(sliding_window.ringbuffer.len(), 3);
//
//     assert!(sliding_window.can_fill());
//     assert!(sliding_window.read(&mut window[..]));
//     assert_eq!(window, vec![1]);
//     assert_eq!(sliding_window.ringbuffer.len(), 2);
//
//     assert!(sliding_window.can_fill());
//     assert!(sliding_window.read(&mut window[..]));
//     assert_eq!(window, vec![2]);
//     assert_eq!(sliding_window.ringbuffer.len(), 1);
//
//     assert!(sliding_window.can_fill());
//     assert!(sliding_window.read(&mut window[..]));
//     assert_eq!(window, vec![3]);
//     assert_eq!(sliding_window.ringbuffer.len(), 0);
//     assert!(!sliding_window.can_fill());
// }
//
// #[test]
// fn test_window_4_step_2() {
//     // 50% overlap
//
//     let mut sliding_window: SlidingWindow<i32> = SlidingWindow::new(4, 2);
//     assert!(!sliding_window.can_fill());
//     let samples: Vec<i32> = vec![];
//     sliding_window.write(&samples[..]);
//     assert!(!sliding_window.can_fill());
//     assert_eq!(sliding_window.ringbuffer.len(), 0);
//
//     let mut window: Vec<i32> = vec![0];
//     assert!(!sliding_window.read(&mut window[..]));
//     assert_eq!(window, vec![0]);
//
//     {
//         let mut ring: VecDeque<i32> = VecDeque::new();
//         let mut window: Vec<i32> = vec![0, 0, 0, 0];
//         let samples: Vec<i32> = vec![1, 2, 3];
//         // println!("samples {:?}", samples);
//         let mut calls = 0;
//         window!(2, ring, &samples[..], &mut window[..], {
//             calls += 1;
//         });
//         assert_eq!(calls, 0);
//         assert_eq!(window, vec![0, 0, 0, 0]);
//         assert_eq!(ring.len(), 3);
//     }
//
//     {
//         let mut ring: VecDeque<i32> = VecDeque::new();
//         let mut window: Vec<i32> = vec![0, 0, 0, 0];
//         let samples: Vec<i32> = vec![1, 2, 3, 4];
//         // println!("samples {:?}", samples);
//         let mut calls = 0;
//         window!(2, ring, &samples[..], &mut window[..], {
//             assert_eq!(ring.len(), 2);
//             assert_eq!(window, vec![1, 2, 3, 4]);
//             calls += 1;
//         });
//         assert_eq!(calls, 1);
//         assert_eq!(window, vec![1, 2, 3, 4]);
//         assert_eq!(ring.len(), 2);
//     }
//
//     {
//         let mut ring: VecDeque<i32> = VecDeque::new();
//         let mut window: Vec<i32> = vec![0, 0, 0, 0];
//         let samples: Vec<i32> = vec![1, 2, 3 ,4, 5, 6, 7, 8, 9, 10, 11];
//         let mut calls = 0;
//         window!(2, ring, &samples[..], &mut window[..], {
//             match calls {
//                 0 => { assert_eq!(window, vec![1, 2, 3, 4]) },
//                 1 => { assert_eq!(window, vec![3, 4, 5, 6]) },
//                 2 => { assert_eq!(window, vec![5, 6, 7, 8]) },
//                 3 => { assert_eq!(window, vec![7, 8, 9, 10]) },
//                 _ => { assert!(false) },
//             }
//             calls += 1;
//         });
//         assert_eq!(calls, 4);
//         assert_eq!(window, vec![7, 8, 9, 10]);
//         assert_eq!(ring.len(), 3);
//
//         let samples: Vec<i32> = vec![12, 13, 14, 15, 16];
//         let mut calls = 0;
//         window!(2, ring, &samples[..], &mut window[..], {
//             match calls {
//                 0 => { assert_eq!(window, vec![9, 10, 11, 12]) },
//                 1 => { assert_eq!(window, vec![11, 12, 13, 14]) },
//                 2 => { assert_eq!(window, vec![13, 14, 15, 16]) },
//                 _ => { assert!(false) },
//             }
//             calls += 1;
//         });
//         assert_eq!(calls, 3);
//         assert_eq!(window, vec![13, 14, 15, 16]);
//         assert_eq!(ring.len(), 2);
//     }
// }
//
// #[test]
// fn test_window_1024_step_512() {
//     // 50% overlap
//     use std::collections::VecDeque;
//     use std::iter;
//
//     {
//         let mut ring: VecDeque<i32> = VecDeque::new();
//         let mut window: Vec<i32> = vec![0; 1024];
//         let samples: Vec<i32> =
//             iter::repeat(1).take(1024)
//             .chain(iter::repeat(2).take(1024))
//             .chain(iter::repeat(3).take(1024))
//             .chain(iter::repeat(4).take(1024))
//             .collect();
//         assert_eq!(samples.len(), 1024 * 4);
//         let mut calls = 0;
//         window!(512, ring, &samples[..], &mut window[..], {
//             match calls {
//                 0 => {
//                     assert_eq!(
//                         window,
//                         iter::repeat(1).take(1024).collect::<Vec<i32>>()
//                     );
//                 },
//                 1 => {
//                     assert_eq!(
//                         window,
//                         iter::repeat(1).take(512)
//                         .chain(iter::repeat(2).take(512))
//                         .collect::<Vec<i32>>()
//                     );
//                 },
//                 2 => {
//                     assert_eq!(
//                         window,
//                         iter::repeat(2).take(1024).collect::<Vec<i32>>()
//                     );
//                 },
//                 3 => {
//                     assert_eq!(
//                         window,
//                         iter::repeat(2).take(512)
//                         .chain(iter::repeat(3).take(512))
//                         .collect::<Vec<i32>>()
//                     );
//                 },
//                 4 => {
//                     assert_eq!(
//                         window,
//                         iter::repeat(3).take(1024).collect::<Vec<i32>>()
//                     );
//                 },
//                 5 => {
//                     assert_eq!(
//                         window,
//                         iter::repeat(3).take(512)
//                         .chain(iter::repeat(4).take(512))
//                         .collect::<Vec<i32>>()
//                     );
//                 },
//                 6 => {
//                     assert_eq!(
//                         window,
//                         iter::repeat(4).take(1024).collect::<Vec<i32>>()
//                     );
//                 },
//                 _ => {}
//             }
//             calls += 1;
//         });
//         assert_eq!(calls, 7);
//         assert_eq!(ring.len(), 512);
//     }
// }

// 4048 and 512
<commit_msg>remove old outcommented tests we have no use for anymore<commit_after>use std::collections::VecDeque;

#[macro_use]
extern crate sliding_window;
use sliding_window::SliceRing;
use sliding_window::OptimizedSliceRing;

#[test]
fn test_slice_ring_unoptimized() {
    test_slice_ring!(VecDeque::<i32>::new());
}

#[test]
fn test_slice_ring_optimized() {
    test_slice_ring!(OptimizedSliceRing::<i32>::new());
}
<|endoftext|>


<commit_before><commit_msg>Validate NIF attributes (#279)<commit_after><|endoftext|>


<commit_before>// http://dev.w3.org/csswg/css3-syntax/#tokenization
//
// The output of the tokenization step is a series of zero or more
// of the following tokens:
// ident, function, at-keyword, hash, string, bad-string, url, bad-url,
// delim, number, percentage, dimension, unicode-range, whitespace, comment,
// cdo, cdc, colon, semicolon, [, ], (, ), {, }.
//
// ident, function, at-keyword, hash, string, and url tokens
// have a value composed of zero or more characters.
// Delim tokens have a value composed of a single character.
// Number, percentage, and dimension tokens have a representation
// composed of 1 or more character, a numeric value,
// and a type flag set to either "integer" or "number".
// The type flag defaults to "integer" if not otherwise set.
// Dimension tokens additionally have a unit
// composed of one or more characters.
// Unicode-range tokens have a range of characters.

use cssparser;

const MAX_UNICODE: uint = 0x10FFFF;

#[deriving_eq]
enum NumericValue {
    Integer(int),
    // The spec calls this "number".
    // Use "float" instead to reduce term overloading with "number token".
    Float(float),
}
// TODO: add a NumberValue.as_float() method.

#[deriving_eq]
enum Token {
    Ident(~str),
    Function(~str),
    AtKeyword(~str),
    Hash(~str),
    String(~str),
    BadString,
    URL(~str),
    BadURL,
    Delim(char),
    Number(NumericValue, ~str),  // value, representation
    Percentage(NumericValue, ~str),  // value, representation
    Dimension(NumericValue, ~str, ~str),  // value, representation, unit
    UnicodeRange(char, char),  // start, end. XXX what about empty ranges?
    WhiteSpace,
    Comment,
    CDO,  // <!--
    CDC,  // -->
    Colon,  // ,
    Semicolon,  // ;
    OpenBraket, // [
    OpenParen, // (
    OpenBrace, // {
    CloseBraket, // ]
    CloseParen, // )
    CloseBrace, // }
}

struct State {
    input: ~str,
    length: uint,
    transform_function_whitespace: bool,
    quirks_mode: bool,
    mut position: uint,
    mut errors: ~[~str]
}

fn is_eof(state: &State) -> bool {
    state.position >= state.length
}

fn current_char(state: &State) -> char {
    str::char_at(state.input, state.position)
}

fn consume_char(state: &State) -> char {
    let range = str::char_range_at(state.input, state.position);
    state.position = range.next;
    range.ch
}

// http://dev.w3.org/csswg/css3-syntax/#tokenization
pub fn tokenize(input: &str, transform_function_whitespace: bool,
            quirks_mode: bool) -> {tokens: ~[Token], parse_errors: ~[~str]} {
    let input = cssparser::preprocess(input);
    let state = &State {
        input: input, length: input.len(), quirks_mode: quirks_mode,
        transform_function_whitespace: transform_function_whitespace,
        position: 0, errors: ~[] };
    let mut tokens: ~[Token] = ~[];

    while !is_eof(state) {
        tokens.push(consume_token(state))
    }

    // Work around `error: moving out of mutable field`
    // TODO: find a cleaner way.
    let mut errors: ~[~str] = ~[];
    errors <-> state.errors;
    {tokens: tokens, parse_errors: errors}
}

// 3.3.4. Data state
fn consume_token(state: &State) -> Token {
    let c = consume_char(state);
    match c {
        '\t' | '\n' | '\x0C' | ' ' => {
            while !is_eof(state) {
                match current_char(state) {
                    '\t' | '\n' | '\x0C' | ' ' => state.position += 1,
                    _ => break,
                }
            }
            WhiteSpace
        },
        '"' => consume_quoted_string(state, false),
        '\'' => consume_quoted_string(state, true),
        '/' if !is_eof(state) && current_char(state) == '*'
            => consume_comment(state),
        ',' => Colon,
        ';' => Semicolon,
        '[' => OpenBraket,
        '(' => OpenParen,
        '{' => OpenBrace,
        ']' => CloseBraket,
        ')' => CloseParen,
        '}' => CloseBrace,
        'a'..'z' | 'A'..'Z' | '_' => consume_ident(state, c),
        c if c >= '\xA0' => consume_ident(state, c),  // Non-ASCII
        _ => Delim(c),
    }
}

// 3.3.5. Double-quote-string state
// 3.3.6. Single-quote-string state
fn consume_quoted_string(state: &State, single_quote: bool) -> Token {
    let mut string: ~str = ~"";
    loop {
        if is_eof(state) {
            state.errors.push(~"EOF in quoted string");
            return String(string);
        }
        match consume_char(state) {
            '"' if !single_quote => return String(string),
            '\'' if single_quote => return String(string),
            '\n' | '\x0C' => {
                state.errors.push(~"Newline in quoted string");
                return BadString
            },
            '\\' => {
                if is_eof(state) {
                    state.errors.push(~"EOF in quoted string");
                    return BadString
                }
                match current_char(state) {
                    // Consume quoted newline
                    '\n' | '\x0C' => state.position += 1,
                    _ =>  str::push_char(&mut string, consume_escape(state))
                }
            }
            c => str::push_char(&mut string, c),
        }
    }
}

// 3.3.9. Comment state
fn consume_comment(state: &State) -> Token {
    state.position += 1;  // consume the * in /*
    match str::find_str_from(state.input, "*/", state.position) {
        Some(end_position) => state.position = end_position + 2,
        None => {
            state.errors.push(~"EOF in comment");
            state.position = state.input.len();
        }
    }
    Comment
}

// 3.3.12. Ident state
// 3.3.13. Ident-rest state
fn consume_ident(state: &State, initial_char: char) -> Token {
    let mut string = str::from_char(initial_char);
    while !is_eof(state) {
        let c = current_char(state);
        let cc = match c {
            'a'..'z' | 'A'..'Z' | '0'..'9' | '_' | '-'  => {
                state.position += 1; c },
            _ if c >= '\xA0' => consume_char(state),  // Non-ASCII
            '\\' => {
                state.position += 1;
                if is_eof(state) { state.position -= 1; break }
                match current_char(state) {
                    '\n' | '\x0C' => { state.position -= 1; break },
                    _ => consume_escape(state)
                }
            },
            '\t' | '\n' | '\x0C' | ' ' if state.transform_function_whitespace
            => {
                state.position += 1;
                return handle_transform_function_whitespace(state, string)
            }
            '(' => {
                state.position += 1;
                // TODO: URLs
                return Function(string)
            },
            _ => break
        };
        str::push_char(&mut string, cc)
    }
    Ident(string)
}

// 3.3.14. Transform-function-whitespace state
fn handle_transform_function_whitespace(state: &State, string: ~str) -> Token {
    while !is_eof(state) {
        match current_char(state) {
            '\t' | '\n' | '\x0C' | ' ' => state.position += 1,
            '(' => { state.position += 1; return Function(string) }
            _ => break,
        }
    }
    // XXX I think the spec is wrong here.
    // See http://lists.w3.org/Archives/Public/www-style/2013Jan/0266.html
    // Go back for one whitespace character.
    state.position -= 1;
    return Ident(string)
}

// 3.3.27. Consume an escaped character
// Assumes that the U+005C REVERSE SOLIDUS (\) has already been consumed
// and that the next input character has already been verified
// to not be a newline or EOF.
fn consume_escape(state: &State) -> char {
    let c = consume_char(state);
    match c {
        '0'..'9' | 'A'..'F' | 'a'..'f' => {
            let mut hex = ~[c];
            while hex.len() < 6 && !is_eof(state) {
                let c = current_char(state);
                match c {
                    '0'..'9' | 'A'..'F' | 'a'..'f' => {
                        hex.push(c); state.position += 1 },
                    _ => break
                }
            }
            if !is_eof(state) {
                match current_char(state) {
                    '\t' | '\n' | '\x0C' | ' ' => state.position += 1,
                    _ => ()
                }
            }
            char_from_hex(hex)
        },
        c => c
    }
}

fn char_from_hex(hex: &[char]) -> char {
    let mut value = 0u;
    for hex.each |cc| {
        let c = *cc;
        value = value * 16 + c as uint - match c {
            '0'..'9' => '0' as uint,
            'A'..'F' => 'A' as uint - 10,
            'a'..'f' => 'a' as uint - 10,
            _ => fail
        }
    }
    if value == 0 || value > MAX_UNICODE {
        '\uFFFD'  // Replacement character
    } else {
        value as char
    }
}

#[test]
fn test_tokenizer() {

    fn assert_tokens(input: &str, expected_tokens: &[Token],
                     expected_errors: &[~str]) {
        assert_tokens_flags(
            input, false, false, expected_tokens, expected_errors)
    }

    fn assert_tokens_flags(
            input: &str,
            transform_function_whitespace: bool, quirks_mode: bool,
            expected_tokens: &[Token], expected_errors: &[~str]) {
        let result = tokenize(
            input, transform_function_whitespace, quirks_mode);
        let tokens: &[Token] = result.tokens;
        let parse_errors: &[~str] = result.parse_errors;
        if tokens != expected_tokens {
            fail fmt!("%? != %?", tokens, expected_tokens);
        }
        if parse_errors != expected_errors {
            fail fmt!("%? != %?", parse_errors, expected_errors);
        }
    }
    assert_tokens("", [], []);
    assert_tokens("?/", [Delim('?'), Delim('/')], []);
    assert_tokens("?/* Li/*psum… */", [Delim('?'), Comment], []);
    assert_tokens("?/* Li/*psum… *//", [Delim('?'), Comment, Delim('/')], []);
    assert_tokens("?/* Lipsum", [Delim('?'), Comment], [~"EOF in comment"]);
    assert_tokens("?/*", [Delim('?'), Comment], [~"EOF in comment"]);
    assert_tokens("[?}{)",
        [OpenBraket, Delim('?'), CloseBrace, OpenBrace, CloseParen], []);

    assert_tokens("(\n \t'Lore\\6d \"ipsu\\6D'",
        [OpenParen, WhiteSpace, String(~"Lorem\"ipsum")], []);
    assert_tokens("'\\''", [String(~"'")], []);
    assert_tokens("\"\\\"\"", [String(~"\"")], []);
    assert_tokens("\"\\\"", [String(~"\"")], [~"EOF in quoted string"]);
    assert_tokens("'\\", [BadString], [~"EOF in quoted string"]);
    assert_tokens("\"0\\0000000\"", [String(~"0\uFFFD0")], []);
    assert_tokens("\"0\\000000 0\"", [String(~"0\uFFFD0")], []);
    assert_tokens("'z\n'a", [BadString, String(~"a")],
        [~"Newline in quoted string", ~"EOF in quoted string"]);

    assert_tokens("Lorem\\ ipsu\\6D dolor sit",
        [Ident(~"Lorem ipsumdolor"), WhiteSpace, Ident(~"sit")], []);
    assert_tokens("foo\\", [Ident(~"foo"), Delim('\\')], []);
    assert_tokens("foo\\\nbar",
        [Ident(~"foo"), Delim('\\'), WhiteSpace, Ident(~"bar")], []);
    assert_tokens("func()", [Function(~"func"), CloseParen], []);
    assert_tokens("func ()",
        [Ident(~"func"), WhiteSpace, OpenParen, CloseParen], []);
    assert_tokens_flags("func ()", true, false,
        [Function(~"func"), CloseParen], []);
}
<commit_msg>Handle idents starting with \ or -<commit_after>// http://dev.w3.org/csswg/css3-syntax/#tokenization
//
// The output of the tokenization step is a series of zero or more
// of the following tokens:
// ident, function, at-keyword, hash, string, bad-string, url, bad-url,
// delim, number, percentage, dimension, unicode-range, whitespace, comment,
// cdo, cdc, colon, semicolon, [, ], (, ), {, }.
//
// ident, function, at-keyword, hash, string, and url tokens
// have a value composed of zero or more characters.
// Delim tokens have a value composed of a single character.
// Number, percentage, and dimension tokens have a representation
// composed of 1 or more character, a numeric value,
// and a type flag set to either "integer" or "number".
// The type flag defaults to "integer" if not otherwise set.
// Dimension tokens additionally have a unit
// composed of one or more characters.
// Unicode-range tokens have a range of characters.

use cssparser;

const MAX_UNICODE: uint = 0x10FFFF;

#[deriving_eq]
enum NumericValue {
    Integer(int),
    // The spec calls this "number".
    // Use "float" instead to reduce term overloading with "number token".
    Float(float),
}
// TODO: add a NumberValue.as_float() method.

#[deriving_eq]
enum Token {
    Ident(~str),
    Function(~str),
    AtKeyword(~str),
    Hash(~str),
    String(~str),
    BadString,
    URL(~str),
    BadURL,
    Delim(char),
    Number(NumericValue, ~str),  // value, representation
    Percentage(NumericValue, ~str),  // value, representation
    Dimension(NumericValue, ~str, ~str),  // value, representation, unit
    UnicodeRange(char, char),  // start, end. XXX what about empty ranges?
    WhiteSpace,
    Comment,
    CDO,  // <!--
    CDC,  // -->
    Colon,  // :
    Semicolon,  // ;
    OpenBraket, // [
    OpenParen, // (
    OpenBrace, // {
    CloseBraket, // ]
    CloseParen, // )
    CloseBrace, // }
}

struct State {
    input: ~str,
    length: uint,
    transform_function_whitespace: bool,
    quirks_mode: bool,
    mut position: uint,
    mut errors: ~[~str]
}

fn is_eof(state: &State) -> bool {
    state.position >= state.length
}

fn current_char(state: &State) -> char {
    str::char_at(state.input, state.position)
}

fn consume_char(state: &State) -> char {
    let range = str::char_range_at(state.input, state.position);
    state.position = range.next;
    range.ch
}

// http://dev.w3.org/csswg/css3-syntax/#tokenization
pub fn tokenize(input: &str, transform_function_whitespace: bool,
            quirks_mode: bool) -> {tokens: ~[Token], parse_errors: ~[~str]} {
    let input = cssparser::preprocess(input);
    let state = &State {
        input: input, length: input.len(), quirks_mode: quirks_mode,
        transform_function_whitespace: transform_function_whitespace,
        position: 0, errors: ~[] };
    let mut tokens: ~[Token] = ~[];

    while !is_eof(state) {
        tokens.push(consume_token(state))
    }

    // Work around `error: moving out of mutable field`
    // TODO: find a cleaner way.
    let mut errors: ~[~str] = ~[];
    errors <-> state.errors;
    {tokens: tokens, parse_errors: errors}
}

// 3.3.4. Data state
fn consume_token(state: &State) -> Token {
    let c = consume_char(state);
    match c {
        '\t' | '\n' | '\x0C' | ' ' => {
            while !is_eof(state) {
                match current_char(state) {
                    '\t' | '\n' | '\x0C' | ' ' => state.position += 1,
                    _ => break,
                }
            }
            WhiteSpace
        },
        '"' => consume_quoted_string(state, false),
        '\'' => consume_quoted_string(state, true),
        '(' => OpenParen,
        ')' => CloseParen,
        '-' => {
            if is_eof(state) { Delim('-') } else {
                // TODO: CDC, negative numbers
                match current_char(state) {
                    '\\' => {
                        state.position += 1;
                        if is_eof(state) { state.position -= 1; Delim('-') }
                        else {
                            let c = current_char(state);
                            state.position -= 1;
                            match c {
                                '\n' | '\x0C' => Delim('-'),
                                _ => consume_ident(state, '-')
                            }
                        }
                    },
                    'a'..'z' | 'A'..'Z' | '_' => consume_ident(state, c),
                    c if c >= '\xA0' => consume_ident(state, c),  // Non-ASCII
                    _ => Delim('-')
                }
            }
        }
        '/' if !is_eof(state) && current_char(state) == '*'
            => consume_comment(state),
        ':' => Colon,
        ';' => Semicolon,
        '[' => OpenBraket,
        '\\' => {
            if is_eof(state) {
                state.errors.push(~"Invalid escape");
                Delim('\\')
            } else {
                match current_char(state) {
                    '\n' | '\x0C' => {
                        state.errors.push(~"Invalid escape"); Delim('\\') },
                    _ => consume_ident(state, consume_escape(state))
                }
            }
        }
        ']' => CloseBraket,
        '{' => OpenBrace,
        '}' => CloseBrace,
        'a'..'z' | 'A'..'Z' | '_' => consume_ident(state, c),
        c if c >= '\xA0' => consume_ident(state, c),  // Non-ASCII
        _ => Delim(c),
    }
}

// 3.3.5. Double-quote-string state
// 3.3.6. Single-quote-string state
fn consume_quoted_string(state: &State, single_quote: bool) -> Token {
    let mut string: ~str = ~"";
    loop {
        if is_eof(state) {
            state.errors.push(~"EOF in quoted string");
            return String(string);
        }
        match consume_char(state) {
            '"' if !single_quote => return String(string),
            '\'' if single_quote => return String(string),
            '\n' | '\x0C' => {
                state.errors.push(~"Newline in quoted string");
                return BadString
            },
            '\\' => {
                if is_eof(state) {
                    state.errors.push(~"EOF in quoted string");
                    return BadString
                }
                match current_char(state) {
                    // Consume quoted newline
                    '\n' | '\x0C' => state.position += 1,
                    _ =>  str::push_char(&mut string, consume_escape(state))
                }
            }
            c => str::push_char(&mut string, c),
        }
    }
}

// 3.3.9. Comment state
fn consume_comment(state: &State) -> Token {
    state.position += 1;  // consume the * in /*
    match str::find_str_from(state.input, "*/", state.position) {
        Some(end_position) => state.position = end_position + 2,
        None => {
            state.errors.push(~"EOF in comment");
            state.position = state.input.len();
        }
    }
    Comment
}

// 3.3.12. Ident state
// 3.3.13. Ident-rest state
fn consume_ident(state: &State, initial_char: char) -> Token {
    let mut string = str::from_char(initial_char);
    while !is_eof(state) {
        let c = current_char(state);
        let cc = match c {
            'a'..'z' | 'A'..'Z' | '0'..'9' | '_' | '-'  => {
                state.position += 1; c },
            _ if c >= '\xA0' => consume_char(state),  // Non-ASCII
            '\\' => {
                state.position += 1;
                if is_eof(state) { state.position -= 1; break }
                match current_char(state) {
                    '\n' | '\x0C' => { state.position -= 1; break },
                    _ => consume_escape(state)
                }
            },
            '\t' | '\n' | '\x0C' | ' ' if state.transform_function_whitespace
            => {
                state.position += 1;
                return handle_transform_function_whitespace(state, string)
            }
            '(' => {
                state.position += 1;
                // TODO: URLs
                return Function(string)
            },
            _ => break
        };
        str::push_char(&mut string, cc)
    }
    Ident(string)
}

// 3.3.14. Transform-function-whitespace state
fn handle_transform_function_whitespace(state: &State, string: ~str) -> Token {
    while !is_eof(state) {
        match current_char(state) {
            '\t' | '\n' | '\x0C' | ' ' => state.position += 1,
            '(' => { state.position += 1; return Function(string) }
            _ => break,
        }
    }
    // XXX I think the spec is wrong here.
    // See http://lists.w3.org/Archives/Public/www-style/2013Jan/0266.html
    // Go back for one whitespace character.
    state.position -= 1;
    return Ident(string)
}

// 3.3.27. Consume an escaped character
// Assumes that the U+005C REVERSE SOLIDUS (\) has already been consumed
// and that the next input character has already been verified
// to not be a newline or EOF.
fn consume_escape(state: &State) -> char {
    let c = consume_char(state);
    match c {
        '0'..'9' | 'A'..'F' | 'a'..'f' => {
            let mut hex = ~[c];
            while hex.len() < 6 && !is_eof(state) {
                let c = current_char(state);
                match c {
                    '0'..'9' | 'A'..'F' | 'a'..'f' => {
                        hex.push(c); state.position += 1 },
                    _ => break
                }
            }
            if !is_eof(state) {
                match current_char(state) {
                    '\t' | '\n' | '\x0C' | ' ' => state.position += 1,
                    _ => ()
                }
            }
            char_from_hex(hex)
        },
        c => c
    }
}

fn char_from_hex(hex: &[char]) -> char {
    let mut value = 0u;
    for hex.each |cc| {
        let c = *cc;
        value = value * 16 + c as uint - match c {
            '0'..'9' => '0' as uint,
            'A'..'F' => 'A' as uint - 10,
            'a'..'f' => 'a' as uint - 10,
            _ => fail
        }
    }
    if value == 0 || value > MAX_UNICODE {
        '\uFFFD'  // Replacement character
    } else {
        value as char
    }
}

#[test]
fn test_tokenizer() {

    fn assert_tokens(input: &str, expected_tokens: &[Token],
                     expected_errors: &[~str]) {
        assert_tokens_flags(
            input, false, false, expected_tokens, expected_errors)
    }

    fn assert_tokens_flags(
            input: &str,
            transform_function_whitespace: bool, quirks_mode: bool,
            expected_tokens: &[Token], expected_errors: &[~str]) {
        let result = tokenize(
            input, transform_function_whitespace, quirks_mode);
        let tokens: &[Token] = result.tokens;
        let parse_errors: &[~str] = result.parse_errors;
        if tokens != expected_tokens {
            fail fmt!("%? != %?", tokens, expected_tokens);
        }
        if parse_errors != expected_errors {
            fail fmt!("%? != %?", parse_errors, expected_errors);
        }
    }
    assert_tokens("", [], []);
    assert_tokens("?/", [Delim('?'), Delim('/')], []);
    assert_tokens("?/* Li/*psum… */", [Delim('?'), Comment], []);
    assert_tokens("?/* Li/*psum… *//", [Delim('?'), Comment, Delim('/')], []);
    assert_tokens("?/* Lipsum", [Delim('?'), Comment], [~"EOF in comment"]);
    assert_tokens("?/*", [Delim('?'), Comment], [~"EOF in comment"]);
    assert_tokens("[?}{)",
        [OpenBraket, Delim('?'), CloseBrace, OpenBrace, CloseParen], []);

    assert_tokens("(\n \t'Lore\\6d \"ipsu\\6D'",
        [OpenParen, WhiteSpace, String(~"Lorem\"ipsum")], []);
    assert_tokens("'\\''", [String(~"'")], []);
    assert_tokens("\"\\\"\"", [String(~"\"")], []);
    assert_tokens("\"\\\"", [String(~"\"")], [~"EOF in quoted string"]);
    assert_tokens("'\\", [BadString], [~"EOF in quoted string"]);
    assert_tokens("\"0\\0000000\"", [String(~"0\uFFFD0")], []);
    assert_tokens("\"0\\000000 0\"", [String(~"0\uFFFD0")], []);
    assert_tokens("'z\n'a", [BadString, String(~"a")],
        [~"Newline in quoted string", ~"EOF in quoted string"]);

    assert_tokens("Lorem\\ ipsu\\6D dolor \\sit",
        [Ident(~"Lorem ipsumdolor"), WhiteSpace, Ident(~"sit")], []);
    assert_tokens("foo\\", [Ident(~"foo"), Delim('\\')], [~"Invalid escape"]);
    assert_tokens("foo\\\nbar",
        [Ident(~"foo"), Delim('\\'), WhiteSpace, Ident(~"bar")],
        [~"Invalid escape"]);
    assert_tokens("-Lipsum", [Ident(~"-Lipsum")], []);
    assert_tokens("-\\Lipsum", [Ident(~"-Lipsum")], []);
    assert_tokens("func()", [Function(~"func"), CloseParen], []);
    assert_tokens("func ()",
        [Ident(~"func"), WhiteSpace, OpenParen, CloseParen], []);
    assert_tokens_flags("func ()", true, false,
        [Function(~"func"), CloseParen], []);
}
<|endoftext|>


<commit_before>use std::collections::hashmap::HashMap;
use std::char::is_digit;

#[deriving(PartialEq,Show,Clone)]
pub enum XPathToken {
    And,
    AtSign,
    CurrentNode,
    Divide,
    DollarSign,
    DoubleColon,
    DoubleSlash,
    Equal,
    GreaterThan,
    GreaterThanOrEqual,
    LeftBracket,
    LeftParen,
    LessThan,
    LessThanOrEqual,
    Literal(String),
    MinusSign,
    Multiply,
    NotEqual,
    Number(f64),
    Or,
    ParentNode,
    Pipe,
    PlusSign,
    PrefixedName(String, String),
    Remainder,
    RightBracket,
    RightParen,
    Slash,
    String(String),
}

impl XPathToken {
    fn precedes_node_test(& self) -> bool {
        match *self {
            AtSign |
            DoubleColon => true,
            _ => false,
        }
    }

    fn precedes_expression(& self) -> bool {
        match *self {
            LeftParen |
            LeftBracket => true,
            _ => false,
        }
    }

    fn is_operator(& self) -> bool {
        match *self {
            Slash |
            DoubleSlash |
            PlusSign |
            MinusSign |
            Pipe |
            Equal |
            NotEqual |
            LessThan |
            LessThanOrEqual |
            GreaterThan |
            GreaterThanOrEqual |
            And |
            Or |
            Remainder |
            Divide |
            Multiply => true,
            _ => false,
        }
    }
}

pub struct XPathTokenizer {
    xpath: Vec<char>,
    start: uint,
    prefer_recognition_of_operator_names: bool,
}

pub type TokenResult = Result<XPathToken, & 'static str>;

impl XPathTokenizer {
    pub fn new(xpath: & str) -> XPathTokenizer {
        XPathTokenizer {
            xpath: xpath.chars().collect(),
            start: 0,
            prefer_recognition_of_operator_names: false,
        }
    }

    fn str_at_is(& self, offset: uint, needle: &[char]) -> bool {
        let s_len = needle.len();

        if self.xpath.len() < offset + s_len { return false; }

        let xpath_chars = self.xpath.slice(offset, offset + s_len);

        needle == xpath_chars
    }

    fn valid_ncname_start_char(& self, offset: uint) -> bool {
        let c = self.xpath[offset];
        if c >= 'A' && c <= 'Z' { return true }
        if c == '_' { return true }
        if c >= 'a' && c <= 'z' { return true }
        // TODO: All non-ASCII codepoints
        return false;
    }

    fn valid_ncname_follow_char(& self, offset: uint) -> bool {
        let c = self.xpath[offset];
        if self.valid_ncname_start_char(offset) { return true }
        if c == '-' { return true }
        if c == '.' { return true }
        if c >= '0' && c <= '9' { return true }
        // TODO: All non-ASCII codepoints
        return false;
    }

    fn while_valid_string(& self, offset: uint) -> uint {
        let mut offset = offset;

        if offset < self.xpath.len() && self.valid_ncname_start_char(offset) {
            offset += 1;

            while offset < self.xpath.len() && self.valid_ncname_follow_char(offset) {
                offset += 1;
            }
        }

        return offset;
    }

    fn while_valid_number(& self, offset: uint) -> uint {
        let mut offset = offset;

        while offset < self.xpath.len() && is_number_char(self.xpath[offset]) {
            offset += 1;
        }

        return offset;
    }

    fn while_not_character(& self, offset: uint, end_char: char) -> uint {
        let mut offset = offset;

        while offset < self.xpath.len() && self.xpath[offset] != end_char {
            offset += 1;
        }

        return offset;
    }

    fn substr(& self, start: uint, end: uint) -> String {
        String::from_chars(self.xpath.slice(start, end))
    }

    fn safe_substr(& self, start: uint, end: uint) -> Option<String> {
        if self.xpath.len() >= end {
            Some(self.substr(start, end))
        } else {
            None
        }
    }

    fn char_at(&self, offset: uint) -> char {
        self.xpath[offset]
    }

    fn char_at_is(&self, offset: uint, c: char) -> bool {
        let has_one_more = self.xpath.len() >= offset + 1;

        has_one_more && self.xpath[offset] == c
    }

    fn char_at_is_not(&self, offset: uint, c: char) -> bool {
        let has_one_more = self.xpath.len() >= offset + 1;

        ! has_one_more || self.xpath[offset] != c
    }

    fn char_at_is_not_digit(& self, offset: uint) -> bool {
        let has_more_chars = self.xpath.len() >= offset + 1;

        ! has_more_chars || ! is_digit(self.xpath[offset])
    }

    fn end_of_whitespace(& self, offset: uint) -> uint {
        let mut offset = offset;

        while offset < self.xpath.len() {
            let c = self.xpath[offset];

            if ! is_xml_space(c) {
                break;
            }

            offset += 1;
        }

        offset
    }

    pub fn has_more_tokens(& self) -> bool {
        self.xpath.len() > self.start
    }

    fn two_char_tokens(& self) -> HashMap<String, XPathToken> {
        let mut m = HashMap::new();
        m.insert("<=".to_string(), LessThanOrEqual);
        m.insert(">=".to_string(), GreaterThanOrEqual);
        m.insert("!=".to_string(), NotEqual);
        m.insert("::".to_string(), DoubleColon);
        m.insert("//".to_string(), DoubleSlash);
        m.insert("..".to_string(), ParentNode);
        m
    }

    fn single_char_tokens(&self) -> HashMap<char, XPathToken> {
        let mut m = HashMap::new();
        m.insert('/', Slash);
        m.insert('(', LeftParen);
        m.insert(')', RightParen);
        m.insert('[', LeftBracket);
        m.insert(']', RightBracket);
        m.insert('@', AtSign);
        m.insert('$', DollarSign);
        m.insert('+', PlusSign);
        m.insert('-', MinusSign);
        m.insert('|', Pipe);
        m.insert('=', Equal);
        m.insert('<', LessThan);
        m.insert('>', GreaterThan);
        m
    }

    fn named_operators(& self) -> Vec<(& 'static str, XPathToken)> {
        vec!(("and", And),
             ("or",  Or),
             ("mod", Remainder),
             ("div", Divide),
             ("*",   Multiply))
    }

    fn tokenize_literal(& mut self, quote_char: char) -> Result<XPathToken, & 'static str> {
        let mut offset = self.start;

        offset += 1; // Skip over the starting quote
        let start_of_string = offset;

        offset = self.while_not_character(offset, quote_char);
        let end_of_string = offset;

        if self.char_at_is_not(offset, quote_char) {
            return Err("found mismatched quote characters");
        }
        offset += 1; // Skip over ending quote

        self.start = offset;
        return Ok(Literal(self.substr(start_of_string, end_of_string)));
    }

    fn raw_next_token(& mut self) -> Result<XPathToken, & 'static str> {
        match self.safe_substr(self.start, self.start + 2) {
            Some(first_two) => {
                match self.two_char_tokens().find(&first_two) {
                    Some(token) => {
                        self.start += 2;
                        return Ok(token.clone());
                    }
                    _ => {}
                }
            },
            _ => {}
        }

        let c = self.char_at(self.start);

        match self.single_char_tokens().find(&c) {
            Some(token) => {
                self.start += 1;
                return Ok(token.clone());
            }
            _ => {}
        }

        for quote_char in QUOTE_CHARS.iter() {
            if *quote_char == c {
                return self.tokenize_literal(*quote_char);
            }
        }

        if '.' == c {
            if self.char_at_is_not_digit(self.start + 1) {
                // Ugly. Should we use START / FOLLOW constructs?
                self.start += 1;
                return Ok(CurrentNode);
            }
        }

        if is_number_char(c) {
            let mut offset = self.start;
            let current_start = self.start;

            offset = self.while_valid_number(offset);

            self.start = offset;
            let substr = self.substr(current_start, offset);
            match from_str(substr.as_slice()) {
                Some(value) => Ok(Number(value)),
                None => fail!("Not really a number!")
            }
        } else {
            let mut offset = self.start;
            let current_start = self.start;

            if self.prefer_recognition_of_operator_names {
                for &(ref name, ref token) in self.named_operators().iter() {
                    let name_chars: Vec<char> = name.chars().collect();
                    let name_chars_slice = name_chars.as_slice();

                    if self.str_at_is(offset, name_chars_slice) {
                        self.start += name_chars.len();
                        return Ok(token.clone());
                    }
                }
            }

            if self.char_at_is(offset, '*') {
                self.start = offset + 1;
                return Ok(String("*".to_string()));
            }

            offset = self.while_valid_string(offset);

            if self.char_at_is(offset, ':') && self.char_at_is_not(offset + 1, ':') {
                let prefix = self.substr(current_start, offset);

                offset += 1;

                let current_start = offset;
                offset = self.while_valid_string(offset);

                if current_start == offset {
                    return Err("The XPath is missing a local name");
                }

                let name = self.substr(current_start, offset);

                self.start = offset;
                return Ok(PrefixedName(prefix, name));

            } else {
                self.start = offset;
                return Ok(String(self.substr(current_start, offset)));
            }
        }
    }

    fn consume_whitespace(& mut self) {
        self.start = self.end_of_whitespace(self.start);
    }

    fn next_token(& mut self) -> TokenResult {
        self.consume_whitespace();

        let old_start = self.start;
        let token = self.raw_next_token();
        if token.is_err() { return token; }

        let token = token.unwrap();

        if old_start == self.start {
            return Err("Unable to create a token");
        }

        self.consume_whitespace();

        if ! (token.precedes_node_test() ||
              token.precedes_expression() ||
              token.is_operator()) {
            // See http://www.w3.org/TR/xpath/#exprlex
            self.prefer_recognition_of_operator_names = true;
        } else {
            self.prefer_recognition_of_operator_names = false;
        }

        return Ok(token);
    }
}

impl Iterator<TokenResult> for XPathTokenizer {
    fn next(&mut self) -> Option<TokenResult> {
        if self.has_more_tokens() {
            Some(self.next_token())
        } else {
            None
        }
    }
}

fn is_xml_space(c: char) -> bool {
    return
        c == ' '  ||
        c == '\t' ||
        c == '\n' ||
        c == '\r';
}

fn is_number_char(c: char) -> bool {
    return is_digit(c) || '.' == c;
}

// broken nesting double in single
// open paren / suqre brace causes indent
static QUOTE_CHARS: [char, .. 2] =  ['\'', '"'];
<commit_msg>Refactor into new object<commit_after>use std::collections::hashmap::HashMap;
use std::char::is_digit;

#[deriving(PartialEq,Show,Clone)]
pub enum XPathToken {
    And,
    AtSign,
    CurrentNode,
    Divide,
    DollarSign,
    DoubleColon,
    DoubleSlash,
    Equal,
    GreaterThan,
    GreaterThanOrEqual,
    LeftBracket,
    LeftParen,
    LessThan,
    LessThanOrEqual,
    Literal(String),
    MinusSign,
    Multiply,
    NotEqual,
    Number(f64),
    Or,
    ParentNode,
    Pipe,
    PlusSign,
    PrefixedName(String, String),
    Remainder,
    RightBracket,
    RightParen,
    Slash,
    String(String),
}

impl XPathToken {
    fn precedes_node_test(& self) -> bool {
        match *self {
            AtSign |
            DoubleColon => true,
            _ => false,
        }
    }

    fn precedes_expression(& self) -> bool {
        match *self {
            LeftParen |
            LeftBracket => true,
            _ => false,
        }
    }

    fn is_operator(& self) -> bool {
        match *self {
            Slash |
            DoubleSlash |
            PlusSign |
            MinusSign |
            Pipe |
            Equal |
            NotEqual |
            LessThan |
            LessThanOrEqual |
            GreaterThan |
            GreaterThanOrEqual |
            And |
            Or |
            Remainder |
            Divide |
            Multiply => true,
            _ => false,
        }
    }
}

pub struct XPathTokenizer {
    xpath: XPathString,
    start: uint,
    prefer_recognition_of_operator_names: bool,
}

pub type TokenResult = Result<XPathToken, & 'static str>;

struct XPathString {
    xpath: Vec<char>,
}

impl XPathString {
    fn new(xpath: &str) -> XPathString {
        XPathString {
            xpath: xpath.chars().collect(),
        }
    }

    fn len(& self) -> uint {
        self.xpath.len()
    }

    fn str_at_is(& self, offset: uint, needle: &[char]) -> bool {
        let s_len = needle.len();

        if self.xpath.len() < offset + s_len { return false; }

        let xpath_chars = self.xpath.slice(offset, offset + s_len);

        needle == xpath_chars
    }

    fn valid_ncname_start_char(& self, offset: uint) -> bool {
        let c = self.xpath[offset];
        if c >= 'A' && c <= 'Z' { return true }
        if c == '_' { return true }
        if c >= 'a' && c <= 'z' { return true }
        // TODO: All non-ASCII codepoints
        return false;
    }

    fn valid_ncname_follow_char(& self, offset: uint) -> bool {
        let c = self.xpath[offset];
        if self.valid_ncname_start_char(offset) { return true }
        if c == '-' { return true }
        if c == '.' { return true }
        if c >= '0' && c <= '9' { return true }
        // TODO: All non-ASCII codepoints
        return false;
    }

    fn while_valid_string(& self, offset: uint) -> uint {
        let mut offset = offset;

        if offset < self.xpath.len() && self.valid_ncname_start_char(offset) {
            offset += 1;

            while offset < self.xpath.len() && self.valid_ncname_follow_char(offset) {
                offset += 1;
            }
        }

        return offset;
    }

    fn while_valid_number(& self, offset: uint) -> uint {
        let mut offset = offset;

        while offset < self.xpath.len() && is_number_char(self.xpath[offset]) {
            offset += 1;
        }

        return offset;
    }

    fn while_not_character(& self, offset: uint, end_char: char) -> uint {
        let mut offset = offset;

        while offset < self.xpath.len() && self.xpath[offset] != end_char {
            offset += 1;
        }

        return offset;
    }

    fn substr(& self, start: uint, end: uint) -> String {
        String::from_chars(self.xpath.slice(start, end))
    }

    fn safe_substr(& self, start: uint, end: uint) -> Option<String> {
        if self.xpath.len() >= end {
            Some(self.substr(start, end))
        } else {
            None
        }
    }

    fn char_at(&self, offset: uint) -> char {
        self.xpath[offset]
    }

    fn char_at_is(&self, offset: uint, c: char) -> bool {
        let has_one_more = self.xpath.len() >= offset + 1;

        has_one_more && self.xpath[offset] == c
    }

    fn char_at_is_not(&self, offset: uint, c: char) -> bool {
        let has_one_more = self.xpath.len() >= offset + 1;

        ! has_one_more || self.xpath[offset] != c
    }

    fn char_at_is_not_digit(& self, offset: uint) -> bool {
        let has_more_chars = self.xpath.len() >= offset + 1;

        ! has_more_chars || ! is_digit(self.xpath[offset])
    }

    fn is_xml_space(&self, offset: uint) -> bool {
        let c = self.xpath[offset];

        return
            c == ' '  ||
            c == '\t' ||
            c == '\n' ||
            c == '\r';
    }

    fn end_of_whitespace(& self, offset: uint) -> uint {
        let mut offset = offset;

        while offset < self.xpath.len() && self.is_xml_space(offset) {
            offset += 1;
        }

        offset
    }
}

static QUOTE_CHARS: [char, .. 2] =  ['\'', '\"'];

impl XPathTokenizer {
    pub fn new(xpath: & str) -> XPathTokenizer {
        XPathTokenizer {
            xpath: XPathString::new(xpath),
            start: 0,
            prefer_recognition_of_operator_names: false,
        }
    }

    pub fn has_more_tokens(& self) -> bool {
        self.xpath.len() > self.start
    }

    fn two_char_tokens(& self) -> HashMap<String, XPathToken> {
        let mut m = HashMap::new();
        m.insert("<=".to_string(), LessThanOrEqual);
        m.insert(">=".to_string(), GreaterThanOrEqual);
        m.insert("!=".to_string(), NotEqual);
        m.insert("::".to_string(), DoubleColon);
        m.insert("//".to_string(), DoubleSlash);
        m.insert("..".to_string(), ParentNode);
        m
    }

    fn single_char_tokens(&self) -> HashMap<char, XPathToken> {
        let mut m = HashMap::new();
        m.insert('/', Slash);
        m.insert('(', LeftParen);
        m.insert(')', RightParen);
        m.insert('[', LeftBracket);
        m.insert(']', RightBracket);
        m.insert('@', AtSign);
        m.insert('$', DollarSign);
        m.insert('+', PlusSign);
        m.insert('-', MinusSign);
        m.insert('|', Pipe);
        m.insert('=', Equal);
        m.insert('<', LessThan);
        m.insert('>', GreaterThan);
        m
    }

    fn named_operators(& self) -> Vec<(& 'static str, XPathToken)> {
        vec!(("and", And),
             ("or",  Or),
             ("mod", Remainder),
             ("div", Divide),
             ("*",   Multiply))
    }

    fn tokenize_literal(& mut self, quote_char: char) -> Result<XPathToken, & 'static str> {
        let mut offset = self.start;

        offset += 1; // Skip over the starting quote
        let start_of_string = offset;

        offset = self.xpath.while_not_character(offset, quote_char);
        let end_of_string = offset;

        if self.xpath.char_at_is_not(offset, quote_char) {
            return Err("found mismatched quote characters");
        }
        offset += 1; // Skip over ending quote

        self.start = offset;
        return Ok(Literal(self.xpath.substr(start_of_string, end_of_string)));
    }

    fn raw_next_token(& mut self) -> Result<XPathToken, & 'static str> {
        match self.xpath.safe_substr(self.start, self.start + 2) {
            Some(first_two) => {
                match self.two_char_tokens().find(&first_two) {
                    Some(token) => {
                        self.start += 2;
                        return Ok(token.clone());
                    }
                    _ => {}
                }
            },
            _ => {}
        }

        let c = self.xpath.char_at(self.start);

        match self.single_char_tokens().find(&c) {
            Some(token) => {
                self.start += 1;
                return Ok(token.clone());
            }
            _ => {}
        }

        for quote_char in QUOTE_CHARS.iter() {
            if *quote_char == c {
                return self.tokenize_literal(*quote_char);
            }
        }

        if '.' == c {
            if self.xpath.char_at_is_not_digit(self.start + 1) {
                // Ugly. Should we use START / FOLLOW constructs?
                self.start += 1;
                return Ok(CurrentNode);
            }
        }

        if is_number_char(c) {
            let mut offset = self.start;
            let current_start = self.start;

            offset = self.xpath.while_valid_number(offset);

            self.start = offset;
            let substr = self.xpath.substr(current_start, offset);
            match from_str(substr.as_slice()) {
                Some(value) => Ok(Number(value)),
                None => fail!("Not really a number!")
            }
        } else {
            let mut offset = self.start;
            let current_start = self.start;

            if self.prefer_recognition_of_operator_names {
                for &(ref name, ref token) in self.named_operators().iter() {
                    let name_chars: Vec<char> = name.chars().collect();
                    let name_chars_slice = name_chars.as_slice();

                    if self.xpath.str_at_is(offset, name_chars_slice) {
                        self.start += name_chars.len();
                        return Ok(token.clone());
                    }
                }
            }

            if self.xpath.char_at_is(offset, '*') {
                self.start = offset + 1;
                return Ok(String("*".to_string()));
            }

            offset = self.xpath.while_valid_string(offset);

            if self.xpath.char_at_is(offset, ':') && self.xpath.char_at_is_not(offset + 1, ':') {
                let prefix = self.xpath.substr(current_start, offset);

                offset += 1;

                let current_start = offset;
                offset = self.xpath.while_valid_string(offset);

                if current_start == offset {
                    return Err("The XPath is missing a local name");
                }

                let name = self.xpath.substr(current_start, offset);

                self.start = offset;
                return Ok(PrefixedName(prefix, name));

            } else {
                self.start = offset;
                return Ok(String(self.xpath.substr(current_start, offset)));
            }
        }
    }

    fn consume_whitespace(& mut self) {
        self.start = self.xpath.end_of_whitespace(self.start);
    }

    fn next_token(& mut self) -> TokenResult {
        self.consume_whitespace();

        let old_start = self.start;
        let token = self.raw_next_token();
        if token.is_err() { return token; }

        let token = token.unwrap();

        if old_start == self.start {
            return Err("Unable to create a token");
        }

        self.consume_whitespace();

        if ! (token.precedes_node_test() ||
              token.precedes_expression() ||
              token.is_operator()) {
            // See http://www.w3.org/TR/xpath/#exprlex
            self.prefer_recognition_of_operator_names = true;
        } else {
            self.prefer_recognition_of_operator_names = false;
        }

        return Ok(token);
    }
}

impl Iterator<TokenResult> for XPathTokenizer {
    fn next(&mut self) -> Option<TokenResult> {
        if self.has_more_tokens() {
            Some(self.next_token())
        } else {
            None
        }
    }
}

fn is_number_char(c: char) -> bool {
    return is_digit(c) || '.' == c;
}
<|endoftext|>


<commit_before>// Copyright (c) 2021-present, Gregory Szorc
// All rights reserved.
//
// This software may be modified and distributed under the terms
// of the BSD license. See the LICENSE file for details.

use {
    crate::{
        decompressor::DCtx,
        exceptions::ZstdError,
        stream::{make_in_buffer_source, InBufferSource},
    },
    pyo3::{exceptions::PyValueError, prelude::*, types::PyBytes, PyIterProtocol},
    std::{cmp::min, sync::Arc},
};

#[pyclass(module = "zstandard.backend_rust")]
pub struct ZstdDecompressorIterator {
    dctx: Arc<DCtx<'static>>,
    source: Box<dyn InBufferSource + Send>,
    write_size: usize,
    finished_output: bool,
}

impl ZstdDecompressorIterator {
    pub fn new(
        py: Python,
        dctx: Arc<DCtx<'static>>,
        reader: &PyAny,
        read_size: usize,
        write_size: usize,
        skip_bytes: usize,
    ) -> PyResult<Self> {
        let mut source = make_in_buffer_source(py, reader, read_size)?;

        let mut skip_bytes = skip_bytes;
        while skip_bytes > 0 {
            let in_buffer = source
                .input_buffer(py)?
                .ok_or_else(|| PyValueError::new_err("skip_bytes larger than first input chunk"))?;

            let read = min(skip_bytes, in_buffer.size - in_buffer.pos);
            source.record_bytes_read(read);
            skip_bytes -= read;
        }

        Ok(Self {
            dctx,
            source,
            write_size,
            finished_output: false,
        })
    }
}

#[pyproto]
impl PyIterProtocol for ZstdDecompressorIterator {
    fn __iter__(slf: PyRef<Self>) -> PyRef<Self> {
        slf
    }
    fn __next__(mut slf: PyRefMut<Self>) -> PyResult<Option<PyObject>> {
        if slf.finished_output {
            return Ok(None);
        }

        let py = unsafe { Python::assume_gil_acquired() };

        let mut dest_buffer: Vec<u8> = Vec::with_capacity(slf.write_size);
        let mut out_buffer = zstd_sys::ZSTD_outBuffer {
            dst: dest_buffer.as_mut_ptr() as *mut _,
            size: dest_buffer.capacity(),
            pos: 0,
        };

        // While input is available.
        while let Some(mut in_buffer) = slf.source.input_buffer(py)? {
            let zresult = unsafe {
                zstd_sys::ZSTD_decompressStream(
                    slf.dctx.dctx(),
                    &mut out_buffer as *mut _,
                    &mut in_buffer as *mut _,
                )
            };
            if unsafe { zstd_sys::ZSTD_isError(zresult) } != 0 {
                return Err(ZstdError::new_err(format!(
                    "zstd decompress error: {}",
                    zstd_safe::get_error_name(zresult)
                )));
            }

            slf.source.record_bytes_read(in_buffer.pos);
            unsafe {
                dest_buffer.set_len(out_buffer.pos);
            }

            // Emit chunk if output buffer is full.
            if out_buffer.pos == out_buffer.size {
                // TODO avoid buffer copy.
                let chunk = PyBytes::new(py, &dest_buffer);
                return Ok(Some(chunk.into_py(py)));
            }

            // Try to get more input to fill output buffer.
            continue;
        }

        // Input is exhausted. Emit what we have or finish.
        if out_buffer.pos > 0 {
            // TODO avoid buffer copy.
            let chunk = PyBytes::new(py, &dest_buffer);
            Ok(Some(chunk.into_py(py)))
        } else {
            Ok(None)
        }
    }
}
<commit_msg>rust: emit partial output chunk<commit_after>// Copyright (c) 2021-present, Gregory Szorc
// All rights reserved.
//
// This software may be modified and distributed under the terms
// of the BSD license. See the LICENSE file for details.

use {
    crate::{
        decompressor::DCtx,
        exceptions::ZstdError,
        stream::{make_in_buffer_source, InBufferSource},
    },
    pyo3::{exceptions::PyValueError, prelude::*, types::PyBytes, PyIterProtocol},
    std::{cmp::min, sync::Arc},
};

#[pyclass(module = "zstandard.backend_rust")]
pub struct ZstdDecompressorIterator {
    dctx: Arc<DCtx<'static>>,
    source: Box<dyn InBufferSource + Send>,
    write_size: usize,
    finished_output: bool,
}

impl ZstdDecompressorIterator {
    pub fn new(
        py: Python,
        dctx: Arc<DCtx<'static>>,
        reader: &PyAny,
        read_size: usize,
        write_size: usize,
        skip_bytes: usize,
    ) -> PyResult<Self> {
        let mut source = make_in_buffer_source(py, reader, read_size)?;

        let mut skip_bytes = skip_bytes;
        while skip_bytes > 0 {
            let in_buffer = source
                .input_buffer(py)?
                .ok_or_else(|| PyValueError::new_err("skip_bytes larger than first input chunk"))?;

            let read = min(skip_bytes, in_buffer.size - in_buffer.pos);
            source.record_bytes_read(read);
            skip_bytes -= read;
        }

        Ok(Self {
            dctx,
            source,
            write_size,
            finished_output: false,
        })
    }
}

#[pyproto]
impl PyIterProtocol for ZstdDecompressorIterator {
    fn __iter__(slf: PyRef<Self>) -> PyRef<Self> {
        slf
    }

    fn __next__(mut slf: PyRefMut<Self>) -> PyResult<Option<PyObject>> {
        if slf.finished_output {
            return Ok(None);
        }

        let py = unsafe { Python::assume_gil_acquired() };

        let mut dest_buffer: Vec<u8> = Vec::with_capacity(slf.write_size);
        let mut out_buffer = zstd_sys::ZSTD_outBuffer {
            dst: dest_buffer.as_mut_ptr() as *mut _,
            size: dest_buffer.capacity(),
            pos: 0,
        };

        // While input is available.
        while let Some(mut in_buffer) = slf.source.input_buffer(py)? {
            let zresult = unsafe {
                zstd_sys::ZSTD_decompressStream(
                    slf.dctx.dctx(),
                    &mut out_buffer as *mut _,
                    &mut in_buffer as *mut _,
                )
            };
            if unsafe { zstd_sys::ZSTD_isError(zresult) } != 0 {
                return Err(ZstdError::new_err(format!(
                    "zstd decompress error: {}",
                    zstd_safe::get_error_name(zresult)
                )));
            }

            slf.source.record_bytes_read(in_buffer.pos);
            unsafe {
                dest_buffer.set_len(out_buffer.pos);
            }

            if zresult == 0 {
                slf.finished_output = true;
            }

            // Emit chunk if output buffer has data.
            if out_buffer.pos > 0 {
                // TODO avoid buffer copy.
                let chunk = PyBytes::new(py, &dest_buffer);
                return Ok(Some(chunk.into_py(py)));
            }

            // Repeat loop to collect more input data.
            continue;
        }

        // Input is exhausted. Emit what we have or finish.
        if out_buffer.pos > 0 {
            // TODO avoid buffer copy.
            let chunk = PyBytes::new(py, &dest_buffer);
            Ok(Some(chunk.into_py(py)))
        } else {
            Ok(None)
        }
    }
}
<|endoftext|>