<commit_before>// Copyright (c) 2016-2017 Chef Software Inc. and/or applicable contributors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

use std::fmt;
use std::result;
use std::str::FromStr;

use error::{Error, Result};
use os::system::{Architecture, Platform};
use util::{deserialize_using_from_str, serialize_using_to_string};

pub trait Target: fmt::Display + Into<PackageTarget> {
    fn validate(&self) -> Result<()>;
}

/// Describes the platform (operating system/kernel)
/// and architecture (x86_64, i386, etc..) that a package is built for
#[derive(Deserialize, Serialize, Eq, PartialEq, Debug, Clone, Hash)]
pub struct PackageTarget {
    #[serde(
        deserialize_with = "deserialize_using_from_str",
        serialize_with = "serialize_using_to_string"
    )]
    pub platform: Platform,
    #[serde(
        deserialize_with = "deserialize_using_from_str",
        serialize_with = "serialize_using_to_string"
    )]
    pub architecture: Architecture,
}

impl PackageTarget {
    /// Creates a new package target
    ///
    /// Errors:
    /// * InvalidPlatform
    /// * InvalidArchitecture
    pub fn new(platform: Platform, architecture: Architecture) -> Self {
        PackageTarget {
            platform: platform,
            architecture: architecture,
        }
    }

    pub fn current_platform() -> Platform {
        if cfg!(target_os = "windows") {
            Platform::Windows
        } else if cfg!(target_os = "linux") {
            Platform::Linux
        } else if cfg!(target_os = "macos") {
            Platform::Darwin
        } else {
            unreachable!("binary built for an unknown platform")
        }
    }

    pub fn current_architecture() -> Architecture {
        if cfg!(target_arch = "x86_64") {
            Architecture::X86_64
        } else {
            unreachable!("binary built for an unknown architecture")
        }
    }
}

impl Target for PackageTarget {
    fn validate(&self) -> Result<()> {
        let default = PackageTarget::default();
        if self.architecture == default.architecture && self.platform == default.platform {
            Ok(())
        } else {
            Err(Error::TargetMatchError(format!("Package target ({}-{}) does not match system \
                                                 target ({}-{}).",
                                                self.architecture,
                                                self.platform,
                                                default.architecture,
                                                default.platform,)))
        }
    }
}

impl Default for PackageTarget {
    fn default() -> PackageTarget {
        PackageTarget {
            platform: Self::current_platform(),
            architecture: Self::current_architecture(),
        }
    }
}

impl fmt::Display for PackageTarget {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        write!(f, "{}-{}", self.architecture, self.platform)
    }
}

impl FromStr for PackageTarget {
    type Err = Error;

    fn from_str(value: &str) -> result::Result<Self, Self::Err> {
        let items: Vec<&str> = value.split("-").collect();
        let (architecture, platform) = match items.len() {
            2 => {
                (try!(Architecture::from_str(items[0].into())),
                 try!(Platform::from_str(items[1]).into()))
            }
            _ => return Err(Error::InvalidPackageTarget(value.to_string())),
        };
        Ok(PackageTarget::new(platform, architecture))
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use std::str::FromStr;
    use os::system::{Architecture, Platform};

    #[test]
    fn package_target_matches_current_operating_system() {
        let target = PackageTarget::default();
        if cfg!(target_os = "windows") {
            assert_eq!(target.platform, Platform::Windows);
        } else if cfg!(target_os = "linux") {
            assert_eq!(target.platform, Platform::Linux);
        } else if cfg!(target_os = "macos") {
            assert_eq!(target.platform, Platform::Darwin);
        } else {
            unreachable!("Platform not defined for target_os! Fix this by adding a conditional \
                          compilation to PackageTarget::current_platform()");
        }
        assert_eq!(target.architecture, Architecture::X86_64);
    }

    #[test]
    fn package_target_from_string() {
        let target = PackageTarget::from_str("x86_64-windows").unwrap();
        assert_eq!(target.platform, Platform::Windows);
        assert_eq!(target.architecture, Architecture::X86_64);
    }

    #[test]
    #[should_panic]
    fn package_target_with_reversed_target_string() {
        PackageTarget::from_str("linux-x86_64").unwrap();
    }

    #[test]
    #[should_panic]
    fn package_target_with_invalid_platform() {
        PackageTarget::from_str("x86_64-intermezzos").unwrap();
    }

    #[test]
    #[should_panic]
    fn package_target_with_invalid_architecture() {
        let _ = PackageTarget::from_str("i986-linux").unwrap();
    }

    #[test]
    fn package_target_validate_matching_platform_and_architecture() {
        current_platform_target().validate().unwrap();
    }

    #[test]
    #[should_panic]
    fn package_target_does_not_validate_different_platform() {
        unsupported_platform_target().validate().unwrap();
    }

    fn current_platform_target() -> PackageTarget {
        if cfg!(target_os = "windows") {
            return PackageTarget::from_str("x86_64-windows").unwrap();
        } else if cfg!(target_os = "linux") {
            PackageTarget::from_str("x86_64-linux").unwrap()
        } else if cfg!(target_os = "macos") {
            PackageTarget::from_str("x86_64-macos").unwrap()
        } else {
            unreachable!("Test case not defined for target_os! Fix this by adding a conditional \
                          compilation to tests::current_platform_target()");
        }
    }

    fn unsupported_platform_target() -> PackageTarget {
        if cfg!(target_os = "windows") {
            PackageTarget::from_str("x86_64-linux").unwrap()
        } else if cfg!(target_os = "linux") {
            PackageTarget::from_str("x86_64-windows").unwrap()
        } else if cfg!(target_os = "macos") {
            PackageTarget::from_str("x86_64-windows").unwrap()
        } else {
            unreachable!("Test case not defined for target_os! Fix this by adding a conditional \
                          compilation to tests::unsupported_platform_target()");
        }
    }
}
<commit_msg>Fix typo that caused macOS builds to fail<commit_after>// Copyright (c) 2016-2017 Chef Software Inc. and/or applicable contributors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

use std::fmt;
use std::result;
use std::str::FromStr;

use error::{Error, Result};
use os::system::{Architecture, Platform};
use util::{deserialize_using_from_str, serialize_using_to_string};

pub trait Target: fmt::Display + Into<PackageTarget> {
    fn validate(&self) -> Result<()>;
}

/// Describes the platform (operating system/kernel)
/// and architecture (x86_64, i386, etc..) that a package is built for
#[derive(Deserialize, Serialize, Eq, PartialEq, Debug, Clone, Hash)]
pub struct PackageTarget {
    #[serde(
        deserialize_with = "deserialize_using_from_str",
        serialize_with = "serialize_using_to_string"
    )]
    pub platform: Platform,
    #[serde(
        deserialize_with = "deserialize_using_from_str",
        serialize_with = "serialize_using_to_string"
    )]
    pub architecture: Architecture,
}

impl PackageTarget {
    /// Creates a new package target
    ///
    /// Errors:
    /// * InvalidPlatform
    /// * InvalidArchitecture
    pub fn new(platform: Platform, architecture: Architecture) -> Self {
        PackageTarget {
            platform: platform,
            architecture: architecture,
        }
    }

    pub fn current_platform() -> Platform {
        if cfg!(target_os = "windows") {
            Platform::Windows
        } else if cfg!(target_os = "linux") {
            Platform::Linux
        } else if cfg!(target_os = "macos") {
            Platform::Darwin
        } else {
            unreachable!("binary built for an unknown platform")
        }
    }

    pub fn current_architecture() -> Architecture {
        if cfg!(target_arch = "x86_64") {
            Architecture::X86_64
        } else {
            unreachable!("binary built for an unknown architecture")
        }
    }
}

impl Target for PackageTarget {
    fn validate(&self) -> Result<()> {
        let default = PackageTarget::default();
        if self.architecture == default.architecture && self.platform == default.platform {
            Ok(())
        } else {
            Err(Error::TargetMatchError(format!("Package target ({}-{}) does not match system \
                                                 target ({}-{}).",
                                                self.architecture,
                                                self.platform,
                                                default.architecture,
                                                default.platform,)))
        }
    }
}

impl Default for PackageTarget {
    fn default() -> PackageTarget {
        PackageTarget {
            platform: Self::current_platform(),
            architecture: Self::current_architecture(),
        }
    }
}

impl fmt::Display for PackageTarget {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        write!(f, "{}-{}", self.architecture, self.platform)
    }
}

impl FromStr for PackageTarget {
    type Err = Error;

    fn from_str(value: &str) -> result::Result<Self, Self::Err> {
        let items: Vec<&str> = value.split("-").collect();
        let (architecture, platform) = match items.len() {
            2 => {
                (try!(Architecture::from_str(items[0].into())),
                 try!(Platform::from_str(items[1]).into()))
            }
            _ => return Err(Error::InvalidPackageTarget(value.to_string())),
        };
        Ok(PackageTarget::new(platform, architecture))
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use std::str::FromStr;
    use os::system::{Architecture, Platform};

    #[test]
    fn package_target_matches_current_operating_system() {
        let target = PackageTarget::default();
        if cfg!(target_os = "windows") {
            assert_eq!(target.platform, Platform::Windows);
        } else if cfg!(target_os = "linux") {
            assert_eq!(target.platform, Platform::Linux);
        } else if cfg!(target_os = "macos") {
            assert_eq!(target.platform, Platform::Darwin);
        } else {
            unreachable!("Platform not defined for target_os! Fix this by adding a conditional \
                          compilation to PackageTarget::current_platform()");
        }
        assert_eq!(target.architecture, Architecture::X86_64);
    }

    #[test]
    fn package_target_from_string() {
        let target = PackageTarget::from_str("x86_64-windows").unwrap();
        assert_eq!(target.platform, Platform::Windows);
        assert_eq!(target.architecture, Architecture::X86_64);
    }

    #[test]
    #[should_panic]
    fn package_target_with_reversed_target_string() {
        PackageTarget::from_str("linux-x86_64").unwrap();
    }

    #[test]
    #[should_panic]
    fn package_target_with_invalid_platform() {
        PackageTarget::from_str("x86_64-intermezzos").unwrap();
    }

    #[test]
    #[should_panic]
    fn package_target_with_invalid_architecture() {
        let _ = PackageTarget::from_str("i986-linux").unwrap();
    }

    #[test]
    fn package_target_validate_matching_platform_and_architecture() {
        current_platform_target().validate().unwrap();
    }

    #[test]
    #[should_panic]
    fn package_target_does_not_validate_different_platform() {
        unsupported_platform_target().validate().unwrap();
    }

    fn current_platform_target() -> PackageTarget {
        if cfg!(target_os = "windows") {
            return PackageTarget::from_str("x86_64-windows").unwrap();
        } else if cfg!(target_os = "linux") {
            PackageTarget::from_str("x86_64-linux").unwrap()
        } else if cfg!(target_os = "macos") {
            PackageTarget::from_str("x86_64-darwin").unwrap()
        } else {
            unreachable!("Test case not defined for target_os! Fix this by adding a conditional \
                          compilation to tests::current_platform_target()");
        }
    }

    fn unsupported_platform_target() -> PackageTarget {
        if cfg!(target_os = "windows") {
            PackageTarget::from_str("x86_64-linux").unwrap()
        } else if cfg!(target_os = "linux") {
            PackageTarget::from_str("x86_64-windows").unwrap()
        } else if cfg!(target_os = "macos") {
            PackageTarget::from_str("x86_64-windows").unwrap()
        } else {
            unreachable!("Test case not defined for target_os! Fix this by adding a conditional \
                          compilation to tests::unsupported_platform_target()");
        }
    }
}
<|endoftext|>


<commit_before>
use std::ops::Fn;
use regex::{Captures, Regex};

/// A Parser is designed to take an input type and turn part of it into an
/// output type.  Probably in every real-life scenario, the input type is a
/// slice of some type.  Thus a parser returns an output type along with another
/// input type, which for slices would be the rest of the data.
pub trait Parser<'a, I, O> {

  fn parse(&self, data: I) -> ParseResult<'a, I, O>;

}

pub type ParseResult<'a, I:'a, O> = Result<(O, I), String>;

pub fn literal<'a, T:'a + Eq + Clone>(literal: T) -> LiteralParser<'a, T>{
  LiteralParser{literal: literal}
}

/// A parser that consumes one item in a slice and only returns ok if it equals
/// the given literal
pub struct LiteralParser<'a, T:'a + Eq + Clone> {
  pub literal: T,
}

impl<'a, T: 'a + Eq + Clone> Parser<'a,  &'a [T], T> for LiteralParser<'a, T> {
  fn parse(&self, data: &'a[T]) -> ParseResult<'a, &'a[T], T> {
    if data.len() < 1 {
      return Err(format!("ran out of data"))
    }
    if data[0] == self.literal {
      Ok((data[0].clone(), data.slice_from(1)))
    } else {
      Err(format!("Literal mismatch"))
    }
  }
}

/// A string Parser that attempts to consume the given regex
pub struct RegexLiteralParser<'a> {
  pub regex: Regex,
}

impl<'a> Parser<'a, &'a str, ()> for RegexLiteralParser<'a> {
  fn parse(&self, data: &'a str) -> ParseResult<'a, &'a str, ()> {
    self.regex.find(data).map(|(_, e)| ((), data.slice_from(e))).ok_or(format!("regex literal match fail"))
  }
}

pub struct RegexCapturesParser<'a> {
  pub regex: Regex,
}
impl<'a> Parser<'a, &'a str, Captures<'a>> for RegexCapturesParser<'a> {
  fn parse(&self, data: &'a str) -> ParseResult<'a, &'a str, Captures<'a>> {
    match self.regex.captures(data) {
      Some(caps) => match caps.pos(0) {
        Some((_, e)) => Ok((caps, data.slice_from(e))),
        None => Err(format!("No Match"))
      },
      None => Err(format!("No Match"))
    }
  }
}

/// A slice Parser that matches against the first item in the slice
pub struct MatchParser<'a, I, O> {
  pub matcher: Box< Fn<(&'a I,), Result<O, String>> +'a>
}
impl<'a, I: Clone, O> Parser<'a, &'a [I], O> for MatchParser<'a, I, O> {
  fn parse(&self, data: &'a[I]) -> ParseResult<'a, &'a[I], O> {
    if data.len() < 1 {
      Err(format!("Unexpected End!"))
    } else {
      self.matcher.call((&data[0],)).map(|res| (res, data.slice_from(1)))
    }
  }
}

/// A Parser that will keep repeating the given parser until it returns an
/// error.  The accumulated results are returned.
pub struct RepParser<'a, I, O, P: Parser<'a, I, O>>{
  pub parser: P
}

impl<'a, I: Clone, O, P: Parser<'a, I, O>> Parser<'a, I, Vec<O>> for RepParser<'a, I, O, P> {
  fn parse(&self, data: I) -> ParseResult<'a, I, Vec<O>> {
    let mut remain = data;
    let mut v: Vec<O> = Vec::new();
    loop {
      match self.parser.parse(remain.clone()) {
        Ok((result, rest)) => {
          v.push(result);
          remain = rest;
        }
        Err(_) => {
          return Ok((v, remain));
        }
      }
    }
  }
}

/// A Parser that will repeatedly parse `rep` and `sep` in sequence until `sep`
/// returns an error.  The accumulated `rep` results are returned.  If `rep`
/// returns an error at any time, the error is escelated.
pub struct RepSepParser<'a, I, O, U, A: Parser<'a, I, O>, B: Parser<'a, I, U>> {
  pub rep: A,
  pub sep: B,
  pub min_reps: uint,
}
impl<'a, I: Clone, O, U, A: Parser<'a, I, O>, B: Parser<'a, I, U>> Parser<'a, I, Vec<O>> for RepSepParser<'a, I, O, U, A, B> {
  fn parse(&self, data: I) -> ParseResult<'a, I, Vec<O>> {
    let mut remain = data;
    let mut v: Vec<O> = Vec::new();
    loop {
      match self.rep.parse(remain) {
        Ok((result, rest)) => {
          v.push(result);
          match self.sep.parse(rest.clone()) {
            Ok((_, rest2)) => {
              remain = rest2
            }
            Err(_) => {
              if v.len() < self.min_reps {
                return Err(format!("Not enough reps: required {}, got {}", self.min_reps, v.len()))
              } else {
                return Ok((v, rest))
              }
            }
          }
        }
        Err(err) => {
          return Err(format!("Error on rep: {}", err));
        }
      }
    }
  }
}

/// A Parser that will combine two parsers into a tuple of results.  If either
/// parser returns an error, the error is escelated (ie partial successes are
/// not returned)
pub struct DualParser<'a, I, A, B, X: Parser<'a, I, A>, Y: Parser<'a, I, B>> {
  pub first: X,
  pub second: Y,
}

impl <'a, I, A, B, X: Parser<'a, I, A>, Y: Parser<'a, I, B>> Parser<'a, I, (A,B)> for DualParser<'a, I, A, B, X, Y> {
  fn parse(&self, data: I) -> ParseResult<'a, I, (A, B)> {
    match self.first.parse(data) {
      Ok((a, d2)) => match self.second.parse(d2) {
        Ok((b, remain)) => Ok(((a, b), remain)),
        Err(err) => Err(err)
      },
      Err(err) => Err(err)
    }
  }
}

//memoization coming sooooon
pub struct LazyParser<'a, I, O> {
  pub generator: Box<Fn() -> Box<Parser<'a, I, O> + 'a> + 'a>
}
impl<'a, I, O> Parser<'a, I, O> for LazyParser<'a, I, O> {
  fn parse(&self, input: I) -> ParseResult<'a, I, O> {
    self.generator.call(()).parse(input)
  }
}

/// A parser that will attempt to parse using parser `a`, and then `b` if
/// the first fails.  The contained parsers are lazy so that we can support recursive
/// grammars.
///
/// In general, the "greedier" parser should be `a`.
pub struct OrParser<'a, I, O, A: Parser<'a, I, O>, B: Parser<'a, I, O>> {
  pub a: A,
  pub b: B,
}

impl<'a, I: Clone, O, A: Parser<'a, I, O>, B: Parser<'a, I, O>> Parser<'a, I, O> for OrParser<'a, I, O, A, B> {
  fn parse(&self, data: I) -> ParseResult<'a, I, O> {
    match self.a.parse(data.clone()) {
      Ok((a, d2)) => Ok((a, d2)),
      Err(_) => match self.b.parse(data.clone()) {
        Ok((b, remain)) => Ok((b, remain)),
        Err(err) => Err(err)
      }
    }
  }
}

/// A Parser that can map the successful result of a parser to another type
pub struct MapParser<'a, I, O, U, P: Parser<'a, I, O>> {
  pub parser: P,
  pub mapper: Box<Fn<(O,), U> + 'a>, //this has to be a &Fn and not a regular lambda since it must be immutable
}
impl<'a, I, O, U, P: Parser<'a, I, O>> Parser<'a, I, U> for MapParser<'a, I, O, U, P> {
  fn parse(&self, data: I) -> ParseResult<'a, I, U> {
    self.parser.parse(data).map(|(output, input)| ((self.mapper.call((output,)), input)))
  }
}

/// A Parser that will attempt to use a parser, returning an option of the contained parser's result
pub struct OptionParser<'a, I, O, P: Parser<'a, I, O>> {
  pub parser: P
}
impl<'a, I: Clone, O, P: Parser<'a, I, O>> Parser<'a, I, Option<O>> for OptionParser<'a, I, O, P> {
  fn parse(&self, data: I) -> ParseResult<'a, I, Option<O>> {
    match self.parser.parse(data.clone()) {
      Ok((result, rest))  => Ok((Some(result), rest)),
      Err(_)              => Ok((None, data)),
    }
  }
}

<commit_msg>'uint' -> 'usize'<commit_after>
use std::ops::Fn;
use regex::{Captures, Regex};

/// A Parser is designed to take an input type and turn part of it into an
/// output type.  Probably in every real-life scenario, the input type is a
/// slice of some type.  Thus a parser returns an output type along with another
/// input type, which for slices would be the rest of the data.
pub trait Parser<'a, I, O> {

  fn parse(&self, data: I) -> ParseResult<'a, I, O>;

}

pub type ParseResult<'a, I:'a, O> = Result<(O, I), String>;

pub fn literal<'a, T:'a + Eq + Clone>(literal: T) -> LiteralParser<'a, T>{
  LiteralParser{literal: literal}
}

/// A parser that consumes one item in a slice and only returns ok if it equals
/// the given literal
pub struct LiteralParser<'a, T:'a + Eq + Clone> {
  pub literal: T,
}

impl<'a, T: 'a + Eq + Clone> Parser<'a,  &'a [T], T> for LiteralParser<'a, T> {
  fn parse(&self, data: &'a[T]) -> ParseResult<'a, &'a[T], T> {
    if data.len() < 1 {
      return Err(format!("ran out of data"))
    }
    if data[0] == self.literal {
      Ok((data[0].clone(), data.slice_from(1)))
    } else {
      Err(format!("Literal mismatch"))
    }
  }
}

/// A string Parser that attempts to consume the given regex
pub struct RegexLiteralParser<'a> {
  pub regex: Regex,
}

impl<'a> Parser<'a, &'a str, ()> for RegexLiteralParser<'a> {
  fn parse(&self, data: &'a str) -> ParseResult<'a, &'a str, ()> {
    self.regex.find(data).map(|(_, e)| ((), data.slice_from(e))).ok_or(format!("regex literal match fail"))
  }
}

pub struct RegexCapturesParser<'a> {
  pub regex: Regex,
}
impl<'a> Parser<'a, &'a str, Captures<'a>> for RegexCapturesParser<'a> {
  fn parse(&self, data: &'a str) -> ParseResult<'a, &'a str, Captures<'a>> {
    match self.regex.captures(data) {
      Some(caps) => match caps.pos(0) {
        Some((_, e)) => Ok((caps, data.slice_from(e))),
        None => Err(format!("No Match"))
      },
      None => Err(format!("No Match"))
    }
  }
}

/// A slice Parser that matches against the first item in the slice
pub struct MatchParser<'a, I, O> {
  pub matcher: Box< Fn<(&'a I,), Result<O, String>> +'a>
}
impl<'a, I: Clone, O> Parser<'a, &'a [I], O> for MatchParser<'a, I, O> {
  fn parse(&self, data: &'a[I]) -> ParseResult<'a, &'a[I], O> {
    if data.len() < 1 {
      Err(format!("Unexpected End!"))
    } else {
      self.matcher.call((&data[0],)).map(|res| (res, data.slice_from(1)))
    }
  }
}

/// A Parser that will keep repeating the given parser until it returns an
/// error.  The accumulated results are returned.
pub struct RepParser<'a, I, O, P: Parser<'a, I, O>>{
  pub parser: P
}

impl<'a, I: Clone, O, P: Parser<'a, I, O>> Parser<'a, I, Vec<O>> for RepParser<'a, I, O, P> {
  fn parse(&self, data: I) -> ParseResult<'a, I, Vec<O>> {
    let mut remain = data;
    let mut v: Vec<O> = Vec::new();
    loop {
      match self.parser.parse(remain.clone()) {
        Ok((result, rest)) => {
          v.push(result);
          remain = rest;
        }
        Err(_) => {
          return Ok((v, remain));
        }
      }
    }
  }
}

/// A Parser that will repeatedly parse `rep` and `sep` in sequence until `sep`
/// returns an error.  The accumulated `rep` results are returned.  If `rep`
/// returns an error at any time, the error is escelated.
pub struct RepSepParser<'a, I, O, U, A: Parser<'a, I, O>, B: Parser<'a, I, U>> {
  pub rep: A,
  pub sep: B,
  pub min_reps: usize,
}
impl<'a, I: Clone, O, U, A: Parser<'a, I, O>, B: Parser<'a, I, U>> Parser<'a, I, Vec<O>> for RepSepParser<'a, I, O, U, A, B> {
  fn parse(&self, data: I) -> ParseResult<'a, I, Vec<O>> {
    let mut remain = data;
    let mut v: Vec<O> = Vec::new();
    loop {
      match self.rep.parse(remain) {
        Ok((result, rest)) => {
          v.push(result);
          match self.sep.parse(rest.clone()) {
            Ok((_, rest2)) => {
              remain = rest2
            }
            Err(_) => {
              if v.len() < self.min_reps {
                return Err(format!("Not enough reps: required {}, got {}", self.min_reps, v.len()))
              } else {
                return Ok((v, rest))
              }
            }
          }
        }
        Err(err) => {
          return Err(format!("Error on rep: {}", err));
        }
      }
    }
  }
}

/// A Parser that will combine two parsers into a tuple of results.  If either
/// parser returns an error, the error is escelated (ie partial successes are
/// not returned)
pub struct DualParser<'a, I, A, B, X: Parser<'a, I, A>, Y: Parser<'a, I, B>> {
  pub first: X,
  pub second: Y,
}

impl <'a, I, A, B, X: Parser<'a, I, A>, Y: Parser<'a, I, B>> Parser<'a, I, (A,B)> for DualParser<'a, I, A, B, X, Y> {
  fn parse(&self, data: I) -> ParseResult<'a, I, (A, B)> {
    match self.first.parse(data) {
      Ok((a, d2)) => match self.second.parse(d2) {
        Ok((b, remain)) => Ok(((a, b), remain)),
        Err(err) => Err(err)
      },
      Err(err) => Err(err)
    }
  }
}

//memoization coming sooooon
pub struct LazyParser<'a, I, O> {
  pub generator: Box<Fn() -> Box<Parser<'a, I, O> + 'a> + 'a>
}
impl<'a, I, O> Parser<'a, I, O> for LazyParser<'a, I, O> {
  fn parse(&self, input: I) -> ParseResult<'a, I, O> {
    self.generator.call(()).parse(input)
  }
}

/// A parser that will attempt to parse using parser `a`, and then `b` if
/// the first fails.  The contained parsers are lazy so that we can support recursive
/// grammars.
///
/// In general, the "greedier" parser should be `a`.
pub struct OrParser<'a, I, O, A: Parser<'a, I, O>, B: Parser<'a, I, O>> {
  pub a: A,
  pub b: B,
}

impl<'a, I: Clone, O, A: Parser<'a, I, O>, B: Parser<'a, I, O>> Parser<'a, I, O> for OrParser<'a, I, O, A, B> {
  fn parse(&self, data: I) -> ParseResult<'a, I, O> {
    match self.a.parse(data.clone()) {
      Ok((a, d2)) => Ok((a, d2)),
      Err(_) => match self.b.parse(data.clone()) {
        Ok((b, remain)) => Ok((b, remain)),
        Err(err) => Err(err)
      }
    }
  }
}

/// A Parser that can map the successful result of a parser to another type
pub struct MapParser<'a, I, O, U, P: Parser<'a, I, O>> {
  pub parser: P,
  pub mapper: Box<Fn<(O,), U> + 'a>, //this has to be a &Fn and not a regular lambda since it must be immutable
}
impl<'a, I, O, U, P: Parser<'a, I, O>> Parser<'a, I, U> for MapParser<'a, I, O, U, P> {
  fn parse(&self, data: I) -> ParseResult<'a, I, U> {
    self.parser.parse(data).map(|(output, input)| ((self.mapper.call((output,)), input)))
  }
}

/// A Parser that will attempt to use a parser, returning an option of the contained parser's result
pub struct OptionParser<'a, I, O, P: Parser<'a, I, O>> {
  pub parser: P
}
impl<'a, I: Clone, O, P: Parser<'a, I, O>> Parser<'a, I, Option<O>> for OptionParser<'a, I, O, P> {
  fn parse(&self, data: I) -> ParseResult<'a, I, Option<O>> {
    match self.parser.parse(data.clone()) {
      Ok((result, rest))  => Ok((Some(result), rest)),
      Err(_)              => Ok((None, data)),
    }
  }
}

<|endoftext|>


<commit_before>/// The tokenizer for the query interpreter

use error::*;
use std::rc::Rc;

#[derive(Debug, PartialEq, Eq)]
pub enum Token {
    Identifier {
        ident: String,
        next: Option<Rc<Token>>
    },

    Index {
        idx: usize,
        next: Option<Rc<Token>>
    }
}

impl Token {

    pub fn set_next(&mut self, token: Token) {
        match self {
            &mut Token::Identifier { next: ref mut next, .. } => *next = Some(Rc::new(token)),
            &mut Token::Index { next: ref mut next, .. }      => *next = Some(Rc::new(token)),
        }
    }

    pub fn identifier(&self) -> Option<&String> {
        match self {
            &Token::Identifier { ident: ref ident, .. } => Some(&ident),
            _ => None,
        }
    }

    #[cfg(test)]
    pub fn idx(&self) -> usize {
        match self {
            &Token::Index { idx: i, .. } => i,
            _ => unreachable!(),
        }
    }
}

pub fn tokenize_with_seperator(query: &String, seperator: char) -> Result<Token> {
    use std::str::Split;

    /// Creates a Token object from a string
    ///
    /// # Panics
    ///
    /// * If the internal regex does not compile (should never happen)
    /// * If the token is non-valid (that is, a array index with a non-i64)
    /// * If the regex does not find anything
    /// * If the integer in the brackets (`[]`) cannot be parsed to a valid i64
    ///
    /// # Incorrect behaviour
    ///
    /// * If the regex finds multiple captures
    ///
    /// # Returns
    ///
    /// The `Token` object with the correct identifier/index for this token and no next token.
    ///
    fn mk_token_object(s: &str) -> Result<Token> {
        use regex::Regex;
        use std::str::FromStr;

        lazy_static! {
            static ref RE: Regex = Regex::new(r"^\[-?\d+\]$").unwrap();
        }

        if !has_array_brackets(s) {
            return Ok(Token::Identifier { ident: String::from(s), next: None });
        }

        match RE.captures(s) {
            None => return Err(Error::from(ErrorKind::ArrayAccessWithoutIndex)),
            Some(captures) => {
                match captures.get(0) {
                    None => Ok(Token::Identifier { ident: String::from(s), next: None }),
                    Some(mtch) => {
                        let mtch = without_array_brackets(mtch.as_str());
                        let i : usize = FromStr::from_str(&mtch).unwrap(); // save because regex
                        Ok(Token::Index {
                            idx: i,
                            next: None,
                        })
                    }
                }
            }
        }
    }

    /// Check whether a str begins with '[' and ends with ']'
    fn has_array_brackets(s: &str) -> bool {
        s.as_bytes()[0] == b'[' && s.as_bytes()[s.len() - 1] == b']'
    }

    /// Remove '[' and ']' from a str
    fn without_array_brackets(s: &str) -> String {
        s.replace("[","").replace("]","")
    }

    fn build_token_tree(split: &mut Split<char>, last: &mut Token) -> Result<()> {
        match split.next() {
            None        => { /* No more tokens */ }
            Some(token) => {
                if token.len() == 0 {
                    return Err(Error::from(ErrorKind::EmptyIdentifier));
                }
                let mut token = try!(mk_token_object(token));
                try!(build_token_tree(split, &mut token));
                last.set_next(token);
            }
        }
        Ok(())
    }

    if query.is_empty() {
        return Err(Error::from(ErrorKind::EmptyQueryError));
    }

    let mut tokens = query.split(seperator);

    match tokens.next() {
        None        => Err(Error::from(ErrorKind::EmptyQueryError)),
        Some(token) => {
            if token.len() == 0 {
                return Err(Error::from(ErrorKind::EmptyIdentifier));
            }
            let mut tok = try!(mk_token_object(token));
            let _       = try!(build_token_tree(&mut tokens, &mut tok));
            Ok(tok)
        }
    }
}

#[cfg(test)]
mod test {
    use error::*;
    use super::*;

    use std::ops::Deref;

    #[test]
    fn test_tokenize_empty_query_to_error() {
        let tokens = tokenize_with_seperator(&String::from(""), '.');
        assert!(tokens.is_err());
        let tokens = tokens.unwrap_err();

        let errkind = tokens.kind();
        assert!(is_match!(errkind, &ErrorKind::EmptyQueryError { .. }));
    }

    #[test]
    fn test_tokenize_seperator_only() {
        let tokens = tokenize_with_seperator(&String::from("."), '.');
        assert!(tokens.is_err());
        let tokens = tokens.unwrap_err();

        let errkind = tokens.kind();
        assert!(is_match!(errkind, &ErrorKind::EmptyIdentifier { .. }));
    }

    #[test]
    fn test_tokenize_array_brackets_only() {
        let tokens = tokenize_with_seperator(&String::from("[]"), '.');
        assert!(tokens.is_err());
        let tokens = tokens.unwrap_err();

        let errkind = tokens.kind();
        assert!(is_match!(errkind, &ErrorKind::ArrayAccessWithoutIndex { .. }));
    }

    #[test]
    fn test_tokenize_identifiers_with_array_brackets_only() {
        let tokens = tokenize_with_seperator(&String::from("a.b.c.[]"), '.');
        assert!(tokens.is_err());
        let tokens = tokens.unwrap_err();

        let errkind = tokens.kind();
        assert!(is_match!(errkind, &ErrorKind::ArrayAccessWithoutIndex { .. }));
    }

    #[test]
    fn test_tokenize_identifiers_in_array_brackets() {
        let tokens = tokenize_with_seperator(&String::from("[a]"), '.');
        assert!(tokens.is_err());
        let tokens = tokens.unwrap_err();

        let errkind = tokens.kind();
        assert!(is_match!(errkind, &ErrorKind::ArrayAccessWithoutIndex { .. }));
    }

    #[test]
    fn test_tokenize_single_token_query() {
        let tokens = tokenize_with_seperator(&String::from("example"), '.');
        assert!(tokens.is_ok());
        let tokens = tokens.unwrap();

        assert!(match tokens {
            Token::Identifier {
                ident: ident,
                next: None
            } => {
                assert_eq!(String::from("example"), ident);
                true
            },
            _ => false,
        });
    }

    #[test]
    fn test_tokenize_double_token_query() {
        let tokens = tokenize_with_seperator(&String::from("a.b"), '.');
        assert!(tokens.is_ok());
        let tokens = tokens.unwrap();

        assert!(match tokens {
            Token::Identifier { next: Some(ref next), .. } => {
                assert_eq!(1, Rc::strong_count(&next));
                assert_eq!("b", next.deref().identifier().unwrap());
                match next.deref() {
                    &Token::Identifier { next: None, .. } => true,
                    _ => false
                }
            },
            _ => false,
        });
        assert_eq!("a", tokens.identifier().unwrap());
    }

    #[test]
    fn test_tokenize_ident_then_array_query() {
        let tokens = tokenize_with_seperator(&String::from("a.[0]"), '.');
        assert!(tokens.is_ok());
        let tokens = tokens.unwrap();

        assert_eq!("a", tokens.identifier().unwrap());
        assert!(match tokens {
            Token::Identifier { next: Some(ref next), .. } => {
                assert_eq!(1, Rc::strong_count(&next));
                match next.deref() {
                    &Token::Index { idx: 0, next: None } => true,
                    _ => false
                }
            },
            _ => false,
        });
    }

    #[test]
    fn test_tokenize_many_idents_then_array_query() {
        use std::ops::Deref;

        let tokens = tokenize_with_seperator(&String::from("a.b.c.[1000]"), '.');
        assert!(tokens.is_ok());
        let tokens = tokens.unwrap();

        assert_eq!("a", tokens.identifier().unwrap());

        let expected =
            Token::Identifier {
                ident: String::from("a"),
                next: Some(Rc::new(Token::Identifier {
                    ident: String::from("b"),
                    next: Some(Rc::new(Token::Identifier {
                        ident: String::from("c"),
                        next: Some(Rc::new(Token::Index {
                            idx: 1000,
                            next: None,
                        })),
                    })),
                })),
            };

        assert_eq!(expected, tokens);
    }

    #[test]
    fn test_tokenize_empty_token_after_good_token() {
        let tokens = tokenize_with_seperator(&String::from("a..b"), '.');
        assert!(tokens.is_err());
        let tokens = tokens.unwrap_err();

        let errkind = tokens.kind();
        assert!(is_match!(errkind, &ErrorKind::EmptyIdentifier { .. }));
    }

    quickcheck! {
        fn test_array_index(i: usize) -> bool {
            match tokenize_with_seperator(&format!("[{}]", i), '.') {
                Ok(Token::Index {
                    idx: i,
                    next: None,
                }) => true,
                _ => false,
            }
        }
    }

}
<commit_msg>Change: Array index regex does not need to allow negative values<commit_after>/// The tokenizer for the query interpreter

use error::*;
use std::rc::Rc;

#[derive(Debug, PartialEq, Eq)]
pub enum Token {
    Identifier {
        ident: String,
        next: Option<Rc<Token>>
    },

    Index {
        idx: usize,
        next: Option<Rc<Token>>
    }
}

impl Token {

    pub fn set_next(&mut self, token: Token) {
        match self {
            &mut Token::Identifier { next: ref mut next, .. } => *next = Some(Rc::new(token)),
            &mut Token::Index { next: ref mut next, .. }      => *next = Some(Rc::new(token)),
        }
    }

    pub fn identifier(&self) -> Option<&String> {
        match self {
            &Token::Identifier { ident: ref ident, .. } => Some(&ident),
            _ => None,
        }
    }

    #[cfg(test)]
    pub fn idx(&self) -> usize {
        match self {
            &Token::Index { idx: i, .. } => i,
            _ => unreachable!(),
        }
    }
}

pub fn tokenize_with_seperator(query: &String, seperator: char) -> Result<Token> {
    use std::str::Split;

    /// Creates a Token object from a string
    ///
    /// # Panics
    ///
    /// * If the internal regex does not compile (should never happen)
    /// * If the token is non-valid (that is, a array index with a non-i64)
    /// * If the regex does not find anything
    /// * If the integer in the brackets (`[]`) cannot be parsed to a valid i64
    ///
    /// # Incorrect behaviour
    ///
    /// * If the regex finds multiple captures
    ///
    /// # Returns
    ///
    /// The `Token` object with the correct identifier/index for this token and no next token.
    ///
    fn mk_token_object(s: &str) -> Result<Token> {
        use regex::Regex;
        use std::str::FromStr;

        lazy_static! {
            static ref RE: Regex = Regex::new(r"^\[\d+\]$").unwrap();
        }

        if !has_array_brackets(s) {
            return Ok(Token::Identifier { ident: String::from(s), next: None });
        }

        match RE.captures(s) {
            None => return Err(Error::from(ErrorKind::ArrayAccessWithoutIndex)),
            Some(captures) => {
                match captures.get(0) {
                    None => Ok(Token::Identifier { ident: String::from(s), next: None }),
                    Some(mtch) => {
                        let mtch = without_array_brackets(mtch.as_str());
                        let i : usize = FromStr::from_str(&mtch).unwrap(); // save because regex
                        Ok(Token::Index {
                            idx: i,
                            next: None,
                        })
                    }
                }
            }
        }
    }

    /// Check whether a str begins with '[' and ends with ']'
    fn has_array_brackets(s: &str) -> bool {
        s.as_bytes()[0] == b'[' && s.as_bytes()[s.len() - 1] == b']'
    }

    /// Remove '[' and ']' from a str
    fn without_array_brackets(s: &str) -> String {
        s.replace("[","").replace("]","")
    }

    fn build_token_tree(split: &mut Split<char>, last: &mut Token) -> Result<()> {
        match split.next() {
            None        => { /* No more tokens */ }
            Some(token) => {
                if token.len() == 0 {
                    return Err(Error::from(ErrorKind::EmptyIdentifier));
                }
                let mut token = try!(mk_token_object(token));
                try!(build_token_tree(split, &mut token));
                last.set_next(token);
            }
        }
        Ok(())
    }

    if query.is_empty() {
        return Err(Error::from(ErrorKind::EmptyQueryError));
    }

    let mut tokens = query.split(seperator);

    match tokens.next() {
        None        => Err(Error::from(ErrorKind::EmptyQueryError)),
        Some(token) => {
            if token.len() == 0 {
                return Err(Error::from(ErrorKind::EmptyIdentifier));
            }
            let mut tok = try!(mk_token_object(token));
            let _       = try!(build_token_tree(&mut tokens, &mut tok));
            Ok(tok)
        }
    }
}

#[cfg(test)]
mod test {
    use error::*;
    use super::*;

    use std::ops::Deref;

    #[test]
    fn test_tokenize_empty_query_to_error() {
        let tokens = tokenize_with_seperator(&String::from(""), '.');
        assert!(tokens.is_err());
        let tokens = tokens.unwrap_err();

        let errkind = tokens.kind();
        assert!(is_match!(errkind, &ErrorKind::EmptyQueryError { .. }));
    }

    #[test]
    fn test_tokenize_seperator_only() {
        let tokens = tokenize_with_seperator(&String::from("."), '.');
        assert!(tokens.is_err());
        let tokens = tokens.unwrap_err();

        let errkind = tokens.kind();
        assert!(is_match!(errkind, &ErrorKind::EmptyIdentifier { .. }));
    }

    #[test]
    fn test_tokenize_array_brackets_only() {
        let tokens = tokenize_with_seperator(&String::from("[]"), '.');
        assert!(tokens.is_err());
        let tokens = tokens.unwrap_err();

        let errkind = tokens.kind();
        assert!(is_match!(errkind, &ErrorKind::ArrayAccessWithoutIndex { .. }));
    }

    #[test]
    fn test_tokenize_identifiers_with_array_brackets_only() {
        let tokens = tokenize_with_seperator(&String::from("a.b.c.[]"), '.');
        assert!(tokens.is_err());
        let tokens = tokens.unwrap_err();

        let errkind = tokens.kind();
        assert!(is_match!(errkind, &ErrorKind::ArrayAccessWithoutIndex { .. }));
    }

    #[test]
    fn test_tokenize_identifiers_in_array_brackets() {
        let tokens = tokenize_with_seperator(&String::from("[a]"), '.');
        assert!(tokens.is_err());
        let tokens = tokens.unwrap_err();

        let errkind = tokens.kind();
        assert!(is_match!(errkind, &ErrorKind::ArrayAccessWithoutIndex { .. }));
    }

    #[test]
    fn test_tokenize_single_token_query() {
        let tokens = tokenize_with_seperator(&String::from("example"), '.');
        assert!(tokens.is_ok());
        let tokens = tokens.unwrap();

        assert!(match tokens {
            Token::Identifier {
                ident: ident,
                next: None
            } => {
                assert_eq!(String::from("example"), ident);
                true
            },
            _ => false,
        });
    }

    #[test]
    fn test_tokenize_double_token_query() {
        let tokens = tokenize_with_seperator(&String::from("a.b"), '.');
        assert!(tokens.is_ok());
        let tokens = tokens.unwrap();

        assert!(match tokens {
            Token::Identifier { next: Some(ref next), .. } => {
                assert_eq!(1, Rc::strong_count(&next));
                assert_eq!("b", next.deref().identifier().unwrap());
                match next.deref() {
                    &Token::Identifier { next: None, .. } => true,
                    _ => false
                }
            },
            _ => false,
        });
        assert_eq!("a", tokens.identifier().unwrap());
    }

    #[test]
    fn test_tokenize_ident_then_array_query() {
        let tokens = tokenize_with_seperator(&String::from("a.[0]"), '.');
        assert!(tokens.is_ok());
        let tokens = tokens.unwrap();

        assert_eq!("a", tokens.identifier().unwrap());
        assert!(match tokens {
            Token::Identifier { next: Some(ref next), .. } => {
                assert_eq!(1, Rc::strong_count(&next));
                match next.deref() {
                    &Token::Index { idx: 0, next: None } => true,
                    _ => false
                }
            },
            _ => false,
        });
    }

    #[test]
    fn test_tokenize_many_idents_then_array_query() {
        use std::ops::Deref;

        let tokens = tokenize_with_seperator(&String::from("a.b.c.[1000]"), '.');
        assert!(tokens.is_ok());
        let tokens = tokens.unwrap();

        assert_eq!("a", tokens.identifier().unwrap());

        let expected =
            Token::Identifier {
                ident: String::from("a"),
                next: Some(Rc::new(Token::Identifier {
                    ident: String::from("b"),
                    next: Some(Rc::new(Token::Identifier {
                        ident: String::from("c"),
                        next: Some(Rc::new(Token::Index {
                            idx: 1000,
                            next: None,
                        })),
                    })),
                })),
            };

        assert_eq!(expected, tokens);
    }

    #[test]
    fn test_tokenize_empty_token_after_good_token() {
        let tokens = tokenize_with_seperator(&String::from("a..b"), '.');
        assert!(tokens.is_err());
        let tokens = tokens.unwrap_err();

        let errkind = tokens.kind();
        assert!(is_match!(errkind, &ErrorKind::EmptyIdentifier { .. }));
    }

    quickcheck! {
        fn test_array_index(i: usize) -> bool {
            match tokenize_with_seperator(&format!("[{}]", i), '.') {
                Ok(Token::Index {
                    idx: i,
                    next: None,
                }) => true,
                _ => false,
            }
        }
    }

}
<|endoftext|>


<commit_before>//! Code to tokenize a `BufRead` instance into an iterator of `LineType`s.

use std::io::BufRead;

use types::LineType;

#[derive(Debug)]
pub struct Tokenizer<R: BufRead> {
    reader: R,
    current_line: String,
}

impl<R> Tokenizer<R> where R: BufRead {
    pub fn new(reader: R) -> Tokenizer<R> {
        Tokenizer {
            reader: reader,
            current_line: String::new(),
        }
    }

    pub fn next_token(&mut self) -> Option<LineType> {
        self.current_line.clear();
        let bytes_read = self.reader.read_line(&mut self.current_line);
        match bytes_read {
            Ok(0) => None,
            Err(e) => { warn!("Could not read line from token reader: {:?}", e); None},
            Ok(_) => Some(LineType::from(&self.current_line[..])),
        }
    }
}
<commit_msg>Support new title format<commit_after>//! Code to tokenize a `BufRead` instance into an iterator of `LineType`s.

use std::io::BufRead;

use types::LineType;

/// A tokenizer is initialized with a BufReader instance that contains the
/// entire Tldr page. It then returns tokens as `Option<LineType>`.
#[derive(Debug)]
pub struct Tokenizer<R: BufRead> {
    /// An instance of `R: BufRead`.
    reader: R,
    /// Whether the first line has already been tokenized or not.
    first_line: bool,
    /// Buffer for the current line. Used internally.
    current_line: String,
}

impl<R> Tokenizer<R> where R: BufRead {
    pub fn new(reader: R) -> Tokenizer<R> {
        Tokenizer {
            reader: reader,
            first_line: true,
            current_line: String::new(),
        }
    }

    pub fn next_token(&mut self) -> Option<LineType> {
        self.current_line.clear();
        let bytes_read = self.reader.read_line(&mut self.current_line);
        match bytes_read {
            Ok(0) => None,
            Err(e) => {
                warn!("Could not read line from token reader: {:?}", e);
                None
            },
            Ok(_) => {
                // Handle new titles
                if self.first_line && !self.current_line.starts_with("#") {
                    // It's the new format! Drop next line.
                    // (Hmm, is there a way to do this without an allocation?)
                    let mut devnull = String::new();
                    if let Err(e) = self.reader.read_line(&mut devnull) {
                        warn!("Could not read line from token reader: {:?}", e);
                        return None;
                    }
                    self.first_line = false;
                    return Some(LineType::Title(self.current_line.trim_right().to_string()));
                }

                // Clear `first_line` flag
                self.first_line = false;

                // Convert line to a `LineType` instance
                Some(LineType::from(&self.current_line[..]))
            }
        }
    }
}

#[cfg(test)]
mod test {
    use super::Tokenizer;
    use types::LineType;

    #[test]
    fn test_first_line_old_format() {
        let input = "# The Title\n\n";
        let mut tokenizer = Tokenizer::new(input.as_bytes());
        let title = tokenizer.next_token().unwrap();
        assert_eq!(title, LineType::Title("The Title".to_string()));
        let empty = tokenizer.next_token().unwrap();
        assert_eq!(empty, LineType::Empty);
    }

    #[test]
    fn test_first_line_new_format() {
        let input = "The Title\n=========\n\n";
        let mut tokenizer = Tokenizer::new(input.as_bytes());
        let title = tokenizer.next_token().unwrap();
        assert_eq!(title, LineType::Title("The Title".to_string()));
        let empty = tokenizer.next_token().unwrap();
        assert_eq!(empty, LineType::Empty);
    }
}
<|endoftext|>


<commit_before>// #![feature(phase)]
// #[phase(plugin, link)] extern crate log;

use racer::codecleaner::{code_chunks,CodeIndicesIter};
// use codecleaner::{code_chunks,CodeIndicesIter};
// mod codecleaner;

#[cfg(test)] use racer::testutils::{rejustify, slice};

pub struct StmtIndicesIter<'a> {
    src: &'a str,
    it: CodeIndicesIter<'a>,
    pos: uint,
    start: uint,
    end: uint,
    bracelevel: int,
    parenlevel: int,
    is_macro: bool,
    enddelim: u8
}

impl<'a> Iterator<(uint, uint)> for StmtIndicesIter<'a> {
    #[inline]
    fn next(&mut self) -> Option<(uint, uint)> {
        let semicolon: u8 = ";".as_bytes()[0];
        let hash: u8 = "#".as_bytes()[0];
        let colon: u8 = ":".as_bytes()[0];
        let openbrace: u8 = "{".as_bytes()[0];
        let closebrace: u8 = "}".as_bytes()[0];
        let openparen: u8 = "(".as_bytes()[0];
        let closeparen: u8 = ")".as_bytes()[0];
        let closesqbrace: u8 = "]".as_bytes()[0];
        let whitespace = " \t\n".as_bytes();

        let src_bytes = self.src.as_bytes();

        loop {

            if self.end <= self.pos {
                // get the next chunk of code
                match self.it.next() {
                    Some((start, end)) => {
                        self.end = end;
                        if self.start == self.pos {
                            self.start = start;
                        }
                        self.pos = start;
                    }
                    None => {
                        return None
                    }
                }
            }

            // if this is a new stmt block, skip the whitespace
            if self.pos == self.start {
                while self.pos < self.end {
                    if !whitespace.contains(&src_bytes[self.pos]) {
                        break;
                    } else {
                        self.pos += 1;
                    }
                }
                self.start = self.pos;
            }

            // if the statement starts with 'macro_rules!' then we're in a macro
            // We need to know this because a closeparen can terminate a macro
            if (self.end - self.pos) > 12 &&
               self.src.slice(self.pos, self.pos+12) == "macro_rules!" {
                self.is_macro = true;
            }

            // iterate through the chunk, looking for stmt end
            while self.pos < self.end {
                if src_bytes[self.pos] == openparen {

                    // macros can be terminated by closeparen if opened by one
                    if self.is_macro &&
                        self.bracelevel == 0 &&
                        self.parenlevel == 0 {
                        self.enddelim = closeparen;
                    }
                    self.parenlevel += 1;

                } else if src_bytes[self.pos] == closeparen {
                    self.parenlevel -= 1;

                } else if src_bytes[self.pos] == openbrace {
                    // if we are top level and stmt is not a ::{Foo,Bar}; then
                    // closebrace finishes the stmt
                    if self.bracelevel == 0 &&
                        self.parenlevel == 0 &&
                       !(self.pos > 1 &&
                         src_bytes[self.pos-1] == colon &&
                         src_bytes[self.pos-2] == colon) {
                           self.enddelim = closebrace;
                    }
                    self.bracelevel += 1;
                } else if src_bytes[self.pos] == closebrace {
                    self.bracelevel -= 1;
                    // have we reached the end of the scope?
                    if self.bracelevel < 0 {
                        return None;
                    }
                }

                // attribute   #[foo = bar]
                if self.bracelevel == 0 && self.start == self.pos &&
                    src_bytes[self.pos] == hash {
                    self.enddelim = closesqbrace;
                }

                if self.bracelevel == 0 && self.parenlevel == 0 &&
                   src_bytes[self.pos] == self.enddelim {
                    let start = self.start;
                    self.start = self.pos+1;
                    self.pos = self.pos+1;
                    self.enddelim = semicolon;
                    self.is_macro = false;
                    return Some((start, self.pos));
                }

                self.pos += 1;
            }
        }
    }
}

pub fn iter_stmts<'a>(src: &'a str) -> StmtIndicesIter<'a> {
    let semicolon: u8 = ";".as_bytes()[0];
    StmtIndicesIter{src: src, it: code_chunks(src),
                    pos: 0, start: 0, end: 0, bracelevel: 0,
                    parenlevel: 0, enddelim: semicolon, is_macro: false}
}

#[test]
fn iterates_single_use_stmts() {
    let src = rejustify("
    use std::Foo; // a comment
    use std::Bar;
    ");
    let mut it = iter_stmts(src.as_slice());
    assert_eq!("use std::Foo;", slice(src.as_slice(), it.next().unwrap()));
    assert_eq!("use std::Bar;", slice(src.as_slice(), it.next().unwrap()));
}

#[test]
fn iterates_use_stmt_over_two_lines() {
    let src = rejustify("
    use std::{Foo,
              Bar}; // a comment
    ");
    let mut it = iter_stmts(src.as_slice());
    assert_eq!("use std::{Foo,
          Bar};", slice(src.as_slice(), it.next().unwrap()));
}

#[test]
fn iterates_use_stmt_without_the_prefix() {
    let src = rejustify("
    use {Foo,
              Bar}; // this is also legit apparently
    ");
    let mut it = iter_stmts(src.as_slice());
    assert_eq!("use {Foo,
          Bar};", slice(src.as_slice(), it.next().unwrap()));
}

#[test]
fn iterates_while_stmt() {
    let src = rejustify("
    while self.pos < 3 { }
    ");
    let mut it = iter_stmts(src.as_slice());
    assert_eq!("while self.pos < 3 { }", slice(src.as_slice(), it.next().unwrap()));
}

#[test]
fn iterates_lambda_arg() {
    let src = rejustify("
    myfn(|n|{});
    ");
    let mut it = iter_stmts(src.as_slice());
    assert_eq!("myfn(|n|{});", slice(src.as_slice(), it.next().unwrap()));
}

#[test]
fn iterates_macro() {
    let src = "
    mod foo;
    macro_rules! otry(
        ($e:expr) => (match $e { Some(e) => e, None => return })
    )
    mod bar;
    ";
    let mut it = iter_stmts(src.as_slice());
    assert_eq!("mod foo;", slice(src.as_slice(), it.next().unwrap()));
    assert_eq!("macro_rules! otry(
        ($e:expr) => (match $e { Some(e) => e, None => return })
    )", slice(src.as_slice(), it.next().unwrap()));
    assert_eq!("mod bar;", slice(src.as_slice(), it.next().unwrap()));
}

// #[test]
// fn iterates_if_else_stmt() {
//     let src = rejustify("
//     if self.pos < 3 { } else { }
//     ");
//     let mut it = iter_stmts(src.as_slice());
//     assert_eq!("if self.pos < 3 { } else { }", slice(src.as_slice(), it.next().unwrap()));
// }

#[test]
fn iterates_inner_scope() {
    let src = "
    while self.pos < 3 {
       let a = 35;
       return a + 35;  // should iterate this
    }
    {
       b = foo;       // but not this
    }
    ";

    let scope = src.as_slice().slice_from(25);
    //println!("blah{}",scope);
    let mut it = iter_stmts(scope);

    assert_eq!("let a = 35;", slice(scope, it.next().unwrap()));
    assert_eq!("return a + 35;", slice(scope, it.next().unwrap()));
    assert_eq!(None, it.next());
}

#[test]
fn iterates_module_attribute() {
    let src = rejustify("
    #![license = \"BSD\"]
    #[test]
    ");
    let mut it = iter_stmts(src.as_slice());
    assert_eq!("#![license = \"BSD\"]", slice(src.as_slice(), it.next().unwrap()));
    assert_eq!("#[test]", slice(src.as_slice(), it.next().unwrap()));
}

// fn main() {
//     use std::io::BufferedReader;
//     use std::io::File;
//     use std::str;

//     //let filetxt = BufferedReader::new(File::open(&Path::new("/usr/local/src/rust/src/libstd/prelude.rs"))).read_to_end().unwrap();
//     //let filetxt = BufferedReader::new(File::open(&Path::new("/usr/local/src/rust/src/libstd/prelude.rs"))).read_to_end().unwrap();
//     let filetxt = BufferedReader::new(File::open(&Path::new("/usr/local/src/rust/src/libcollections/str.rs"))).read_to_end().unwrap();
//     let src = str::from_utf8(filetxt.as_slice()).unwrap();

//     for (start,end) in iter_stmts(src) {
//         println!("BLOB |{}|",src.slice(start,end));
//     }

// }
<commit_msg>codeiter: handle  'use {Foo,Bar};'<commit_after>// #![feature(phase)]
// #[phase(plugin, link)] extern crate log;

use racer::codecleaner::{code_chunks,CodeIndicesIter};
// use codecleaner::{code_chunks,CodeIndicesIter};
// mod codecleaner;

#[cfg(test)] use racer::testutils::{rejustify, slice};

pub struct StmtIndicesIter<'a> {
    src: &'a str,
    it: CodeIndicesIter<'a>,
    pos: uint,
    start: uint,
    end: uint,
    bracelevel: int,
    parenlevel: int,
    is_macro: bool,
    enddelim: u8
}

impl<'a> Iterator<(uint, uint)> for StmtIndicesIter<'a> {
    #[inline]
    fn next(&mut self) -> Option<(uint, uint)> {
        let semicolon: u8 = ";".as_bytes()[0];
        let hash: u8 = "#".as_bytes()[0];
        let openbrace: u8 = "{".as_bytes()[0];
        let closebrace: u8 = "}".as_bytes()[0];
        let openparen: u8 = "(".as_bytes()[0];
        let closeparen: u8 = ")".as_bytes()[0];
        let closesqbrace: u8 = "]".as_bytes()[0];
        let whitespace = " \t\n".as_bytes();

        let __u: u8 = "u".as_bytes()[0];
        let __s: u8 = "s".as_bytes()[0];
        let __e: u8 = "e".as_bytes()[0];

        let src_bytes = self.src.as_bytes();

        loop {

            if self.end <= self.pos {
                // get the next chunk of code
                match self.it.next() {
                    Some((start, end)) => {
                        self.end = end;
                        if self.start == self.pos {
                            self.start = start;
                        }
                        self.pos = start;
                    }
                    None => {
                        return None
                    }
                }
            }

            // if this is a new stmt block, skip the whitespace
            if self.pos == self.start {
                while self.pos < self.end {
                    if !whitespace.contains(&src_bytes[self.pos]) {
                        break;
                    } else {
                        self.pos += 1;
                    }
                }
                self.start = self.pos;
            }

            // if the statement starts with 'macro_rules!' then we're in a macro
            // We need to know this because a closeparen can terminate a macro
            if (self.end - self.pos) > 12 &&
               self.src.slice(self.pos, self.pos+12) == "macro_rules!" {
                self.is_macro = true;
            }

            // iterate through the chunk, looking for stmt end
            while self.pos < self.end {
                if src_bytes[self.pos] == openparen {

                    // macros can be terminated by closeparen if opened by one
                    if self.is_macro &&
                        self.bracelevel == 0 &&
                        self.parenlevel == 0 {
                        self.enddelim = closeparen;
                    }
                    self.parenlevel += 1;

                } else if src_bytes[self.pos] == closeparen {
                    self.parenlevel -= 1;

                } else if src_bytes[self.pos] == openbrace {
                    // if we are top level and stmt is not a 'use' then
                    // closebrace finishes the stmt
                    if self.bracelevel == 0 &&
                        self.parenlevel == 0 &&
                       !(is_a_use_stmt(self.src, self.start, self.pos)) {
                           self.enddelim = closebrace;
                    }
                    self.bracelevel += 1;
                } else if src_bytes[self.pos] == closebrace {
                    self.bracelevel -= 1;
                    // have we reached the end of the scope?
                    if self.bracelevel < 0 {
                        return None;
                    }
                }

                // attribute   #[foo = bar]
                if self.bracelevel == 0 && self.start == self.pos &&
                    src_bytes[self.pos] == hash {
                    self.enddelim = closesqbrace;
                }

                if self.bracelevel == 0 && self.parenlevel == 0 &&
                   src_bytes[self.pos] == self.enddelim {
                    let start = self.start;
                    self.start = self.pos+1;
                    self.pos = self.pos+1;
                    self.enddelim = semicolon;
                    self.is_macro = false;
                    return Some((start, self.pos));
                }

                self.pos += 1;
            }
        }
    }
}

fn is_a_use_stmt(src: &str, start: uint, pos: uint) -> bool {
    let src_bytes = src.as_bytes();
    let whitespace = " {\t\n".as_bytes();
    (pos > 3 && src_bytes.slice(start, start+3) == "use".as_bytes() &&
     whitespace.contains(&src_bytes[start+3])) ||
        (pos > 7 && src_bytes.slice(start, start+7) == "pub use".as_bytes() &&
                      whitespace.contains(&src_bytes[start+7]))
}

pub fn iter_stmts<'a>(src: &'a str) -> StmtIndicesIter<'a> {
    let semicolon: u8 = ";".as_bytes()[0];
    StmtIndicesIter{src: src, it: code_chunks(src),
                    pos: 0, start: 0, end: 0, bracelevel: 0,
                    parenlevel: 0, enddelim: semicolon, is_macro: false}
}

#[test]
fn iterates_single_use_stmts() {
    let src = rejustify("
    use std::Foo; // a comment
    use std::Bar;
    ");
    let mut it = iter_stmts(src.as_slice());
    assert_eq!("use std::Foo;", slice(src.as_slice(), it.next().unwrap()));
    assert_eq!("use std::Bar;", slice(src.as_slice(), it.next().unwrap()));
}

#[test]
fn iterates_use_stmt_over_two_lines() {
    let src = rejustify("
    use std::{Foo,
              Bar}; // a comment
    ");
    let mut it = iter_stmts(src.as_slice());
    assert_eq!("use std::{Foo,
          Bar};", slice(src.as_slice(), it.next().unwrap()));
}

#[test]
fn iterates_use_stmt_without_the_prefix() {
    let src = rejustify("
    pub use {Foo,
              Bar}; // this is also legit apparently
    ");
    let mut it = iter_stmts(src.as_slice());
    assert_eq!("pub use {Foo,
          Bar};", slice(src.as_slice(), it.next().unwrap()));
}

#[test]
fn iterates_while_stmt() {
    let src = rejustify("
    while self.pos < 3 { }
    ");
    let mut it = iter_stmts(src.as_slice());
    assert_eq!("while self.pos < 3 { }", slice(src.as_slice(), it.next().unwrap()));
}

#[test]
fn iterates_lambda_arg() {
    let src = rejustify("
    myfn(|n|{});
    ");
    let mut it = iter_stmts(src.as_slice());
    assert_eq!("myfn(|n|{});", slice(src.as_slice(), it.next().unwrap()));
}

#[test]
fn iterates_macro() {
    let src = "
    mod foo;
    macro_rules! otry(
        ($e:expr) => (match $e { Some(e) => e, None => return })
    )
    mod bar;
    ";
    let mut it = iter_stmts(src.as_slice());
    assert_eq!("mod foo;", slice(src.as_slice(), it.next().unwrap()));
    assert_eq!("macro_rules! otry(
        ($e:expr) => (match $e { Some(e) => e, None => return })
    )", slice(src.as_slice(), it.next().unwrap()));
    assert_eq!("mod bar;", slice(src.as_slice(), it.next().unwrap()));
}

// #[test]
// fn iterates_if_else_stmt() {
//     let src = rejustify("
//     if self.pos < 3 { } else { }
//     ");
//     let mut it = iter_stmts(src.as_slice());
//     assert_eq!("if self.pos < 3 { } else { }", slice(src.as_slice(), it.next().unwrap()));
// }

#[test]
fn iterates_inner_scope() {
    let src = "
    while self.pos < 3 {
       let a = 35;
       return a + 35;  // should iterate this
    }
    {
       b = foo;       // but not this
    }
    ";

    let scope = src.as_slice().slice_from(25);
    //println!("blah{}",scope);
    let mut it = iter_stmts(scope);

    assert_eq!("let a = 35;", slice(scope, it.next().unwrap()));
    assert_eq!("return a + 35;", slice(scope, it.next().unwrap()));
    assert_eq!(None, it.next());
}

#[test]
fn iterates_module_attribute() {
    let src = rejustify("
    #![license = \"BSD\"]
    #[test]
    ");
    let mut it = iter_stmts(src.as_slice());
    assert_eq!("#![license = \"BSD\"]", slice(src.as_slice(), it.next().unwrap()));
    assert_eq!("#[test]", slice(src.as_slice(), it.next().unwrap()));
}

// fn main() {
//     use std::io::BufferedReader;
//     use std::io::File;
//     use std::str;

//     //let filetxt = BufferedReader::new(File::open(&Path::new("/usr/local/src/rust/src/libstd/prelude.rs"))).read_to_end().unwrap();
//     //let filetxt = BufferedReader::new(File::open(&Path::new("/usr/local/src/rust/src/libstd/prelude.rs"))).read_to_end().unwrap();
//     let filetxt = BufferedReader::new(File::open(&Path::new("/usr/local/src/rust/src/libcollections/str.rs"))).read_to_end().unwrap();
//     let src = str::from_utf8(filetxt.as_slice()).unwrap();

//     for (start,end) in iter_stmts(src) {
//         println!("BLOB |{}|",src.slice(start,end));
//     }

// }
<|endoftext|>


<commit_before>// #![feature(phase)]
// #[phase(plugin, link)] extern crate log;

use racer::codecleaner::{code_chunks,CodeIndicesIter};
//use codecleaner::{code_chunks,CodeIndicesIter};
//mod codecleaner;

#[cfg(test)] use racer::testutils::{rejustify, slice};

pub struct StmtIndicesIter<'a> {
    src: &'a str,
    it: CodeIndicesIter<'a>,
    pos: uint,
    start: uint,
    end: uint,
    level: int,
    enddelim: u8
}

impl<'a> Iterator<(uint, uint)> for StmtIndicesIter<'a> {
    #[inline]
    fn next(&mut self) -> Option<(uint, uint)> {
        let semicolon: u8 = ";".as_bytes()[0];
        let hash: u8 = "#".as_bytes()[0];
        let colon: u8 = ":".as_bytes()[0];
        let openbrace: u8 = "{".as_bytes()[0];
        let closebrace: u8 = "}".as_bytes()[0];
        let closesqbrace: u8 = "]".as_bytes()[0];
        let whitespace = " \t\n".as_bytes();

        let src_bytes = self.src.as_bytes();

        loop {

            if self.end <= self.pos {
                // get the next chunk of code
                match self.it.next() {
                    Some((start, end)) => {
                        self.end = end;
                        if self.start == self.pos {
                            self.start = start;
                        }
                        self.pos = start;
                    }
                    None => {
                        return None
                    }
                }
            }

            // if this is a new stmt block, skip the whitespace
            if self.pos == self.start {
                while self.pos < self.end {
                    if !whitespace.contains(&src_bytes[self.pos]) {
                        break;
                    } else {
                        self.pos += 1;
                    }
                }
                self.start = self.pos;
            }

            // iterate through the chunk, looking for stmt end
            while self.pos < self.end {

                if src_bytes[self.pos] == openbrace {
                    // if is not a ::{Foo,Bar}; then closebrace finishes the stmt
                    if self.level == 0 &&
                       !(self.pos > 1 &&
                         src_bytes[self.pos-1] == colon &&
                         src_bytes[self.pos-2] == colon) {
                           self.enddelim = closebrace;
                    }
                    self.level += 1;
                } else if src_bytes[self.pos] == closebrace {
                    self.level -= 1;
                    // have we reached the end of the scope?
                    if self.level < 0 {
                        self.fuse();
                        return None;
                    }
                }

                // attribute   #[foo = bar]
                if self.level == 0 && self.start == self.pos &&
                    src_bytes[self.pos] == hash {
                    self.enddelim = closesqbrace;
                }

                if self.level == 0 && src_bytes[self.pos] == self.enddelim {
                    let start = self.start;
                    self.start = self.pos+1;
                    self.pos = self.pos+1;
                    self.enddelim = semicolon;
                    return Some((start, self.pos));
                }

                self.pos += 1;
            }
        }
    }
}

pub fn iter_stmts<'a>(src: &'a str) -> StmtIndicesIter<'a> {
    let semicolon: u8 = ";".as_bytes()[0];
    StmtIndicesIter{src: src, it: code_chunks(src),
                    pos: 0, start: 0, end: 0, level: 0, enddelim: semicolon}
}

#[test]
fn iterates_single_use_stmts() {
    let src = rejustify("
    use std::Foo; // a comment
    use std::Bar;
    ");
    let mut it = iter_stmts(src.as_slice());
    assert_eq!("use std::Foo;", slice(src.as_slice(), it.next().unwrap()));
    assert_eq!("use std::Bar;", slice(src.as_slice(), it.next().unwrap()));
}

#[test]
fn iterates_use_stmt_over_two_lines() {
    let src = rejustify("
    use std::{Foo,
              Bar}; // a comment
    ");
    let mut it = iter_stmts(src.as_slice());
    assert_eq!("use std::{Foo,
          Bar};", slice(src.as_slice(), it.next().unwrap()));
}

#[test]
fn iterates_while_stmt() {
    let src = rejustify("
    while self.pos < 3 { }
    ");
    let mut it = iter_stmts(src.as_slice());
    assert_eq!("while self.pos < 3 { }", slice(src.as_slice(), it.next().unwrap()));
}

// #[test]
// fn iterates_if_else_stmt() {
//     let src = rejustify("
//     if self.pos < 3 { } else { }
//     ");
//     let mut it = iter_stmts(src.as_slice());
//     assert_eq!("if self.pos < 3 { } else { }", slice(src.as_slice(), it.next().unwrap()));
// }

#[test]
fn iterates_inner_scope() {
    let src = rejustify("
    while self.pos < 3 {
       let a = 35;
       return a + 35;  // should iterate this
    }
    {
       b = foo;       // but not this
    }
    ");

    let scope = src.as_slice().slice_from(25);
    debug!("blah{}",scope);
    let mut it = iter_stmts(scope);

    assert_eq!("let a = 35;", slice(scope, it.next().unwrap()));
    assert_eq!("return a + 35;", slice(scope, it.next().unwrap()));
    assert_eq!(None, it.next());
}

#[test]
fn iterates_module_attribute() {
    let src = rejustify("
    #![license = \"BSD\"]
    #[test]
    ");
    let mut it = iter_stmts(src.as_slice());
    assert_eq!("#![license = \"BSD\"]", slice(src.as_slice(), it.next().unwrap()));
    assert_eq!("#[test]", slice(src.as_slice(), it.next().unwrap()));
}

// fn main() {
//     use std::io::BufferedReader;
//     use std::io::File;
//     use std::str;

//     let filetxt = BufferedReader::new(File::open(&Path::new("/usr/local/src/rust/src/libstd/prelude.rs"))).read_to_end().unwrap();
//     let src = str::from_utf8(filetxt.as_slice()).unwrap();

//     for (start,end) in iter_stmts(src) {
//         println!("BLOB |{}|",src.slice(start,end));
//     }

// }
<commit_msg>codeiter.rs - handle macro-rules! definitions<commit_after>// #![feature(phase)]
// #[phase(plugin, link)] extern crate log;

use racer::codecleaner::{code_chunks,CodeIndicesIter};
//use codecleaner::{code_chunks,CodeIndicesIter};
//mod codecleaner;

#[cfg(test)] use racer::testutils::{rejustify, slice};

pub struct StmtIndicesIter<'a> {
    src: &'a str,
    it: CodeIndicesIter<'a>,
    pos: uint,
    start: uint,
    end: uint,
    bracelevel: int,
    parenlevel: int,
    is_macro: bool,
    enddelim: u8
}

impl<'a> Iterator<(uint, uint)> for StmtIndicesIter<'a> {
    #[inline]
    fn next(&mut self) -> Option<(uint, uint)> {
        let semicolon: u8 = ";".as_bytes()[0];
        let hash: u8 = "#".as_bytes()[0];
        let colon: u8 = ":".as_bytes()[0];
        let openbrace: u8 = "{".as_bytes()[0];
        let closebrace: u8 = "}".as_bytes()[0];
        let openparen: u8 = "(".as_bytes()[0];
        let closeparen: u8 = ")".as_bytes()[0];
        let closesqbrace: u8 = "]".as_bytes()[0];
        let whitespace = " \t\n".as_bytes();

        let src_bytes = self.src.as_bytes();

        loop {

            if self.end <= self.pos {
                // get the next chunk of code
                match self.it.next() {
                    Some((start, end)) => {
                        self.end = end;
                        if self.start == self.pos {
                            self.start = start;
                        }
                        self.pos = start;
                    }
                    None => {
                        return None
                    }
                }
            }

            // if this is a new stmt block, skip the whitespace
            if self.pos == self.start {
                while self.pos < self.end {
                    if !whitespace.contains(&src_bytes[self.pos]) {
                        break;
                    } else {
                        self.pos += 1;
                    }
                }
                self.start = self.pos;
            }

            // if the statement starts with 'macro_rules!' then we're in a macro
            // We need to know this because a closeparen can terminate a macro
            if (self.end - self.pos) > 12 &&
               self.src.slice(self.pos, self.pos+12) == "macro_rules!" {
                self.is_macro = true;
            }

            // iterate through the chunk, looking for stmt end
            while self.pos < self.end {
                if src_bytes[self.pos] == openparen {

                    // macros can be terminated by closeparen if opened by one
                    if self.is_macro &&
                        self.bracelevel == 0 &&
                        self.parenlevel == 0 {
                        self.enddelim = closeparen;
                    }
                    self.parenlevel += 1;

                } else if src_bytes[self.pos] == closeparen {
                    self.parenlevel -= 1;

                } else if src_bytes[self.pos] == openbrace {
                    // if we are top level and stmt is not a ::{Foo,Bar}; then
                    // closebrace finishes the stmt
                    if self.bracelevel == 0 &&
                        self.parenlevel == 0 &&
                       !(self.pos > 1 &&
                         src_bytes[self.pos-1] == colon &&
                         src_bytes[self.pos-2] == colon) {
                           self.enddelim = closebrace;
                    }
                    self.bracelevel += 1;
                } else if src_bytes[self.pos] == closebrace {
                    self.bracelevel -= 1;
                    // have we reached the end of the scope?
                    if self.bracelevel < 0 {
                        self.fuse();
                        return None;
                    }
                }

                // attribute   #[foo = bar]
                if self.bracelevel == 0 && self.start == self.pos &&
                    src_bytes[self.pos] == hash {
                    self.enddelim = closesqbrace;
                }

                if self.bracelevel == 0 && self.parenlevel == 0 &&
                   src_bytes[self.pos] == self.enddelim {
                    let start = self.start;
                    self.start = self.pos+1;
                    self.pos = self.pos+1;
                    self.enddelim = semicolon;
                    self.is_macro = false;
                    return Some((start, self.pos));
                }

                self.pos += 1;
            }
        }
    }
}

pub fn iter_stmts<'a>(src: &'a str) -> StmtIndicesIter<'a> {
    let semicolon: u8 = ";".as_bytes()[0];
    StmtIndicesIter{src: src, it: code_chunks(src),
                    pos: 0, start: 0, end: 0, bracelevel: 0,
                    parenlevel: 0, enddelim: semicolon, is_macro: false}
}

#[test]
fn iterates_single_use_stmts() {
    let src = rejustify("
    use std::Foo; // a comment
    use std::Bar;
    ");
    let mut it = iter_stmts(src.as_slice());
    assert_eq!("use std::Foo;", slice(src.as_slice(), it.next().unwrap()));
    assert_eq!("use std::Bar;", slice(src.as_slice(), it.next().unwrap()));
}

#[test]
fn iterates_use_stmt_over_two_lines() {
    let src = rejustify("
    use std::{Foo,
              Bar}; // a comment
    ");
    let mut it = iter_stmts(src.as_slice());
    assert_eq!("use std::{Foo,
          Bar};", slice(src.as_slice(), it.next().unwrap()));
}

#[test]
fn iterates_while_stmt() {
    let src = rejustify("
    while self.pos < 3 { }
    ");
    let mut it = iter_stmts(src.as_slice());
    assert_eq!("while self.pos < 3 { }", slice(src.as_slice(), it.next().unwrap()));
}

#[test]
fn iterates_lambda_arg() {
    let src = rejustify("
    myfn(|n|{});
    ");
    let mut it = iter_stmts(src.as_slice());
    assert_eq!("myfn(|n|{});", slice(src.as_slice(), it.next().unwrap()));
}

#[test]
fn iterates_macro() {
    let src = "
    mod foo;
    macro_rules! otry(
        ($e:expr) => (match $e { Some(e) => e, None => return })
    )
    mod bar;
    ";
    let mut it = iter_stmts(src.as_slice());
    assert_eq!("mod foo;", slice(src.as_slice(), it.next().unwrap()));
    assert_eq!("macro_rules! otry(
        ($e:expr) => (match $e { Some(e) => e, None => return })
    )", slice(src.as_slice(), it.next().unwrap()));
    assert_eq!("mod bar;", slice(src.as_slice(), it.next().unwrap()));
}

// #[test]
// fn iterates_if_else_stmt() {
//     let src = rejustify("
//     if self.pos < 3 { } else { }
//     ");
//     let mut it = iter_stmts(src.as_slice());
//     assert_eq!("if self.pos < 3 { } else { }", slice(src.as_slice(), it.next().unwrap()));
// }

#[test]
fn iterates_inner_scope() {
    let src = "
    while self.pos < 3 {
       let a = 35;
       return a + 35;  // should iterate this
    }
    {
       b = foo;       // but not this
    }
    ";

    let scope = src.as_slice().slice_from(25);
    //println!("blah{}",scope);
    let mut it = iter_stmts(scope);

    assert_eq!("let a = 35;", slice(scope, it.next().unwrap()));
    assert_eq!("return a + 35;", slice(scope, it.next().unwrap()));
    assert_eq!(None, it.next());
}

#[test]
fn iterates_module_attribute() {
    let src = rejustify("
    #![license = \"BSD\"]
    #[test]
    ");
    let mut it = iter_stmts(src.as_slice());
    assert_eq!("#![license = \"BSD\"]", slice(src.as_slice(), it.next().unwrap()));
    assert_eq!("#[test]", slice(src.as_slice(), it.next().unwrap()));
}

// fn main() {
//     use std::io::BufferedReader;
//     use std::io::File;
//     use std::str;

//     let filetxt = BufferedReader::new(File::open(&Path::new("/usr/local/src/rust/src/libstd/prelude.rs"))).read_to_end().unwrap();
//     let src = str::from_utf8(filetxt.as_slice()).unwrap();

//     for (start,end) in iter_stmts(src) {
//         println!("BLOB |{}|",src.slice(start,end));
//     }

// }
<|endoftext|>


<commit_before>use mapgen::{MapGenerator, CHUNK_SIZE_usize, CHUNK_SIZE};
pub use core::{Vec2, Tile, TileMap};
use std::mem;

/// A chunk grid, i.e. a grid of chunk size cotaining Block
pub type ChunkGrid<'a> = [[Tile<'a>; CHUNK_SIZE_usize]; CHUNK_SIZE_usize];
/// A cache of the relevant chunks
///
/// The ordering of the chunks are as follows.
/// 1---2
/// |   |
/// 3---4
pub struct Cache<'a> {
  /// The offset of the cache, i.e. the place where you find chunk1
  offset: Vec2<i64>,
  chunk1: ChunkGrid<'a>,
  chunk2: ChunkGrid<'a>,
  chunk3: ChunkGrid<'a>,
  chunk4: ChunkGrid<'a>,
}
// NOTE: When moving chunks around remember to use mem::replace

impl<'a> Cache<'a> {
  fn update<'b, T>(&mut self, new_offset: Vec2<i64>, mapgen: T)
            where T: TileMap<'b>, 'b: 'a {
    // TODO: Optimize this by using parts of the old cache which just should be
    //       moved.
    // TODO: Consume iters?
    for (y, row) in self.chunk1.iter_mut().enumerate() {
      for (x, block_ptr) in row.iter_mut().enumerate() {
        *block_ptr = mapgen.get_tile(new_offset); // Lifetime 'a
      }
    }
    for (y, row) in self.chunk2.iter_mut().enumerate() {
      for (x, block_ptr) in row.iter_mut().enumerate() {
        *block_ptr = mapgen.get_tile(new_offset
                                      + Vec2(CHUNK_SIZE, 0));
      }
    }
    for (y, row) in self.chunk3.iter_mut().enumerate() {
      for (x, block_ptr) in row.iter_mut().enumerate() {
        *block_ptr = mapgen.get_tile(new_offset
                                      + Vec2(0, CHUNK_SIZE));
      }
    }
    for (y, row) in self.chunk4.iter_mut().enumerate() {
      for (x, block_ptr) in row.iter_mut().enumerate() {
        *block_ptr = mapgen.get_tile(new_offset
                                      + Vec2(CHUNK_SIZE, CHUNK_SIZE));
      }
    }
    self.offset = new_offset;
  }

  fn get_block(&self) -> Tile {
    unimplemented!();
  }
}

// TODO: Implement tilemap for cache.
<commit_msg>add loader<commit_after><|endoftext|>


<commit_before>use chrono::UTC;
use chrono::datetime::DateTime;

use params::*;

use rs_es::Client;
use rs_es::query::{Filter, Query};
use rs_es::units::JsonVal;
use rs_es::operations::search::{Sort, SortField, Order};

use terms::VectorOfTerms;

pub struct Talent;

impl Talent {
  /// Return a `Vec<Filter>` with visibility criteria for the talents.
  /// The `epoch` must be given as `I64` (UNIX time in seconds) and is
  /// the range in which batches are searched.
  /// If `presented_talents` is provided, talents who match the IDs
  /// contained there skip the standard visibility criteria.
  ///
  /// Basically, the talents must be accepted into the platform and must be
  /// inside a living batch to match the visibility criteria.
  fn visibility_filters(epoch: i64, presented_talents: Vec<i32>) -> Vec<Filter> {
    let visibility_rules = Filter::build_bool()
                                  .with_must(
                                    vec![
                                      Filter::build_term("accepted", true)
                                             .build(),
                                      Filter::build_range("batch_start_at")
                                             .with_lte(JsonVal::from(epoch))
                                             .with_format("epoch_second")
                                             .build(),
                                      Filter::build_range("batch_end_at")
                                             .with_gte(JsonVal::from(epoch))
                                             .with_format("epoch_second")
                                             .build()
                                    ])
                                  .build();

    if presented_talents.len() > 0 { // preferred over !_.is_empty()
      let presented_talents_filters = Filter::build_bool()
                                             .with_must(
                                               vec![
                                                 <Filter as VectorOfTerms<i32>>::build_terms(
                                                   "ids", &presented_talents)
                                               ].into_iter()
                                                .flat_map(|x| x)
                                                .collect::<Vec<Filter>>())
                                             .build();
      vec![
        Filter::build_bool()
               .with_should(vec![visibility_rules, presented_talents_filters])
               .build()
      ]
    }
    else {
      vec![visibility_rules]
    }
  }

  /// Given parameters inside the query string mapped inside a `Map`,
  /// and the `epoch` (defined as UNIX time in seconds) for batches,
  /// return a `Query` for ElasticSearch.
  ///
  /// `VectorOfTerms` are ORred, while `Filter`s are ANDed.
  /// I.e.: given ["Fullstack", "DevOps"] as `work_roles`, found talents
  /// will present at least one of these roles), but both `work_roles`
  /// and `work_languages`, if provided, must not be empty.
  fn search_filters(params: &Map, epoch: i64) -> Query {
    let company_id = i32_vec_from_params!(params, "company_id");

    Query::build_filtered(Filter::build_bool()
                                 .with_must(
                                   vec![
                                     <Filter as VectorOfTerms<String>>::build_terms(
                                       "work_roles", &vec_from_params!(params, "work_roles")),

                                     <Filter as VectorOfTerms<String>>::build_terms(
                                       "work_languages", &vec_from_params!(params, "work_languages")),

                                     <Filter as VectorOfTerms<String>>::build_terms(
                                       "work_experience", &vec_from_params!(params, "work_experience")),

                                     <Filter as VectorOfTerms<String>>::build_terms(
                                       "work_locations", &vec_from_params!(params, "work_locations")),

                                     <Filter as VectorOfTerms<String>>::build_terms(
                                      "work_authorization", &vec_from_params!(params, "work_authorization")),

                                     Talent::visibility_filters(epoch,
                                       i32_vec_from_params!(params, "presented_talents"))
                                   ].into_iter()
                                    .flat_map(|x| x)
                                    .collect::<Vec<Filter>>())
                                 .with_must_not(
                                   vec![
                                     <Filter as VectorOfTerms<i32>>::build_terms(
                                       "company_ids", &company_id),

                                     <Filter as VectorOfTerms<i32>>::build_terms(
                                       "blocked_companies", &company_id)
                                   ].into_iter()
                                    .flat_map(|x| x)
                                    .collect::<Vec<Filter>>())
                                 .build())
          .build()
  }

  /// Query ElasticSearch on given `indexes` and `params` and return the IDs of
  /// the found talents.
  pub fn search(mut es: &mut Client, default_indexes: &[&str], params: &Map) -> Vec<u32> {
    let epoch = match params.find(&["epoch"]) {
      Some(&Value::I64(epoch)) => epoch,
      _ => DateTime::timestamp(&UTC::now())
    };

    let indexes: Vec<&str> = match params.find(&["index"]) {
      Some(&Value::String(ref index)) => vec![&index[..]],
      _ => default_indexes.to_vec()
    };

    let result = es.search_query()
                   .with_indexes(&indexes)
                   .with_query(&Talent::search_filters(params, epoch))
                   .with_sort(&Talent::sorting_criteria())
                   .send();

    match result {
      Ok(result) => {
        result.hits.hits.into_iter()
                        .map(|hit| hit.source.unwrap()["id"]
                                             .as_u64()
                                             .unwrap() as u32)
                        .collect::<Vec<u32>>()
      },
      Err(err) => {
        println!("{:?}", err);
        vec![]
      }
    }
  }

  /// Return a `Sort` that makes values be sorted for `updated_at`, descendently.
  fn sorting_criteria() -> Sort {
    Sort::new(
      vec![
        SortField::new("updated_at", Some(Order::Desc)).build()
      ])
  }
}
<commit_msg>resources/user: dedup results of search<commit_after>use chrono::UTC;
use chrono::datetime::DateTime;

use params::*;

use rs_es::Client;
use rs_es::query::{Filter, Query};
use rs_es::units::JsonVal;
use rs_es::operations::search::{Sort, SortField, Order};

use terms::VectorOfTerms;

pub struct Talent;

impl Talent {
  /// Return a `Vec<Filter>` with visibility criteria for the talents.
  /// The `epoch` must be given as `I64` (UNIX time in seconds) and is
  /// the range in which batches are searched.
  /// If `presented_talents` is provided, talents who match the IDs
  /// contained there skip the standard visibility criteria.
  ///
  /// Basically, the talents must be accepted into the platform and must be
  /// inside a living batch to match the visibility criteria.
  fn visibility_filters(epoch: i64, presented_talents: Vec<i32>) -> Vec<Filter> {
    let visibility_rules = Filter::build_bool()
                                  .with_must(
                                    vec![
                                      Filter::build_term("accepted", true)
                                             .build(),
                                      Filter::build_range("batch_start_at")
                                             .with_lte(JsonVal::from(epoch))
                                             .with_format("epoch_second")
                                             .build(),
                                      Filter::build_range("batch_end_at")
                                             .with_gte(JsonVal::from(epoch))
                                             .with_format("epoch_second")
                                             .build()
                                    ])
                                  .build();

    if presented_talents.len() > 0 { // preferred over !_.is_empty()
      let presented_talents_filters = Filter::build_bool()
                                             .with_must(
                                               vec![
                                                 <Filter as VectorOfTerms<i32>>::build_terms(
                                                   "ids", &presented_talents)
                                               ].into_iter()
                                                .flat_map(|x| x)
                                                .collect::<Vec<Filter>>())
                                             .build();
      vec![
        Filter::build_bool()
               .with_should(vec![visibility_rules, presented_talents_filters])
               .build()
      ]
    }
    else {
      vec![visibility_rules]
    }
  }

  /// Given parameters inside the query string mapped inside a `Map`,
  /// and the `epoch` (defined as UNIX time in seconds) for batches,
  /// return a `Query` for ElasticSearch.
  ///
  /// `VectorOfTerms` are ORred, while `Filter`s are ANDed.
  /// I.e.: given ["Fullstack", "DevOps"] as `work_roles`, found talents
  /// will present at least one of these roles), but both `work_roles`
  /// and `work_languages`, if provided, must not be empty.
  fn search_filters(params: &Map, epoch: i64) -> Query {
    let company_id = i32_vec_from_params!(params, "company_id");

    Query::build_filtered(Filter::build_bool()
                                 .with_must(
                                   vec![
                                     <Filter as VectorOfTerms<String>>::build_terms(
                                       "work_roles", &vec_from_params!(params, "work_roles")),

                                     <Filter as VectorOfTerms<String>>::build_terms(
                                       "work_languages", &vec_from_params!(params, "work_languages")),

                                     <Filter as VectorOfTerms<String>>::build_terms(
                                       "work_experience", &vec_from_params!(params, "work_experience")),

                                     <Filter as VectorOfTerms<String>>::build_terms(
                                       "work_locations", &vec_from_params!(params, "work_locations")),

                                     <Filter as VectorOfTerms<String>>::build_terms(
                                      "work_authorization", &vec_from_params!(params, "work_authorization")),

                                     Talent::visibility_filters(epoch,
                                       i32_vec_from_params!(params, "presented_talents"))
                                   ].into_iter()
                                    .flat_map(|x| x)
                                    .collect::<Vec<Filter>>())
                                 .with_must_not(
                                   vec![
                                     <Filter as VectorOfTerms<i32>>::build_terms(
                                       "company_ids", &company_id),

                                     <Filter as VectorOfTerms<i32>>::build_terms(
                                       "blocked_companies", &company_id)
                                   ].into_iter()
                                    .flat_map(|x| x)
                                    .collect::<Vec<Filter>>())
                                 .build())
          .build()
  }

  /// Query ElasticSearch on given `indexes` and `params` and return the IDs of
  /// the found talents.
  pub fn search(mut es: &mut Client, default_indexes: &[&str], params: &Map) -> Vec<u32> {
    let epoch = match params.find(&["epoch"]) {
      Some(&Value::I64(epoch)) => epoch,
      _ => DateTime::timestamp(&UTC::now())
    };

    let indexes: Vec<&str> = match params.find(&["index"]) {
      Some(&Value::String(ref index)) => vec![&index[..]],
      _ => default_indexes.to_vec()
    };

    let result = es.search_query()
                   .with_indexes(&indexes)
                   .with_query(&Talent::search_filters(params, epoch))
                   .with_sort(&Talent::sorting_criteria())
                   .send();

    match result {
      Ok(result) => {
        let mut results = result.hits.hits.into_iter()
                                          .map(|hit| hit.source.unwrap()["id"]
                                                               .as_u64()
                                                               .unwrap() as u32)
                                          .collect::<Vec<u32>>();
        results.dedup();
        results
      },
      Err(err) => {
        println!("{:?}", err);
        vec![]
      }
    }
  }

  /// Return a `Sort` that makes values be sorted for `updated_at`, descendently.
  fn sorting_criteria() -> Sort {
    Sort::new(
      vec![
        SortField::new("updated_at", Some(Order::Desc)).build()
      ])
  }
}
<|endoftext|>