<commit_before>use internal::{IResult,Err};
use std::collections::HashMap;

pub trait HexDisplay {
  fn offset(&self, second:&[u8]) -> usize;

  /// Converts the value of `self` to a hex value, returning the owned
  /// string.
  fn to_hex(&self, chunk_size: usize) -> String;

  fn to_hex_from(&self, chunk_size: usize, from: usize) -> String;
}

static CHARS: &'static[u8] = b"0123456789abcdef";

impl HexDisplay for [u8] {
  fn offset(&self, second:&[u8]) -> usize {
    let fst = self.as_ptr();
    let snd = second.as_ptr();

    snd as usize - fst as usize
  }

  #[allow(unused_variables)]
  fn to_hex(&self, chunk_size: usize) -> String {
    self.to_hex_from(chunk_size, 0)
  }

  #[allow(unused_variables)]
  fn to_hex_from(&self, chunk_size: usize, from: usize) -> String {
    let mut v = Vec::with_capacity(self.len() * 3);
    let mut i = from;
    for chunk in self.chunks(chunk_size) {
      let s = format!("{:08x}", i);
      for &ch in s.as_bytes().iter() {
        v.push(ch);
      }
      v.push('\t' as u8);

      i = i + chunk_size;

      for &byte in chunk {
        v.push(CHARS[(byte >> 4) as usize]);
        v.push(CHARS[(byte & 0xf) as usize]);
        v.push(' ' as u8);
      }
      if chunk_size > chunk.len() {
        for j in 0..(chunk_size - chunk.len()) {
          v.push(' ' as u8);
          v.push(' ' as u8);
          v.push(' ' as u8);
        }
      }
      v.push('\t' as u8);

      for &byte in chunk {
        if (byte >=32 && byte <= 126) || byte >= 128 {
          v.push(byte);
        } else {
          v.push('.' as u8);
        }
      }
      v.push('\n' as u8);
    }

    unsafe {
      String::from_utf8_unchecked(v)
    }
  }
}

/// Prints a message if the parser fails
///
/// The message prints the `Error` or `Incomplete`
/// and the parser's calling code
///
/// ```
/// # #[macro_use] extern crate nom;
/// # fn main() {
///    named!(f, dbg!( tag!( "abcd" ) ) );
///
///    let a = &b"efgh"[..];
///
///    // Will print the following message:
///    // Error(Position(0, [101, 102, 103, 104])) at l.5 by ' tag ! ( "abcd" ) '
///    f(a);
/// # }
/// ```
#[macro_export]
macro_rules! dbg (
  ($i: expr, $submac:ident!( $($args:tt)* )) => (
    {
      let l = line!();
      match $submac!($i, $($args)*) {
        $crate::IResult::Error(a) => {
          println!("Error({:?}) at l.{} by ' {} '", a, l, stringify!($submac!($($args)*)));
          $crate::IResult::Error(a)
        },
        $crate::IResult::Incomplete(a) => {
          println!("Incomplete({:?}) at {} by ' {} '", a, l, stringify!($submac!($($args)*)));
          $crate::IResult::Incomplete(a)
        },
        a => a
      }
    }
  );
);

/// Prints a message and the input if the parser fails
///
/// The message prints the `Error` or `Incomplete`
/// and the parser's calling code.
///
/// It also displays the input in hexdump format
///
/// ```
/// # #[macro_use] extern crate nom;
/// # fn main() {
///    named!(f, dbg_dmp!( tag!( "abcd" ) ) );
///
///    let a = &b"efghijkl"[..];
///
///    // Will print the following message:
///    // Error(Position(0, [101, 102, 103, 104, 105, 106, 107, 108])) at l.5 by ' tag ! ( "abcd" ) '
///    // 00000000        65 66 67 68 69 6a 6b 6c         efghijkl
///    f(a);
/// # }
#[macro_export]
macro_rules! dbg_dmp (
  ($i: expr, $submac:ident!( $($args:tt)* )) => (
    {
      use $crate::util::HexDisplay;
      let l = line!();
      match $submac!($i, $($args)*) {
        $crate::IResult::Error(a) => {
          println!("Error({:?}) at l.{} by ' {} '\n{}", a, l, stringify!($submac!($($args)*)), $i.to_hex(8));
          $crate::IResult::Error(a)
        },
        $crate::IResult::Incomplete(a) => {
          println!("Incomplete({:?}) at {} by ' {} '\n{}", a, l, stringify!($submac!($($args)*)), $i.to_hex(8));
          $crate::IResult::Incomplete(a)
        },
        a => a
      }
    }
  );
);

pub fn error_to_list(e:Err) -> Vec<u32> {
  let mut v:Vec<u32> = Vec::new();
  let mut err = e;
  loop {
    match err {
      Err::Code(i) | Err::Position(i,_)                  => {
        v.push(i);
        return v;
      },
      Err::Node(i, next) | Err::NodePosition(i, _, next) => {
        v.push(i);
        err = *next;
      }
    }
  }
}

pub fn compare_error_paths(e1:Err, e2:Err) -> bool {
  return error_to_list(e1) == error_to_list(e2)
}

pub fn add_error_pattern<'a,'b,I,O>(h: &mut HashMap<Vec<u32>, &'b str>, res: IResult<'a,I,O>, message: &'b str) -> bool {
  if let IResult::Error(e) = res {
    h.insert(error_to_list(e), message);
    true
  } else {
    false
  }
}

pub fn slice_to_offsets(input: &[u8], s: &[u8]) -> (usize, usize) {
  let start = input.as_ptr();
  let off1  = s.as_ptr() as usize - start as usize;
  let off2  = off1 + s.len();
  return (off1, off2);
}

pub fn print_error<I,O>(input: &[u8], res: IResult<I,O>) {
  if let IResult::Error(e) = res {
    let mut v:Vec<(u32, usize, usize)> = Vec::new();
    let mut err = e.clone();
    loop {
      match err {
        Err::Position(i,s)            => {
          let (o1, o2) = slice_to_offsets(input, s);
          v.push((i, o1, o2));
          //println!("v is: {:?}", v);
          break;
        },
        Err::NodePosition(i, s, next) => {
          let (o1, o2) = slice_to_offsets(input, s);
          v.push((i, o1, o2));
          err = *next;
        },
        Err::Node(_, next)            => {
          err = *next;
        },
        Err::Code(_)                  => {
          break;
        }
      }
    }
    v.sort_by(|a, b| a.1.cmp(&b.1));
    //println!("{:?}", print_offsets(input, 0, &v));
    println!("{}",   print_offsets(input, 0, &v));
    //println!("{:?}", e);

  } else {
    println!("not an error");
  }
}

pub fn generate_colors(v: &Vec<(u32, usize, usize)>) -> HashMap<u32, u8> {
  let mut h: HashMap<u32, u8> = HashMap::new();
  let mut color = 0;

  for &(c,_,_) in v.iter() {
    h.insert(c, color + 31);
    color = color + 1 % 7;
  }

  h
}

pub fn code_from_offset(v: &Vec<(u32, usize, usize)>, offset: usize) -> Option<u32> {
  let mut acc: Option<(u32, usize, usize)> = None;
  for &(c, s, e) in v.iter() {
    if s <= offset && offset <=e {
      if let Some((_, start, end)) = acc {
        if start <= s && e <= end {
          acc = Some((c, s, e));
        }
      } else {
        acc = Some((c, s, e));
      }
    }
  }
  if let Some((code, _, _)) = acc {
    return Some(code);
  } else {
    return None;
  }
}

pub fn reset_color(v: &mut Vec<u8>) {
  v.push(0x1B);
  v.push('[' as u8);
  v.push(0);
  v.push('m' as u8);
}

pub fn write_color(v: &mut Vec<u8>, color: u8) {
  v.push(0x1B);
  v.push('[' as u8);
  v.push(1);
  v.push(';' as u8);
  let s = color.to_string();
  let bytes = s.as_bytes();
  v.extend(bytes.iter().cloned());
  v.push('m' as u8);
}

pub fn print_offsets(input: &[u8], from: usize, offsets: &Vec<(u32, usize, usize)>) -> String {
  let mut v = Vec::with_capacity(input.len() * 3);
  let mut i = from;
  let chunk_size = 8;
  let mut current_code:  Option<u32> = None;
  let mut current_code2: Option<u32> = None;

  let colors = generate_colors(&offsets);

  let prefix = "parser codes: ";
  v.extend(prefix.as_bytes().iter().cloned());

  for (code, &color) in &colors {
    let s = code.to_string();
    let bytes = s.as_bytes();
    write_color(&mut v, color);
    v.extend(bytes.iter().cloned());
    reset_color(&mut v);
    v.push(' ' as u8);
  }
  reset_color(&mut v);
  v.push('\n' as u8);

  for chunk in input.chunks(chunk_size) {
    let s = format!("{:08x}", i);
    for &ch in s.as_bytes().iter() {
      v.push(ch);
    }
    v.push('\t' as u8);

    let mut k = i;
    let mut l = i;
    for &byte in chunk {
      if let Some(code) = code_from_offset(&offsets, k) {
        if let Some(current) = current_code {
          if current != code {
            reset_color(&mut v);
            current_code = Some(code);
            if let Some(&color) = colors.get(&code) {
              write_color(&mut v, color);
            }
          }
        } else {
          current_code = Some(code);
          if let Some(&color) = colors.get(&code) {
            write_color(&mut v, color);
          }
        }
      }
      v.push(CHARS[(byte >> 4) as usize]);
      v.push(CHARS[(byte & 0xf) as usize]);
      v.push(' ' as u8);
      k = k + 1;
    }

    reset_color(&mut v);

    if chunk_size > chunk.len() {
      for _ in 0..(chunk_size - chunk.len()) {
        v.push(' ' as u8);
        v.push(' ' as u8);
        v.push(' ' as u8);
      }
    }
    v.push('\t' as u8);

    for &byte in chunk {
      if let Some(code) = code_from_offset(&offsets, l) {
        if let Some(current) = current_code2 {
          if current != code {
            reset_color(&mut v);
            current_code2 = Some(code);
            if let Some(&color) = colors.get(&code) {
              write_color(&mut v, color);
            }
          }
        } else {
          current_code2 = Some(code);
          if let Some(&color) = colors.get(&code) {
            write_color(&mut v, color);
          }
        }
      }
      if (byte >=32 && byte <= 126) || byte >= 128 {
        v.push(byte);
      } else {
        v.push('.' as u8);
      }
      l = l + 1;
    }
    reset_color(&mut v);

    v.push('\n' as u8);
    i = i + chunk_size;
  }

  unsafe {
    String::from_utf8_unchecked(v)
  }

}

pub trait AsBytes {
  fn as_bytes(&self) -> &[u8];
}

impl<'a> AsBytes for &'a str {
  #[inline(always)]
  fn as_bytes(&self) -> &[u8] {
    str::as_bytes(self)
  }
}

impl AsBytes for str {
  #[inline(always)]
  fn as_bytes(&self) -> &[u8] {
    str::as_bytes(self)
  }
}

impl<'a> AsBytes for &'a [u8] {
  #[inline(always)]
  fn as_bytes(&self) -> &[u8] {
    *self
  }
}

impl AsBytes for [u8] {
  #[inline(always)]
  fn as_bytes(&self) -> &[u8] {
    self
  }
}

pub enum ErrorCode {
  Tag,
  MapRes,
  MapOpt,
  Alt,
  IsNot,
  IsA,
  Filter,
  SeparatedList,
  SeparatedNonEmptyList,
  Many1,
  Count,
  TakeUntilAndConsume,
  TakeUntil,
  TakeUntilEitherAndConsume,
  TakeUntilEither,
  LengthValue,
  TagClosure,
  Alpha,
  Digit,
  AlphaNumeric,
  Space,
  MultiSpace,
  LengthValueFn,
  Eof
}
<commit_msg>make the code more modular<commit_after>use internal::{IResult,Err};
use std::collections::HashMap;

pub trait HexDisplay {
  fn offset(&self, second:&[u8]) -> usize;

  /// Converts the value of `self` to a hex value, returning the owned
  /// string.
  fn to_hex(&self, chunk_size: usize) -> String;

  fn to_hex_from(&self, chunk_size: usize, from: usize) -> String;
}

static CHARS: &'static[u8] = b"0123456789abcdef";

impl HexDisplay for [u8] {
  fn offset(&self, second:&[u8]) -> usize {
    let fst = self.as_ptr();
    let snd = second.as_ptr();

    snd as usize - fst as usize
  }

  #[allow(unused_variables)]
  fn to_hex(&self, chunk_size: usize) -> String {
    self.to_hex_from(chunk_size, 0)
  }

  #[allow(unused_variables)]
  fn to_hex_from(&self, chunk_size: usize, from: usize) -> String {
    let mut v = Vec::with_capacity(self.len() * 3);
    let mut i = from;
    for chunk in self.chunks(chunk_size) {
      let s = format!("{:08x}", i);
      for &ch in s.as_bytes().iter() {
        v.push(ch);
      }
      v.push('\t' as u8);

      i = i + chunk_size;

      for &byte in chunk {
        v.push(CHARS[(byte >> 4) as usize]);
        v.push(CHARS[(byte & 0xf) as usize]);
        v.push(' ' as u8);
      }
      if chunk_size > chunk.len() {
        for j in 0..(chunk_size - chunk.len()) {
          v.push(' ' as u8);
          v.push(' ' as u8);
          v.push(' ' as u8);
        }
      }
      v.push('\t' as u8);

      for &byte in chunk {
        if (byte >=32 && byte <= 126) || byte >= 128 {
          v.push(byte);
        } else {
          v.push('.' as u8);
        }
      }
      v.push('\n' as u8);
    }

    unsafe {
      String::from_utf8_unchecked(v)
    }
  }
}

/// Prints a message if the parser fails
///
/// The message prints the `Error` or `Incomplete`
/// and the parser's calling code
///
/// ```
/// # #[macro_use] extern crate nom;
/// # fn main() {
///    named!(f, dbg!( tag!( "abcd" ) ) );
///
///    let a = &b"efgh"[..];
///
///    // Will print the following message:
///    // Error(Position(0, [101, 102, 103, 104])) at l.5 by ' tag ! ( "abcd" ) '
///    f(a);
/// # }
/// ```
#[macro_export]
macro_rules! dbg (
  ($i: expr, $submac:ident!( $($args:tt)* )) => (
    {
      let l = line!();
      match $submac!($i, $($args)*) {
        $crate::IResult::Error(a) => {
          println!("Error({:?}) at l.{} by ' {} '", a, l, stringify!($submac!($($args)*)));
          $crate::IResult::Error(a)
        },
        $crate::IResult::Incomplete(a) => {
          println!("Incomplete({:?}) at {} by ' {} '", a, l, stringify!($submac!($($args)*)));
          $crate::IResult::Incomplete(a)
        },
        a => a
      }
    }
  );
);

/// Prints a message and the input if the parser fails
///
/// The message prints the `Error` or `Incomplete`
/// and the parser's calling code.
///
/// It also displays the input in hexdump format
///
/// ```
/// # #[macro_use] extern crate nom;
/// # fn main() {
///    named!(f, dbg_dmp!( tag!( "abcd" ) ) );
///
///    let a = &b"efghijkl"[..];
///
///    // Will print the following message:
///    // Error(Position(0, [101, 102, 103, 104, 105, 106, 107, 108])) at l.5 by ' tag ! ( "abcd" ) '
///    // 00000000        65 66 67 68 69 6a 6b 6c         efghijkl
///    f(a);
/// # }
#[macro_export]
macro_rules! dbg_dmp (
  ($i: expr, $submac:ident!( $($args:tt)* )) => (
    {
      use $crate::util::HexDisplay;
      let l = line!();
      match $submac!($i, $($args)*) {
        $crate::IResult::Error(a) => {
          println!("Error({:?}) at l.{} by ' {} '\n{}", a, l, stringify!($submac!($($args)*)), $i.to_hex(8));
          $crate::IResult::Error(a)
        },
        $crate::IResult::Incomplete(a) => {
          println!("Incomplete({:?}) at {} by ' {} '\n{}", a, l, stringify!($submac!($($args)*)), $i.to_hex(8));
          $crate::IResult::Incomplete(a)
        },
        a => a
      }
    }
  );
);

pub fn error_to_list(e:Err) -> Vec<u32> {
  let mut v:Vec<u32> = Vec::new();
  let mut err = e;
  loop {
    match err {
      Err::Code(i) | Err::Position(i,_)                  => {
        v.push(i);
        return v;
      },
      Err::Node(i, next) | Err::NodePosition(i, _, next) => {
        v.push(i);
        err = *next;
      }
    }
  }
}

pub fn compare_error_paths(e1:Err, e2:Err) -> bool {
  return error_to_list(e1) == error_to_list(e2)
}

pub fn add_error_pattern<'a,'b,I,O>(h: &mut HashMap<Vec<u32>, &'b str>, res: IResult<'a,I,O>, message: &'b str) -> bool {
  if let IResult::Error(e) = res {
    h.insert(error_to_list(e), message);
    true
  } else {
    false
  }
}

pub fn slice_to_offsets(input: &[u8], s: &[u8]) -> (usize, usize) {
  let start = input.as_ptr();
  let off1  = s.as_ptr() as usize - start as usize;
  let off2  = off1 + s.len();
  return (off1, off2);
}

pub fn prepare_errors<I,O>(input: &[u8], res: IResult<I,O>) -> Option<Vec<(u32, usize, usize)> > {
  if let IResult::Error(e) = res {
    let mut v:Vec<(u32, usize, usize)> = Vec::new();
    let mut err = e.clone();
    loop {
      match err {
        Err::Position(i,s)            => {
          let (o1, o2) = slice_to_offsets(input, s);
          v.push((i, o1, o2));
          //println!("v is: {:?}", v);
          break;
        },
        Err::NodePosition(i, s, next) => {
          let (o1, o2) = slice_to_offsets(input, s);
          v.push((i, o1, o2));
          err = *next;
        },
        Err::Node(_, next)            => {
          err = *next;
        },
        Err::Code(_)                  => {
          break;
        }
      }
    }
    v.sort_by(|a, b| a.1.cmp(&b.1));
    Some(v)
  } else {
    None
  }
}

pub fn print_error<I,O>(input: &[u8], res: IResult<I,O>) {
  if let Some(v) = prepare_errors(input, res) {
    let colors = generate_colors(&v);
    println!("parser codes: {}",   print_codes(colors, HashMap::new()));
    println!("{}",   print_offsets(input, 0, &v));

  } else {
    println!("not an error");
  }
}

pub fn generate_colors(v: &Vec<(u32, usize, usize)>) -> HashMap<u32, u8> {
  let mut h: HashMap<u32, u8> = HashMap::new();
  let mut color = 0;

  for &(c,_,_) in v.iter() {
    h.insert(c, color + 31);
    color = color + 1 % 7;
  }

  h
}

pub fn code_from_offset(v: &Vec<(u32, usize, usize)>, offset: usize) -> Option<u32> {
  let mut acc: Option<(u32, usize, usize)> = None;
  for &(c, s, e) in v.iter() {
    if s <= offset && offset <=e {
      if let Some((_, start, end)) = acc {
        if start <= s && e <= end {
          acc = Some((c, s, e));
        }
      } else {
        acc = Some((c, s, e));
      }
    }
  }
  if let Some((code, _, _)) = acc {
    return Some(code);
  } else {
    return None;
  }
}

pub fn reset_color(v: &mut Vec<u8>) {
  v.push(0x1B);
  v.push('[' as u8);
  v.push(0);
  v.push('m' as u8);
}

pub fn write_color(v: &mut Vec<u8>, color: u8) {
  v.push(0x1B);
  v.push('[' as u8);
  v.push(1);
  v.push(';' as u8);
  let s = color.to_string();
  let bytes = s.as_bytes();
  v.extend(bytes.iter().cloned());
  v.push('m' as u8);
}

pub fn print_codes(colors: HashMap<u32, u8>, names: HashMap<u32, &str>) -> String {
  let mut v = Vec::new();
  for (code, &color) in &colors {
    if let Some(&s) = names.get(&code) {
      let bytes = s.as_bytes();
      write_color(&mut v, color);
      v.extend(bytes.iter().cloned());
    } else {
      let s = code.to_string();
      let bytes = s.as_bytes();
      write_color(&mut v, color);
      v.extend(bytes.iter().cloned());
    }
    reset_color(&mut v);
    v.push(' ' as u8);
  }
  reset_color(&mut v);

  unsafe {
    String::from_utf8_unchecked(v)
  }

}

pub fn print_offsets(input: &[u8], from: usize, offsets: &Vec<(u32, usize, usize)>) -> String {
  let mut v = Vec::with_capacity(input.len() * 3);
  let mut i = from;
  let chunk_size = 8;
  let mut current_code:  Option<u32> = None;
  let mut current_code2: Option<u32> = None;

  let colors = generate_colors(&offsets);

  for chunk in input.chunks(chunk_size) {
    let s = format!("{:08x}", i);
    for &ch in s.as_bytes().iter() {
      v.push(ch);
    }
    v.push('\t' as u8);

    let mut k = i;
    let mut l = i;
    for &byte in chunk {
      if let Some(code) = code_from_offset(&offsets, k) {
        if let Some(current) = current_code {
          if current != code {
            reset_color(&mut v);
            current_code = Some(code);
            if let Some(&color) = colors.get(&code) {
              write_color(&mut v, color);
            }
          }
        } else {
          current_code = Some(code);
          if let Some(&color) = colors.get(&code) {
            write_color(&mut v, color);
          }
        }
      }
      v.push(CHARS[(byte >> 4) as usize]);
      v.push(CHARS[(byte & 0xf) as usize]);
      v.push(' ' as u8);
      k = k + 1;
    }

    reset_color(&mut v);

    if chunk_size > chunk.len() {
      for _ in 0..(chunk_size - chunk.len()) {
        v.push(' ' as u8);
        v.push(' ' as u8);
        v.push(' ' as u8);
      }
    }
    v.push('\t' as u8);

    for &byte in chunk {
      if let Some(code) = code_from_offset(&offsets, l) {
        if let Some(current) = current_code2 {
          if current != code {
            reset_color(&mut v);
            current_code2 = Some(code);
            if let Some(&color) = colors.get(&code) {
              write_color(&mut v, color);
            }
          }
        } else {
          current_code2 = Some(code);
          if let Some(&color) = colors.get(&code) {
            write_color(&mut v, color);
          }
        }
      }
      if (byte >=32 && byte <= 126) || byte >= 128 {
        v.push(byte);
      } else {
        v.push('.' as u8);
      }
      l = l + 1;
    }
    reset_color(&mut v);

    v.push('\n' as u8);
    i = i + chunk_size;
  }

  unsafe {
    String::from_utf8_unchecked(v)
  }

}

pub trait AsBytes {
  fn as_bytes(&self) -> &[u8];
}

impl<'a> AsBytes for &'a str {
  #[inline(always)]
  fn as_bytes(&self) -> &[u8] {
    str::as_bytes(self)
  }
}

impl AsBytes for str {
  #[inline(always)]
  fn as_bytes(&self) -> &[u8] {
    str::as_bytes(self)
  }
}

impl<'a> AsBytes for &'a [u8] {
  #[inline(always)]
  fn as_bytes(&self) -> &[u8] {
    *self
  }
}

impl AsBytes for [u8] {
  #[inline(always)]
  fn as_bytes(&self) -> &[u8] {
    self
  }
}

pub enum ErrorCode {
  Tag,
  MapRes,
  MapOpt,
  Alt,
  IsNot,
  IsA,
  Filter,
  SeparatedList,
  SeparatedNonEmptyList,
  Many1,
  Count,
  TakeUntilAndConsume,
  TakeUntil,
  TakeUntilEitherAndConsume,
  TakeUntilEither,
  LengthValue,
  TagClosure,
  Alpha,
  Digit,
  AlphaNumeric,
  Space,
  MultiSpace,
  LengthValueFn,
  Eof
}
<|endoftext|>


<commit_before>use std::borrow::Cow;
use std::env;
use std::ffi::OsStr;
use std::fs::{FileType, Metadata};
use std::io;
use std::path::{Path, PathBuf};
use std::process;
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::mpsc::{channel, Receiver, Sender};
use std::sync::{Arc, Mutex};
use std::thread;
use std::time;

use anyhow::{anyhow, Result};
use ignore::overrides::OverrideBuilder;
use ignore::{self, WalkBuilder};
use regex::bytes::Regex;

use crate::error::print_error;
use crate::exec;
use crate::exit_codes::{merge_exitcodes, ExitCode};
use crate::filesystem;
use crate::options::Options;
use crate::output;

/// The receiver thread can either be buffering results or directly streaming to the console.
enum ReceiverMode {
    /// Receiver is still buffering in order to sort the results, if the search finishes fast
    /// enough.
    Buffering,

    /// Receiver is directly printing results to the output.
    Streaming,
}

/// The Worker threads can result in a valid entry having PathBuf or an error.
pub enum WorkerResult {
    Entry(PathBuf),
    Error(ignore::Error),
}

/// Maximum size of the output buffer before flushing results to the console
pub const MAX_BUFFER_LENGTH: usize = 1000;

/// Recursively scan the given search path for files / pathnames matching the pattern.
///
/// If the `--exec` argument was supplied, this will create a thread pool for executing
/// jobs in parallel from a given command line and the discovered paths. Otherwise, each
/// path will simply be written to standard output.
pub fn scan(path_vec: &[PathBuf], pattern: Arc<Regex>, config: Arc<Options>) -> Result<ExitCode> {
    let mut path_iter = path_vec.iter();
    let first_path_buf = path_iter
        .next()
        .expect("Error: Path vector can not be empty");
    let (tx, rx) = channel();

    let mut override_builder = OverrideBuilder::new(first_path_buf.as_path());

    for pattern in &config.exclude_patterns {
        override_builder
            .add(pattern)
            .map_err(|e| anyhow!("Malformed exclude pattern: {}", e))?;
    }
    let overrides = override_builder
        .build()
        .map_err(|_| anyhow!("Mismatch in exclude patterns"))?;

    let mut walker = WalkBuilder::new(first_path_buf.as_path());
    walker
        .hidden(config.ignore_hidden)
        .ignore(config.read_fdignore)
        .parents(config.read_fdignore || config.read_vcsignore)
        .git_ignore(config.read_vcsignore)
        .git_global(config.read_vcsignore)
        .git_exclude(config.read_vcsignore)
        .overrides(overrides)
        .follow_links(config.follow_links)
        // No need to check for supported platforms, option is unavailable on unsupported ones
        .same_file_system(config.one_file_system)
        .max_depth(config.max_depth);

    if config.read_fdignore {
        walker.add_custom_ignore_filename(".fdignore");
    }

    if config.read_global_ignore {
        #[cfg(target_os = "macos")]
        let config_dir_op = env::var_os("XDG_CONFIG_HOME")
            .map(PathBuf::from)
            .filter(|p| p.is_absolute())
            .or_else(|| dirs::home_dir().map(|d| d.join(".config")));

        #[cfg(not(target_os = "macos"))]
        let config_dir_op = dirs::config_dir();

        if let Some(global_ignore_file) = config_dir_op
            .map(|p| p.join("fd").join("ignore"))
            .filter(|p| p.is_file())
        {
            let result = walker.add_ignore(global_ignore_file);
            match result {
                Some(ignore::Error::Partial(_)) => (),
                Some(err) => {
                    print_error(format!(
                        "Malformed pattern in global ignore file. {}.",
                        err.to_string()
                    ));
                }
                None => (),
            }
        }
    }

    for ignore_file in &config.ignore_files {
        let result = walker.add_ignore(ignore_file);
        match result {
            Some(ignore::Error::Partial(_)) => (),
            Some(err) => {
                print_error(format!(
                    "Malformed pattern in custom ignore file. {}.",
                    err.to_string()
                ));
            }
            None => (),
        }
    }

    for path_entry in path_iter {
        walker.add(path_entry.as_path());
    }

    let parallel_walker = walker.threads(config.threads).build_parallel();

    let wants_to_quit = Arc::new(AtomicBool::new(false));
    if config.ls_colors.is_some() && config.command.is_none() {
        let wq = Arc::clone(&wants_to_quit);
        ctrlc::set_handler(move || {
            if wq.load(Ordering::Relaxed) {
                // Ctrl-C has been pressed twice, exit NOW
                process::exit(ExitCode::KilledBySigint.into());
            } else {
                wq.store(true, Ordering::Relaxed);
            }
        })
        .unwrap();
    }

    // Spawn the thread that receives all results through the channel.
    let receiver_thread = spawn_receiver(&config, &wants_to_quit, rx);

    // Spawn the sender threads.
    spawn_senders(&config, &wants_to_quit, pattern, parallel_walker, tx);

    // Wait for the receiver thread to print out all results.
    let exit_code = receiver_thread.join().unwrap();

    if wants_to_quit.load(Ordering::Relaxed) {
        Ok(ExitCode::KilledBySigint)
    } else {
        Ok(exit_code)
    }
}

fn spawn_receiver(
    config: &Arc<Options>,
    wants_to_quit: &Arc<AtomicBool>,
    rx: Receiver<WorkerResult>,
) -> thread::JoinHandle<ExitCode> {
    let config = Arc::clone(config);
    let wants_to_quit = Arc::clone(wants_to_quit);

    let show_filesystem_errors = config.show_filesystem_errors;
    let threads = config.threads;

    thread::spawn(move || {
        // This will be set to `Some` if the `--exec` argument was supplied.
        if let Some(ref cmd) = config.command {
            if cmd.in_batch_mode() {
                exec::batch(rx, cmd, show_filesystem_errors)
            } else {
                let shared_rx = Arc::new(Mutex::new(rx));

                let out_perm = Arc::new(Mutex::new(()));

                // Each spawned job will store it's thread handle in here.
                let mut handles = Vec::with_capacity(threads);
                for _ in 0..threads {
                    let rx = Arc::clone(&shared_rx);
                    let cmd = Arc::clone(cmd);
                    let out_perm = Arc::clone(&out_perm);

                    // Spawn a job thread that will listen for and execute inputs.
                    let handle =
                        thread::spawn(move || exec::job(rx, cmd, out_perm, show_filesystem_errors));

                    // Push the handle of the spawned thread into the vector for later joining.
                    handles.push(handle);
                }

                // Wait for all threads to exit before exiting the program.
                let mut results: Vec<ExitCode> = Vec::new();
                for h in handles {
                    results.push(h.join().unwrap());
                }

                merge_exitcodes(&results)
            }
        } else {
            let start = time::Instant::now();

            let mut buffer = vec![];

            // Start in buffering mode
            let mut mode = ReceiverMode::Buffering;

            // Maximum time to wait before we start streaming to the console.
            let max_buffer_time = config
                .max_buffer_time
                .unwrap_or_else(|| time::Duration::from_millis(100));

            let stdout = io::stdout();
            let mut stdout = stdout.lock();

            let mut num_results = 0;

            for worker_result in rx {
                match worker_result {
                    WorkerResult::Entry(value) => {
                        match mode {
                            ReceiverMode::Buffering => {
                                buffer.push(value);

                                // Have we reached the maximum buffer size or maximum buffering time?
                                if buffer.len() > MAX_BUFFER_LENGTH
                                    || time::Instant::now() - start > max_buffer_time
                                {
                                    // Flush the buffer
                                    for v in &buffer {
                                        output::print_entry(
                                            &mut stdout,
                                            v,
                                            &config,
                                            &wants_to_quit,
                                        );
                                    }
                                    buffer.clear();

                                    // Start streaming
                                    mode = ReceiverMode::Streaming;
                                }
                            }
                            ReceiverMode::Streaming => {
                                output::print_entry(&mut stdout, &value, &config, &wants_to_quit);
                            }
                        }

                        num_results += 1;
                    }
                    WorkerResult::Error(err) => {
                        if show_filesystem_errors {
                            print_error(err.to_string());
                        }
                    }
                }

                if let Some(max_results) = config.max_results {
                    if num_results >= max_results {
                        break;
                    }
                }
            }

            // If we have finished fast enough (faster than max_buffer_time), we haven't streamed
            // anything to the console, yet. In this case, sort the results and print them:
            if !buffer.is_empty() {
                buffer.sort();
                for value in buffer {
                    output::print_entry(&mut stdout, &value, &config, &wants_to_quit);
                }
            }

            ExitCode::Success
        }
    })
}

pub enum DirEntry {
    Normal(ignore::DirEntry),
    BrokenSymlink(PathBuf),
}

impl DirEntry {
    pub fn path(&self) -> &Path {
        match self {
            DirEntry::Normal(e) => e.path(),
            DirEntry::BrokenSymlink(pathbuf) => pathbuf.as_path(),
        }
    }

    pub fn file_type(&self) -> Option<FileType> {
        match self {
            DirEntry::Normal(e) => e.file_type(),
            DirEntry::BrokenSymlink(pathbuf) => {
                pathbuf.symlink_metadata().map(|m| m.file_type()).ok()
            }
        }
    }

    pub fn metadata(&self) -> Option<Metadata> {
        match self {
            DirEntry::Normal(e) => e.metadata().ok(),
            DirEntry::BrokenSymlink(_) => None,
        }
    }

    pub fn depth(&self) -> Option<usize> {
        match self {
            DirEntry::Normal(e) => Some(e.depth()),
            DirEntry::BrokenSymlink(_) => None,
        }
    }
}

fn spawn_senders(
    config: &Arc<Options>,
    wants_to_quit: &Arc<AtomicBool>,
    pattern: Arc<Regex>,
    parallel_walker: ignore::WalkParallel,
    tx: Sender<WorkerResult>,
) {
    parallel_walker.run(|| {
        let config = Arc::clone(config);
        let pattern = Arc::clone(&pattern);
        let tx_thread = tx.clone();
        let wants_to_quit = Arc::clone(wants_to_quit);

        Box::new(move |entry_o| {
            if wants_to_quit.load(Ordering::Relaxed) {
                return ignore::WalkState::Quit;
            }

            let entry = match entry_o {
                Ok(ref e) if e.depth() == 0 => {
                    // Skip the root directory entry.
                    return ignore::WalkState::Continue;
                }
                Ok(e) => DirEntry::Normal(e),
                Err(ignore::Error::WithPath {
                    path,
                    err: inner_err,
                }) => match inner_err.as_ref() {
                    ignore::Error::Io(io_error)
                        if io_error.kind() == io::ErrorKind::NotFound
                            && path
                                .symlink_metadata()
                                .ok()
                                .map_or(false, |m| m.file_type().is_symlink()) =>
                    {
                        DirEntry::BrokenSymlink(path)
                    }
                    _ => {
                        tx_thread
                            .send(WorkerResult::Error(ignore::Error::WithPath {
                                path,
                                err: inner_err,
                            }))
                            .unwrap();
                        return ignore::WalkState::Continue;
                    }
                },
                Err(err) => {
                    tx_thread.send(WorkerResult::Error(err)).unwrap();
                    return ignore::WalkState::Continue;
                }
            };

            if let Some(min_depth) = config.min_depth {
                if entry.depth().map_or(true, |d| d < min_depth) {
                    return ignore::WalkState::Continue;
                }
            }

            // Check the name first, since it doesn't require metadata
            let entry_path = entry.path();

            let search_str: Cow<OsStr> = if config.search_full_path {
                let path_abs_buf = filesystem::path_absolute_form(entry_path)
                    .expect("Retrieving absolute path succeeds");
                Cow::Owned(path_abs_buf.as_os_str().to_os_string())
            } else {
                match entry_path.file_name() {
                    Some(filename) => Cow::Borrowed(filename),
                    None => unreachable!(
                        "Encountered file system entry without a file name. This should only \
                         happen for paths like 'foo/bar/..' or '/' which are not supposed to \
                         appear in a file system traversal."
                    ),
                }
            };

            if !pattern.is_match(&filesystem::osstr_to_bytes(search_str.as_ref())) {
                return ignore::WalkState::Continue;
            }

            // Filter out unwanted extensions.
            if let Some(ref exts_regex) = config.extensions {
                if let Some(path_str) = entry_path.file_name() {
                    if !exts_regex.is_match(&filesystem::osstr_to_bytes(path_str)) {
                        return ignore::WalkState::Continue;
                    }
                } else {
                    return ignore::WalkState::Continue;
                }
            }

            // Filter out unwanted file types.
            if let Some(ref file_types) = config.file_types {
                if let Some(ref entry_type) = entry.file_type() {
                    if (!file_types.files && entry_type.is_file())
                        || (!file_types.directories && entry_type.is_dir())
                        || (!file_types.symlinks && entry_type.is_symlink())
                        || (!file_types.sockets && filesystem::is_socket(entry_type))
                        || (!file_types.pipes && filesystem::is_pipe(entry_type))
                        || (file_types.executables_only
                            && !entry
                                .metadata()
                                .map(|m| filesystem::is_executable(&m))
                                .unwrap_or(false))
                        || (file_types.empty_only && !filesystem::is_empty(&entry))
                        || !(entry_type.is_file()
                            || entry_type.is_dir()
                            || entry_type.is_symlink()
                            || filesystem::is_socket(entry_type)
                            || filesystem::is_pipe(entry_type))
                    {
                        return ignore::WalkState::Continue;
                    }
                } else {
                    return ignore::WalkState::Continue;
                }
            }

            #[cfg(unix)]
            {
                if let Some(ref owner_constraint) = config.owner_constraint {
                    if let Ok(ref metadata) = entry_path.metadata() {
                        if !owner_constraint.matches(&metadata) {
                            return ignore::WalkState::Continue;
                        }
                    } else {
                        return ignore::WalkState::Continue;
                    }
                }
            }

            // Filter out unwanted sizes if it is a file and we have been given size constraints.
            if !config.size_constraints.is_empty() {
                if entry_path.is_file() {
                    if let Ok(metadata) = entry_path.metadata() {
                        let file_size = metadata.len();
                        if config
                            .size_constraints
                            .iter()
                            .any(|sc| !sc.is_within(file_size))
                        {
                            return ignore::WalkState::Continue;
                        }
                    } else {
                        return ignore::WalkState::Continue;
                    }
                } else {
                    return ignore::WalkState::Continue;
                }
            }

            // Filter out unwanted modification times
            if !config.time_constraints.is_empty() {
                let mut matched = false;
                if let Ok(metadata) = entry_path.metadata() {
                    if let Ok(modified) = metadata.modified() {
                        matched = config
                            .time_constraints
                            .iter()
                            .all(|tf| tf.applies_to(&modified));
                    }
                }
                if !matched {
                    return ignore::WalkState::Continue;
                }
            }

            let send_result = tx_thread.send(WorkerResult::Entry(entry_path.to_owned()));

            if send_result.is_err() {
                return ignore::WalkState::Quit;
            }

            ignore::WalkState::Continue
        })
    });
}
<commit_msg>Fix warning<commit_after>use std::borrow::Cow;
use std::ffi::OsStr;
use std::fs::{FileType, Metadata};
use std::io;
use std::path::{Path, PathBuf};
use std::process;
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::mpsc::{channel, Receiver, Sender};
use std::sync::{Arc, Mutex};
use std::thread;
use std::time;

use anyhow::{anyhow, Result};
use ignore::overrides::OverrideBuilder;
use ignore::{self, WalkBuilder};
use regex::bytes::Regex;

use crate::error::print_error;
use crate::exec;
use crate::exit_codes::{merge_exitcodes, ExitCode};
use crate::filesystem;
use crate::options::Options;
use crate::output;

/// The receiver thread can either be buffering results or directly streaming to the console.
enum ReceiverMode {
    /// Receiver is still buffering in order to sort the results, if the search finishes fast
    /// enough.
    Buffering,

    /// Receiver is directly printing results to the output.
    Streaming,
}

/// The Worker threads can result in a valid entry having PathBuf or an error.
pub enum WorkerResult {
    Entry(PathBuf),
    Error(ignore::Error),
}

/// Maximum size of the output buffer before flushing results to the console
pub const MAX_BUFFER_LENGTH: usize = 1000;

/// Recursively scan the given search path for files / pathnames matching the pattern.
///
/// If the `--exec` argument was supplied, this will create a thread pool for executing
/// jobs in parallel from a given command line and the discovered paths. Otherwise, each
/// path will simply be written to standard output.
pub fn scan(path_vec: &[PathBuf], pattern: Arc<Regex>, config: Arc<Options>) -> Result<ExitCode> {
    let mut path_iter = path_vec.iter();
    let first_path_buf = path_iter
        .next()
        .expect("Error: Path vector can not be empty");
    let (tx, rx) = channel();

    let mut override_builder = OverrideBuilder::new(first_path_buf.as_path());

    for pattern in &config.exclude_patterns {
        override_builder
            .add(pattern)
            .map_err(|e| anyhow!("Malformed exclude pattern: {}", e))?;
    }
    let overrides = override_builder
        .build()
        .map_err(|_| anyhow!("Mismatch in exclude patterns"))?;

    let mut walker = WalkBuilder::new(first_path_buf.as_path());
    walker
        .hidden(config.ignore_hidden)
        .ignore(config.read_fdignore)
        .parents(config.read_fdignore || config.read_vcsignore)
        .git_ignore(config.read_vcsignore)
        .git_global(config.read_vcsignore)
        .git_exclude(config.read_vcsignore)
        .overrides(overrides)
        .follow_links(config.follow_links)
        // No need to check for supported platforms, option is unavailable on unsupported ones
        .same_file_system(config.one_file_system)
        .max_depth(config.max_depth);

    if config.read_fdignore {
        walker.add_custom_ignore_filename(".fdignore");
    }

    if config.read_global_ignore {
        #[cfg(target_os = "macos")]
        let config_dir_op = std::env::var_os("XDG_CONFIG_HOME")
            .map(PathBuf::from)
            .filter(|p| p.is_absolute())
            .or_else(|| dirs::home_dir().map(|d| d.join(".config")));

        #[cfg(not(target_os = "macos"))]
        let config_dir_op = dirs::config_dir();

        if let Some(global_ignore_file) = config_dir_op
            .map(|p| p.join("fd").join("ignore"))
            .filter(|p| p.is_file())
        {
            let result = walker.add_ignore(global_ignore_file);
            match result {
                Some(ignore::Error::Partial(_)) => (),
                Some(err) => {
                    print_error(format!(
                        "Malformed pattern in global ignore file. {}.",
                        err.to_string()
                    ));
                }
                None => (),
            }
        }
    }

    for ignore_file in &config.ignore_files {
        let result = walker.add_ignore(ignore_file);
        match result {
            Some(ignore::Error::Partial(_)) => (),
            Some(err) => {
                print_error(format!(
                    "Malformed pattern in custom ignore file. {}.",
                    err.to_string()
                ));
            }
            None => (),
        }
    }

    for path_entry in path_iter {
        walker.add(path_entry.as_path());
    }

    let parallel_walker = walker.threads(config.threads).build_parallel();

    let wants_to_quit = Arc::new(AtomicBool::new(false));
    if config.ls_colors.is_some() && config.command.is_none() {
        let wq = Arc::clone(&wants_to_quit);
        ctrlc::set_handler(move || {
            if wq.load(Ordering::Relaxed) {
                // Ctrl-C has been pressed twice, exit NOW
                process::exit(ExitCode::KilledBySigint.into());
            } else {
                wq.store(true, Ordering::Relaxed);
            }
        })
        .unwrap();
    }

    // Spawn the thread that receives all results through the channel.
    let receiver_thread = spawn_receiver(&config, &wants_to_quit, rx);

    // Spawn the sender threads.
    spawn_senders(&config, &wants_to_quit, pattern, parallel_walker, tx);

    // Wait for the receiver thread to print out all results.
    let exit_code = receiver_thread.join().unwrap();

    if wants_to_quit.load(Ordering::Relaxed) {
        Ok(ExitCode::KilledBySigint)
    } else {
        Ok(exit_code)
    }
}

fn spawn_receiver(
    config: &Arc<Options>,
    wants_to_quit: &Arc<AtomicBool>,
    rx: Receiver<WorkerResult>,
) -> thread::JoinHandle<ExitCode> {
    let config = Arc::clone(config);
    let wants_to_quit = Arc::clone(wants_to_quit);

    let show_filesystem_errors = config.show_filesystem_errors;
    let threads = config.threads;

    thread::spawn(move || {
        // This will be set to `Some` if the `--exec` argument was supplied.
        if let Some(ref cmd) = config.command {
            if cmd.in_batch_mode() {
                exec::batch(rx, cmd, show_filesystem_errors)
            } else {
                let shared_rx = Arc::new(Mutex::new(rx));

                let out_perm = Arc::new(Mutex::new(()));

                // Each spawned job will store it's thread handle in here.
                let mut handles = Vec::with_capacity(threads);
                for _ in 0..threads {
                    let rx = Arc::clone(&shared_rx);
                    let cmd = Arc::clone(cmd);
                    let out_perm = Arc::clone(&out_perm);

                    // Spawn a job thread that will listen for and execute inputs.
                    let handle =
                        thread::spawn(move || exec::job(rx, cmd, out_perm, show_filesystem_errors));

                    // Push the handle of the spawned thread into the vector for later joining.
                    handles.push(handle);
                }

                // Wait for all threads to exit before exiting the program.
                let mut results: Vec<ExitCode> = Vec::new();
                for h in handles {
                    results.push(h.join().unwrap());
                }

                merge_exitcodes(&results)
            }
        } else {
            let start = time::Instant::now();

            let mut buffer = vec![];

            // Start in buffering mode
            let mut mode = ReceiverMode::Buffering;

            // Maximum time to wait before we start streaming to the console.
            let max_buffer_time = config
                .max_buffer_time
                .unwrap_or_else(|| time::Duration::from_millis(100));

            let stdout = io::stdout();
            let mut stdout = stdout.lock();

            let mut num_results = 0;

            for worker_result in rx {
                match worker_result {
                    WorkerResult::Entry(value) => {
                        match mode {
                            ReceiverMode::Buffering => {
                                buffer.push(value);

                                // Have we reached the maximum buffer size or maximum buffering time?
                                if buffer.len() > MAX_BUFFER_LENGTH
                                    || time::Instant::now() - start > max_buffer_time
                                {
                                    // Flush the buffer
                                    for v in &buffer {
                                        output::print_entry(
                                            &mut stdout,
                                            v,
                                            &config,
                                            &wants_to_quit,
                                        );
                                    }
                                    buffer.clear();

                                    // Start streaming
                                    mode = ReceiverMode::Streaming;
                                }
                            }
                            ReceiverMode::Streaming => {
                                output::print_entry(&mut stdout, &value, &config, &wants_to_quit);
                            }
                        }

                        num_results += 1;
                    }
                    WorkerResult::Error(err) => {
                        if show_filesystem_errors {
                            print_error(err.to_string());
                        }
                    }
                }

                if let Some(max_results) = config.max_results {
                    if num_results >= max_results {
                        break;
                    }
                }
            }

            // If we have finished fast enough (faster than max_buffer_time), we haven't streamed
            // anything to the console, yet. In this case, sort the results and print them:
            if !buffer.is_empty() {
                buffer.sort();
                for value in buffer {
                    output::print_entry(&mut stdout, &value, &config, &wants_to_quit);
                }
            }

            ExitCode::Success
        }
    })
}

pub enum DirEntry {
    Normal(ignore::DirEntry),
    BrokenSymlink(PathBuf),
}

impl DirEntry {
    pub fn path(&self) -> &Path {
        match self {
            DirEntry::Normal(e) => e.path(),
            DirEntry::BrokenSymlink(pathbuf) => pathbuf.as_path(),
        }
    }

    pub fn file_type(&self) -> Option<FileType> {
        match self {
            DirEntry::Normal(e) => e.file_type(),
            DirEntry::BrokenSymlink(pathbuf) => {
                pathbuf.symlink_metadata().map(|m| m.file_type()).ok()
            }
        }
    }

    pub fn metadata(&self) -> Option<Metadata> {
        match self {
            DirEntry::Normal(e) => e.metadata().ok(),
            DirEntry::BrokenSymlink(_) => None,
        }
    }

    pub fn depth(&self) -> Option<usize> {
        match self {
            DirEntry::Normal(e) => Some(e.depth()),
            DirEntry::BrokenSymlink(_) => None,
        }
    }
}

fn spawn_senders(
    config: &Arc<Options>,
    wants_to_quit: &Arc<AtomicBool>,
    pattern: Arc<Regex>,
    parallel_walker: ignore::WalkParallel,
    tx: Sender<WorkerResult>,
) {
    parallel_walker.run(|| {
        let config = Arc::clone(config);
        let pattern = Arc::clone(&pattern);
        let tx_thread = tx.clone();
        let wants_to_quit = Arc::clone(wants_to_quit);

        Box::new(move |entry_o| {
            if wants_to_quit.load(Ordering::Relaxed) {
                return ignore::WalkState::Quit;
            }

            let entry = match entry_o {
                Ok(ref e) if e.depth() == 0 => {
                    // Skip the root directory entry.
                    return ignore::WalkState::Continue;
                }
                Ok(e) => DirEntry::Normal(e),
                Err(ignore::Error::WithPath {
                    path,
                    err: inner_err,
                }) => match inner_err.as_ref() {
                    ignore::Error::Io(io_error)
                        if io_error.kind() == io::ErrorKind::NotFound
                            && path
                                .symlink_metadata()
                                .ok()
                                .map_or(false, |m| m.file_type().is_symlink()) =>
                    {
                        DirEntry::BrokenSymlink(path)
                    }
                    _ => {
                        tx_thread
                            .send(WorkerResult::Error(ignore::Error::WithPath {
                                path,
                                err: inner_err,
                            }))
                            .unwrap();
                        return ignore::WalkState::Continue;
                    }
                },
                Err(err) => {
                    tx_thread.send(WorkerResult::Error(err)).unwrap();
                    return ignore::WalkState::Continue;
                }
            };

            if let Some(min_depth) = config.min_depth {
                if entry.depth().map_or(true, |d| d < min_depth) {
                    return ignore::WalkState::Continue;
                }
            }

            // Check the name first, since it doesn't require metadata
            let entry_path = entry.path();

            let search_str: Cow<OsStr> = if config.search_full_path {
                let path_abs_buf = filesystem::path_absolute_form(entry_path)
                    .expect("Retrieving absolute path succeeds");
                Cow::Owned(path_abs_buf.as_os_str().to_os_string())
            } else {
                match entry_path.file_name() {
                    Some(filename) => Cow::Borrowed(filename),
                    None => unreachable!(
                        "Encountered file system entry without a file name. This should only \
                         happen for paths like 'foo/bar/..' or '/' which are not supposed to \
                         appear in a file system traversal."
                    ),
                }
            };

            if !pattern.is_match(&filesystem::osstr_to_bytes(search_str.as_ref())) {
                return ignore::WalkState::Continue;
            }

            // Filter out unwanted extensions.
            if let Some(ref exts_regex) = config.extensions {
                if let Some(path_str) = entry_path.file_name() {
                    if !exts_regex.is_match(&filesystem::osstr_to_bytes(path_str)) {
                        return ignore::WalkState::Continue;
                    }
                } else {
                    return ignore::WalkState::Continue;
                }
            }

            // Filter out unwanted file types.
            if let Some(ref file_types) = config.file_types {
                if let Some(ref entry_type) = entry.file_type() {
                    if (!file_types.files && entry_type.is_file())
                        || (!file_types.directories && entry_type.is_dir())
                        || (!file_types.symlinks && entry_type.is_symlink())
                        || (!file_types.sockets && filesystem::is_socket(entry_type))
                        || (!file_types.pipes && filesystem::is_pipe(entry_type))
                        || (file_types.executables_only
                            && !entry
                                .metadata()
                                .map(|m| filesystem::is_executable(&m))
                                .unwrap_or(false))
                        || (file_types.empty_only && !filesystem::is_empty(&entry))
                        || !(entry_type.is_file()
                            || entry_type.is_dir()
                            || entry_type.is_symlink()
                            || filesystem::is_socket(entry_type)
                            || filesystem::is_pipe(entry_type))
                    {
                        return ignore::WalkState::Continue;
                    }
                } else {
                    return ignore::WalkState::Continue;
                }
            }

            #[cfg(unix)]
            {
                if let Some(ref owner_constraint) = config.owner_constraint {
                    if let Ok(ref metadata) = entry_path.metadata() {
                        if !owner_constraint.matches(&metadata) {
                            return ignore::WalkState::Continue;
                        }
                    } else {
                        return ignore::WalkState::Continue;
                    }
                }
            }

            // Filter out unwanted sizes if it is a file and we have been given size constraints.
            if !config.size_constraints.is_empty() {
                if entry_path.is_file() {
                    if let Ok(metadata) = entry_path.metadata() {
                        let file_size = metadata.len();
                        if config
                            .size_constraints
                            .iter()
                            .any(|sc| !sc.is_within(file_size))
                        {
                            return ignore::WalkState::Continue;
                        }
                    } else {
                        return ignore::WalkState::Continue;
                    }
                } else {
                    return ignore::WalkState::Continue;
                }
            }

            // Filter out unwanted modification times
            if !config.time_constraints.is_empty() {
                let mut matched = false;
                if let Ok(metadata) = entry_path.metadata() {
                    if let Ok(modified) = metadata.modified() {
                        matched = config
                            .time_constraints
                            .iter()
                            .all(|tf| tf.applies_to(&modified));
                    }
                }
                if !matched {
                    return ignore::WalkState::Continue;
                }
            }

            let send_result = tx_thread.send(WorkerResult::Entry(entry_path.to_owned()));

            if send_result.is_err() {
                return ignore::WalkState::Quit;
            }

            ignore::WalkState::Continue
        })
    });
}
<|endoftext|>


<commit_before>//! CLI tool to use the functions provided by the [cretonne-wasm](../cton_wasm/index.html) crate.
//!
//! Reads Wasm binary files, translates the functions' code to Cretonne IL.

use cton_wasm::{translate_module, DummyEnvironment, ModuleEnvironment};
use std::path::PathBuf;
use cretonne::Context;
use cretonne::settings::FlagsOrIsa;
use std::fs::File;
use std::error::Error;
use std::io;
use std::path::Path;
use std::process::Command;
use tempdir::TempDir;
use term;
use utils::{pretty_verifier_error, pretty_error, parse_sets_and_isa, read_to_end};

macro_rules! vprintln {
    ($x: expr, $($tts:tt)*) => {
        if $x {
            println!($($tts)*);
        }
    }
}

macro_rules! vprint {
    ($x: expr, $($tts:tt)*) => {
        if $x {
            print!($($tts)*);
        }
    }
}

pub fn run(
    files: Vec<String>,
    flag_verbose: bool,
    flag_just_decode: bool,
    flag_check_translation: bool,
    flag_print: bool,
    flag_set: Vec<String>,
    flag_isa: String,
) -> Result<(), String> {
    let parsed = parse_sets_and_isa(flag_set, flag_isa)?;

    for filename in files {
        let path = Path::new(&filename);
        let name = String::from(path.as_os_str().to_string_lossy());
        handle_module(
            flag_verbose,
            flag_just_decode,
            flag_check_translation,
            flag_print,
            path.to_path_buf(),
            name,
            parsed.as_fisa(),
        )?;
    }
    Ok(())
}

fn handle_module(
    flag_verbose: bool,
    flag_just_decode: bool,
    flag_check_translation: bool,
    flag_print: bool,
    path: PathBuf,
    name: String,
    fisa: FlagsOrIsa,
) -> Result<(), String> {
    let mut terminal = term::stdout().unwrap();
    terminal.fg(term::color::YELLOW).unwrap();
    vprint!(flag_verbose, "Handling: ");
    terminal.reset().unwrap();
    vprintln!(flag_verbose, "\"{}\"", name);
    terminal.fg(term::color::MAGENTA).unwrap();
    vprint!(flag_verbose, "Translating... ");
    terminal.reset().unwrap();
    let mut data = read_to_end(path.clone()).map_err(|err| {
        String::from(err.description())
    })?;
    if !data.starts_with(&[b'\0', b'a', b's', b'm']) {
        let tmp_dir = TempDir::new("cretonne-wasm").unwrap();
        let file_path = tmp_dir.path().join("module.wasm");
        File::create(file_path.clone()).unwrap();
        Command::new("wat2wasm")
            .arg(path.clone())
            .arg("-o")
            .arg(file_path.to_str().unwrap())
            .output()
            .or_else(|e| if let io::ErrorKind::NotFound = e.kind() {
                return Err(String::from("wat2wasm not found"));
            } else {
                return Err(String::from(e.description()));
            })?;
        data = read_to_end(file_path).map_err(
            |err| String::from(err.description()),
        )?;
    }
    let mut dummy_environ = DummyEnvironment::with_flags(fisa.flags.clone());
    translate_module(&data, &mut dummy_environ)?;
    terminal.fg(term::color::GREEN).unwrap();
    vprintln!(flag_verbose, "ok");
    terminal.reset().unwrap();
    if flag_just_decode {
        return Ok(());
    }
    terminal.fg(term::color::MAGENTA).unwrap();
    if flag_check_translation {
        vprint!(flag_verbose, "Checking... ");
    } else {
        vprint!(flag_verbose, "Compiling... ");
    }
    terminal.reset().unwrap();
    let num_func_imports = dummy_environ.get_num_func_imports();
    for (def_index, func) in dummy_environ.info.function_bodies.iter().enumerate() {
        let func_index = num_func_imports + def_index;
        let mut context = Context::new();
        context.func = func.clone();
        if flag_check_translation {
            context.verify(fisa).map_err(|err| {
                pretty_verifier_error(&context.func, fisa.isa, err)
            })?;
        } else {
            if let Some(isa) = fisa.isa {
                context.compile(isa).map_err(|err| {
                    pretty_error(&context.func, fisa.isa, err)
                })?;
            } else {
                return Err(String::from("compilation requires a target isa"));
            }
        }
        if flag_print {
            vprintln!(flag_verbose, "");
            if let Some(start_func) = dummy_environ.info.start_func {
                if func_index == start_func {
                    println!("; Selected as wasm start function");
                }
            }
            for export_name in &dummy_environ.info.functions[func_index].export_names {
                println!("; Exported as \"{}\"", export_name);
            }
            println!("{}", context.func.display(fisa.isa));
            vprintln!(flag_verbose, "");
        }
    }
    terminal.fg(term::color::GREEN).unwrap();
    vprintln!(flag_verbose, "ok");
    terminal.reset().unwrap();
    Ok(())
}
<commit_msg>Allow to print translated wasm file.<commit_after>//! CLI tool to use the functions provided by the [cretonne-wasm](../cton_wasm/index.html) crate.
//!
//! Reads Wasm binary files, translates the functions' code to Cretonne IL.

use cton_wasm::{translate_module, DummyEnvironment, ModuleEnvironment};
use std::path::PathBuf;
use cretonne::Context;
use cretonne::settings::FlagsOrIsa;
use std::fs::File;
use std::error::Error;
use std::io;
use std::path::Path;
use std::process::Command;
use tempdir::TempDir;
use term;
use utils::{pretty_verifier_error, pretty_error, parse_sets_and_isa, read_to_end};

macro_rules! vprintln {
    ($x: expr, $($tts:tt)*) => {
        if $x {
            println!($($tts)*);
        }
    }
}

macro_rules! vprint {
    ($x: expr, $($tts:tt)*) => {
        if $x {
            print!($($tts)*);
        }
    }
}

pub fn run(
    files: Vec<String>,
    flag_verbose: bool,
    flag_just_decode: bool,
    flag_check_translation: bool,
    flag_print: bool,
    flag_set: Vec<String>,
    flag_isa: String,
) -> Result<(), String> {
    let parsed = parse_sets_and_isa(flag_set, flag_isa)?;

    for filename in files {
        let path = Path::new(&filename);
        let name = String::from(path.as_os_str().to_string_lossy());
        handle_module(
            flag_verbose,
            flag_just_decode,
            flag_check_translation,
            flag_print,
            path.to_path_buf(),
            name,
            parsed.as_fisa(),
        )?;
    }
    Ok(())
}

fn handle_module(
    flag_verbose: bool,
    flag_just_decode: bool,
    flag_check_translation: bool,
    flag_print: bool,
    path: PathBuf,
    name: String,
    fisa: FlagsOrIsa,
) -> Result<(), String> {
    let mut terminal = term::stdout().unwrap();
    terminal.fg(term::color::YELLOW).unwrap();
    vprint!(flag_verbose, "Handling: ");
    terminal.reset().unwrap();
    vprintln!(flag_verbose, "\"{}\"", name);
    terminal.fg(term::color::MAGENTA).unwrap();
    vprint!(flag_verbose, "Translating... ");
    terminal.reset().unwrap();
    let mut data = read_to_end(path.clone()).map_err(|err| {
        String::from(err.description())
    })?;
    if !data.starts_with(&[b'\0', b'a', b's', b'm']) {
        let tmp_dir = TempDir::new("cretonne-wasm").unwrap();
        let file_path = tmp_dir.path().join("module.wasm");
        File::create(file_path.clone()).unwrap();
        Command::new("wat2wasm")
            .arg(path.clone())
            .arg("-o")
            .arg(file_path.to_str().unwrap())
            .output()
            .or_else(|e| if let io::ErrorKind::NotFound = e.kind() {
                return Err(String::from("wat2wasm not found"));
            } else {
                return Err(String::from(e.description()));
            })?;
        data = read_to_end(file_path).map_err(
            |err| String::from(err.description()),
        )?;
    }
    let mut dummy_environ = DummyEnvironment::with_flags(fisa.flags.clone());
    translate_module(&data, &mut dummy_environ)?;
    terminal.fg(term::color::GREEN).unwrap();
    vprintln!(flag_verbose, "ok");
    terminal.reset().unwrap();
    if flag_just_decode {
        if flag_print {
            let num_func_imports = dummy_environ.get_num_func_imports();
            for (def_index, func) in dummy_environ.info.function_bodies.iter().enumerate() {
                let func_index = num_func_imports + def_index;
                let mut context = Context::new();
                context.func = func.clone();
                if let Some(start_func) = dummy_environ.info.start_func {
                    if func_index == start_func {
                        println!("; Selected as wasm start function");
                    }
                }
                vprintln!(flag_verbose, "");
                for export_name in &dummy_environ.info.functions[func_index].export_names {
                    println!("; Exported as \"{}\"", export_name);
                }
                println!("{}", context.func.display(None));
                vprintln!(flag_verbose, "");
            }
            terminal.reset().unwrap();
        }
        return Ok(());
    }
    terminal.fg(term::color::MAGENTA).unwrap();
    if flag_check_translation {
        vprint!(flag_verbose, "Checking... ");
    } else {
        vprint!(flag_verbose, "Compiling... ");
    }
    terminal.reset().unwrap();
    let num_func_imports = dummy_environ.get_num_func_imports();
    for (def_index, func) in dummy_environ.info.function_bodies.iter().enumerate() {
        let func_index = num_func_imports + def_index;
        let mut context = Context::new();
        context.func = func.clone();
        if flag_check_translation {
            context.verify(fisa).map_err(|err| {
                pretty_verifier_error(&context.func, fisa.isa, err)
            })?;
        } else {
            if let Some(isa) = fisa.isa {
                context.compile(isa).map_err(|err| {
                    pretty_error(&context.func, fisa.isa, err)
                })?;
            } else {
                return Err(String::from("compilation requires a target isa"));
            }
        }
        if flag_print {
            vprintln!(flag_verbose, "");
            if let Some(start_func) = dummy_environ.info.start_func {
                if func_index == start_func {
                    println!("; Selected as wasm start function");
                }
            }
            for export_name in &dummy_environ.info.functions[func_index].export_names {
                println!("; Exported as \"{}\"", export_name);
            }
            println!("{}", context.func.display(fisa.isa));
            vprintln!(flag_verbose, "");
        }
    }
    terminal.fg(term::color::GREEN).unwrap();
    vprintln!(flag_verbose, "ok");
    terminal.reset().unwrap();
    Ok(())
}
<|endoftext|>


<commit_before>use native;
use std::comm::channel;

struct WorkUnit {
    arg: int,
    callback: Sender<int>,
}

enum MessageToWorker {
    Work(WorkUnit),
    Halt,
}

enum MessageToDispatcher {
    Dispatch(WorkUnit),
    HaltAll,
    RegisterWorker(Sender<Sender<Sender<MessageToWorker>>>),
}

struct WorkQueue {
    dispatcher: Sender<MessageToDispatcher>,
}

impl WorkQueue {
    pub fn create() -> WorkQueue {
        let (dispatcher, dispatcher_inbox) = channel::<MessageToDispatcher>();

        // dispatcher
        native::task::spawn(proc() {
            let (want_work, idle_worker) = channel::<Sender<MessageToWorker>>();
            let mut worker_count = 0;
            let inbox = dispatcher_inbox;
            let idle_worker = idle_worker;
            loop {
                match inbox.recv() {
                    Dispatch(work_item) => {
                        idle_worker.recv().send(Work(work_item));
                    },
                    RegisterWorker(want_idle_sender) => {
                        worker_count += 1;
                        want_idle_sender.send(want_work.clone());
                    }
                    HaltAll => {
                        while worker_count > 0 {
                            idle_worker.recv().send(Halt);
                            worker_count -= 1;
                        }
                        return;
                    },
                };
            }
        });
        return WorkQueue{dispatcher: dispatcher};
    }

    pub fn register_worker(&self) -> Sender<Sender<MessageToWorker>> {
        let (reg_s, reg_r) = channel::<Sender<Sender<MessageToWorker>>>();
        self.dispatcher.send(RegisterWorker(reg_s));
        return reg_r.recv();
    }

    pub fn execute(&self, arg: int) -> Receiver<int> {
        let (callback, wait_for_rv) = channel::<int>();
        self.dispatcher.send(Dispatch(WorkUnit{arg: arg, callback: callback}));
        return wait_for_rv;
    }
}

impl Drop for WorkQueue {
    fn drop(&mut self) {
        self.dispatcher.send(HaltAll);
    }
}

#[test]
fn test_queue() {
    let queue = WorkQueue::create();
    for _ in range(0, 3) {
        let want_work = queue.register_worker();
        native::task::spawn(proc() {
            let want_work = want_work;
            loop {
                let (idle, get_work_unit) = channel::<MessageToWorker>();
                want_work.send(idle);
                let work_unit = match get_work_unit.recv() {
                    Work(wu) => wu,
                    Halt     => return
                };
                let rv = work_unit.arg * 2;
                work_unit.callback.send(rv);
            }
        });
    }
    let mut promise_list: ~[Receiver<int>] = ~[];
    for c in range(0, 10) {
        let rv = queue.execute(c);
        promise_list.push(rv);
    }
    let return_list = promise_list.map(|promise| promise.recv());
    assert_eq!(return_list, ~[0, 2, 4, 6, 8, 10, 12, 14, 16, 18]);
}
<commit_msg>parametrize the queue<commit_after>use native;
use std::comm::channel;

struct WorkUnit<ARG, RV> {
    arg: ARG,
    callback: Sender<RV>,
}

enum MessageToWorker<ARG, RV> {
    Work(WorkUnit<ARG, RV>),
    Halt,
}

enum MessageToDispatcher<ARG, RV> {
    Dispatch(WorkUnit<ARG, RV>),
    HaltAll,
    RegisterWorker(Sender<Sender<Sender<MessageToWorker<ARG, RV>>>>),
}

struct WorkQueue<ARG, RV> {
    dispatcher: Sender<MessageToDispatcher<ARG, RV>>,
}

impl<ARG:Send, RV:Send> WorkQueue<ARG, RV> {
    pub fn create() -> WorkQueue<ARG, RV> {
        let (dispatcher, dispatcher_inbox) = channel::<MessageToDispatcher<ARG, RV>>();

        // dispatcher
        native::task::spawn(proc() {
            let (want_work, idle_worker) = channel::<Sender<MessageToWorker<ARG, RV>>>();
            let mut worker_count = 0;
            let inbox = dispatcher_inbox;
            let idle_worker = idle_worker;
            loop {
                match inbox.recv() {
                    Dispatch(work_item) => {
                        idle_worker.recv().send(Work(work_item));
                    },
                    RegisterWorker(want_idle_sender) => {
                        worker_count += 1;
                        want_idle_sender.send(want_work.clone());
                    }
                    HaltAll => {
                        while worker_count > 0 {
                            idle_worker.recv().send(Halt);
                            worker_count -= 1;
                        }
                        return;
                    },
                };
            }
        });
        return WorkQueue{dispatcher: dispatcher};
    }

    pub fn register_worker(&self) -> Sender<Sender<MessageToWorker<ARG, RV>>> {
        let (reg_s, reg_r) = channel::<Sender<Sender<MessageToWorker<ARG, RV>>>>();
        self.dispatcher.send(RegisterWorker(reg_s));
        return reg_r.recv();
    }

    pub fn execute(&self, arg: ARG) -> Receiver<RV> {
        let (callback, wait_for_rv) = channel::<RV>();
        self.dispatcher.send(Dispatch(WorkUnit{arg: arg, callback: callback}));
        return wait_for_rv;
    }
}

// rustc complais "cannot implement a destructor on a structure with
// type parameters", but our destruction is safe, we only send
// a simple message on a channel.
#[unsafe_destructor]
impl<ARG:Send, RV:Send> Drop for WorkQueue<ARG, RV> {
    fn drop(&mut self) {
        self.dispatcher.send(HaltAll);
    }
}

#[test]
fn test_queue() {
    let queue = WorkQueue::<int, int>::create();
    for _ in range(0, 3) {
        let want_work = queue.register_worker();
        native::task::spawn(proc() {
            let want_work = want_work;
            loop {
                let (idle, get_work_unit) = channel::<MessageToWorker<int, int>>();
                want_work.send(idle);
                let work_unit = match get_work_unit.recv() {
                    Work(wu) => wu,
                    Halt     => return
                };
                let rv = work_unit.arg * 2;
                work_unit.callback.send(rv);
            }
        });
    }
    let mut promise_list: ~[Receiver<int>] = ~[];
    for c in range(0, 10) {
        let rv = queue.execute(c);
        promise_list.push(rv);
    }
    let return_list = promise_list.map(|promise| promise.recv());
    assert_eq!(return_list, ~[0, 2, 4, 6, 8, 10, 12, 14, 16, 18]);
}
<|endoftext|>


<commit_before>//! This module contains safe rust functions for glk functions

use ::gen::*;

pub fn main() { unsafe { glk_main() } }
pub fn exit() { unsafe { glk_exit() } }
// TODO: Interrupt Handler
pub fn tick() { unsafe { glk_tick() } }
pub fn gestalt(sel: u32, val: u32) -> u32 { unsafe { glk_gestalt(sel, val) } }
// TODO: gestalt_ext
pub fn char_to_lower(input: u8) -> u8 { unsafe { glk_char_to_lower(input) } }
pub fn char_to_upper(input: u8) -> u8 { unsafe { glk_char_to_upper(input) } }
// TODO: lots
<commit_msg>Implemented safe wrappers for a large chunk of the glk functions<commit_after>//! This module contains safe rust functions for glk functions. Function
//! and struct names have removed the glk prefix (which is now the
//! namespace) and been modified to match nominal rust name formatting.

use ::gen::*;

pub type WinId = winid_t;
pub type StrId = strid_t;
pub type FRefId = frefid_t;
pub type SChanId = schanid_t;

pub fn main() { unsafe { glk_main() } }
pub fn exit() { unsafe { glk_exit() } }
// TODO: Interrupt Handler
pub fn tick() { unsafe { glk_tick() } }
pub fn gestalt(sel: u32, val: u32) -> u32 { unsafe { glk_gestalt(sel, val) } }
// TODO: gestalt_ext
pub fn char_to_lower(input: u8) -> u8 { unsafe { glk_char_to_lower(input) } }
pub fn char_to_upper(input: u8) -> u8 { unsafe { glk_char_to_upper(input) } }

// Line Echo

pub fn set_echo_line_event(win: WinId, val: u32) {
    unsafe { glk_set_echo_line_event(win, val) }
}

// Line Terminators

// TODO: pub fn set_terminators_line_event

// Unicode

// Unicode Norm

// Image

pub fn image_draw(win: WinId, image: u32, val1: i32, val2: i32) -> u32 {
    unsafe { glk_image_draw(win, image, val1, val2) }
}
pub fn image_draw_scaled(
    win: WinId,
    image: u32,
    val1: i32,
    val2: i32,
    width: u32,
    height: u32,
) -> u32 {
    unsafe { glk_image_draw_scaled(win, image, val1, val2, width, height) }
}
// TODO: image_get_info
pub fn window_flow_break(win: WinId) { unsafe { glk_window_flow_break(win) } }
pub fn window_erase_rect(
    win: WinId,
    left: i32,
    top: i32,
    width: u32,
    height: u32,
) {
    unsafe { glk_window_erase_rect(win, left, top, width, height) }
}
pub fn window_fill_rect(
    win: WinId,
    color: u32,
    left: i32,
    top: i32,
    width: u32,
    height: u32,
) {
    unsafe { glk_window_fill_rect(win, color, left, top, width, height) }
}
pub fn window_set_background_color(win: WinId, color: u32) {
    unsafe { glk_window_set_background_color(win, color) }
}

// Sound

pub fn schannel_create(rock: u32) -> SChanId {
    unsafe { glk_schannel_create(rock) }
}
pub fn schannel_destroy(chan: SChanId) {
    unsafe { glk_schannel_destroy(chan) }
}
// TODO: schannel_iterate
pub fn schannel_get_rock(chan: SChanId) -> u32 {
    unsafe { glk_schannel_get_rock(chan) }
}
pub fn schannel_play(chan: SChanId, snd: u32) -> u32 {
    unsafe { glk_schannel_play(chan, snd) }
}
pub fn schannel_play_ext(
    chan: SChanId,
    snd: u32,
    repeats: u32,
    notify: u32,
) -> u32 {
    unsafe { glk_schannel_play_ext(chan, snd, repeats, notify) }
}
pub fn schannel_stop(chan: SChanId) {
    unsafe { glk_schannel_stop(chan) }
}
pub fn schannel_set_volume(chan: SChanId, vol: u32) {
    unsafe { glk_schannel_set_volume(chan, vol) }
}
pub fn sound_load_hint(snd: u32, flag: u32) {
    unsafe { glk_sound_load_hint(snd, flag) }
}

// Sound 2

pub fn schannel_create_ext(rock: u32, volume: u32) -> SChanId {
    unsafe { glk_schannel_create_ext(rock, volume) }
}
// TODO: schannel_play_multi
pub fn schannel_pause(chan: SChanId) {
    unsafe { glk_schannel_pause(chan) }
}
pub fn schannel_unpause(chan: SChanId) {
    unsafe { glk_schannel_unpause(chan) }
}
pub fn schannel_set_volume_ext(
    chan: SChanId,
    volume: u32,
    duration: u32,
    notify: u32,
) {
    unsafe { glk_schannel_set_volume_ext(chan, volume, duration, notify) }
}

// Hyperlinks

pub fn set_hyperlink(link: u32) { unsafe { glk_set_hyperlink(link) } }
pub fn set_hyperlink_stream(stream: StrId, link: u32) {
    unsafe { glk_set_hyperlink_stream(stream, link) }
}
pub fn request_hyperlink_event(win: WinId) {
    unsafe { glk_request_hyperlink_event(win) }
}
pub fn cancel_hyperlink_event(win: WinId) {
    unsafe { glk_cancel_hyperlink_event(win) }
}

// Datetime

pub type Time = glktimeval_t;

impl Time {
    pub fn new(high_sec: i32, low_sec: u32, microsec: i32) -> Time {
        Time{
            high_sec: high_sec,
            low_sec: low_sec,
            microsec:  microsec,
        }
    }
}

pub type Date = glkdate_t;

impl Date {
    pub fn new(
        year: i32,
        month: i32,
        day: i32,
        weekday: i32,
        hour: i32,
        minute: i32,
        second: i32,
        microsec: i32,
    ) -> Date {
        Date{
            year: year,
            month: month,
            day: day,
            weekday: weekday,
            hour: hour,
            minute: minute,
            second: second,
            microsec: microsec,
        }
    }
}

pub fn current_time(time: &mut Time) {
    unsafe { glk_current_time(time) }
}
pub fn current_simple_time(factor: u32) -> i32 {
    unsafe { glk_current_simple_time(factor) }
}
pub fn time_to_date_utc(time: &mut Time, date: &mut Date) {
    unsafe { glk_time_to_date_utc(time, date) }
}
pub fn time_to_date_local(time: &mut Time, date: &mut Date) {
    unsafe { glk_time_to_date_local(time, date) }
}
pub fn simple_time_to_date_utc(time: i32, factor: u32, date: &mut Date) {
    unsafe { glk_simple_time_to_date_utc(time, factor, date) }
}
pub fn simple_time_to_date_local(time: i32, factor: u32, date: &mut Date) {
    unsafe { glk_simple_time_to_date_local(time, factor, date) }
}
pub fn date_to_time_utc(date: &mut Date, time: &mut Time) {
    unsafe { glk_date_to_time_utc(date, time) }
}
pub fn date_to_time_local(date: &mut Date, time: &mut Time) {
    unsafe { glk_date_to_time_local(date, time) }
}
pub fn date_to_simple_time_utc(date: &mut Date, factor: u32) -> i32 {
    unsafe { glk_date_to_simple_time_utc(date, factor) }
}
pub fn date_to_simple_time_local(date: &mut Date, factor: u32) -> i32 {
    unsafe { glk_date_to_simple_time_local(date, factor) }
}

// Resource Stream

pub fn stream_open_resource(filenum: u32, rock: u32) -> StrId {
    unsafe { glk_stream_open_resource(filenum, rock) }
}
pub fn stream_open_resource_uni(filenum: u32, rock: u32) -> StrId {
    unsafe { glk_stream_open_resource_uni(filenum, rock) }
}
<|endoftext|>


<commit_before>extern crate deeplearn;
extern crate matrix;

use deeplearn::Graph;
use deeplearn::op::{Add, MatMul, Mse, Relu};
use matrix::Matrix;

fn main() {
    let ctx = matrix::Context::new();

    // Setup the graph. It's going to have 2 inputs, 3 hidden nodes in it's 1 hidden layer, and 1
    // output node
    let mut graph = Graph::new();

    //////////////////////////
    // Layer 1

    // Input to xor gate. 5 batches of 2 inputs each.
    let a = graph.add_variable(&ctx, (5, 2));
    // Weights for layer 1: [2 inputs x 3 nodes]
    let l1_w = graph.add_variable(&ctx, (2, 3));
    // Use matrix multiplication to do a fully connected layer
    let l1_mat_mul = graph.add_node(&ctx,
                                    MatMul::new(&ctx, (5, 2), (2, 3)),
                                    vec![a, l1_w],
                                    &[(5, 3)]); // out shape: [5x2]*[2x3] = [5 batches x 3 outputs]
    // Grab VarIndex for l1_mat_mul's output
    let l1_mat_mul_out = l1_mat_mul.get(&graph).outputs[0];
    // 3 biases; one for each node in layer 1
    let l1_b = graph.add_variable(&ctx, (1, 3));
    // Here we add the biases to the matrix multiplication output
    let l1_biased = graph.add_node(&ctx,
                               Add::new(0),
                               vec![l1_mat_mul_out, l1_b],
                               &[(5, 3)]);
    // Grab VarIndex for l1_biased's output
    let l1_biased_out = l1_biased.get(&graph).outputs[0];
    // Run the biased input*weight sums through an ReLU activation
    let l1_relu = graph.add_node(&ctx,
                                 Relu::new(),
                                 vec![l1_biased_out],
                                 &[(5, 3)]);
    let l1_relu_out = l1_relu.get(&graph).outputs[0];

    //////////////////////////
    // Layer 2

    // Weights for layer 2: [3 inputs x 1 nodes]
    let l2_w = graph.add_variable(&ctx, (3, 1));
    // Fully connected layer 2. Use layer 1's output as layer 2's input
    let l2_mat_mul = graph.add_node(&ctx,
                                    MatMul::new(&ctx, (5, 3), (3, 1)),
                                    vec![l1_relu_out, l2_w],
                                    &[(5, 1)]); // out shape: [5x3]*[3x1] = [5 batches x 1 output]
    // Grab VarIndex for l2_mat_mul's output
    let l2_mat_mul_out = l2_mat_mul.get(&graph).outputs[0];
    // 1 bias for 1 output node
    let l2_b = graph.add_variable(&ctx, (1, 1));
    // Here we add the bias to the matrix multiplication output
    let l2_biased = graph.add_node(&ctx,
                               Add::new(0),
                               vec![l2_mat_mul_out, l2_b],
                               &[(5, 1)]);
    // Grab VarIndex for l2_biased's output
    let l2_biased_out = l2_biased.get(&graph).outputs[0];
    // Run the biased input*weight sums through an ReLU activation
    let l2_relu = graph.add_node(&ctx,
                                 Relu::new(),
                                 vec![l2_biased_out],
                                 &[(5, 1)]);
    // Grab VarIndex for l2_relu's output
    let l2_relu_out = l2_relu.get(&graph).outputs[0];

    //////////////////////////
    // Loss

    // Add a variable for the training outputs: [5 batches x 1 output]
    let train_out = graph.add_variable(&ctx, (5, 1));
    // Use mean squared error loss function
    let loss = graph.add_node(&ctx,
                              Mse::new(),
                              vec![l2_relu_out, train_out],
                              &[(1, 1)]);
    // Grab the VarIndex for loss's output
    let loss_out = loss.get(&graph).outputs[0];
    // Create a gradient to apply to the loss function
    let loss_d = graph.add_gradient(&ctx, loss, 0);

    ////////////////////////////////////////////////////////////////////////////////////////////////

    // Send some input data
    let a1_cpu = Matrix::from_vec(5, 2, vec![1.0, 0.0,
                                             0.0, 1.0,
                                             1.0, 0.0,
                                             0.0, 1.0,
                                             0.5, 0.0]);
    let a2_cpu = Matrix::from_vec(5, 2, vec![1.0, 1.0,
                                             1.0, 1.0,
                                             0.0, 0.0,
                                             0.0, 0.0,
                                             0.5, 0.5]);
    let train_out1_cpu = Matrix::from_vec(5, 1, vec![1.0, 1.0, 1.0, 1.0, 0.5]);
    let train_out2_cpu = Matrix::from_vec(5, 1, vec![0.0, 0.0, 0.0, 0.0, 0.0]);
    let l1_w_cpu = Matrix::from_vec(2, 3, vec![1.0, 0.3, 0.0,
                                               0.0, 0.8, 0.6]);
    let l1_b_cpu = Matrix::from_vec(1, 3, vec![0.3, -0.5, 0.1]);
    let l2_w_cpu = Matrix::from_vec(3, 1, vec![1.0, 0.5, 1.0]);
    let l2_b_cpu = Matrix::from_vec(1, 1, vec![0.2]);
    // We apply a gradient of -0.1 to the loss function
    let loss_d_cpu = Matrix::from_vec(1, 1, vec![-0.1]);

    // Upload all the data to the gpu
    l1_w.get(&graph).set(&ctx, &l1_w_cpu);
    l1_b.get(&graph).set(&ctx, &l1_b_cpu);
    l2_w.get(&graph).set(&ctx, &l2_w_cpu);
    l2_b.get(&graph).set(&ctx, &l2_b_cpu);
    loss_d.get(&graph).set(&ctx, &loss_d_cpu);

    /////////////////////////
    // Run/Train the network
    for epoch in 0..1000 {
        if epoch % 2 == 0 {
            a.get(&graph).set(&ctx, &a1_cpu);
            train_out.get(&graph).set(&ctx, &train_out1_cpu);
        } else {
            a.get(&graph).set(&ctx, &a2_cpu);
            train_out.get(&graph).set(&ctx, &train_out2_cpu);
        }
        graph.run(&ctx);
        let out = l2_relu_out.get(&graph).get(&ctx);
        let out_d = l2_relu.get(&graph)
                           .out_grad[0]
                           .gradient()
                           .get(&graph)
                           .get(&ctx);
        let l = loss_out.get(&graph).get(&ctx);
        let l1_w_d = graph.get_input_gradient(l1_w).unwrap().get(&graph);
        let l2_w_d = graph.get_input_gradient(l1_w).unwrap().get(&graph);
        l1_w.get(&graph).add(&ctx, -1, &*l1_w_d, &*l1_w.get(&graph));
        l2_w.get(&graph).add(&ctx, -1, &*l2_w_d, &*l2_w.get(&graph));
        if epoch % 100 == 0 || epoch % 100 == 1 {
            println!("===================");
            println!("Epoch: {}", epoch);
            println!("out = {:?}", out);
            println!("out_d = {:?}", out_d);
            println!("loss = {:?}", l);
            println!("l1_w = {:?}", l1_w.get(&graph).get(&ctx));
            println!("l1_b = {:?}", l1_b.get(&graph).get(&ctx));
            println!("l2_w = {:?}", l2_w.get(&graph).get(&ctx));
            println!("l2_b = {:?}", l2_b.get(&graph).get(&ctx));
        }
    }
}
<commit_msg>tweaked xor training data<commit_after>extern crate deeplearn;
extern crate matrix;

use deeplearn::Graph;
use deeplearn::op::{Add, MatMul, Mse, Relu};
use matrix::Matrix;

fn main() {
    let ctx = matrix::Context::new();

    // Setup the graph. It's going to have 2 inputs, 3 hidden nodes in it's 1 hidden layer, and 1
    // output node
    let mut graph = Graph::new();

    //////////////////////////
    // Layer 1

    // Input to xor gate. 5 batches of 2 inputs each.
    let a = graph.add_variable(&ctx, (5, 2));
    // Weights for layer 1: [2 inputs x 3 nodes]
    let l1_w = graph.add_variable(&ctx, (2, 3));
    // Use matrix multiplication to do a fully connected layer
    let l1_mat_mul = graph.add_node(&ctx,
                                    MatMul::new(&ctx, (5, 2), (2, 3)),
                                    vec![a, l1_w],
                                    &[(5, 3)]); // out shape: [5x2]*[2x3] = [5 batches x 3 outputs]
    // Grab VarIndex for l1_mat_mul's output
    let l1_mat_mul_out = l1_mat_mul.get(&graph).outputs[0];
    // 3 biases; one for each node in layer 1
    let l1_b = graph.add_variable(&ctx, (1, 3));
    // Here we add the biases to the matrix multiplication output
    let l1_biased = graph.add_node(&ctx,
                               Add::new(0),
                               vec![l1_mat_mul_out, l1_b],
                               &[(5, 3)]);
    // Grab VarIndex for l1_biased's output
    let l1_biased_out = l1_biased.get(&graph).outputs[0];
    // Run the biased input*weight sums through an ReLU activation
    let l1_relu = graph.add_node(&ctx,
                                 Relu::new(),
                                 vec![l1_biased_out],
                                 &[(5, 3)]);
    let l1_relu_out = l1_relu.get(&graph).outputs[0];

    //////////////////////////
    // Layer 2

    // Weights for layer 2: [3 inputs x 1 nodes]
    let l2_w = graph.add_variable(&ctx, (3, 1));
    // Fully connected layer 2. Use layer 1's output as layer 2's input
    let l2_mat_mul = graph.add_node(&ctx,
                                    MatMul::new(&ctx, (5, 3), (3, 1)),
                                    vec![l1_relu_out, l2_w],
                                    &[(5, 1)]); // out shape: [5x3]*[3x1] = [5 batches x 1 output]
    // Grab VarIndex for l2_mat_mul's output
    let l2_mat_mul_out = l2_mat_mul.get(&graph).outputs[0];
    // 1 bias for 1 output node
    let l2_b = graph.add_variable(&ctx, (1, 1));
    // Here we add the bias to the matrix multiplication output
    let l2_biased = graph.add_node(&ctx,
                               Add::new(0),
                               vec![l2_mat_mul_out, l2_b],
                               &[(5, 1)]);
    // Grab VarIndex for l2_biased's output
    let l2_biased_out = l2_biased.get(&graph).outputs[0];
    // Run the biased input*weight sums through an ReLU activation
    let l2_relu = graph.add_node(&ctx,
                                 Relu::new(),
                                 vec![l2_biased_out],
                                 &[(5, 1)]);
    // Grab VarIndex for l2_relu's output
    let l2_relu_out = l2_relu.get(&graph).outputs[0];

    //////////////////////////
    // Loss

    // Add a variable for the training outputs: [5 batches x 1 output]
    let train_out = graph.add_variable(&ctx, (5, 1));
    // Use mean squared error loss function
    let loss = graph.add_node(&ctx,
                              Mse::new(),
                              vec![l2_relu_out, train_out],
                              &[(1, 1)]);
    // Grab the VarIndex for loss's output
    let loss_out = loss.get(&graph).outputs[0];
    // Create a gradient to apply to the loss function
    let loss_d = graph.add_gradient(&ctx, loss, 0);

    ////////////////////////////////////////////////////////////////////////////////////////////////

    // Send some input data
    let a1_cpu = Matrix::from_vec(5, 2, vec![1.0, 0.0,
                                             0.0, 1.0,
                                             1.0, 0.0,
                                             0.0, 1.0,
                                             0.0, 1.0]);
    let a2_cpu = Matrix::from_vec(5, 2, vec![1.0, 1.0,
                                             1.0, 1.0,
                                             0.0, 0.0,
                                             0.0, 0.0,
                                             1.0, 1.0]);
    let train_out1_cpu = Matrix::from_vec(5, 1, vec![1.0, 1.0, 1.0, 1.0, 1.0]);
    let train_out2_cpu = Matrix::from_vec(5, 1, vec![0.1, 0.1, 0.1, 0.1, 0.1]);
    let l1_w_cpu = Matrix::from_vec(2, 3, vec![1.0, 1.0, 0.0,
                                               0.0, 1.0, 1.0]);
    let l1_b_cpu = Matrix::from_vec(1, 3, vec![0.0, -1.5, 0.0]);
    let l2_w_cpu = Matrix::from_vec(3, 1, vec![1.0, -2.0, 1.0]);
    let l2_b_cpu = Matrix::from_vec(1, 1, vec![0.5]);
    // We apply a gradient of -0.1 to the loss function
    let loss_d_cpu = Matrix::from_vec(1, 1, vec![-0.1]);

    // Upload all the data to the gpu
    l1_w.get(&graph).set(&ctx, &l1_w_cpu);
    l1_b.get(&graph).set(&ctx, &l1_b_cpu);
    l2_w.get(&graph).set(&ctx, &l2_w_cpu);
    l2_b.get(&graph).set(&ctx, &l2_b_cpu);
    loss_d.get(&graph).set(&ctx, &loss_d_cpu);

    /////////////////////////
    // Run/Train the network
    for epoch in 0..1000 {
        if epoch % 2 == 0 {
            a.get(&graph).set(&ctx, &a1_cpu);
            train_out.get(&graph).set(&ctx, &train_out1_cpu);
        } else {
            a.get(&graph).set(&ctx, &a2_cpu);
            train_out.get(&graph).set(&ctx, &train_out2_cpu);
        }
        graph.run(&ctx);
        let out = l2_relu_out.get(&graph).get(&ctx);
        let out_d = l2_relu.get(&graph)
                           .out_grad[0]
                           .gradient()
                           .get(&graph)
                           .get(&ctx);
        let l = loss_out.get(&graph).get(&ctx);
        let l1_w_d = graph.get_input_gradient(l1_w).unwrap().get(&graph);
        let l2_w_d = graph.get_input_gradient(l1_w).unwrap().get(&graph);
        l1_w.get(&graph).add(&ctx, -1, &*l1_w_d, &*l1_w.get(&graph));
        l2_w.get(&graph).add(&ctx, -1, &*l2_w_d, &*l2_w.get(&graph));
        if epoch % 100 == 0 || epoch % 100 == 1 {
            println!("===================");
            println!("Epoch: {}", epoch);
            println!("out = {:?}", out);
            println!("out_d = {:?}", out_d);
            println!("loss = {:?}", l);
            println!("l1_w = {:?}", l1_w.get(&graph).get(&ctx));
            println!("l1_b = {:?}", l1_b.get(&graph).get(&ctx));
            println!("l2_w = {:?}", l2_w.get(&graph).get(&ctx));
            println!("l2_b = {:?}", l2_b.get(&graph).get(&ctx));
        }
    }
}
<|endoftext|>