<commit_before>use std::io;
use std::io::fs::mkdir_recursive;
use std::os;

use csv;
use csv::index::Indexed;

use CliResult;
use config::{Config, Delimiter};
use util;

static USAGE: &'static str = "
Splits the given CSV data into chunks.

The files are written to the directory given with the name '{start}.csv',
where {start} is the index of the first record of the chunk (starting at 0).

Usage:
    xsv split [options] <outdir> [<input>]
    xsv split --help

split options:
    -s, --size <arg>       The number of records to write into each chunk.
                           [default: 500]
    -j, --jobs <arg>       The number of spliting jobs to run in parallel.
                           This only works when the given CSV data has
                           an index already created. Note that a file handle
                           is opened for each job.
                           When set to '0', the number of jobs is set to the
                           number of CPUs detected.
                           [default: 0]

Common options:
    -h, --help             Display this message
    -o, --output <file>    Write output to <file> instead of stdout.
    -n, --no-headers       When set, the first row will NOT be interpreted
                           as column names. Otherwise, the first row will
                           appear in all chunks as the header row.
    -d, --delimiter <arg>  The field delimiter for reading CSV data.
                           Must be a single character. (default: ,)
";

#[deriving(Clone, Decodable)]
struct Args {
    arg_input: Option<String>,
    arg_outdir: String,
    flag_size: uint,
    flag_jobs: uint,
    flag_output: Option<String>,
    flag_no_headers: bool,
    flag_delimiter: Option<Delimiter>,
}

pub fn run(argv: &[&str]) -> CliResult<()> {
    let args: Args = try!(util::get_args(USAGE, argv));
    if args.flag_size == 0 {
        return fail!("--size must be greater than 0.");
    }
    try!(mkdir_recursive(&Path::new(args.arg_outdir[]), io::ALL_PERMISSIONS));

    match try!(args.rconfig().indexed()) {
        Some(idx) => args.parallel_split(idx),
        None => args.sequential_split(),
    }
}

impl Args {
    fn sequential_split(&self) -> CliResult<()> {
        let rconfig = self.rconfig();
        let mut rdr = try!(rconfig.reader());
        let headers = try!(rdr.byte_headers());

        let mut wtr = try!(self.new_writer(headers[], 0));
        for (i, row) in rdr.byte_records().enumerate() {
            if i > 0 && i % self.flag_size == 0 {
                try!(wtr.flush());
                wtr = try!(self.new_writer(headers[], i));
            }
            let row = try!(row);
            try!(wtr.write(row.into_iter()));
        }
        try!(wtr.flush());
        Ok(())
    }

    fn parallel_split(&self, idx: Indexed<io::File, io::File>)
                     -> CliResult<()> {
        use std::sync::TaskPool;

        let nchunks = util::num_of_chunks(idx.count() as uint, self.flag_size);
        let pool = TaskPool::new(self.njobs());
        for i in range(0, nchunks) {
            let args = self.clone();
            pool.execute(move || {
                let conf = args.rconfig();
                let mut idx = conf.indexed().unwrap().unwrap();
                let headers = idx.csv().byte_headers().unwrap();
                let mut wtr = args.new_writer(headers[], i * args.flag_size)
                                  .unwrap();

                idx.seek((i * args.flag_size) as u64).unwrap();
                for row in idx.csv().byte_records().take(args.flag_size) {
                    let row = row.unwrap();
                    wtr.write(row.into_iter()).unwrap();
                }
                wtr.flush().unwrap();
            });
        }
        Ok(())
    }

    fn new_writer(&self, headers: &[csv::ByteString], start: uint)
                 -> CliResult<csv::Writer<Box<io::Writer+'static>>> {
        let dir = Path::new(self.arg_outdir.clone());
        let path = dir.join(format!("{}.csv", start));
        let spath = Some(path.display().to_string());
        let mut wtr = try!(Config::new(&spath).writer());
        if !self.rconfig().no_headers {
            try!(wtr.write(headers.iter().map(|f| f[])));
        }
        Ok(wtr)
    }

    fn rconfig(&self) -> Config {
        Config::new(&self.arg_input)
               .delimiter(self.flag_delimiter)
               .no_headers(self.flag_no_headers)
    }

    fn njobs(&self) -> uint {
        if self.flag_jobs == 0 { os::num_cpus() } else { self.flag_jobs }
    }
}
<commit_msg>SUBTLE CHANGE! TaskPool no longer blocks until all tasks are completed.<commit_after>use std::io;
use std::io::fs::mkdir_recursive;
use std::os;

use csv;
use csv::index::Indexed;

use CliResult;
use config::{Config, Delimiter};
use util;

static USAGE: &'static str = "
Splits the given CSV data into chunks.

The files are written to the directory given with the name '{start}.csv',
where {start} is the index of the first record of the chunk (starting at 0).

Usage:
    xsv split [options] <outdir> [<input>]
    xsv split --help

split options:
    -s, --size <arg>       The number of records to write into each chunk.
                           [default: 500]
    -j, --jobs <arg>       The number of spliting jobs to run in parallel.
                           This only works when the given CSV data has
                           an index already created. Note that a file handle
                           is opened for each job.
                           When set to '0', the number of jobs is set to the
                           number of CPUs detected.
                           [default: 0]

Common options:
    -h, --help             Display this message
    -o, --output <file>    Write output to <file> instead of stdout.
    -n, --no-headers       When set, the first row will NOT be interpreted
                           as column names. Otherwise, the first row will
                           appear in all chunks as the header row.
    -d, --delimiter <arg>  The field delimiter for reading CSV data.
                           Must be a single character. (default: ,)
";

#[deriving(Clone, Decodable)]
struct Args {
    arg_input: Option<String>,
    arg_outdir: String,
    flag_size: uint,
    flag_jobs: uint,
    flag_output: Option<String>,
    flag_no_headers: bool,
    flag_delimiter: Option<Delimiter>,
}

pub fn run(argv: &[&str]) -> CliResult<()> {
    let args: Args = try!(util::get_args(USAGE, argv));
    if args.flag_size == 0 {
        return fail!("--size must be greater than 0.");
    }
    try!(mkdir_recursive(&Path::new(args.arg_outdir[]), io::ALL_PERMISSIONS));

    match try!(args.rconfig().indexed()) {
        Some(idx) => args.parallel_split(idx),
        None => args.sequential_split(),
    }
}

impl Args {
    fn sequential_split(&self) -> CliResult<()> {
        let rconfig = self.rconfig();
        let mut rdr = try!(rconfig.reader());
        let headers = try!(rdr.byte_headers());

        let mut wtr = try!(self.new_writer(headers[], 0));
        for (i, row) in rdr.byte_records().enumerate() {
            if i > 0 && i % self.flag_size == 0 {
                try!(wtr.flush());
                wtr = try!(self.new_writer(headers[], i));
            }
            let row = try!(row);
            try!(wtr.write(row.into_iter()));
        }
        try!(wtr.flush());
        Ok(())
    }

    fn parallel_split(&self, idx: Indexed<io::File, io::File>)
                     -> CliResult<()> {
        use std::comm::channel;
        use std::sync::TaskPool;

        let nchunks = util::num_of_chunks(idx.count() as uint, self.flag_size);
        let pool = TaskPool::new(self.njobs());
        let (tx, rx) = channel();
        for i in range(0, nchunks) {
            let args = self.clone();
            let tx = tx.clone();
            pool.execute(move || {
                let conf = args.rconfig();
                let mut idx = conf.indexed().unwrap().unwrap();
                let headers = idx.csv().byte_headers().unwrap();
                let mut wtr = args.new_writer(headers[], i * args.flag_size)
                                  .unwrap();

                idx.seek((i * args.flag_size) as u64).unwrap();
                for row in idx.csv().byte_records().take(args.flag_size) {
                    let row = row.unwrap();
                    wtr.write(row.into_iter()).unwrap();
                }
                wtr.flush().unwrap();
                tx.send(());
            });
        }
        drop(tx);
        for _ in rx.iter() {}
        Ok(())
    }

    fn new_writer(&self, headers: &[csv::ByteString], start: uint)
                 -> CliResult<csv::Writer<Box<io::Writer+'static>>> {
        let dir = Path::new(self.arg_outdir.clone());
        let path = dir.join(format!("{}.csv", start));
        let spath = Some(path.display().to_string());
        let mut wtr = try!(Config::new(&spath).writer());
        if !self.rconfig().no_headers {
            try!(wtr.write(headers.iter().map(|f| f[])));
        }
        Ok(wtr)
    }

    fn rconfig(&self) -> Config {
        Config::new(&self.arg_input)
               .delimiter(self.flag_delimiter)
               .no_headers(self.flag_no_headers)
    }

    fn njobs(&self) -> uint {
        if self.flag_jobs == 0 { os::num_cpus() } else { self.flag_jobs }
    }
}
<|endoftext|>


<commit_before>//! Contains functions for computing SCCs and serializing graphs.
use std::io::{stderr, Read, Write, BufRead, BufReader, BufWriter};
use petgraph::{Graph};
use petgraph::algo::scc;
use petgraph::graph::NodeIndex;
use std::collections::HashMap;

fn load_graph<R>(source: &mut BufReader<R>) -> (Graph<u64, ()>, HashMap<u64, NodeIndex>) where R: Read {
    // read graph size
    let mut size = String::new();
    source.read_line(&mut size).ok().expect("failed to read graph size");

    let sz = size.trim().parse::<i64>().unwrap();
    writeln!(&mut stderr(), "Graph size is {} edges. Loading...", sz).unwrap();

    let mut graph = Graph::<u64, ()>::new();
    let mut nodes = HashMap::new();

    for line in source.lines() {
        let l = line.unwrap();
        let (s, e, o) = scan_fmt!(&l, "{} {} {}", u64, u64, u8);
        let (start, end, oneway) = (s.unwrap(), e.unwrap(), o.unwrap());

        if start == end {
            continue
        }

        // adding a node erases the index, so for each unique index we need to keep
        // a map of the corresponding node <-> internal node id mappings. shitty,
        // but it seems to work.
        if !nodes.contains_key(&start) {
            let sn = graph.add_node(0);
            nodes.insert(start, sn);
        }

        if !nodes.contains_key(&end) {
            let en = graph.add_node(0);
            nodes.insert(end, en);
        }

        let si = nodes.get(&start).unwrap();
        let ei = nodes.get(&end).unwrap();

        graph.update_edge(*si, *ei, ());

        if oneway != 1 {
            graph.update_edge(*ei, *si, ());
        }
    }

    (graph, nodes)
}

/// Computes the strongly connected components. Takes a graph stream as an
/// input where the format is as follows:
///
/// ```
/// num_edges
/// id1 id2 oneway
/// id1 id2 oneway
/// ...
/// id1 id2 oneway
/// ```
///
/// where `id1` is the start of the edge, `id2` is the end of the edge,
/// and `oneway` is **0** if the edge is bidirectional (id2->id1 is permitted)
/// or **1** if `id2` -> `id1` is not allowed. `num_edges` is the number of edges in the
/// file.
pub fn compute<R>(source: &mut BufReader<R>) -> Vec<Vec<u64>> where R: Read {
    let (graph, nodes) = load_graph(source);
    writeln!(&mut stderr(), "Graph loaded. Computing SCCs.").unwrap();
    let mut comps: Vec<Vec<_>> = scc(&graph);
    writeln!(&mut stderr(), "Calculated {} components.", comps.len()).unwrap();

    // sort by size, and reverse
    comps.sort_by(|v, w| v.len().cmp(&w.len()));
    comps.reverse();
    // find out the real nodes
    let inverse_nodes: HashMap<_,_> = nodes.into_iter().map(|(k, v)| (v, k)).collect();
    let mut real_comps = Vec::new();
    for comp in comps {
        let real_nodes = comp.into_iter().map(|n| *inverse_nodes.get(&n).unwrap()).collect();
        real_comps.push(real_nodes);
    }
    real_comps
}

/// Serializes the contents of the components into a stream where each node is
/// separated by `delim`. Panics if target cannot be opened.
pub fn serialize<W>(comps: Vec<Vec<u64>>, target: &mut BufWriter<W>, delim: u8, include_biggest: bool) where W: Write {
    let sccs = match include_biggest {
        true => comps.iter(),
        false => comps[1..].iter(),
    };
    for comp in sccs {
        for node in comp {
            target.write(node.to_string().as_bytes()).unwrap();
            target.write(&[delim]).unwrap();
        }
    }
}
<commit_msg>Ignore the doc test.<commit_after>//! Contains functions for computing SCCs and serializing graphs.
use std::io::{stderr, Read, Write, BufRead, BufReader, BufWriter};
use petgraph::{Graph};
use petgraph::algo::scc;
use petgraph::graph::NodeIndex;
use std::collections::HashMap;

fn load_graph<R>(source: &mut BufReader<R>) -> (Graph<u64, ()>, HashMap<u64, NodeIndex>) where R: Read {
    // read graph size
    let mut size = String::new();
    source.read_line(&mut size).ok().expect("failed to read graph size");

    let sz = size.trim().parse::<i64>().unwrap();
    writeln!(&mut stderr(), "Graph size is {} edges. Loading...", sz).unwrap();

    let mut graph = Graph::<u64, ()>::new();
    let mut nodes = HashMap::new();

    for line in source.lines() {
        let l = line.unwrap();
        let (s, e, o) = scan_fmt!(&l, "{} {} {}", u64, u64, u8);
        let (start, end, oneway) = (s.unwrap(), e.unwrap(), o.unwrap());

        if start == end {
            continue
        }

        // adding a node erases the index, so for each unique index we need to keep
        // a map of the corresponding node <-> internal node id mappings. shitty,
        // but it seems to work.
        if !nodes.contains_key(&start) {
            let sn = graph.add_node(0);
            nodes.insert(start, sn);
        }

        if !nodes.contains_key(&end) {
            let en = graph.add_node(0);
            nodes.insert(end, en);
        }

        let si = nodes.get(&start).unwrap();
        let ei = nodes.get(&end).unwrap();

        graph.update_edge(*si, *ei, ());

        if oneway != 1 {
            graph.update_edge(*ei, *si, ());
        }
    }

    (graph, nodes)
}

/// Computes the strongly connected components. Takes a graph stream as an
/// input where the format is as follows:
///
/// ```ignore
/// num_edges
/// id1 id2 oneway
/// id1 id2 oneway
/// ...
/// id1 id2 oneway
/// ```
///
/// where `id1` is the start of the edge, `id2` is the end of the edge,
/// and `oneway` is **0** if the edge is bidirectional (id2->id1 is permitted)
/// or **1** if `id2` -> `id1` is not allowed. `num_edges` is the number of edges in the
/// file.
pub fn compute<R>(source: &mut BufReader<R>) -> Vec<Vec<u64>> where R: Read {
    let (graph, nodes) = load_graph(source);
    writeln!(&mut stderr(), "Graph loaded. Computing SCCs.").unwrap();
    let mut comps: Vec<Vec<_>> = scc(&graph);
    writeln!(&mut stderr(), "Calculated {} components.", comps.len()).unwrap();

    // sort by size, and reverse
    comps.sort_by(|v, w| v.len().cmp(&w.len()));
    comps.reverse();
    // find out the real nodes
    let inverse_nodes: HashMap<_,_> = nodes.into_iter().map(|(k, v)| (v, k)).collect();
    let mut real_comps = Vec::new();
    for comp in comps {
        let real_nodes = comp.into_iter().map(|n| *inverse_nodes.get(&n).unwrap()).collect();
        real_comps.push(real_nodes);
    }
    real_comps
}

/// Serializes the contents of the components into a stream where each node is
/// separated by `delim`. Panics if target cannot be opened.
pub fn serialize<W>(comps: Vec<Vec<u64>>, target: &mut BufWriter<W>, delim: u8, include_biggest: bool) where W: Write {
    let sccs = match include_biggest {
        true => comps.iter(),
        false => comps[1..].iter(),
    };
    for comp in sccs {
        for node in comp {
            target.write(node.to_string().as_bytes()).unwrap();
            target.write(&[delim]).unwrap();
        }
    }
}
<|endoftext|>


<commit_before>
use std::str::CharIndices;
use std::iter::Peekable;

#[derive(PartialEq, Debug)]
pub enum Token<'a> {
  Eof(usize),
  Punctuator(Punctuator, usize),
  Name(&'a str, usize, usize),
  IntValue(&'a str, usize, usize),
  FloatValue(&'a str, usize, usize),
  StringValue(&'a str, usize, usize),
}

#[derive(PartialEq, Debug)]
pub enum Punctuator {
  Bang,
  Dollar,
  LeftParen,
  RightParen,
  Spread(usize),
  Colon,
  Equals,
  At,
  LeftBracket,
  RightBracket,
  LeftBrace,
  RightBrace,
  Pipe,
}

macro_rules! take {
  ($lexer: ident, $($p: pat)|*) => ({
    let mut is_taken = false;
    if let Some(&(p, c)) = $lexer.iter.peek() {
      match c {
        $(
          $p => {
            $lexer.iter.next();
            $lexer.hi = p;
            is_taken = true;
          }
        )*
        _ => {}
      }
    }
    is_taken
  });
}

macro_rules! take_while {
  ($lexer: ident, $($p: pat)|*) => ({
    let mut is_taken = false;
    while let Some(&(p, c)) = $lexer.iter.peek() {
      match c {
        $(
          $p => {
            $lexer.iter.next();
            is_taken = true;
          }
        )*
        _ => {
          $lexer.hi = p;
          break;
        }
      }
    }
    is_taken
  });
}

macro_rules! take_while_not {
  ($lexer: ident, $($p: pat)|*) => ({
    let mut is_taken = false;
    while let Some(&(p, c)) = $lexer.iter.peek() {
      match c {
        $(
          $p => {
            $lexer.hi = p;
            break;
          }
        )*
        _ => {
            $lexer.iter.next();
            is_taken = true;
        }
      }
    }
    is_taken
  });
}

macro_rules! peek {
  ($lexer: ident, $($p: pat)|*) => ({
    let mut is_match = false;
    if let Some(&(_, c)) = $lexer.iter.peek() {
      match c {
        $(
          $p => {
            is_match = true;
          }
        )*
        _ => {}
      }
    }
    is_match
  });
}

macro_rules! take_eof {
  ($lexer: ident) => ({
    if $lexer.iter.peek().is_none() {
      $lexer.hi = $lexer.input_len;
      true
    } else {
      false
    }
  });
}

pub struct Lexer<'a> {
  input: &'a str,
  input_len: usize,
  iter: Peekable<CharIndices<'a>>,
  is_eof: bool,
  lo: usize,
  hi: usize,
}

impl<'a> Lexer<'a> {
  pub fn new(input: &'a str) -> Lexer<'a> {
    Lexer {
      input: input,
      input_len: input.len(),
      iter: input.char_indices().peekable(),
      is_eof: false,
      lo: 0,
      hi: 0,
    }
  }
}

fn skip_ignored_token(lexer: &mut Lexer) {
  loop {
    if take!(lexer, '#') {
      take_while_not!(lexer, '\r' | '\n');
    }
    if !take_while!(lexer, '\u{feff}' | ' ' | '\t' | '\r' | '\n' | ',') {
      break;
    }
  }
}

fn scan_spread(lexer: &mut Lexer) -> bool {
  if take!(lexer, '.') {
    if take!(lexer, '.') {
      if take!(lexer, '.') {
        return true;
      }
    }
  }
  return false;
}

fn scan_name(lexer: &mut Lexer) {
  take_while!(lexer, '_' | 'a'...'z' | 'A'...'Z' | '0'...'9');
  take_eof!(lexer);
}

// Return the raw string without unescape character
fn scan_string(lexer: &mut Lexer) -> (bool, bool) {
  let mut is_bad_escape = false;
  let mut is_terminated = false;
  if take!(lexer, '"') {
    lexer.lo = lexer.lo + 1;
  }
  loop {
    if take!(lexer, '\\') {
      if !take!(lexer, '\\' | '"' | 'b' | 'f' | 'n' | 'r' | 't' | '/' | 'u') {
        is_bad_escape = true;
        break;
      }
    }
    take_while_not!(lexer, '"' | '\\');
    if take!(lexer, '"') {
      is_terminated = true;
      break;
    }
  }
  (is_terminated, is_bad_escape)
}

// IntValue:   (+|-)?(0|[1-9][0-9]*)
// FloatValue: (+|-)?(0|[1-9][0-9]*)(\.[0-9]+)?((E|e)(+|-)?[0-9]+)?
fn scan_number(lexer: &mut Lexer) -> (bool, bool) {
  let mut is_float = false;
  take!(lexer, '+' | '-');
  // Integer part
  if take!(lexer, '1'...'9') {
    take_while!(lexer, '0'...'9');
    if take!(lexer, ' ' | '\t' | '\r' | '\n' | ',') || take_eof!(lexer) {
      return (true, is_float);
    }
  } else if take!(lexer, '0') {
    if take!(lexer, ' ' | '\t' | '\r' | '\n' | ',') || take_eof!(lexer) {
      return (true, is_float);
    }
  }
  // Fractional part
  if take!(lexer, '.') {
    if !take_while!(lexer, '0'...'9') {
      return (false, is_float);
    }
    is_float = true;
  }
  // Exponent part
  if !is_float && peek!(lexer, 'E' | 'e') {
    return (false, is_float);
  }
  if take!(lexer, 'E' | 'e') {
    if take!(lexer, '+' | '-') {
      if !take_while!(lexer, '0'...'9') {
        return (false, is_float);
      }
    } else {
      take_while!(lexer, '0'...'9');
    }
  }

  if take!(lexer, ' ' | '\t' | '\r' | '\n' | ',') || take_eof!(lexer) {
    return (true, is_float);
  }
  return (false, is_float);
}

impl<'a> Iterator for Lexer<'a> {
  type Item = Result<Token<'a>, &'a str>;
  fn next(&mut self) -> Option<Result<Token<'a>, &'a str>> {
    use self::Token::*;
    use self::Punctuator::*;

    if !self.is_eof && take_eof!(self) {
      self.is_eof = true;
      return Some(Ok(Token::Eof(self.hi)));
    }
    if self.is_eof {
      return None;
    }
    skip_ignored_token(self);

    let mut item = None;
    if let Some(&(p, c)) = self.iter.peek() {
      self.lo = p;
      self.hi = p;
      item = match c {
        '!' => {
          self.iter.next();
          Some(Ok(Punctuator(Bang, self.lo)))
        }
        '$' => {
          self.iter.next();
          Some(Ok(Punctuator(Dollar, self.lo)))
        }
        '(' => {
          self.iter.next();
          Some(Ok(Punctuator(LeftParen, self.lo)))
        }
        ')' => {
          self.iter.next();
          Some(Ok(Punctuator(RightParen, self.lo)))
        }
        ':' => {
          self.iter.next();
          Some(Ok(Punctuator(Colon, self.lo)))
        }
        '=' => {
          self.iter.next();
          Some(Ok(Punctuator(Equals, self.lo)))
        }
        '@' => {
          self.iter.next();
          Some(Ok(Punctuator(At, self.lo)))
        }
        '[' => {
          self.iter.next();
          Some(Ok(Punctuator(LeftBracket, self.lo)))
        }
        ']' => {
          self.iter.next();
          Some(Ok(Punctuator(RightBracket, self.lo)))
        }
        '{' => {
          self.iter.next();
          Some(Ok(Punctuator(LeftBrace, self.lo)))
        }
        '}' => {
          self.iter.next();
          Some(Ok(Punctuator(RightBrace, self.lo)))
        }
        '|' => {
          self.iter.next();
          Some(Ok(Punctuator(Pipe, self.lo)))
        }
        '.' => {
          if scan_spread(self) {
            Some(Ok(Punctuator(Spread(self.lo), self.hi)))
          } else {
            Some(Err("Unexpected character."))
          }
        }
        '_' | 'a'...'z' | 'A'...'Z' => {
          scan_name(self);
          Some(Ok(Name(&self.input[self.lo..self.hi], self.lo, self.hi)))
        }
        '-' | '0'...'9' => {
          match scan_number(self) {
            (false, false) => Some(Err("Invalid integer number.")),
            (false, true) => Some(Err("Invalid float number.")),
            (true, false) => Some(Ok(IntValue(&self.input[self.lo..self.hi], self.lo, self.hi))),
            (true, true) => Some(Ok(FloatValue(&self.input[self.lo..self.hi], self.lo, self.hi))),
          }
        }
        '"' => {
          match scan_string(self) {
            (false, _) => Some(Err("Unterminated string.")),
            (_, true) => Some(Err("Bad character escape sequence.")),
            _ => Some(Ok(StringValue(&self.input[self.lo..self.hi], self.lo, self.hi))),
          }
        }
        _ => Some(Err("Unexpected character.")),
      }
    }
    item
  }
}

#[cfg(test)]
mod tests {
  use super::Punctuator::*;
  use super::Token::*;
  use super::{Lexer, Token};

  fn test_next_token(lexer: &mut Lexer, expected: Token) {
    let next = lexer.next();
    assert_eq!(Some(Ok(expected)), next);
  }

  fn test_with_lex(input: &str, expeced: Token) {
    let mut lexer = Lexer::new(&input);
    test_next_token(&mut lexer, expeced);
  }

  fn test_str(input: &str, expeced: &str) {
    let len = expeced.len();
    test_with_lex(input, StringValue(expeced, 1, len + 1))
  }

  #[test]
  fn test_string() {
    test_str(r#""simple""#, r#"simple"#);
    test_str(r#"" white space ""#, r#" white space "#);
    test_str(r#""quote \"""#, r#"quote \""#);
    test_str(r#""slashes \\ \/""#, r#"slashes \\ \/"#);
    test_str(r#""unicode \u1234\u5678\u90AB\uCDEF""#,
             r#"unicode \u1234\u5678\u90AB\uCDEF"#);
  }

  #[test]
  fn test_name() {
    let mut lexer = Lexer::new("     name1    name2   \"simple\" \"  white space   \" other_name");
    test_next_token(&mut lexer, Name("name1", 5, 10));
    test_next_token(&mut lexer, Name("name2", 14, 19));
    test_next_token(&mut lexer, StringValue("simple", 23, 29));
    test_next_token(&mut lexer, StringValue("  white space   ", 32, 48));
    test_next_token(&mut lexer, Name("other_name", 50, 60));
    test_next_token(&mut lexer, Eof(60));
    assert_eq!(None, lexer.next());
  }

  #[test]
  fn test_number() {
    let mut lexer = Lexer::new("-9 0 9 -0 -9 1234 -1234 0.0 -0.0 1.0 -1.012 -0.101 1.0e2 -1.0E2 \
                                0.00e-002 -0.00E+002 9.1E+0");
    test_next_token(&mut lexer, IntValue("-9", 0, 2));
    test_next_token(&mut lexer, IntValue("0", 3, 4));
    test_next_token(&mut lexer, IntValue("9", 5, 6));
    test_next_token(&mut lexer, IntValue("-0", 7, 9));
    test_next_token(&mut lexer, IntValue("-9", 10, 12));
    test_next_token(&mut lexer, IntValue("1234", 13, 17));
    test_next_token(&mut lexer, IntValue("-1234", 18, 23));
    test_next_token(&mut lexer, FloatValue("0.0", 24, 27));
    test_next_token(&mut lexer, FloatValue("-0.0", 28, 32));
    test_next_token(&mut lexer, FloatValue("1.0", 33, 36));
    test_next_token(&mut lexer, FloatValue("-1.012", 37, 43));
    test_next_token(&mut lexer, FloatValue("-0.101", 44, 50));
    test_next_token(&mut lexer, FloatValue("1.0e2", 51, 56));
    test_next_token(&mut lexer, FloatValue("-1.0E2", 57, 63));
    test_next_token(&mut lexer, FloatValue("0.00e-002", 64, 73));
    test_next_token(&mut lexer, FloatValue("-0.00E+002", 74, 84));
    test_next_token(&mut lexer, FloatValue("9.1E+0", 85, 91));
    test_next_token(&mut lexer, Eof(91));
    assert_eq!(None, lexer.next());
  }

  #[test]
  fn test_symbol() {
    let mut lexer = Lexer::new("@ !  :$ =    [  {   (|] } )      ...");
    test_next_token(&mut lexer, Punctuator(At, 0));
    test_next_token(&mut lexer, Punctuator(Bang, 2));
    test_next_token(&mut lexer, Punctuator(Colon, 5));
    test_next_token(&mut lexer, Punctuator(Dollar, 6));
    test_next_token(&mut lexer, Punctuator(Equals, 8));
    test_next_token(&mut lexer, Punctuator(LeftBracket, 13));
    test_next_token(&mut lexer, Punctuator(LeftBrace, 16));
    test_next_token(&mut lexer, Punctuator(LeftParen, 20));
    test_next_token(&mut lexer, Punctuator(Pipe, 21));
    test_next_token(&mut lexer, Punctuator(RightBracket, 22));
    test_next_token(&mut lexer, Punctuator(RightBrace, 24));
    test_next_token(&mut lexer, Punctuator(RightParen, 26));
    test_next_token(&mut lexer, Punctuator(Spread(33), 35));
    test_next_token(&mut lexer, Eof(36));
    assert_eq!(None, lexer.next());
  }
}
<commit_msg>feat: Punctuator has lo and hi, again<commit_after>
use std::str::CharIndices;
use std::iter::Peekable;

#[derive(PartialEq, Debug)]
pub enum Token<'a> {
  Eof,
  Punctuator(Punctuator, usize, usize),
  Name(&'a str, usize, usize),
  IntValue(&'a str, usize, usize),
  FloatValue(&'a str, usize, usize),
  StringValue(&'a str, usize, usize),
}

#[derive(PartialEq, Debug)]
pub enum Punctuator {
  Bang,
  Dollar,
  LeftParen,
  RightParen,
  Spread,
  Colon,
  Equals,
  At,
  LeftBracket,
  RightBracket,
  LeftBrace,
  RightBrace,
  Pipe,
}

macro_rules! take {
  ($lexer: ident, $($p: pat)|*) => ({
    let mut is_taken = false;
    if let Some(&(p, c)) = $lexer.iter.peek() {
      match c {
        $(
          $p => {
            $lexer.iter.next();
            $lexer.hi = p;
            is_taken = true;
          }
        )*
        _ => {}
      }
    }
    is_taken
  });
}

macro_rules! take_while {
  ($lexer: ident, $($p: pat)|*) => ({
    let mut is_taken = false;
    while let Some(&(p, c)) = $lexer.iter.peek() {
      match c {
        $(
          $p => {
            $lexer.iter.next();
            is_taken = true;
          }
        )*
        _ => {
          $lexer.hi = p;
          break;
        }
      }
    }
    is_taken
  });
}

macro_rules! take_while_not {
  ($lexer: ident, $($p: pat)|*) => ({
    let mut is_taken = false;
    while let Some(&(p, c)) = $lexer.iter.peek() {
      match c {
        $(
          $p => {
            $lexer.hi = p;
            break;
          }
        )*
        _ => {
            $lexer.iter.next();
            is_taken = true;
        }
      }
    }
    is_taken
  });
}

macro_rules! peek {
  ($lexer: ident, $($p: pat)|*) => ({
    let mut is_match = false;
    if let Some(&(_, c)) = $lexer.iter.peek() {
      match c {
        $(
          $p => {
            is_match = true;
          }
        )*
        _ => {}
      }
    }
    is_match
  });
}

macro_rules! take_eof {
  ($lexer: ident) => ({
    if $lexer.iter.peek().is_none() {
      $lexer.hi = $lexer.input_len;
      true
    } else {
      false
    }
  });
}

pub struct Lexer<'a> {
  input: &'a str,
  input_len: usize,
  iter: Peekable<CharIndices<'a>>,
  is_eof: bool,
  lo: usize,
  hi: usize,
}

impl<'a> Lexer<'a> {
  pub fn new(input: &'a str) -> Lexer<'a> {
    Lexer {
      input: input,
      input_len: input.len(),
      iter: input.char_indices().peekable(),
      is_eof: false,
      lo: 0,
      hi: 0,
    }
  }
}

fn skip_ignored_token(lexer: &mut Lexer) {
  loop {
    if take!(lexer, '#') {
      take_while_not!(lexer, '\r' | '\n');
    }
    if !take_while!(lexer, '\u{feff}' | ' ' | '\t' | '\r' | '\n' | ',') {
      break;
    }
  }
}

fn scan_spread(lexer: &mut Lexer) -> bool {
  if take!(lexer, '.') {
    if take!(lexer, '.') {
      if take!(lexer, '.') {
        return true;
      }
    }
  }
  return false;
}

fn scan_name(lexer: &mut Lexer) {
  take_while!(lexer, '_' | 'a'...'z' | 'A'...'Z' | '0'...'9');
  take_eof!(lexer);
}

// Return the raw string without unescape character
fn scan_string(lexer: &mut Lexer) -> (bool, bool) {
  let mut is_bad_escape = false;
  let mut is_terminated = false;
  if take!(lexer, '"') {
    lexer.lo = lexer.lo + 1;
  }
  loop {
    if take!(lexer, '\\') {
      if !take!(lexer, '\\' | '"' | 'b' | 'f' | 'n' | 'r' | 't' | '/' | 'u') {
        is_bad_escape = true;
        break;
      }
    }
    take_while_not!(lexer, '"' | '\\');
    if take!(lexer, '"') {
      is_terminated = true;
      break;
    }
  }
  (is_terminated, is_bad_escape)
}

// IntValue:   (+|-)?(0|[1-9][0-9]*)
// FloatValue: (+|-)?(0|[1-9][0-9]*)(\.[0-9]+)?((E|e)(+|-)?[0-9]+)?
fn scan_number(lexer: &mut Lexer) -> (bool, bool) {
  let mut is_float = false;
  take!(lexer, '+' | '-');
  // Integer part
  if take!(lexer, '1'...'9') {
    take_while!(lexer, '0'...'9');
    if take!(lexer, ' ' | '\t' | '\r' | '\n' | ',') || take_eof!(lexer) {
      return (true, is_float);
    }
  } else if take!(lexer, '0') {
    if take!(lexer, ' ' | '\t' | '\r' | '\n' | ',') || take_eof!(lexer) {
      return (true, is_float);
    }
  }
  // Fractional part
  if take!(lexer, '.') {
    if !take_while!(lexer, '0'...'9') {
      return (false, is_float);
    }
    is_float = true;
  }
  // Exponent part
  if !is_float && peek!(lexer, 'E' | 'e') {
    return (false, is_float);
  }
  if take!(lexer, 'E' | 'e') {
    if take!(lexer, '+' | '-') {
      if !take_while!(lexer, '0'...'9') {
        return (false, is_float);
      }
    } else {
      take_while!(lexer, '0'...'9');
    }
  }

  if take!(lexer, ' ' | '\t' | '\r' | '\n' | ',') || take_eof!(lexer) {
    return (true, is_float);
  }
  return (false, is_float);
}

impl<'a> Iterator for Lexer<'a> {
  type Item = Result<Token<'a>, &'a str>;
  fn next(&mut self) -> Option<Result<Token<'a>, &'a str>> {
    use self::Token::*;
    use self::Punctuator::*;

    if !self.is_eof && take_eof!(self) {
      self.is_eof = true;
      return Some(Ok(Token::Eof));
    }
    if self.is_eof {
      return None;
    }
    skip_ignored_token(self);

    let mut item = None;
    if let Some(&(p, c)) = self.iter.peek() {
      self.lo = p;
      self.hi = p;
      item = match c {
        '!' => {
          self.iter.next();
          Some(Ok(Punctuator(Bang, self.lo, self.hi)))
        }
        '$' => {
          self.iter.next();
          Some(Ok(Punctuator(Dollar, self.lo, self.hi)))
        }
        '(' => {
          self.iter.next();
          Some(Ok(Punctuator(LeftParen, self.lo, self.hi)))
        }
        ')' => {
          self.iter.next();
          Some(Ok(Punctuator(RightParen, self.lo, self.hi)))
        }
        ':' => {
          self.iter.next();
          Some(Ok(Punctuator(Colon, self.lo, self.hi)))
        }
        '=' => {
          self.iter.next();
          Some(Ok(Punctuator(Equals, self.lo, self.hi)))
        }
        '@' => {
          self.iter.next();
          Some(Ok(Punctuator(At, self.lo, self.hi)))
        }
        '[' => {
          self.iter.next();
          Some(Ok(Punctuator(LeftBracket, self.lo, self.hi)))
        }
        ']' => {
          self.iter.next();
          Some(Ok(Punctuator(RightBracket, self.lo, self.hi)))
        }
        '{' => {
          self.iter.next();
          Some(Ok(Punctuator(LeftBrace, self.lo, self.hi)))
        }
        '}' => {
          self.iter.next();
          Some(Ok(Punctuator(RightBrace, self.lo, self.hi)))
        }
        '|' => {
          self.iter.next();
          Some(Ok(Punctuator(Pipe, self.lo, self.hi)))
        }
        '.' => {
          if scan_spread(self) {
            Some(Ok(Punctuator(Spread, self.lo, self.hi)))
          } else {
            Some(Err("Unexpected character."))
          }
        }
        '_' | 'a'...'z' | 'A'...'Z' => {
          scan_name(self);
          Some(Ok(Name(&self.input[self.lo..self.hi], self.lo, self.hi)))
        }
        '-' | '0'...'9' => {
          match scan_number(self) {
            (false, false) => Some(Err("Invalid integer number.")),
            (false, true) => Some(Err("Invalid float number.")),
            (true, false) => Some(Ok(IntValue(&self.input[self.lo..self.hi], self.lo, self.hi))),
            (true, true) => Some(Ok(FloatValue(&self.input[self.lo..self.hi], self.lo, self.hi))),
          }
        }
        '"' => {
          match scan_string(self) {
            (false, _) => Some(Err("Unterminated string.")),
            (_, true) => Some(Err("Bad character escape sequence.")),
            _ => Some(Ok(StringValue(&self.input[self.lo..self.hi], self.lo, self.hi))),
          }
        }
        _ => Some(Err("Unexpected character.")),
      }
    }
    item
  }
}

#[cfg(test)]
mod tests {
  use super::Punctuator::*;
  use super::Token::*;
  use super::{Lexer, Token};

  fn test_next_token(lexer: &mut Lexer, expected: Token) {
    let next = lexer.next();
    assert_eq!(Some(Ok(expected)), next);
  }

  fn test_with_lex(input: &str, expeced: Token) {
    let mut lexer = Lexer::new(&input);
    test_next_token(&mut lexer, expeced);
  }

  fn test_str(input: &str, expeced: &str) {
    let len = expeced.len();
    test_with_lex(input, StringValue(expeced, 1, len + 1))
  }

  #[test]
  fn test_string() {
    test_str(r#""simple""#, r#"simple"#);
    test_str(r#"" white space ""#, r#" white space "#);
    test_str(r#""quote \"""#, r#"quote \""#);
    test_str(r#""slashes \\ \/""#, r#"slashes \\ \/"#);
    test_str(r#""unicode \u1234\u5678\u90AB\uCDEF""#,
             r#"unicode \u1234\u5678\u90AB\uCDEF"#);
  }

  #[test]
  fn test_name() {
    let mut lexer = Lexer::new("     name1    name2   \"simple\" \"  white space   \" other_name");
    test_next_token(&mut lexer, Name("name1", 5, 10));
    test_next_token(&mut lexer, Name("name2", 14, 19));
    test_next_token(&mut lexer, StringValue("simple", 23, 29));
    test_next_token(&mut lexer, StringValue("  white space   ", 32, 48));
    test_next_token(&mut lexer, Name("other_name", 50, 60));
    test_next_token(&mut lexer, Eof);
    assert_eq!(None, lexer.next());
  }

  #[test]
  fn test_number() {
    let mut lexer = Lexer::new("-9 0 9 -0 -9 1234 -1234 0.0 -0.0 1.0 -1.012 -0.101 1.0e2 -1.0E2 \
                                0.00e-002 -0.00E+002 9.1E+0");
    test_next_token(&mut lexer, IntValue("-9", 0, 2));
    test_next_token(&mut lexer, IntValue("0", 3, 4));
    test_next_token(&mut lexer, IntValue("9", 5, 6));
    test_next_token(&mut lexer, IntValue("-0", 7, 9));
    test_next_token(&mut lexer, IntValue("-9", 10, 12));
    test_next_token(&mut lexer, IntValue("1234", 13, 17));
    test_next_token(&mut lexer, IntValue("-1234", 18, 23));
    test_next_token(&mut lexer, FloatValue("0.0", 24, 27));
    test_next_token(&mut lexer, FloatValue("-0.0", 28, 32));
    test_next_token(&mut lexer, FloatValue("1.0", 33, 36));
    test_next_token(&mut lexer, FloatValue("-1.012", 37, 43));
    test_next_token(&mut lexer, FloatValue("-0.101", 44, 50));
    test_next_token(&mut lexer, FloatValue("1.0e2", 51, 56));
    test_next_token(&mut lexer, FloatValue("-1.0E2", 57, 63));
    test_next_token(&mut lexer, FloatValue("0.00e-002", 64, 73));
    test_next_token(&mut lexer, FloatValue("-0.00E+002", 74, 84));
    test_next_token(&mut lexer, FloatValue("9.1E+0", 85, 91));
    test_next_token(&mut lexer, Eof);
    assert_eq!(None, lexer.next());
  }

  #[test]
  fn test_symbol() {
    let mut lexer = Lexer::new("@ !  :$ =    [  {   (|] } )      ...");
    test_next_token(&mut lexer, Punctuator(At, 0, 0));
    test_next_token(&mut lexer, Punctuator(Bang, 2, 2));
    test_next_token(&mut lexer, Punctuator(Colon, 5, 5));
    test_next_token(&mut lexer, Punctuator(Dollar, 6, 6));
    test_next_token(&mut lexer, Punctuator(Equals, 8, 8));
    test_next_token(&mut lexer, Punctuator(LeftBracket, 13, 13));
    test_next_token(&mut lexer, Punctuator(LeftBrace, 16, 16));
    test_next_token(&mut lexer, Punctuator(LeftParen, 20, 20));
    test_next_token(&mut lexer, Punctuator(Pipe, 21, 21));
    test_next_token(&mut lexer, Punctuator(RightBracket, 22, 22));
    test_next_token(&mut lexer, Punctuator(RightBrace, 24, 24));
    test_next_token(&mut lexer, Punctuator(RightParen, 26, 26));
    test_next_token(&mut lexer, Punctuator(Spread, 33, 35));
    test_next_token(&mut lexer, Eof);
    assert_eq!(None, lexer.next());
  }
}
<|endoftext|>


<commit_before>use crate::result::*;
use crate::sign::Signature;
use chrono::Utc;
use rand::{thread_rng, Rng};
use reqwest;
use reqwest::header::HeaderMap;
pub use reqwest::{Method, Request, RequestBuilder, Response, StatusCode, Url};
use std::sync::RwLock;
use std::time::{Duration, Instant};

const BASE_URL: &'static str = "https://marketplace.walmartapis.com";

#[derive(Debug, Clone, Copy)]
pub enum WalmartMarketplace {
  USA,
  Canada,
}

pub enum WalmartCredential {
  TokenApi {
    client_id: String,
    client_secret: String,
  },
  Signature {
    channel_type: String,
    consumer_id: String,
    private_key: String,
  },
}

enum AuthState {
  TokenApi {
    client_id: String,
    client_secret: String,
    bearer_token: RwLock<Option<BearerToken>>,
  },
  Signature {
    channel_type: String,
    signature: Signature,
  },
}

struct BearerToken {
  access_token: String,
  expires_at: Instant,
}

pub trait ExtendUrlParams {
  fn extend_url_params(self, url: &mut Url);
}

impl ExtendUrlParams for () {
  fn extend_url_params(self, _: &mut Url) {}
}

impl<'a> ExtendUrlParams for &'a str {
  fn extend_url_params(self, url: &mut Url) {
    url.set_query(Some(self));
  }
}

impl<'a> ExtendUrlParams for String {
  fn extend_url_params(self, url: &mut Url) {
    if !self.is_empty() {
      url.set_query(Some(self.as_ref()));
    }
  }
}

impl<T1: AsRef<str>, T2: AsRef<str>> ExtendUrlParams for Vec<(T1, T2)> {
  fn extend_url_params(self, url: &mut Url) {
    url.query_pairs_mut().extend_pairs(self);
  }
}

pub struct Client {
  marketplace: WalmartMarketplace,
  base_url: Url,
  auth_state: AuthState,
  http: reqwest::Client,
}

impl Client {
  pub fn new(
    marketplace: WalmartMarketplace,
    credential: WalmartCredential,
  ) -> WalmartResult<Client> {
    let http = reqwest::Client::new();
    Client::with_http_client(marketplace, credential, http)
  }

  pub fn with_http_client(
    marketplace: WalmartMarketplace,
    credential: WalmartCredential,
    http: reqwest::Client,
  ) -> WalmartResult<Client> {
    Ok(Client {
      marketplace,
      base_url: Url::parse(BASE_URL)?,
      auth_state: match credential {
        WalmartCredential::Signature {
          consumer_id,
          private_key,
          channel_type,
        } => AuthState::Signature {
          signature: Signature::new(&consumer_id, &private_key)?,
          channel_type,
        },
        WalmartCredential::TokenApi {
          client_id,
          client_secret,
        } => AuthState::TokenApi {
          client_id,
          client_secret,
          bearer_token: RwLock::new(None),
        },
      },
      http,
    })
  }

  fn request<P>(&self, method: Method, path: &str, params: P) -> WalmartResult<RequestBuilder>
  where
    P: ExtendUrlParams,
  {
    let mut url = match self.marketplace {
      WalmartMarketplace::USA => self.base_url.join(path)?,
      WalmartMarketplace::Canada => {
        // add `ca` to url
        let path = path
          .split('/')
          .enumerate()
          .map(|(i, seg)| {
            if i == 1 {
              format!("{}/ca", seg)
            } else {
              seg.to_string()
            }
          })
          .collect::<Vec<String>>()
          .join("/");
        self.base_url.join(&path)?
      }
    };
    params.extend_url_params(&mut url);

    debug!("request: method = {}, url = {}", method, url);

    let timestamp = Utc::now();
    let timestamp = timestamp.timestamp() * 1000 + timestamp.timestamp_subsec_millis() as i64;

    let mut req = self.http.request(method.clone(), url.as_str());

    let mut headers = HeaderMap::new();
    let rid: String = thread_rng().gen_ascii_chars().take(10).collect();
    headers.insert("WM_SVC.NAME", "Walmart Marketplace".parse()?);
    headers.insert("WM_QOS.CORRELATION_ID", rid.parse()?);
    headers.insert("WM_SEC.TIMESTAMP", timestamp.to_string().parse()?);

    match self.auth_state {
      AuthState::Signature {
        ref channel_type,
        ref signature,
      } => {
        let sign = signature.sign(url.as_str(), method.clone(), timestamp)?;
        debug!("auth: Signature: sign = {}", sign);
        headers.insert("WM_CONSUMER.CHANNEL.TYPE", channel_type.parse()?);
        headers.insert("WM_CONSUMER.ID", signature.consumer_id().parse()?);
        headers.insert("WM_SEC.AUTH_SIGNATURE", sign.parse()?);
      }
      AuthState::TokenApi {
        ref client_id,
        ref client_secret,
        ..
      } => {
        let access_token = self.get_access_token(false)?;
        debug!("auth: TokenApi: access_token = {}", access_token);
        headers.insert("WM_SEC.ACCESS_TOKEN", access_token.parse()?);
        req = req.basic_auth(client_id, Some(client_secret));
      }
    }
    let req = req.headers(headers);
    Ok(req)
  }

  fn clear_access_token(&self) {
    match self.auth_state {
      AuthState::TokenApi {
        ref bearer_token, ..
      } => {
        bearer_token.write().unwrap().take();
      }
      _ => {}
    }
  }

  fn get_access_token(&self, force_renew: bool) -> WalmartResult<String> {
    use std::collections::HashMap;
    #[derive(Debug, Deserialize)]
    struct WalmartBearerToken {
      access_token: String,
      token_type: String,
      expires_in: u64,
    }

    match self.auth_state {
      AuthState::TokenApi {
        ref bearer_token,
        ref client_id,
        ref client_secret,
      } => {
        if !force_renew {
          let lock = bearer_token.read().unwrap();
          if let Some(ref token) = lock.as_ref() {
            if token.expires_at - Instant::now() > Duration::from_secs(120) {
              return Ok(token.access_token.clone());
            }
          }
          drop(lock);
        }

        let mut form = HashMap::new();
        form.insert("grant_type", "client_credentials");

        let mut headers = HeaderMap::new();
        let rid: String = thread_rng().gen_ascii_chars().take(10).collect();
        headers.insert("WM_SVC.NAME", "Walmart Marketplace".parse()?);
        headers.insert("WM_QOS.CORRELATION_ID", rid.parse()?);
        headers.insert("Accept", "application/json".parse()?);

        let mut res = self
          .http
          .request(Method::POST, &format!("{}/v3/token", BASE_URL))
          .headers(headers)
          .form(&form)
          .basic_auth(client_id, Some(client_secret))
          .send()?;

        let token: WalmartBearerToken = res.json()?;
        let access_token = token.access_token.clone();

        if token.token_type != "Bearer" {
          return Err(WalmartError::Msg(format!(
            "unsupported token type: {}",
            token.token_type
          )));
        }

        debug!("token: {:#?}", token);

        let mut lock = bearer_token.write().unwrap();
        std::mem::replace(
          &mut lock as &mut Option<_>,
          Some(BearerToken {
            access_token: token.access_token,
            expires_at: Instant::now() + Duration::from_secs(token.expires_in),
          }),
        );

        Ok(access_token)
      }
      _ => {
        return Err(WalmartError::Msg(
          "cannot get bearer with Signature Authentication".to_string(),
        ))
      }
    }
  }

  pub fn request_json<P>(
    &self,
    method: Method,
    path: &str,
    params: P,
  ) -> WalmartResult<RequestBuilder>
  where
    P: ExtendUrlParams,
  {
    use reqwest::header::{HeaderValue, ACCEPT};

    self
      .request(method, path, params)
      .map(|req| req.header(ACCEPT, HeaderValue::from_static("application/json")))
  }

  pub fn request_xml<P>(
    &self,
    method: Method,
    path: &str,
    params: P,
  ) -> WalmartResult<RequestBuilder>
  where
    P: ExtendUrlParams,
  {
    use reqwest::header::{HeaderValue, ACCEPT};

    self
      .request(method, path, params)
      .map(|req| req.header(ACCEPT, HeaderValue::from_static("application/xml")))
  }

  pub fn send(&self, req: RequestBuilder) -> WalmartResult<Response> {
    match req.send() {
      Ok(res) => {
        if res.status() == StatusCode::UNAUTHORIZED {
          self.clear_access_token();
        }
        Ok(res)
      }
      Err(err) => Err(err.into()),
    }
  }

  pub(crate) fn get_marketplace(&self) -> WalmartMarketplace {
    self.marketplace
  }
}

// #[cfg(test)]
// mod tests {
//   use super::*;
//   use dotenv::dotenv;
//   use std::env;

//   #[test]
//   fn client() {
//     use std::io::Read;
//     dotenv().ok();

//     let client = Client::new(&env::var("WALMART_CONSUMER_ID").unwrap(), &env::var("WALMART_PRIVATE_KEY").unwrap()).unwrap();
//     let mut res = client.request_json(Method::Get, "/v3/feeds/117E39F0B7654B08A059457FB6E803FF@AQYBAAA", ()).unwrap().send().unwrap();
//     println!("status: {}", res.status());
//     let mut json = String::new();
//     res.read_to_string(&mut json).unwrap();
//     println!("body: {}", json);
//     {
//       use std::fs::File;
//       use std::io::Write;
//       let mut f = File::create("samples/get_feed_aand_item_status.json").unwrap();
//       write!(&mut f, "{}", json).unwrap()
//     }
//   }
// }
<commit_msg>fix: Fix Client::get_access_token panic.<commit_after>use crate::result::*;
use crate::sign::Signature;
use chrono::Utc;
use rand::{thread_rng, Rng};
use reqwest;
use reqwest::header::HeaderMap;
pub use reqwest::{Method, Request, RequestBuilder, Response, StatusCode, Url};
use std::sync::RwLock;
use std::time::{Duration, Instant};

const BASE_URL: &'static str = "https://marketplace.walmartapis.com";

#[derive(Debug, Clone, Copy)]
pub enum WalmartMarketplace {
  USA,
  Canada,
}

pub enum WalmartCredential {
  TokenApi {
    client_id: String,
    client_secret: String,
  },
  Signature {
    channel_type: String,
    consumer_id: String,
    private_key: String,
  },
}

enum AuthState {
  TokenApi {
    client_id: String,
    client_secret: String,
    bearer_token: RwLock<Option<BearerToken>>,
  },
  Signature {
    channel_type: String,
    signature: Signature,
  },
}

struct BearerToken {
  access_token: String,
  expires_at: Instant,
}

pub trait ExtendUrlParams {
  fn extend_url_params(self, url: &mut Url);
}

impl ExtendUrlParams for () {
  fn extend_url_params(self, _: &mut Url) {}
}

impl<'a> ExtendUrlParams for &'a str {
  fn extend_url_params(self, url: &mut Url) {
    url.set_query(Some(self));
  }
}

impl<'a> ExtendUrlParams for String {
  fn extend_url_params(self, url: &mut Url) {
    if !self.is_empty() {
      url.set_query(Some(self.as_ref()));
    }
  }
}

impl<T1: AsRef<str>, T2: AsRef<str>> ExtendUrlParams for Vec<(T1, T2)> {
  fn extend_url_params(self, url: &mut Url) {
    url.query_pairs_mut().extend_pairs(self);
  }
}

pub struct Client {
  marketplace: WalmartMarketplace,
  base_url: Url,
  auth_state: AuthState,
  http: reqwest::Client,
}

impl Client {
  pub fn new(
    marketplace: WalmartMarketplace,
    credential: WalmartCredential,
  ) -> WalmartResult<Client> {
    let http = reqwest::Client::new();
    Client::with_http_client(marketplace, credential, http)
  }

  pub fn with_http_client(
    marketplace: WalmartMarketplace,
    credential: WalmartCredential,
    http: reqwest::Client,
  ) -> WalmartResult<Client> {
    Ok(Client {
      marketplace,
      base_url: Url::parse(BASE_URL)?,
      auth_state: match credential {
        WalmartCredential::Signature {
          consumer_id,
          private_key,
          channel_type,
        } => AuthState::Signature {
          signature: Signature::new(&consumer_id, &private_key)?,
          channel_type,
        },
        WalmartCredential::TokenApi {
          client_id,
          client_secret,
        } => AuthState::TokenApi {
          client_id,
          client_secret,
          bearer_token: RwLock::new(None),
        },
      },
      http,
    })
  }

  fn request<P>(&self, method: Method, path: &str, params: P) -> WalmartResult<RequestBuilder>
  where
    P: ExtendUrlParams,
  {
    let mut url = match self.marketplace {
      WalmartMarketplace::USA => self.base_url.join(path)?,
      WalmartMarketplace::Canada => {
        // add `ca` to url
        let path = path
          .split('/')
          .enumerate()
          .map(|(i, seg)| {
            if i == 1 {
              format!("{}/ca", seg)
            } else {
              seg.to_string()
            }
          })
          .collect::<Vec<String>>()
          .join("/");
        self.base_url.join(&path)?
      }
    };
    params.extend_url_params(&mut url);

    debug!("request: method = {}, url = {}", method, url);

    let timestamp = Utc::now();
    let timestamp = timestamp.timestamp() * 1000 + timestamp.timestamp_subsec_millis() as i64;

    let mut req = self.http.request(method.clone(), url.as_str());

    let mut headers = HeaderMap::new();
    let rid: String = thread_rng().gen_ascii_chars().take(10).collect();
    headers.insert("WM_SVC.NAME", "Walmart Marketplace".parse()?);
    headers.insert("WM_QOS.CORRELATION_ID", rid.parse()?);
    headers.insert("WM_SEC.TIMESTAMP", timestamp.to_string().parse()?);

    match self.auth_state {
      AuthState::Signature {
        ref channel_type,
        ref signature,
      } => {
        let sign = signature.sign(url.as_str(), method.clone(), timestamp)?;
        debug!("auth: Signature: sign = {}", sign);
        headers.insert("WM_CONSUMER.CHANNEL.TYPE", channel_type.parse()?);
        headers.insert("WM_CONSUMER.ID", signature.consumer_id().parse()?);
        headers.insert("WM_SEC.AUTH_SIGNATURE", sign.parse()?);
      }
      AuthState::TokenApi {
        ref client_id,
        ref client_secret,
        ..
      } => {
        let access_token = self.get_access_token(false)?;
        debug!("auth: TokenApi: access_token = {}", access_token);
        headers.insert("WM_SEC.ACCESS_TOKEN", access_token.parse()?);
        req = req.basic_auth(client_id, Some(client_secret));
      }
    }
    let req = req.headers(headers);
    Ok(req)
  }

  fn clear_access_token(&self) {
    match self.auth_state {
      AuthState::TokenApi {
        ref bearer_token, ..
      } => {
        bearer_token.write().unwrap().take();
      }
      _ => {}
    }
  }

  fn get_access_token(&self, force_renew: bool) -> WalmartResult<String> {
    use std::collections::HashMap;
    #[derive(Debug, Deserialize)]
    struct WalmartBearerToken {
      access_token: String,
      token_type: String,
      expires_in: u64,
    }

    match self.auth_state {
      AuthState::TokenApi {
        ref bearer_token,
        ref client_id,
        ref client_secret,
      } => {
        if !force_renew {
          let lock = bearer_token.read().unwrap();
          if let Some(ref token) = lock.as_ref() {
            if token.expires_at.saturating_duration_since(Instant::now()) > Duration::from_secs(120)
            {
              return Ok(token.access_token.clone());
            }
          }
          drop(lock);
        }

        let mut form = HashMap::new();
        form.insert("grant_type", "client_credentials");

        let mut headers = HeaderMap::new();
        let rid: String = thread_rng().gen_ascii_chars().take(10).collect();
        headers.insert("WM_SVC.NAME", "Walmart Marketplace".parse()?);
        headers.insert("WM_QOS.CORRELATION_ID", rid.parse()?);
        headers.insert("Accept", "application/json".parse()?);

        let mut res = self
          .http
          .request(Method::POST, &format!("{}/v3/token", BASE_URL))
          .headers(headers)
          .form(&form)
          .basic_auth(client_id, Some(client_secret))
          .send()?;

        let token: WalmartBearerToken = res.json()?;
        let access_token = token.access_token.clone();

        if token.token_type != "Bearer" {
          return Err(WalmartError::Msg(format!(
            "unsupported token type: {}",
            token.token_type
          )));
        }

        debug!("token: {:#?}", token);

        let mut lock = bearer_token.write().unwrap();
        std::mem::replace(
          &mut lock as &mut Option<_>,
          Some(BearerToken {
            access_token: token.access_token,
            expires_at: Instant::now() + Duration::from_secs(token.expires_in),
          }),
        );

        Ok(access_token)
      }
      _ => {
        return Err(WalmartError::Msg(
          "cannot get bearer with Signature Authentication".to_string(),
        ))
      }
    }
  }

  pub fn request_json<P>(
    &self,
    method: Method,
    path: &str,
    params: P,
  ) -> WalmartResult<RequestBuilder>
  where
    P: ExtendUrlParams,
  {
    use reqwest::header::{HeaderValue, ACCEPT};

    self
      .request(method, path, params)
      .map(|req| req.header(ACCEPT, HeaderValue::from_static("application/json")))
  }

  pub fn request_xml<P>(
    &self,
    method: Method,
    path: &str,
    params: P,
  ) -> WalmartResult<RequestBuilder>
  where
    P: ExtendUrlParams,
  {
    use reqwest::header::{HeaderValue, ACCEPT};

    self
      .request(method, path, params)
      .map(|req| req.header(ACCEPT, HeaderValue::from_static("application/xml")))
  }

  pub fn send(&self, req: RequestBuilder) -> WalmartResult<Response> {
    match req.send() {
      Ok(res) => {
        if res.status() == StatusCode::UNAUTHORIZED {
          self.clear_access_token();
        }
        Ok(res)
      }
      Err(err) => Err(err.into()),
    }
  }

  pub(crate) fn get_marketplace(&self) -> WalmartMarketplace {
    self.marketplace
  }
}

// #[cfg(test)]
// mod tests {
//   use super::*;
//   use dotenv::dotenv;
//   use std::env;

//   #[test]
//   fn client() {
//     use std::io::Read;
//     dotenv().ok();

//     let client = Client::new(&env::var("WALMART_CONSUMER_ID").unwrap(), &env::var("WALMART_PRIVATE_KEY").unwrap()).unwrap();
//     let mut res = client.request_json(Method::Get, "/v3/feeds/117E39F0B7654B08A059457FB6E803FF@AQYBAAA", ()).unwrap().send().unwrap();
//     println!("status: {}", res.status());
//     let mut json = String::new();
//     res.read_to_string(&mut json).unwrap();
//     println!("body: {}", json);
//     {
//       use std::fs::File;
//       use std::io::Write;
//       let mut f = File::create("samples/get_feed_aand_item_status.json").unwrap();
//       write!(&mut f, "{}", json).unwrap()
//     }
//   }
// }
<|endoftext|>


<commit_before>//#![feature(macro_rules)]
//use std::any::{Any, TypeId};
use std::cmp::Ordering;
use std::collections::HashMap;
use std::ops::{Add, Sub};

/*
{start: }
*/
type Matrix<T> = HashMap<&'static str, HashMap<&'static str, Option<T>>>;

struct DistanceMatrix<T> {
    length: usize,
    inner_max: Matrix<T>,
}

impl<T> DistanceMatrix<T>
where
    T: Add + Sub + Ord,
{
    fn new(input: Matrix<T>) -> Self {
        DistanceMatrix {
            length: input.len(),
            inner_max: input,
        }
    }

    fn new_dijkstra_on(&mut self, end: &'static str) {}
}

//struct nil;
// type check function template
/*
fn is_nil<T: ?Sized + Any>(o: &T) -> bool {
    TypeId::of::<i32>() == TypeId::of::<T>()
}*/

// new_distance_matrix!()
// cannot check type in macro body. so do not need this macro
macro_rules! new_distance_matrix {
    ($([$($x:expr),*]),*) => {{
        vec![$(vec![$(
            if is_nil($x) {
                None
            }else{
                Some($x)
            }
        ),*]),*]
    }};
    ($($x:expr),*) => {{

    }};
}

fn main() {
    let test0: Matrix<u32> = [(
        "a",
        [("b", Some(6)), ("c", Some(3))] as HashMap<&'static str, Option<u32>>,
    )];
    println!("{:?}", test0);
    //println!("{:?}", new_distance_matrix![[1, 2, 3], [4, 5, 6], [7, 8]]);
}
<commit_msg>a bit<commit_after>//#![feature(macro_rules)]
//use std::any::{Any, TypeId};
use std::cmp::{Ordering, PartialOrd};
use std::collections::{HashMap, HashSet};
use std::ops::{Add, Deref, Sub};

/*
{start: {to: {distance, path}}}
 */
type Detail<Value> = (Value, Vec<&'static str>);
type Matrix<Value> = HashMap<&'static str, HashMap<&'static str, Detail<Value>>>;

/*
impl<T> Deref for Detail<T> {
    type Target = (T, Vec<&'static str>);

    fn deref(&self) -> &Self {
        &(*self.0, *self.1)
    }
}*/

struct DistanceMatrix<T> {
    length: usize,
    inner_max: Matrix<T>,
}

impl<T> DistanceMatrix<T>
where
    T: Add<Output = T> + Sub + Ord + Copy,
{
    fn new(input: Matrix<T>) -> Self {
        DistanceMatrix {
            length: input.len(),
            inner_max: input,
        }
    }

    fn path(&mut self, start: &'static str, to: &'static str) -> Option<Vec<&'static str>> {
        match self.inner_max.get(start) {
            Some(data) => match data.get(to) {
                Some((_, p)) => return Some(p.clone()),
                None => return None,
            },
            None => return None,
        }
    }

    fn val_between(&mut self, start: &'static str, to: &'static str) -> Option<T> {
        match self.inner_max.get(start) {
            Some(data) => match data.get(to) {
                Some((v, _)) => return Some(*v),
                None => return None,
            },
            None => return None,
        }
    }

    fn new_dijkstra_on(&mut self, start: &'static str) {
        let mut all_notes = self
            .inner_max
            .keys()
            .map(|x| *x)
            .collect::<HashSet<&'static str>>();
        all_notes.remove(start);

        let mut smallest_nodes_connected: (&'static str, Detail<T>);
        {
            let this_nodes_data = match self.inner_max.get(start) {
                Some(data) => data.clone(),
                None => {
                    println!("{}", "do not have this node");
                    return;
                }
            };

            // find shortest node of start
            let (a, b) = this_nodes_data
                .iter()
                .map(|x| x)
                .min_by_key(|(_, v)| v.0)
                .unwrap();
            smallest_nodes_connected = (*a, b.clone());
        }

        all_notes.remove(smallest_nodes_connected.0);

        let mut distance_to_now: T;
        let mut path_to_now: Vec<&'static str>;

        println!("{:?}", smallest_nodes_connected.0);
        while (!all_notes.is_empty()) {
            let distance_to_now: T;
            let path_to_now: Vec<&'static str>;

            {
                distance_to_now = self.val_between(start, smallest_nodes_connected.0).unwrap();
                path_to_now = self.path(start, smallest_nodes_connected.0).unwrap();
            }

            let this_nodes_data = match { self.inner_max.get_mut(smallest_nodes_connected.0) } {
                Some(data) => data.clone(),
                None => {
                    println!("{}", "do not have this node");
                    return;
                }
            };

            //update start node
            for (k, v) in this_nodes_data.iter() {
                if let Some(v_old) = self.val_between(start, k) {
                    if (distance_to_now + v.0) < { v_old } {
                        let mut new_path = path_to_now.clone();
                        new_path.push(k);
                        self.inner_max
                            .get_mut(start)
                            .unwrap()
                            .insert(k, (distance_to_now + v.0, new_path));
                    }
                } else {
                    let mut new_path = path_to_now.clone();
                    new_path.push(k);
                    self.inner_max
                        .get_mut(start)
                        .unwrap()
                        .insert(k, (distance_to_now + v.0, new_path));
                }
            }

            all_notes.remove(smallest_nodes_connected.0);
        }
    }
}

//struct nil;
// type check function template
/*
fn is_nil<T: ?Sized + Any>(o: &T) -> bool {
    TypeId::of::<i32>() == TypeId::of::<T>()
}*/

// new_distance_matrix!()
// cannot check type in macro body. so do not need this macro
/*
macro_rules! new_distance_matrix {
    ($([$($x:expr),*]),*) => {{
        vec![$(vec![$(
            if is_nil($x) {
                None
            }else{
                Some($x)
            }
        ),*]),*]
    }};
    ($($x:expr),*) => {{

    }};
}*/

fn main() {
    //let test0: Matrix<u32>;
    //println!("{:?}", test0);
    //println!("{:?}", new_distance_matrix![[1, 2, 3], [4, 5, 6], [7, 8]]);

    let mut testcaseM1: Matrix<u32> = HashMap::new();
    testcaseM1.insert(
        "a",
        [("b", (6 as u32, vec![])), ("c", (3 as u32, vec![]))]
            .iter()
            .cloned()
            .collect(),
    );
    testcaseM1.insert(
        "b",
        [
            ("a", (6 as u32, vec![])),
            ("c", (2 as u32, vec![])),
            ("d", (5 as u32, vec![])),
        ]
            .iter()
            .cloned()
            .collect(),
    );
    println!("{:?}", testcaseM1);

    let mut testcase1: DistanceMatrix<u32>;
    testcase1 = DistanceMatrix {
        length: 4,
        inner_max: testcaseM1,
    };

    //Option<(&&str, &(u32, std::vec::Vec<u32>))>
    println!("{:?}", testcase1.new_dijkstra_on("a"));
}
<|endoftext|>


<commit_before>extern crate chrono;

use self::chrono::{DateTime, Duration, Local};
use self::chrono::offset::TimeZone;

use constant::STORAGE_DIR_NAME;
use error::{CannotHappenError, CliError};
use lib::fs::*;
use lib::io::*;
use std::fs;
use std::io::{Error as IoError, ErrorKind as IoErrorKind};
use std::path::{Path, PathBuf};

pub struct Storage {
    now : DateTime<Local>,
    date: String,
}

impl Storage {
    pub fn path() -> PathBuf {
        path_buf![super::working_dir(), STORAGE_DIR_NAME]
    }

    pub fn exist() -> bool {
        Self::path().is_dir()
    }

    pub fn create() -> Result<(), IoError> {
        print_with_tag(Tag::Info, format!("Create \"{}\" directory", STORAGE_DIR_NAME));

        super::create_essential_dir(Self::path())
    }

    pub fn new() -> Self {
        let now  = Local::now();
        let date = now.format("%Y-%m-%d").to_string();
        Storage { now: now, date: date }
    }

    pub fn path_to_box(&self) -> PathBuf {
        path_buf![Storage::path(), &self.date]
    }

    pub fn path_to_dust_box(&self) -> PathBuf {
        path_buf![self.path_to_box(), "dusts"]
    }

    pub fn create_box(&self) -> Result<(), IoError> {
        print_with_tag(Tag::Info, format!(r#"Create "{}" directory in "{}""#, self.date, STORAGE_DIR_NAME));

        super::create_essential_dir_all(self.path_to_dust_box())
    }

    pub fn squeeze_dusts<P: AsRef<Path>>(&self, paths_to_dust: Vec<P>) -> Result<(), CliError> {
        let path_to_dust_box = self.path_to_dust_box();

        for path_to_dust in &paths_to_dust {
            let path_to_dust = path_to_dust.as_ref();
            let target_file  = try!(path_to_dust.file_name().ok_or(CannotHappenError));
            let target_base  = try!(path_to_dust.parent().ok_or(CannotHappenError));

            let to = path_buf![&path_to_dust_box, target_base];

            let message = format!("Move \"{}\" to \"{}\"", path_to_dust.display(), path_to_dust_box.display());
            print_with_tag(Tag::Info, message);

            try!(super::create_essential_dir_all(&to));

            // forcedly overwrite if the file exists with same name.
            match fs::rename(path_to_dust, path_buf![to, target_file]) {
                Ok(_)  => (),
                Err(e) => match e.kind() {
                    IoErrorKind::PermissionDenied => print_with_tag(Tag::Info, "Interrupted moving file for permission"),
                    _                             => return Err(From::from(e)),
                },
            };
        }

        Ok(())
    }

    pub fn squeeze_empty_dirs_only<P: AsRef<Path>>(&self, paths_to_dir: Vec<P>) -> Result<(), IoError> {
        let path_to_dust_box = self.path_to_dust_box();

        for path_to_dir in &paths_to_dir {
            if is_empty_dir(path_to_dir) {
                let path_to_dir = path_to_dir.as_ref();

                let message = format!("Move empty \"{}\" to \"{}\"", path_to_dir.display(), path_to_dust_box.display());
                print_with_tag(Tag::Info, message);

                match fs::remove_dir(path_to_dir) {
                    Ok(_)  => try!(super::create_essential_dir_all(path_buf![&path_to_dust_box, path_to_dir])),
                    Err(e) => match e.kind() {
                        IoErrorKind::PermissionDenied => print_with_tag(Tag::Info, "Interrupted moving directory for permission"),
                        _                             => return Err(From::from(e)),
                    },
                };
            }
        }

        Ok(())
    }

    pub fn delete_expired_boxes(&self, moratorium: Duration) -> Result<(), IoError> {
        let path_to_storage = Self::path();

        let target_boxes = try!(ls(&path_to_storage))
            .into_iter()
            .filter(|date| match Local.datetime_from_str(format!("{} 00:00:00", date).as_ref(), "%Y-%m-%d %H:%M:%S") {
                Ok(created_date) => created_date + moratorium < self.now,
                Err(_)           => false,
            })
            .map(|dir| path_to_storage.join(dir))
            .collect::<Vec<PathBuf>>();

        for target_box in &target_boxes {
            print_with_tag(Tag::Info, format!("Remove \"{}\"", target_box.display()));
            try!(super::delete_dir_all(target_box));
        };

        Ok(())
    }
}
<commit_msg>[Remedy] Remove duplicated sentences<commit_after>extern crate chrono;

use self::chrono::{DateTime, Duration, Local};
use self::chrono::offset::TimeZone;

use constant::STORAGE_DIR_NAME;
use error::{CannotHappenError, CliError};
use lib::fs::*;
use lib::io::*;
use std::fs;
use std::io::{Error as IoError, ErrorKind as IoErrorKind};
use std::path::{Path, PathBuf};

pub struct Storage {
    now : DateTime<Local>,
    date: String,
}

impl Storage {
    pub fn path() -> PathBuf {
        path_buf![super::working_dir(), STORAGE_DIR_NAME]
    }

    pub fn exist() -> bool {
        Self::path().is_dir()
    }

    pub fn create() -> Result<(), IoError> {
        print_with_tag(Tag::Info, format!("Create \"{}\" directory", STORAGE_DIR_NAME));

        super::create_essential_dir(Self::path())
    }

    pub fn new() -> Self {
        let now  = Local::now();
        let date = now.format("%Y-%m-%d").to_string();
        Storage { now: now, date: date }
    }

    pub fn path_to_box(&self) -> PathBuf {
        path_buf![Storage::path(), &self.date]
    }

    pub fn path_to_dust_box(&self) -> PathBuf {
        path_buf![self.path_to_box(), "dusts"]
    }

    pub fn create_box(&self) -> Result<(), IoError> {
        print_with_tag(Tag::Info, format!(r#"Create "{}" directory in "{}""#, self.date, STORAGE_DIR_NAME));

        super::create_essential_dir_all(self.path_to_dust_box())
    }

    pub fn squeeze_dusts<P: AsRef<Path>>(&self, paths_to_dust: Vec<P>) -> Result<(), CliError> {
        let path_to_dust_box = self.path_to_dust_box();

        print_with_tag(Tag::Info, format!("Move dusts to \"{}\"", path_to_dust_box.display()));

        for path_to_dust in &paths_to_dust {
            let path_to_dust = path_to_dust.as_ref();
            let target_file  = try!(path_to_dust.file_name().ok_or(CannotHappenError));
            let target_base  = try!(path_to_dust.parent().ok_or(CannotHappenError));

            let to = path_buf![&path_to_dust_box, target_base];

            let message = format!("  => \"{}\"", path_to_dust.display());
            print_with_tag(Tag::Info, message);

            try!(super::create_essential_dir_all(&to));

            // forcedly overwrite if the file exists with same name.
            match fs::rename(path_to_dust, path_buf![to, target_file]) {
                Ok(_)  => (),
                Err(e) => match e.kind() {
                    IoErrorKind::PermissionDenied => print_with_tag(Tag::Info, "     Interrupted for permission"),
                    _                             => return Err(From::from(e)),
                },
            };
        }

        Ok(())
    }

    pub fn squeeze_empty_dirs_only<P: AsRef<Path>>(&self, paths_to_dir: Vec<P>) -> Result<(), IoError> {
        let path_to_dust_box = self.path_to_dust_box();

        print_with_tag(Tag::Info, format!("Move empty dirs to \"{}\"", path_to_dust_box.display()));

        for path_to_dir in &paths_to_dir {
            if is_empty_dir(path_to_dir) {
                let path_to_dir = path_to_dir.as_ref();

                let message = format!("  => \"{}\"", path_to_dir.display());
                print_with_tag(Tag::Info, message);

                match fs::remove_dir(path_to_dir) {
                    Ok(_)  => try!(super::create_essential_dir_all(path_buf![&path_to_dust_box, path_to_dir])),
                    Err(e) => match e.kind() {
                        IoErrorKind::PermissionDenied => print_with_tag(Tag::Info, "     Interrupted for permission"),
                        _                             => return Err(From::from(e)),
                    },
                };
            }
        }

        Ok(())
    }

    pub fn delete_expired_boxes(&self, moratorium: Duration) -> Result<(), IoError> {
        let path_to_storage = Self::path();

        print_with_tag(Tag::Info, "Delete expired dusts");

        let target_boxes = try!(ls(&path_to_storage))
            .into_iter()
            .filter(|date| match Local.datetime_from_str(format!("{} 00:00:00", date).as_ref(), "%Y-%m-%d %H:%M:%S") {
                Ok(created_date) => created_date + moratorium < self.now,
                Err(_)           => false,
            })
            .map(|dir| path_to_storage.join(dir))
            .collect::<Vec<PathBuf>>();

        for target_box in &target_boxes {
            print_with_tag(Tag::Info, format!("  => \"{}\"", target_box.display()));
            try!(super::delete_dir_all(target_box));
        };

        Ok(())
    }
}
<|endoftext|>


<commit_before>//! Instruction Set Architectures.
//!
//! The `isa` module provides a `TargetIsa` trait which provides the behavior specialization needed
//! by the ISA-independent code generator. The sub-modules of this module provide definitions for
//! the instruction sets that Cretonne can target. Each sub-module has it's own implementation of
//! `TargetIsa`.
//!
//! # Constructing a `TargetIsa` instance
//!
//! The target ISA is built from the following information:
//!
//! - The name of the target ISA as a string. Cretonne is a cross-compiler, so the ISA to target
//!   can be selected dynamically. Individual ISAs can be left out when Cretonne is compiled, so a
//!   string is used to identify the proper sub-module.
//! - Values for settings that apply to all ISAs. This is represented by a `settings::Flags`
//!   instance.
//! - Values for ISA-specific settings.
//!
//! The `isa::lookup()` function is the main entry point which returns an `isa::Builder`
//! appropriate for the requested ISA:
//!
//! ```
//! use cretonne::settings::{self, Configurable};
//! use cretonne::isa;
//!
//! let shared_builder = settings::builder();
//! let shared_flags = settings::Flags::new(shared_builder);
//!
//! match isa::lookup("riscv") {
//!     None => {
//!         // The RISC-V target ISA is not available.
//!     }
//!     Some(mut isa_builder) => {
//!         isa_builder.set("supports_m", "on");
//!         let isa = isa_builder.finish(shared_flags);
//!     }
//! }
//! ```
//!
//! The configured target ISA trait object is a `Box<TargetIsa>` which can be used for multiple
//! concurrent function compilations.

pub mod riscv;

use settings;
use ir::{Inst, DataFlowGraph};

/// Look for a supported ISA with the given `name`.
/// Return a builder that can create a corresponding `TargetIsa`.
pub fn lookup(name: &str) -> Option<Builder> {
    match name {
        "riscv" => riscv_builder(),
        _ => None,
    }
}

// Make a builder for RISC-V.
fn riscv_builder() -> Option<Builder> {
    Some(riscv::isa_builder())
}

/// Builder for a `TargetIsa`.
/// Modify the ISA-specific settings before creating the `TargetIsa` trait object with `finish`.
pub struct Builder {
    setup: settings::Builder,
    constructor: fn(settings::Flags, settings::Builder) -> Box<TargetIsa>,
}

impl Builder {
    /// Combine the ISA-specific settings with the provided ISA-independent settings and allocate a
    /// fully configured `TargetIsa` trait object.
    pub fn finish(self, shared_flags: settings::Flags) -> Box<TargetIsa> {
        (self.constructor)(shared_flags, self.setup)
    }
}

impl settings::Configurable for Builder {
    fn set(&mut self, name: &str, value: &str) -> settings::Result<()> {
        self.setup.set(name, value)
    }

    fn set_bool(&mut self, name: &str, value: bool) -> settings::Result<()> {
        self.setup.set_bool(name, value)
    }
}

pub trait TargetIsa {
    /// Encode an instruction after determining it is legal.
    ///
    /// If `inst` can legally be encoded in this ISA, produce the corresponding `Encoding` object.
    /// Otherwise, return `None`.
    ///
    /// This is also the main entry point for determining if an instruction is legal.
    fn encode(&self, dfg: &DataFlowGraph, inst: &Inst) -> Option<Encoding>;
}

/// Bits needed to encode an instruction as binary machine code.
///
/// The encoding consists of two parts, both specific to the target ISA: An encoding *recipe*, and
/// encoding *bits*. The recipe determines the native instruction format and the mapping of
/// operands to encoded bits. The encoding bits provide additional information to the recipe,
/// typically parts of the opcode.
pub struct Encoding(u32);

impl Encoding {
    /// Create a new `Encoding` containing `(recipe, bits)`. The `num_bits` parameter is the
    /// ISA-dependent size of `bits`.
    pub fn new(recipe: u32, bits: u32, num_bits: u8) -> Encoding {
        Encoding((recipe << num_bits) | bits)
    }

    /// Split the encoding into two parts: `(recipe, bits)`. Only the target ISA knows how many
    /// bits are in each part.
    pub fn split(&self, num_bits: u8) -> (u32, u32) {
        (self.0 >> num_bits, self.0 & ((1 << num_bits) - 1))
    }
}
<commit_msg>Split the Encoding data type into two u16 values.<commit_after>//! Instruction Set Architectures.
//!
//! The `isa` module provides a `TargetIsa` trait which provides the behavior specialization needed
//! by the ISA-independent code generator. The sub-modules of this module provide definitions for
//! the instruction sets that Cretonne can target. Each sub-module has it's own implementation of
//! `TargetIsa`.
//!
//! # Constructing a `TargetIsa` instance
//!
//! The target ISA is built from the following information:
//!
//! - The name of the target ISA as a string. Cretonne is a cross-compiler, so the ISA to target
//!   can be selected dynamically. Individual ISAs can be left out when Cretonne is compiled, so a
//!   string is used to identify the proper sub-module.
//! - Values for settings that apply to all ISAs. This is represented by a `settings::Flags`
//!   instance.
//! - Values for ISA-specific settings.
//!
//! The `isa::lookup()` function is the main entry point which returns an `isa::Builder`
//! appropriate for the requested ISA:
//!
//! ```
//! use cretonne::settings::{self, Configurable};
//! use cretonne::isa;
//!
//! let shared_builder = settings::builder();
//! let shared_flags = settings::Flags::new(shared_builder);
//!
//! match isa::lookup("riscv") {
//!     None => {
//!         // The RISC-V target ISA is not available.
//!     }
//!     Some(mut isa_builder) => {
//!         isa_builder.set("supports_m", "on");
//!         let isa = isa_builder.finish(shared_flags);
//!     }
//! }
//! ```
//!
//! The configured target ISA trait object is a `Box<TargetIsa>` which can be used for multiple
//! concurrent function compilations.

pub mod riscv;

use settings;
use ir::{Inst, DataFlowGraph};

/// Look for a supported ISA with the given `name`.
/// Return a builder that can create a corresponding `TargetIsa`.
pub fn lookup(name: &str) -> Option<Builder> {
    match name {
        "riscv" => riscv_builder(),
        _ => None,
    }
}

// Make a builder for RISC-V.
fn riscv_builder() -> Option<Builder> {
    Some(riscv::isa_builder())
}

/// Builder for a `TargetIsa`.
/// Modify the ISA-specific settings before creating the `TargetIsa` trait object with `finish`.
pub struct Builder {
    setup: settings::Builder,
    constructor: fn(settings::Flags, settings::Builder) -> Box<TargetIsa>,
}

impl Builder {
    /// Combine the ISA-specific settings with the provided ISA-independent settings and allocate a
    /// fully configured `TargetIsa` trait object.
    pub fn finish(self, shared_flags: settings::Flags) -> Box<TargetIsa> {
        (self.constructor)(shared_flags, self.setup)
    }
}

impl settings::Configurable for Builder {
    fn set(&mut self, name: &str, value: &str) -> settings::Result<()> {
        self.setup.set(name, value)
    }

    fn set_bool(&mut self, name: &str, value: bool) -> settings::Result<()> {
        self.setup.set_bool(name, value)
    }
}

pub trait TargetIsa {
    /// Encode an instruction after determining it is legal.
    ///
    /// If `inst` can legally be encoded in this ISA, produce the corresponding `Encoding` object.
    /// Otherwise, return `None`.
    ///
    /// This is also the main entry point for determining if an instruction is legal.
    fn encode(&self, dfg: &DataFlowGraph, inst: &Inst) -> Option<Encoding>;
}

/// Bits needed to encode an instruction as binary machine code.
///
/// The encoding consists of two parts, both specific to the target ISA: An encoding *recipe*, and
/// encoding *bits*. The recipe determines the native instruction format and the mapping of
/// operands to encoded bits. The encoding bits provide additional information to the recipe,
/// typically parts of the opcode.
pub struct Encoding(u16, u16);

impl Encoding {
    /// Create a new `Encoding` containing `(recipe, bits)`.
    pub fn new(recipe: u16, bits: u16) -> Encoding {
        Encoding(recipe, bits)
    }
}
<|endoftext|>


<commit_before>use encoding::all::UTF_16LE;
use encoding::{Encoding, DecoderTrap};
use byteorder::{ReadBytesExt, LittleEndian};
use lnkpkg::errors::{LnkError};
use lnkpkg::utils;
use lnkpkg::volumeinfo::{VolumeInfo};
use lnkpkg::netshareinfo::{NetworkShareInfo};
use std::io::SeekFrom;
use std::io::Read;
use std::io::Seek;

#[derive(Debug)]
pub struct LocationInfo {
    pub info_size: u32,
    pub header_size: u32,
    pub flags: u32,
    pub offset_vol_info: u32,
    pub offset_loc_path: u32,
    pub offset_net_share: u32,
    pub offset_common_path: u32,
    pub volume_info: VolumeInfo,
    pub netshare_info: NetworkShareInfo
}
impl LocationInfo {
    pub fn new<Rs: Read + Seek>(mut reader: Rs) -> Result<LocationInfo,LnkError> {
        let _offset = reader.seek(
            SeekFrom::Current(0)
        )?;

        let info_size = reader.read_u32::<LittleEndian>()?;
        let header_size = reader.read_u32::<LittleEndian>()?;
        let flags = reader.read_u32::<LittleEndian>()?;
        let offset_vol_info = reader.read_u32::<LittleEndian>()?;
        let offset_loc_path = reader.read_u32::<LittleEndian>()?;
        let offset_net_share = reader.read_u32::<LittleEndian>()?;
        let offset_common_path = reader.read_u32::<LittleEndian>()?;

        // read volume_info
        reader.seek(
            SeekFrom::Start(_offset + offset_vol_info as u64)
        )?;
        let volume_info = VolumeInfo::new(&mut reader)?;

        // read netshare info
        reader.seek(
            SeekFrom::Start(_offset + offset_net_share as u64)
        )?;
        let netshare_info = NetworkShareInfo::new(&mut reader)?;

        // seek to end of location info
        reader.seek(
            SeekFrom::Start(_offset + info_size as u64)
        )?;

        Ok (
            LocationInfo {
                info_size: info_size,
                header_size: header_size,
                flags: flags,
                offset_vol_info: offset_vol_info,
                offset_loc_path: offset_loc_path,
                offset_net_share: offset_net_share,
                offset_common_path: offset_common_path,
                volume_info: volume_info,
                netshare_info: netshare_info
            }
        )
    }
}
<commit_msg>Added local path and common path strings to LocationInfo struct<commit_after>use encoding::all::UTF_16LE;
use encoding::{Encoding, DecoderTrap};
use byteorder::{ReadBytesExt, LittleEndian};
use lnkpkg::errors::{LnkError};
use lnkpkg::utils;
use lnkpkg::volumeinfo::{VolumeInfo};
use lnkpkg::netshareinfo::{NetworkShareInfo};
use std::io::SeekFrom;
use std::io::Read;
use std::io::Seek;

#[derive(Debug)]
pub struct LocationInfo {
    info_size: u32,
    header_size: u32,
    flags: u32,
    offset_vol_info: u32,
    offset_loc_path: u32,
    offset_net_share: u32,
    offset_common_path: u32,
    volume_info: VolumeInfo,
    local_path: String,
    netshare_info: NetworkShareInfo,
    common_path: String
}
impl LocationInfo {
    pub fn new<Rs: Read + Seek>(mut reader: Rs) -> Result<LocationInfo,LnkError> {
        let _offset = reader.seek(
            SeekFrom::Current(0)
        )?;

        let info_size = reader.read_u32::<LittleEndian>()?;
        let header_size = reader.read_u32::<LittleEndian>()?;
        let flags = reader.read_u32::<LittleEndian>()?;
        let offset_vol_info = reader.read_u32::<LittleEndian>()?;
        let offset_loc_path = reader.read_u32::<LittleEndian>()?;
        let offset_net_share = reader.read_u32::<LittleEndian>()?;
        let offset_common_path = reader.read_u32::<LittleEndian>()?;

        // read volume_info
        reader.seek(
            SeekFrom::Start(_offset + offset_vol_info as u64)
        )?;
        let volume_info = VolumeInfo::new(&mut reader)?;

        // read local path
        reader.seek(
            SeekFrom::Start(_offset + offset_loc_path as u64)
        )?;
        let local_path = utils::read_string_u8_till_null(&mut reader)?;

        // read netshare info
        reader.seek(
            SeekFrom::Start(_offset + offset_net_share as u64)
        )?;
        let netshare_info = NetworkShareInfo::new(&mut reader)?;

        // read common path
        reader.seek(
            SeekFrom::Start(_offset + offset_common_path as u64)
        )?;
        let common_path = utils::read_string_u8_till_null(&mut reader)?;

        // seek to end of location info
        reader.seek(
            SeekFrom::Start(_offset + info_size as u64)
        )?;

        Ok (
            LocationInfo {
                info_size: info_size,
                header_size: header_size,
                flags: flags,
                offset_vol_info: offset_vol_info,
                offset_loc_path: offset_loc_path,
                offset_net_share: offset_net_share,
                offset_common_path: offset_common_path,
                volume_info: volume_info,
                local_path: local_path,
                netshare_info: netshare_info,
                common_path: common_path
            }
        )
    }
}
<|endoftext|>


<commit_before>//! A basic output stream example, using an Output AudioUnit to generate a sine wave.

extern crate coreaudio_rs as coreaudio;
extern crate num;

use coreaudio::audio_unit::{AudioUnit, Type, SubType};
use num::Float;
use std::f64::consts::PI;

// NOTE: temporary replacement for unstable `std::iter::iterate`
struct Iter {
    value: f64,
}
impl Iterator for Iter {
    type Item = f64;
    fn next(&mut self) -> Option<f64> {
        self.value += 440.0 / 44_100.0;
        Some(self.value)
    }
}

fn main() {

    // 440hz sine wave generator.
    let mut samples = Iter { value: 0.0 }
        .map(|phase| (phase * PI * 2.0).sin() as f32 * 0.15);

    // Construct an Output audio unit.
    let mut audio_unit = AudioUnit::new(Type::Output, SubType::HalOutput).unwrap();
    audio_unit.render_callback(Some(Box::new(move |buffer, num_frames| {
        for frame in (0..num_frames) {
            let sample = samples.next().unwrap();
            for channel in buffer.iter_mut() {
                channel[frame] = sample;
            }
        }
        Ok(())
    }))).ok();
    audio_unit.start().ok();

    ::std::thread::sleep_ms(3000);
}
<commit_msg>Removed unnecessary parens in example code<commit_after>//! A basic output stream example, using an Output AudioUnit to generate a sine wave.

extern crate coreaudio_rs as coreaudio;
extern crate num;

use coreaudio::audio_unit::{AudioUnit, Type, SubType};
use num::Float;
use std::f64::consts::PI;

// NOTE: temporary replacement for unstable `std::iter::iterate`
struct Iter {
    value: f64,
}
impl Iterator for Iter {
    type Item = f64;
    fn next(&mut self) -> Option<f64> {
        self.value += 440.0 / 44_100.0;
        Some(self.value)
    }
}

fn main() {

    // 440hz sine wave generator.
    let mut samples = Iter { value: 0.0 }
        .map(|phase| (phase * PI * 2.0).sin() as f32 * 0.15);

    // Construct an Output audio unit.
    let mut audio_unit = AudioUnit::new(Type::Output, SubType::HalOutput).unwrap();
    audio_unit.render_callback(Some(Box::new(move |buffer, num_frames| {
        for frame in 0..num_frames {
            let sample = samples.next().unwrap();
            for channel in buffer.iter_mut() {
                channel[frame] = sample;
            }
        }
        Ok(())
    }))).ok();
    audio_unit.start().ok();

    ::std::thread::sleep_ms(3000);
}
<|endoftext|>


<commit_before>#![deny(warnings)]

#[macro_use]
extern crate rust_embed;
extern crate warp;

use std::borrow::Cow;
use warp::{filters::path::Tail, http::Response, Filter, Rejection, Reply};

#[derive(RustEmbed)]
#[folder = "examples/public/"]
struct Asset;

fn main() {
  let index_hml = warp::get2().and(warp::path::end()).and_then(|| serve("index.html"));

  let dist = warp::path("dist").and(warp::path::tail()).and_then(|tail: Tail| serve(tail.as_str()));

  let routes = index_hml.or(dist);

  warp::serve(routes).run(([127, 0, 0, 1], 8080));
}

fn serve(path: &str) -> Result<impl Reply, Rejection> {
  let mime = mime_guess::guess_mime_type(path);

  let asset: Option<Cow<'static, [u8]>> = Asset::get(path);

  let file = asset.ok_or_else(|| warp::reject::not_found())?;

  Ok(Response::builder().header("content-type", mime.to_string()).body(file))
}
<commit_msg>Fix mime_guess error in Warp example<commit_after>#![deny(warnings)]

#[macro_use]
extern crate rust_embed;
extern crate warp;

use std::borrow::Cow;
use warp::{filters::path::Tail, http::Response, Filter, Rejection, Reply};

#[derive(RustEmbed)]
#[folder = "examples/public/"]
struct Asset;

fn main() {
  let index_hml = warp::get2().and(warp::path::end()).and_then(|| serve("index.html"));

  let dist = warp::path("dist").and(warp::path::tail()).and_then(|tail: Tail| serve(tail.as_str()));

  let routes = index_hml.or(dist);

  warp::serve(routes).run(([127, 0, 0, 1], 8080));
}

fn serve(path: &str) -> Result<impl Reply, Rejection> {
  let mime = mime_guess::from_path(path).first_or_octet_stream();

  let asset: Option<Cow<'static, [u8]>> = Asset::get(path);

  let file = asset.ok_or_else(|| warp::reject::not_found())?;

  Ok(Response::builder().header("content-type", mime.to_string()).body(file))
}
<|endoftext|>