<commit_before>use prelude::*;
use super::{CheckpointFormat};
use rw::{ReadBuffer, WriteBuffer};

use csv::{Writer as CsvWriter};
use densearray::{Reshape, ReshapeMut};

use rand::{Rng};
use rustc_serialize::{Encodable};
use std::cmp::{max, min};
use std::fmt::{Debug};
use std::fs::{File, create_dir_all};
use std::io::{Write};
use std::path::{Path, PathBuf};
use std::time::{Instant};

//pub mod adagrad;
pub mod adagrad_new;
//pub mod adam;
pub mod adam_new;
//pub mod rmsprop;
//pub mod sgd;
pub mod sgd_new;
//pub mod shared_sgd;
pub mod shared_sgd_new;

#[derive(Clone, Debug)]
pub struct CheckpointConfig {
  pub prefix:   PathBuf,
  pub trace:    bool,
}

pub struct CheckpointState {
  pub cfg:          CheckpointConfig,
  pub config_file:  Option<File>,
  pub train_file:   Option<CsvWriter<File>>,
  pub valid_file:   Option<CsvWriter<File>>,
  start_time:   Instant,
  lap_time:     Instant,
  elapsed_s:    f64,
}

impl CheckpointState {
  pub fn new(cfg: CheckpointConfig) -> CheckpointState {
    Self::with_extra_fields(cfg, "")
  }

  pub fn with_extra_fields(cfg: CheckpointConfig, extra_fields: &str) -> CheckpointState {
    create_dir_all(&cfg.prefix).ok();
    let mut lo_idx = Some(0);
    let mut hi_idx = None;
    let mut idx = 0;
    while lo_idx != hi_idx {
      match (lo_idx, hi_idx) {
        (Some(lo), None) => {
          idx = lo;
        }
        (Some(lo), Some(hi)) => {
          idx = (lo + hi) / 2;
        }
        _ => unreachable!(),
      }
      let mut config_path = PathBuf::from(&cfg.prefix);
      let mut config_filename = format!("config.{}", idx);
      config_path.push(&config_filename);
      if config_path.exists() {
        match (lo_idx, hi_idx) {
          (Some(lo), None) => {
            lo_idx = Some(lo + max(1, lo));
          }
          (Some(lo), Some(hi)) => {
            lo_idx = Some(min(hi, max(lo + 1, idx)));
          }
          _ => unreachable!(),
        }
      } else {
        match (lo_idx, hi_idx) {
          (Some(_), None) => {
            lo_idx = Some(idx / 2);
            hi_idx = Some(idx);
          }
          (Some(_), Some(_)) => {
            hi_idx = Some(idx);
          }
          _ => unreachable!(),
        }
      }
    }
    idx = hi_idx.unwrap();
    let mut cfg_f = None;
    let mut trace_f = None;
    let mut valid_f = None;
    loop {
      let mut config_path = PathBuf::from(&cfg.prefix);
      let mut config_filename = format!("config.{}", idx);
      config_path.push(&config_filename);
      match File::create(&config_path) {
        Err(_) => {
          idx += 1;
          continue;
        }
        Ok(f) => {
          cfg_f = Some(f);
          break;
        }
      }
    }
    if cfg.trace {
      let mut train_path = PathBuf::from(&cfg.prefix);
      let mut train_filename = format!("train.log.{}", idx);
      train_path.push(&train_filename);
      assert!(!train_path.exists());
      let mut f = File::create(&train_path).unwrap();
      //writeln!(&mut f, "iter,loss,other,elapsed").unwrap();
      if extra_fields.is_empty() {
        writeln!(&mut f, "iter,loss,accuracy,elapsed").unwrap();
      } else {
        writeln!(&mut f, "iter,loss,accuracy,elapsed,{}", extra_fields).unwrap();
      }
      trace_f = Some(f);

      let mut valid_path = PathBuf::from(&cfg.prefix);
      let mut valid_filename = format!("valid.log.{}", idx);
      valid_path.push(&valid_filename);
      assert!(!valid_path.exists());
      let mut f = File::create(&valid_path).unwrap();
      //writeln!(&mut f, "iter,val_loss,val_other,elapsed").unwrap();
      if extra_fields.is_empty() {
        writeln!(&mut f, "iter,loss,accuracy,elapsed").unwrap();
      } else {
        writeln!(&mut f, "iter,loss,accuracy,elapsed,{}", extra_fields).unwrap();
      }
      valid_f = Some(f);
    }
    let init_time = Instant::now();
    CheckpointState{
      cfg:          cfg,
      config_file:  cfg_f,
      train_file:   trace_f.map(|f| CsvWriter::from_writer(f)),
      valid_file:   valid_f.map(|f| CsvWriter::from_writer(f)),
      start_time:   init_time,
      lap_time:     init_time,
      elapsed_s:    0.0,
    }
  }

  pub fn start_timing(&mut self) {
    self.start_time = Instant::now();
  }

  pub fn stop_timing(&mut self) {
    self.lap_time = Instant::now();
    let duration = self.lap_time - self.start_time;
    self.elapsed_s = duration.as_secs() as f64 + 1.0e-9 * duration.subsec_nanos() as f64;
  }

  pub fn elapsed(&self) -> f64 {
    self.elapsed_s
  }

  pub fn append_config_info<U>(&mut self, info: &U) where U: Debug {
    if let Some(ref mut config_file) = self.config_file {
      writeln!(config_file, "{:?}", info).unwrap();
    }
  }

  pub fn append_class_stats_train(&mut self, stats: &ClassLossStats) {
    let elapsed = self.elapsed_s;
    self.append_train_info(&stats.to_record(elapsed));
  }

  pub fn append_train_info<Rec>(&mut self, rec: &Rec) where Rec: Encodable {
    if let Some(ref mut train_file) = self.train_file {
      //writeln!(train_file, "{},{:.6},{:.6},{:.6}", stats.iter_nr, stats.avg_loss(), stats.accuracy(), self.elapsed_s).unwrap();
      train_file.encode(rec).unwrap();
      train_file.flush().unwrap();
    }
  }

  pub fn append_class_stats_valid(&mut self, stats: &ClassLossStats) {
    let elapsed = self.elapsed_s;
    self.append_valid_info(&stats.to_record(elapsed));
  }

  pub fn append_valid_info<Rec>(&mut self, rec: &Rec) where Rec: Encodable {
    if let Some(ref mut valid_file) = self.valid_file {
      //writeln!(valid_file, "{},{:.6},{:.6},{:.6}", stats.iter_nr, stats.avg_loss(), stats.accuracy(), self.elapsed_s).unwrap();
      valid_file.encode(rec).unwrap();
      valid_file.flush().unwrap();
    }
  }
}

pub trait TraceRecord {
  fn loss(&self) -> f32;
  fn other(&self) -> f32;
}

#[derive(Clone, Debug)]
pub enum StepSize {
  Constant(f32),
  Custom{init_step: f32, steps: Vec<(usize, f32)>},
  Decay{init_step: f32, step_decay: f32, decay_iters: usize},
  Adaptive{init_step: f32, test_iters: usize, epoch_iters: usize, sched: AdaptiveStepSizeSchedule},
  BacktrackingLineSearch{decay: f32, c: f32},
  //WeakWolfeLineSearch,
}

#[derive(Clone, Copy, Debug)]
pub enum AdaptiveStepSizeSchedule {
  Pow10,
  Decimal,
}

pub fn adaptive_pow10_step_size_factor(t: usize) -> f32 {
  const F1: [f32; 2] = [0.3,    0.1];
  const F2: [f32; 2] = [0.03,   0.01];
  const F3: [f32; 2] = [0.003,  0.001];
  const F4: [f32; 2] = [0.0003, 0.0001];
  if t == 0 {
    1.0
  } else if t > 0 && t <= 2 {
    F1[t-1]
  } else if t > 2 && t <= 4 {
    F2[t-2-1]
  } else if t > 4 && t <= 6 {
    F3[t-4-1]
  } else if t > 6 && t <= 8 {
    F4[t-6-1]
  } else {
    unimplemented!();
  }
}

pub fn adaptive_decimal_step_size_factor(t: usize) -> f32 {
  const F1: [f32; 9] = [0.9, 0.8, 0.7, 0.6, 0.5, 0.4, 0.3, 0.2, 0.1];
  const F2: [f32; 9] = [0.09, 0.08, 0.07, 0.06, 0.05, 0.04, 0.03, 0.02, 0.01];
  const F3: [f32; 9] = [0.009, 0.008, 0.007, 0.006, 0.005, 0.004, 0.003, 0.002, 0.001];
  if t == 0 {
    1.0
  } else if t > 0 && t <= 9 {
    F1[t-1]
  } else if t > 9 && t <= 18 {
    F2[t-9-1]
  } else if t > 18 && t <= 27 {
    F3[t-18-1]
  } else {
    unimplemented!();
  }
}

#[derive(Clone, Copy, Debug)]
pub enum GradientMomentum {
  HeavyBall(f32),
  Nesterov(f32),
}

#[derive(Clone, Copy, Debug)]
pub enum NesterovParamState {
  Orig,
  PlusMomentum,
}

#[derive(Clone, Copy, Debug)]
pub struct StepStats {
  pub elapsed:  Option<f64>,
}

pub trait OptWorker<T, S> {
  type Rng: Rng;

  //fn init_param(&mut self, rng: &mut Xorshiftplus128Rng);
  fn init_param(&mut self, rng: &mut Self::Rng);
  /*fn load_local_param(&mut self, param_reader: &mut ReadBuffer<T>);
  fn store_local_param(&mut self, param_writer: &mut WriteBuffer<T>);
  fn store_global_param(&mut self, param_writer: &mut WriteBuffer<T>);*/

  fn step(&mut self, samples: &mut Iterator<Item=S>) /* -> StepStats*/;
  fn eval(&mut self, epoch_size: usize, samples: &mut Iterator<Item=S>);
}

pub trait OptCheckpoint<Format> where Format: CheckpointFormat {
  fn save_checkpoint(&mut self, path: &Path) -> Result<(), ()>;
  fn restore_checkpoint(&mut self, path: &Path) -> Result<(), ()>;
}

pub trait OptStats<Stats> {
  fn reset_opt_stats(&mut self);
  fn get_opt_stats(&self) -> &Stats;
}

pub trait OptUpdateStats<Stats> {
  fn update_stats(&mut self, stats: &mut Stats);
}

#[derive(Clone, Default, Debug)]
pub struct ClassOptStats {
  pub iter_count:       usize,
  pub sample_count:     usize,
  pub correct_count:    usize,
  pub avg_loss:         f32,
}

impl ClassOptStats {
  pub fn reset(&mut self) {
    self.iter_count = 0;
    self.sample_count = 0;
    self.correct_count = 0;
    self.avg_loss = 0.0;
  }

  pub fn accuracy(&self) -> f32 {
    self.correct_count as f32 / self.sample_count as f32
  }
}

#[derive(Clone, Default, Debug)]
pub struct RegressOptStats {
  pub avg_loss:         f32,
}
<commit_msg>Allow setting all custom fields for tracing. Add a random delay before creating checkpoint files to avoid filesystem races.<commit_after>use prelude::*;
use super::{CheckpointFormat};
use rw::{ReadBuffer, WriteBuffer};

use csv::{Writer as CsvWriter};
use densearray::{Reshape, ReshapeMut};

use rand::{Rng, thread_rng};
use rustc_serialize::{Encodable};
use std::cmp::{max, min};
use std::fmt::{Debug};
use std::fs::{File, create_dir_all};
use std::io::{Write};
use std::path::{Path, PathBuf};
use std::thread::{sleep};
use std::time::{Duration, Instant};

//pub mod adagrad;
pub mod adagrad_new;
//pub mod adam;
pub mod adam_new;
//pub mod rmsprop;
//pub mod sgd;
pub mod sgd_new;
//pub mod shared_sgd;
pub mod shared_sgd_new;

#[derive(Clone, Debug)]
pub struct CheckpointConfig {
  pub prefix:   PathBuf,
  pub trace:    bool,
}

pub struct CheckpointState {
  pub cfg:          CheckpointConfig,
  pub config_file:  Option<File>,
  pub train_file:   Option<CsvWriter<File>>,
  pub valid_file:   Option<CsvWriter<File>>,
  start_time:   Instant,
  lap_time:     Instant,
  elapsed_s:    f64,
}

impl CheckpointState {
  pub fn new(cfg: CheckpointConfig) -> CheckpointState {
    Self::with_fields(cfg, "iter,loss,accuracy,elapsed")
  }

  pub fn with_fields(cfg: CheckpointConfig, fields: &str) -> CheckpointState {
    let delay_ns = thread_rng().gen_range(1_000_000, 100_000_000);
    sleep(Duration::new(0, delay_ns));
    create_dir_all(&cfg.prefix).ok();
    loop {
      let mut lo_idx = Some(0);
      let mut hi_idx = None;
      let mut idx = 0;
      while lo_idx != hi_idx {
        match (lo_idx, hi_idx) {
          (Some(lo), None) => {
            idx = lo;
          }
          (Some(lo), Some(hi)) => {
            idx = (lo + hi) / 2;
          }
          _ => unreachable!(),
        }
        let mut config_path = PathBuf::from(&cfg.prefix);
        let mut config_filename = format!("config.{}", idx);
        config_path.push(&config_filename);
        if config_path.exists() {
          match (lo_idx, hi_idx) {
            (Some(lo), None) => {
              lo_idx = Some(lo + max(1, lo));
            }
            (Some(lo), Some(hi)) => {
              lo_idx = Some(min(hi, max(lo + 1, idx)));
            }
            _ => unreachable!(),
          }
        } else {
          match (lo_idx, hi_idx) {
            (Some(_), None) => {
              lo_idx = Some(idx / 2);
              hi_idx = Some(idx);
            }
            (Some(_), Some(_)) => {
              hi_idx = Some(idx);
            }
            _ => unreachable!(),
          }
        }
      }
      idx = hi_idx.unwrap();
      let mut cfg_f = None;
      let mut trace_f = None;
      let mut valid_f = None;
      let mut config_path = PathBuf::from(&cfg.prefix);
      let mut config_filename = format!("config.{}", idx);
      config_path.push(&config_filename);
      if config_path.exists() {
        continue;
      }
      match File::create(&config_path) {
        Err(_) => {
          continue;
        }
        Ok(f) => {
          cfg_f = Some(f);
        }
      }
      if cfg.trace {
        let mut train_path = PathBuf::from(&cfg.prefix);
        let mut train_filename = format!("train.log.{}", idx);
        train_path.push(&train_filename);
        assert!(!train_path.exists());
        let mut f = File::create(&train_path).unwrap();
        writeln!(&mut f, "{}", fields).unwrap();
        trace_f = Some(f);

        let mut valid_path = PathBuf::from(&cfg.prefix);
        let mut valid_filename = format!("valid.log.{}", idx);
        valid_path.push(&valid_filename);
        assert!(!valid_path.exists());
        let mut f = File::create(&valid_path).unwrap();
        writeln!(&mut f, "{}", fields).unwrap();
        valid_f = Some(f);
      }
      let init_time = Instant::now();
      return CheckpointState{
        cfg:          cfg,
        config_file:  cfg_f,
        train_file:   trace_f.map(|f| CsvWriter::from_writer(f)),
        valid_file:   valid_f.map(|f| CsvWriter::from_writer(f)),
        start_time:   init_time,
        lap_time:     init_time,
        elapsed_s:    0.0,
      };
    }
  }

  pub fn start_timing(&mut self) {
    self.start_time = Instant::now();
  }

  pub fn stop_timing(&mut self) {
    self.lap_time = Instant::now();
    let duration = self.lap_time - self.start_time;
    self.elapsed_s = duration.as_secs() as f64 + 1.0e-9 * duration.subsec_nanos() as f64;
  }

  pub fn elapsed(&self) -> f64 {
    self.elapsed_s
  }

  pub fn append_config_info<U>(&mut self, info: &U) where U: Debug {
    if let Some(ref mut config_file) = self.config_file {
      writeln!(config_file, "{:?}", info).unwrap();
    }
  }

  pub fn append_class_stats_train(&mut self, stats: &ClassLossStats) {
    let elapsed = self.elapsed_s;
    self.append_train_info(&stats.to_record(elapsed));
  }

  pub fn append_train_info<Rec>(&mut self, rec: &Rec) where Rec: Encodable {
    if let Some(ref mut train_file) = self.train_file {
      train_file.encode(rec).unwrap();
      //train_file.flush().unwrap();
    }
  }

  pub fn append_class_stats_valid(&mut self, stats: &ClassLossStats) {
    let elapsed = self.elapsed_s;
    self.append_valid_info(&stats.to_record(elapsed));
  }

  pub fn append_valid_info<Rec>(&mut self, rec: &Rec) where Rec: Encodable {
    if let Some(ref mut valid_file) = self.valid_file {
      valid_file.encode(rec).unwrap();
      valid_file.flush().unwrap();
    }
  }
}

pub trait TraceRecord {
  fn loss(&self) -> f32;
  fn other(&self) -> f32;
}

#[derive(Clone, Debug)]
pub enum StepSize {
  Constant(f32),
  Custom{init_step: f32, steps: Vec<(usize, f32)>},
  Decay{init_step: f32, step_decay: f32, decay_iters: usize},
  Adaptive{init_step: f32, test_iters: usize, epoch_iters: usize, sched: AdaptiveStepSizeSchedule},
  BacktrackingLineSearch{decay: f32, c: f32},
  //WeakWolfeLineSearch,
}

#[derive(Clone, Copy, Debug)]
pub enum AdaptiveStepSizeSchedule {
  Pow10,
  Decimal,
}

pub fn adaptive_pow10_step_size_factor(t: usize) -> f32 {
  const F1: [f32; 2] = [0.3,    0.1];
  const F2: [f32; 2] = [0.03,   0.01];
  const F3: [f32; 2] = [0.003,  0.001];
  const F4: [f32; 2] = [0.0003, 0.0001];
  if t == 0 {
    1.0
  } else if t > 0 && t <= 2 {
    F1[t-1]
  } else if t > 2 && t <= 4 {
    F2[t-2-1]
  } else if t > 4 && t <= 6 {
    F3[t-4-1]
  } else if t > 6 && t <= 8 {
    F4[t-6-1]
  } else {
    unimplemented!();
  }
}

pub fn adaptive_decimal_step_size_factor(t: usize) -> f32 {
  const F1: [f32; 9] = [0.9, 0.8, 0.7, 0.6, 0.5, 0.4, 0.3, 0.2, 0.1];
  const F2: [f32; 9] = [0.09, 0.08, 0.07, 0.06, 0.05, 0.04, 0.03, 0.02, 0.01];
  const F3: [f32; 9] = [0.009, 0.008, 0.007, 0.006, 0.005, 0.004, 0.003, 0.002, 0.001];
  if t == 0 {
    1.0
  } else if t > 0 && t <= 9 {
    F1[t-1]
  } else if t > 9 && t <= 18 {
    F2[t-9-1]
  } else if t > 18 && t <= 27 {
    F3[t-18-1]
  } else {
    unimplemented!();
  }
}

#[derive(Clone, Copy, Debug)]
pub enum GradientMomentum {
  HeavyBall(f32),
  Nesterov(f32),
}

#[derive(Clone, Copy, Debug)]
pub enum NesterovParamState {
  Orig,
  PlusMomentum,
}

#[derive(Clone, Copy, Debug)]
pub struct StepStats {
  pub elapsed:  Option<f64>,
}

pub trait OptWorker<T, S> {
  type Rng: Rng;

  //fn init_param(&mut self, rng: &mut Xorshiftplus128Rng);
  fn init_param(&mut self, rng: &mut Self::Rng);
  /*fn load_local_param(&mut self, param_reader: &mut ReadBuffer<T>);
  fn store_local_param(&mut self, param_writer: &mut WriteBuffer<T>);
  fn store_global_param(&mut self, param_writer: &mut WriteBuffer<T>);*/

  fn step(&mut self, samples: &mut Iterator<Item=S>) /* -> StepStats*/;
  fn eval(&mut self, epoch_size: usize, samples: &mut Iterator<Item=S>);
}

pub trait OptCheckpoint<Format> where Format: CheckpointFormat {
  fn save_checkpoint(&mut self, path: &Path) -> Result<(), ()>;
  fn restore_checkpoint(&mut self, path: &Path) -> Result<(), ()>;
}

pub trait OptStats<Stats> {
  fn reset_opt_stats(&mut self);
  fn get_opt_stats(&self) -> &Stats;
}

pub trait OptUpdateStats<Stats> {
  fn update_stats(&mut self, stats: &mut Stats);
}

#[derive(Clone, Default, Debug)]
pub struct ClassOptStats {
  pub iter_count:       usize,
  pub sample_count:     usize,
  pub correct_count:    usize,
  pub avg_loss:         f32,
}

impl ClassOptStats {
  pub fn reset(&mut self) {
    self.iter_count = 0;
    self.sample_count = 0;
    self.correct_count = 0;
    self.avg_loss = 0.0;
  }

  pub fn accuracy(&self) -> f32 {
    self.correct_count as f32 / self.sample_count as f32
  }
}

#[derive(Clone, Default, Debug)]
pub struct RegressOptStats {
  pub avg_loss:         f32,
}
<|endoftext|>


<commit_before>/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */

//! Implements traversal over the DOM tree. The traversal starts in sequential
//! mode, and optionally parallelizes as it discovers work.

#![deny(missing_docs)]

use context::{StyleContext, ThreadLocalStyleContext};
use dom::{SendNode, TElement, TNode};
use parallel;
use parallel::{DispatchMode, WORK_UNIT_MAX};
use rayon;
use scoped_tls::ScopedTLS;
use std::collections::VecDeque;
use std::mem;
use time;
use traversal::{DomTraversal, PerLevelTraversalData, PreTraverseToken};

/// Do a DOM traversal for top-down and (optionally) bottom-up processing,
/// generic over `D`.
///
/// We use an adaptive traversal strategy. We start out with simple sequential
/// processing, until we arrive at a wide enough level in the DOM that the
/// parallel traversal would parallelize it. If a thread pool is provided, we
/// then transfer control over to the parallel traversal.
///
/// Returns true if the traversal was parallel, and also returns the statistics
/// object containing information on nodes traversed (on nightly only). Not
/// all of its fields will be initialized since we don't call finish().
pub fn traverse_dom<E, D>(
    traversal: &D,
    token: PreTraverseToken<E>,
    pool: Option<&rayon::ThreadPool>
)
where
    E: TElement,
    D: DomTraversal<E>,
{
    let root =
        token.traversal_root().expect("Should've ensured we needed to traverse");

    let dump_stats = traversal.shared_context().options.dump_style_statistics;
    let is_nightly  = traversal.shared_context().options.is_nightly();
    let start_time = if dump_stats { Some(time::precise_time_s()) } else { None };

    // Declare the main-thread context, as well as the worker-thread contexts,
    // which we may or may not instantiate. It's important to declare the worker-
    // thread contexts first, so that they get dropped second. This matters because:
    //   * ThreadLocalContexts borrow AtomicRefCells in TLS.
    //   * Dropping a ThreadLocalContext can run SequentialTasks.
    //   * Sequential tasks may call into functions like
    //     Servo_StyleSet_GetBaseComputedValuesForElement, which instantiate a
    //     ThreadLocalStyleContext on the main thread. If the main thread
    //     ThreadLocalStyleContext has not released its TLS borrow by that point,
    //     we'll panic on double-borrow.
    let mut maybe_tls: Option<ScopedTLS<ThreadLocalStyleContext<E>>> = None;
    let mut tlc = ThreadLocalStyleContext::new(traversal.shared_context());
    let mut context = StyleContext {
        shared: traversal.shared_context(),
        thread_local: &mut tlc,
    };

    // Process the nodes breadth-first, just like the parallel traversal does.
    // This helps keep similar traversal characteristics for the style sharing
    // cache.
    let mut discovered =
        VecDeque::<SendNode<E::ConcreteNode>>::with_capacity(WORK_UNIT_MAX * 2);
    let mut depth = root.depth();
    let mut nodes_remaining_at_current_depth = 1;
    discovered.push_back(unsafe { SendNode::new(root.as_node()) });
    while let Some(node) = discovered.pop_front() {
        let mut children_to_process = 0isize;
        let traversal_data = PerLevelTraversalData { current_dom_depth: depth };
        traversal.process_preorder(&traversal_data, &mut context, *node, |n| {
            children_to_process += 1;
            discovered.push_back(unsafe { SendNode::new(n) });
        });

        traversal.handle_postorder_traversal(&mut context, root.as_node().opaque(),
                                             *node, children_to_process);

        nodes_remaining_at_current_depth -= 1;
        if nodes_remaining_at_current_depth == 0 {
            depth += 1;
            // If there is enough work to parallelize over, and the caller allows
            // parallelism, switch to the parallel driver. We do this only when
            // moving to the next level in the dom so that we can pass the same
            // depth for all the children.
            if pool.is_some() && discovered.len() > WORK_UNIT_MAX {
                let pool = pool.unwrap();
                maybe_tls = Some(ScopedTLS::<ThreadLocalStyleContext<E>>::new(pool));
                let root_opaque = root.as_node().opaque();
                let drain = discovered.drain(..);
                pool.install(|| {
                    rayon::scope(|scope| {
                        parallel::traverse_nodes(
                            drain,
                            DispatchMode::TailCall,
                            /* recursion_ok = */ true,
                            root_opaque,
                            PerLevelTraversalData { current_dom_depth: depth },
                            scope,
                            pool,
                            traversal,
                            maybe_tls.as_ref().unwrap()
                        );
                    });
                });
                break;
            }
            nodes_remaining_at_current_depth = discovered.len();
        }
    }
    // Accumulate statistics
    if dump_stats || is_nightly {
        let mut aggregate =
            mem::replace(&mut context.thread_local.statistics, Default::default());
        let parallel = maybe_tls.is_some();
        if let Some(ref mut tls) = maybe_tls {
            let slots = unsafe { tls.unsafe_get() };
            aggregate = slots.iter().fold(aggregate, |acc, t| {
                match *t.borrow() {
                    None => acc,
                    Some(ref cx) => &cx.statistics + &acc,
                }
            });
        }

        // dump to stdout if requested
        if dump_stats && aggregate.is_large_traversal() {
            aggregate.finish(traversal, parallel, start_time.unwrap());
             println!("{}", aggregate);
        }
    }
}
<commit_msg>Auto merge of #20247 - bholley:fix_stat_collection, r=Manishearth<commit_after>/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */

//! Implements traversal over the DOM tree. The traversal starts in sequential
//! mode, and optionally parallelizes as it discovers work.

#![deny(missing_docs)]

use context::{StyleContext, ThreadLocalStyleContext};
use dom::{SendNode, TElement, TNode};
use parallel;
use parallel::{DispatchMode, WORK_UNIT_MAX};
use rayon;
use scoped_tls::ScopedTLS;
use std::collections::VecDeque;
use std::mem;
use time;
use traversal::{DomTraversal, PerLevelTraversalData, PreTraverseToken};

/// Do a DOM traversal for top-down and (optionally) bottom-up processing,
/// generic over `D`.
///
/// We use an adaptive traversal strategy. We start out with simple sequential
/// processing, until we arrive at a wide enough level in the DOM that the
/// parallel traversal would parallelize it. If a thread pool is provided, we
/// then transfer control over to the parallel traversal.
///
/// Returns true if the traversal was parallel, and also returns the statistics
/// object containing information on nodes traversed (on nightly only). Not
/// all of its fields will be initialized since we don't call finish().
pub fn traverse_dom<E, D>(
    traversal: &D,
    token: PreTraverseToken<E>,
    pool: Option<&rayon::ThreadPool>
)
where
    E: TElement,
    D: DomTraversal<E>,
{
    let root =
        token.traversal_root().expect("Should've ensured we needed to traverse");

    let dump_stats = traversal.shared_context().options.dump_style_statistics;
    let start_time = if dump_stats { Some(time::precise_time_s()) } else { None };

    // Declare the main-thread context, as well as the worker-thread contexts,
    // which we may or may not instantiate. It's important to declare the worker-
    // thread contexts first, so that they get dropped second. This matters because:
    //   * ThreadLocalContexts borrow AtomicRefCells in TLS.
    //   * Dropping a ThreadLocalContext can run SequentialTasks.
    //   * Sequential tasks may call into functions like
    //     Servo_StyleSet_GetBaseComputedValuesForElement, which instantiate a
    //     ThreadLocalStyleContext on the main thread. If the main thread
    //     ThreadLocalStyleContext has not released its TLS borrow by that point,
    //     we'll panic on double-borrow.
    let mut maybe_tls: Option<ScopedTLS<ThreadLocalStyleContext<E>>> = None;
    let mut tlc = ThreadLocalStyleContext::new(traversal.shared_context());
    let mut context = StyleContext {
        shared: traversal.shared_context(),
        thread_local: &mut tlc,
    };

    // Process the nodes breadth-first, just like the parallel traversal does.
    // This helps keep similar traversal characteristics for the style sharing
    // cache.
    let mut discovered =
        VecDeque::<SendNode<E::ConcreteNode>>::with_capacity(WORK_UNIT_MAX * 2);
    let mut depth = root.depth();
    let mut nodes_remaining_at_current_depth = 1;
    discovered.push_back(unsafe { SendNode::new(root.as_node()) });
    while let Some(node) = discovered.pop_front() {
        let mut children_to_process = 0isize;
        let traversal_data = PerLevelTraversalData { current_dom_depth: depth };
        traversal.process_preorder(&traversal_data, &mut context, *node, |n| {
            children_to_process += 1;
            discovered.push_back(unsafe { SendNode::new(n) });
        });

        traversal.handle_postorder_traversal(&mut context, root.as_node().opaque(),
                                             *node, children_to_process);

        nodes_remaining_at_current_depth -= 1;
        if nodes_remaining_at_current_depth == 0 {
            depth += 1;
            // If there is enough work to parallelize over, and the caller allows
            // parallelism, switch to the parallel driver. We do this only when
            // moving to the next level in the dom so that we can pass the same
            // depth for all the children.
            if pool.is_some() && discovered.len() > WORK_UNIT_MAX {
                let pool = pool.unwrap();
                maybe_tls = Some(ScopedTLS::<ThreadLocalStyleContext<E>>::new(pool));
                let root_opaque = root.as_node().opaque();
                let drain = discovered.drain(..);
                pool.install(|| {
                    rayon::scope(|scope| {
                        parallel::traverse_nodes(
                            drain,
                            DispatchMode::TailCall,
                            /* recursion_ok = */ true,
                            root_opaque,
                            PerLevelTraversalData { current_dom_depth: depth },
                            scope,
                            pool,
                            traversal,
                            maybe_tls.as_ref().unwrap()
                        );
                    });
                });
                break;
            }
            nodes_remaining_at_current_depth = discovered.len();
        }
    }

    // dump statistics to stdout if requested
    if dump_stats {
        let mut aggregate =
            mem::replace(&mut context.thread_local.statistics, Default::default());
        let parallel = maybe_tls.is_some();
        if let Some(ref mut tls) = maybe_tls {
            let slots = unsafe { tls.unsafe_get() };
            aggregate = slots.iter().fold(aggregate, |acc, t| {
                match *t.borrow() {
                    None => acc,
                    Some(ref cx) => &cx.statistics + &acc,
                }
            });
        }

        aggregate.finish(traversal, parallel, start_time.unwrap());
        if aggregate.is_large_traversal() {
             println!("{}", aggregate);
        }
    }
}
<|endoftext|>


<commit_before>use diesel::pg::Pg;
use diesel::prelude::*;

use crate::app::App;
use crate::github;
use crate::util::errors::{cargo_err, AppResult};

use crate::models::{Crate, Team, User};
use crate::schema::{crate_owners, users};
use crate::views::EncodableOwner;

#[derive(Insertable, Associations, Identifiable, Debug, Clone, Copy)]
#[belongs_to(Crate)]
#[belongs_to(User, foreign_key = "owner_id")]
#[belongs_to(Team, foreign_key = "owner_id")]
#[table_name = "crate_owners"]
#[primary_key(crate_id, owner_id, owner_kind)]
pub struct CrateOwner {
    pub crate_id: i32,
    pub owner_id: i32,
    pub created_by: i32,
    pub owner_kind: i32,
    pub email_notifications: bool,
}

type BoxedQuery<'a> = crate_owners::BoxedQuery<'a, Pg, crate_owners::SqlType>;

impl CrateOwner {
    /// Returns a base crate owner query filtered by the owner kind argument. This query also
    /// filters out deleted records.
    pub fn by_owner_kind(kind: OwnerKind) -> BoxedQuery<'static> {
        use self::crate_owners::dsl::*;

        crate_owners
            .filter(deleted.eq(false))
            .filter(owner_kind.eq(kind as i32))
            .into_boxed()
    }
}

#[derive(Debug, Clone, Copy)]
#[repr(u32)]
pub enum OwnerKind {
    User = 0,
    Team = 1,
}

/// Unifies the notion of a User or a Team.
#[derive(Debug)]
pub enum Owner {
    User(User),
    Team(Team),
}

impl Owner {
    /// Finds the owner by name. Always recreates teams to get the most
    /// up-to-date GitHub ID. Fails out if the user isn't found in the
    /// database, the team isn't found on GitHub, or if the user isn't a member
    /// of the team on GitHub.
    /// May be a user's GH login or a full team name. This is case
    /// sensitive.
    pub fn find_or_create_by_login(
        app: &App,
        conn: &PgConnection,
        req_user: &User,
        name: &str,
    ) -> AppResult<Owner> {
        if name.contains(':') {
            Ok(Owner::Team(Team::create_or_update(
                app, conn, name, req_user,
            )?))
        } else {
            users::table
                .filter(users::gh_login.eq(name))
                .filter(users::gh_id.ne(-1))
                .first(conn)
                .map(Owner::User)
                .map_err(|_| cargo_err(&format_args!("could not find user with login `{}`", name)))
        }
    }

    pub fn kind(&self) -> i32 {
        match *self {
            Owner::User(_) => OwnerKind::User as i32,
            Owner::Team(_) => OwnerKind::Team as i32,
        }
    }

    pub fn login(&self) -> &str {
        match *self {
            Owner::User(ref user) => &user.gh_login,
            Owner::Team(ref team) => &team.login,
        }
    }

    pub fn id(&self) -> i32 {
        match *self {
            Owner::User(ref user) => user.id,
            Owner::Team(ref team) => team.id,
        }
    }

    pub fn encodable(self) -> EncodableOwner {
        match self {
            Owner::User(User {
                id,
                name,
                gh_login,
                gh_avatar,
                ..
            }) => {
                let url = format!("https://github.com/{}", gh_login);
                EncodableOwner {
                    id,
                    login: gh_login,
                    avatar: gh_avatar,
                    url: Some(url),
                    name,
                    kind: String::from("user"),
                }
            }
            Owner::Team(Team {
                id,
                name,
                login,
                avatar,
                ..
            }) => {
                let url = github::team_url(&login);
                EncodableOwner {
                    id,
                    login,
                    url: Some(url),
                    avatar,
                    name,
                    kind: String::from("team"),
                }
            }
        }
    }
}
<commit_msg>Invite the account with the highest github id<commit_after>use diesel::pg::Pg;
use diesel::prelude::*;

use crate::app::App;
use crate::github;
use crate::util::errors::{cargo_err, AppResult};

use crate::models::{Crate, Team, User};
use crate::schema::{crate_owners, users};
use crate::views::EncodableOwner;

#[derive(Insertable, Associations, Identifiable, Debug, Clone, Copy)]
#[belongs_to(Crate)]
#[belongs_to(User, foreign_key = "owner_id")]
#[belongs_to(Team, foreign_key = "owner_id")]
#[table_name = "crate_owners"]
#[primary_key(crate_id, owner_id, owner_kind)]
pub struct CrateOwner {
    pub crate_id: i32,
    pub owner_id: i32,
    pub created_by: i32,
    pub owner_kind: i32,
    pub email_notifications: bool,
}

type BoxedQuery<'a> = crate_owners::BoxedQuery<'a, Pg, crate_owners::SqlType>;

impl CrateOwner {
    /// Returns a base crate owner query filtered by the owner kind argument. This query also
    /// filters out deleted records.
    pub fn by_owner_kind(kind: OwnerKind) -> BoxedQuery<'static> {
        use self::crate_owners::dsl::*;

        crate_owners
            .filter(deleted.eq(false))
            .filter(owner_kind.eq(kind as i32))
            .into_boxed()
    }
}

#[derive(Debug, Clone, Copy)]
#[repr(u32)]
pub enum OwnerKind {
    User = 0,
    Team = 1,
}

/// Unifies the notion of a User or a Team.
#[derive(Debug)]
pub enum Owner {
    User(User),
    Team(Team),
}

impl Owner {
    /// Finds the owner by name. Always recreates teams to get the most
    /// up-to-date GitHub ID. Fails out if the user isn't found in the
    /// database, the team isn't found on GitHub, or if the user isn't a member
    /// of the team on GitHub.
    /// May be a user's GH login or a full team name. This is case
    /// sensitive.
    pub fn find_or_create_by_login(
        app: &App,
        conn: &PgConnection,
        req_user: &User,
        name: &str,
    ) -> AppResult<Owner> {
        if name.contains(':') {
            Ok(Owner::Team(Team::create_or_update(
                app, conn, name, req_user,
            )?))
        } else {
            users::table
                .filter(users::gh_login.eq(name))
                .filter(users::gh_id.ne(-1))
                .order(users::gh_id.desc())
                .first(conn)
                .map(Owner::User)
                .map_err(|_| cargo_err(&format_args!("could not find user with login `{}`", name)))
        }
    }

    pub fn kind(&self) -> i32 {
        match *self {
            Owner::User(_) => OwnerKind::User as i32,
            Owner::Team(_) => OwnerKind::Team as i32,
        }
    }

    pub fn login(&self) -> &str {
        match *self {
            Owner::User(ref user) => &user.gh_login,
            Owner::Team(ref team) => &team.login,
        }
    }

    pub fn id(&self) -> i32 {
        match *self {
            Owner::User(ref user) => user.id,
            Owner::Team(ref team) => team.id,
        }
    }

    pub fn encodable(self) -> EncodableOwner {
        match self {
            Owner::User(User {
                id,
                name,
                gh_login,
                gh_avatar,
                ..
            }) => {
                let url = format!("https://github.com/{}", gh_login);
                EncodableOwner {
                    id,
                    login: gh_login,
                    avatar: gh_avatar,
                    url: Some(url),
                    name,
                    kind: String::from("user"),
                }
            }
            Owner::Team(Team {
                id,
                name,
                login,
                avatar,
                ..
            }) => {
                let url = github::team_url(&login);
                EncodableOwner {
                    id,
                    login,
                    url: Some(url),
                    avatar,
                    name,
                    kind: String::from("team"),
                }
            }
        }
    }
}
<|endoftext|>


<commit_before><commit_msg>Make MAL password field public<commit_after><|endoftext|>


<commit_before>// Copyright 2013 The Servo Project Developers. See the COPYRIGHT
// file at the top-level directory of this distribution.
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.

//! Implementation of cross-process surfaces for Android. This uses EGL surface.

use platform::surface::NativeSurfaceMethods;
use texturegl::Texture;

use geom::size::Size2D;
use gleam::gl::{egl_image_target_texture2d_oes, TEXTURE_2D, TexImage2D, BGRA_EXT, UNSIGNED_BYTE};
use egl::egl::EGLDisplay;
use egl::eglext::{EGLImageKHR, DestroyImageKHR};
use libc::c_void;
use std::mem;
use std::ptr;
use std::slice::bytes::copy_memory;
use std::vec::Vec;

/// FIXME(Aydin Kim) :Currently, native surface is consist of 2 types of hybrid image buffer. EGLImageKHR is used to GPU rendering and vector is used to CPU rendering. EGL extension seems not provide simple way to accessing its bitmap directly. In the future, we need to find out the way to integrate them.

#[deriving(Clone, Copy)]
pub struct NativeGraphicsMetadata {
    pub display: EGLDisplay,
}

pub struct NativePaintingGraphicsContext{
    display : EGLDisplay,
}

impl NativePaintingGraphicsContext {
    pub fn from_metadata(metadata: &NativeGraphicsMetadata) -> NativePaintingGraphicsContext {
        NativePaintingGraphicsContext {
            display : metadata.display,
        }
    }
}

impl Drop for NativePaintingGraphicsContext {
    fn drop(&mut self) {}
}

pub struct NativeCompositingGraphicsContext;

impl NativeCompositingGraphicsContext {
    pub fn new() -> NativeCompositingGraphicsContext {
        NativeCompositingGraphicsContext
    }
}

pub struct NativeSurface {
    image: Option<EGLImageKHR>, // For GPU rendering
    bitmap: Option<Vec<u8>>, // For CPU rendering
    will_leak: bool,
}

impl NativeSurface {
    pub fn from_image_khr(image_khr: EGLImageKHR) -> NativeSurface {
        let mut _image: Option<EGLImageKHR> = None;
        if image_khr != ptr::null_mut() {
            _image = Some(image_khr);
        }
        NativeSurface {
            image : _image,
            bitmap: None,
            will_leak: true,
        }
    }
}

impl NativeSurfaceMethods for NativeSurface {
    /// This may only be called on the case of CPU rendering.
    fn new(_native_context: &NativePaintingGraphicsContext, size: Size2D<i32>, _stride: i32) -> NativeSurface {
        let len = size.width * size.height * 4;
        let bitmap: Vec<u8> = Vec::from_elem(len as uint, 0 as u8);

        NativeSurface {
            image: None,
            bitmap: Some(bitmap),
            will_leak : true,
        }
    }

    /// This may only be called on the compositor side.
    fn bind_to_texture(&self,
                       _native_context: &NativeCompositingGraphicsContext,
                       texture: &Texture,
                       _size: Size2D<int>) {
        let _bound = texture.bind();
        match self.image {
            None => match self.bitmap {
                Some(ref bitmap) => {
                    let data = bitmap.as_ptr() as *const c_void;
                    unsafe {
                        TexImage2D(TEXTURE_2D, 0, BGRA_EXT as i32, _size.width as i32, _size.height as i32,
                                   0, BGRA_EXT as u32, UNSIGNED_BYTE, data);
                    }
                }
                None => {
                    debug!("Cannot bind the buffer(CPU rendering), there is no bitmap");
                }
            },
            Some(image_khr) => {
                egl_image_target_texture2d_oes(TEXTURE_2D, image_khr as *const c_void);
            }
        }
    }

    /// This may only be called on the painting side.
    fn upload(&mut self, _graphics_context: &NativePaintingGraphicsContext, data: &[u8]) {
        match self.bitmap {
            Some(ref mut bitmap) => {
                copy_memory(bitmap.as_mut_slice(), data);
            }
            None => {
                debug!("Cannot upload the buffer(CPU rendering), there is no bitmap");
            }
        }
    }

    fn get_id(&self) -> int {
        match self.image {
            None => 0,
            Some(image_khr) => image_khr as int,
        }
    }

    fn destroy(&mut self, graphics_context: &NativePaintingGraphicsContext) {
        match self.image {
            None => {},
            Some(image_khr) => {
                DestroyImageKHR(graphics_context.display, image_khr);
                mem::replace(&mut self.image, None);
            }
        }
        self.mark_wont_leak()
    }

    fn mark_will_leak(&mut self) {
        self.will_leak = true
    }
    fn mark_wont_leak(&mut self) {
        self.will_leak = false
    }
}
<commit_msg>Android build fix for rustup<commit_after>// Copyright 2013 The Servo Project Developers. See the COPYRIGHT
// file at the top-level directory of this distribution.
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.

//! Implementation of cross-process surfaces for Android. This uses EGL surface.

use platform::surface::NativeSurfaceMethods;
use texturegl::Texture;

use geom::size::Size2D;
use gleam::gl::{egl_image_target_texture2d_oes, TEXTURE_2D, TexImage2D, BGRA_EXT, UNSIGNED_BYTE};
use egl::egl::EGLDisplay;
use egl::eglext::{EGLImageKHR, DestroyImageKHR};
use libc::c_void;
use std::mem;
use std::ptr;
use std::slice::bytes::copy_memory;
use std::vec::Vec;

/// FIXME(Aydin Kim) :Currently, native surface is consist of 2 types of hybrid image buffer. EGLImageKHR is used to GPU rendering and vector is used to CPU rendering. EGL extension seems not provide simple way to accessing its bitmap directly. In the future, we need to find out the way to integrate them.

#[deriving(Clone, Copy)]
pub struct NativeGraphicsMetadata {
    pub display: EGLDisplay,
}

pub struct NativePaintingGraphicsContext{
    display : EGLDisplay,
}

impl NativePaintingGraphicsContext {
    pub fn from_metadata(metadata: &NativeGraphicsMetadata) -> NativePaintingGraphicsContext {
        NativePaintingGraphicsContext {
            display : metadata.display,
        }
    }
}

impl Drop for NativePaintingGraphicsContext {
    fn drop(&mut self) {}
}

#[deriving(Copy)]
pub struct NativeCompositingGraphicsContext;

impl NativeCompositingGraphicsContext {
    pub fn new() -> NativeCompositingGraphicsContext {
        NativeCompositingGraphicsContext
    }
}

pub struct NativeSurface {
    image: Option<EGLImageKHR>, // For GPU rendering
    bitmap: Option<Vec<u8>>, // For CPU rendering
    will_leak: bool,
}

impl NativeSurface {
    pub fn from_image_khr(image_khr: EGLImageKHR) -> NativeSurface {
        let mut _image: Option<EGLImageKHR> = None;
        if image_khr != ptr::null_mut() {
            _image = Some(image_khr);
        }
        NativeSurface {
            image : _image,
            bitmap: None,
            will_leak: true,
        }
    }
}

impl NativeSurfaceMethods for NativeSurface {
    /// This may only be called on the case of CPU rendering.
    fn new(_native_context: &NativePaintingGraphicsContext, size: Size2D<i32>, _stride: i32) -> NativeSurface {
        let len = size.width * size.height * 4;
        let bitmap: Vec<u8> = Vec::from_elem(len as uint, 0 as u8);

        NativeSurface {
            image: None,
            bitmap: Some(bitmap),
            will_leak : true,
        }
    }

    /// This may only be called on the compositor side.
    fn bind_to_texture(&self,
                       _native_context: &NativeCompositingGraphicsContext,
                       texture: &Texture,
                       _size: Size2D<int>) {
        let _bound = texture.bind();
        match self.image {
            None => match self.bitmap {
                Some(ref bitmap) => {
                    let data = bitmap.as_ptr() as *const c_void;
                    unsafe {
                        TexImage2D(TEXTURE_2D, 0, BGRA_EXT as i32, _size.width as i32, _size.height as i32,
                                   0, BGRA_EXT as u32, UNSIGNED_BYTE, data);
                    }
                }
                None => {
                    debug!("Cannot bind the buffer(CPU rendering), there is no bitmap");
                }
            },
            Some(image_khr) => {
                egl_image_target_texture2d_oes(TEXTURE_2D, image_khr as *const c_void);
            }
        }
    }

    /// This may only be called on the painting side.
    fn upload(&mut self, _graphics_context: &NativePaintingGraphicsContext, data: &[u8]) {
        match self.bitmap {
            Some(ref mut bitmap) => {
                copy_memory(bitmap.as_mut_slice(), data);
            }
            None => {
                debug!("Cannot upload the buffer(CPU rendering), there is no bitmap");
            }
        }
    }

    fn get_id(&self) -> int {
        match self.image {
            None => 0,
            Some(image_khr) => image_khr as int,
        }
    }

    fn destroy(&mut self, graphics_context: &NativePaintingGraphicsContext) {
        match self.image {
            None => {},
            Some(image_khr) => {
                DestroyImageKHR(graphics_context.display, image_khr);
                mem::replace(&mut self.image, None);
            }
        }
        self.mark_wont_leak()
    }

    fn mark_will_leak(&mut self) {
        self.will_leak = true
    }
    fn mark_wont_leak(&mut self) {
        self.will_leak = false
    }
}
<|endoftext|>


<commit_before>extern crate sdl2;

#[macro_use]
mod events;

use sdl2::render::Renderer;

pub fn spawn<F>(title: &str, init:F) where F: Fn( &mut Phi ) -> Box<View> {

    // initialize SDL2
    let sdl_context =  sdl2::init().unwrap();
    let video = sdl_context.video().unwrap();
    let mut timer = sdl_context.timer().unwrap();

    // create a new window
    let window = video.window(&title, 800, 600)
            .position_centered().opengl()
            .build().unwrap();

    let mut context = Phi{
        events: Events::new( sdl_context.event_pump().unwrap() ),
        renderer: window.renderer().accelerated().build().unwrap(),
    };

    let mut current_view = init(&mut context);

    // Frame timing

    let interval = 1_000 / 60;
    let mut before = timer.ticks();
    let mut last_second = timer.ticks();
    let mut fps = 0u16;

    loop {
        let now = timer.ticks();
        let delta = now - before;
        let elapsed = delta as f64 / 1_000.0;

        if delta < interval {
            timer.delay( interval - delta );
                continue;
        }

        before = now;
        fps += 1;

       if now - last_second > 1_000 {
           println!("FPS: {}", fps);
           last_second = now;
           fps = 0;
       }

        context.events.pump();
        match current_view.render( &mut context, elapsed ){
            ViewAction::None => context.renderer.present(),
            ViewAction::Quit => break
        }
    }

}

struct_events! {
    keyboard: {
        key_escape: Escape,
        key_up: Up,
        key_down: Down
    },
    else: {
        quit: Quit { .. }
    }
}

/// Bundles the Phi abstractions in a single structur
/// to be passed easily
pub struct Phi<'window> {
    pub events: Events,
    pub renderer: Renderer<'window>
}

/// A Way for the currently executed View
/// to communicate with the game loop. It specifies
/// which action should be executed before the next rendering
pub enum ViewAction {
    None,
    Quit,
}

pub trait View {
    /// Called every fram to take care of both the logic
    /// and the rendering of the current view
    ///
    /// `elapsed` is expressed in seconds
    fn render(&mut self, context: &mut Phi, elapsed: f64) -> ViewAction;
}

<commit_msg>add a new event handler for the space bar<commit_after>extern crate sdl2;

#[macro_use]
mod events;

use sdl2::render::Renderer;

pub fn spawn<F>(title: &str, init:F) where F: Fn( &mut Phi ) -> Box<View> {

    // initialize SDL2
    let sdl_context =  sdl2::init().unwrap();
    let video = sdl_context.video().unwrap();
    let mut timer = sdl_context.timer().unwrap();

    // create a new window
    let window = video.window(&title, 800, 600)
            .position_centered().opengl()
            .build().unwrap();

    let mut context = Phi{
        events: Events::new( sdl_context.event_pump().unwrap() ),
        renderer: window.renderer().accelerated().build().unwrap(),
    };

    let mut current_view = init(&mut context);

    // Frame timing

    let interval = 1_000 / 60;
    let mut before = timer.ticks();
    let mut last_second = timer.ticks();
    let mut fps = 0u16;

    loop {
        let now = timer.ticks();
        let delta = now - before;
        let elapsed = delta as f64 / 1_000.0;

        if delta < interval {
            timer.delay( interval - delta );
                continue;
        }

        before = now;
        fps += 1;

       if now - last_second > 1_000 {
           println!("FPS: {}", fps);
           last_second = now;
           fps = 0;
       }

        context.events.pump();
        match current_view.render( &mut context, elapsed ){
            ViewAction::None => context.renderer.present(),
            ViewAction::Quit => break,
            ViewAction::ChangeView(new_view) => current_view = new_view,
        }
    }

}

struct_events! {
    keyboard: {
        key_escape: Escape,
        key_up: Up,
        key_down: Down,
        key_space: Space
    },
    else: {
        quit: Quit { .. }
    }
}

/// Bundles the Phi abstractions in a single structur
/// to be passed easily
pub struct Phi<'window> {
    pub events: Events,
    pub renderer: Renderer<'window>
}

/// A Way for the currently executed View
/// to communicate with the game loop. It specifies
/// which action should be executed before the next rendering
pub enum ViewAction {
    None,
    Quit,
    ChangeView(Box<View>)
}

pub trait View {
    /// Called every fram to take care of both the logic
    /// and the rendering of the current view
    ///
    /// `elapsed` is expressed in seconds
    fn render(&mut self, context: &mut Phi, elapsed: f64) -> ViewAction;
}

<|endoftext|>


<commit_before>use super::location::{Location, Span};

#[derive(Clone)]
struct StringStream {
    input: Vec<char>,
    index: usize,
    line: i32,
    column: i32,
}

impl StringStream {
    fn new(input: &str) -> Self {
        Self {
            input: input.chars().collect(),
            index: 0,
            line: 1,
            column: 1,
        }
    }

    fn read(&mut self) -> char {
        let character = self
            .input
            .get(self.index)
            .map(|value| *value)
            .unwrap_or('\0');
        self.index += 1;
        if character == '\n' {
            self.line += 1;
            self.column = 1;
        } else {
            self.column += 1;
        }
        character
    }

    fn peek(&self) -> char {
        self.input
            .get(self.index)
            .map(|value| *value)
            .unwrap_or('\0')
    }

    fn read_until(&mut self, target: char) {
        while self.peek() != target {
            let character = self.read();
            assert!(character != '\0');
        }
    }

    fn read_if_match(&mut self, target: &[char]) -> bool {
        // Bounds-check first!
        if (self.index + target.len()) > self.input.len() {
            return false;
        }
        let substr = &self.input[self.index..(self.index + target.len())];
        if substr != target {
            return false;
        }
        // Consume if they match.
        for _ in target {
            self.read();
        }
        true
    }

    fn location(&self) -> Location {
        Location::new(self.index as u32, self.line, self.column)
    }
}

#[derive(Clone, Debug, PartialEq)]
pub enum Token {
    Arrow,
    BraceLeft,
    BraceRight,
    Comma,
    Dot,
    EOF,
    Equals,
    Identifier(String, Span),
    Integer(i64),
    Let(Location),
    Minus,
    ParenthesesLeft,
    ParenthesesRight,
    Plus,
    Return,
    Star,
    Terminal(char),
    Var(Location),
}

impl Token {
    pub fn location(&self) -> Option<Location> {
        match self {
            Token::Let(location) | Token::Var(location) => Some(location.clone()),
            Token::Identifier(_, location) => Some(location.start.clone()),
            _ => None,
        }
    }
}

impl Token {
    pub fn newline(self) -> bool {
        match self {
            Token::Terminal(character) => character == '\n',
            _ => false,
        }
    }
}

#[derive(Clone)]
pub struct TokenStream {
    input: StringStream,
    peeking: bool,
    next_token: Option<Token>,
}

impl TokenStream {
    pub fn from_string(input: String) -> Self {
        Self::new(StringStream::new(input.as_str()))
    }

    fn new(input: StringStream) -> Self {
        TokenStream {
            input: input,
            peeking: false,
            next_token: None,
        }
    }

    pub fn backtrack(&mut self, other: &TokenStream) {
        self.input = other.input.clone();
        self.peeking = other.peeking;
        self.next_token = other.next_token.clone();
    }

    pub fn peek(&mut self) -> Token {
        if !self.peeking {
            self.peeking = true;
            self.next_token = Some(self.lex());
        }
        self.next_token.clone().unwrap()
    }

    /// Peeks, and if the peeked token is the same type as the expected then
    /// read that type and return the read (not expected) token.
    pub fn read_if(&mut self, expected: Token) -> Option<Token> {
        use std::mem::discriminant;

        let got = self.peek();
        if discriminant(&expected) == discriminant(&got) {
            Some(self.read())
        } else {
            None
        }
    }

    pub fn read(&mut self) -> Token {
        let token = self.peek();
        self.peeking = false;
        return token;
    }

    fn lex(&mut self) -> Token {
        self.consume_space_and_comments();

        loop {
            let location = self.input.location();
            let character = self.input.peek();

            if identifier_head(character) {
                return self.lex_identifier(location);
            } else if numeric_head(character) {
                return self.lex_arrow_minus_or_numeric();
            } else {
                self.input.read();
                match character {
                    '\0' => return Token::EOF,
                    '{' => return Token::BraceLeft,
                    '}' => return Token::BraceRight,
                    '(' => return Token::ParenthesesLeft,
                    ')' => return Token::ParenthesesRight,
                    ',' => return Token::Comma,
                    '.' => return Token::Dot,
                    '=' => return Token::Equals,
                    '+' => return Token::Plus,
                    '*' => return Token::Star,
                    ';' => return Token::Terminal(character),
                    '\n' => {
                        self.consume_more_newline_terminals();
                        return Token::Terminal(character);
                    }
                    _ => panic!("Unexpected character: {}", character),
                }
            }
        }
    }

    fn lex_identifier(&mut self, start: Location) -> Token {
        let mut identifier = vec![self.input.read()];
        loop {
            let character = self.input.peek();
            if identifier_tail(character) {
                self.input.read();
                identifier.push(character);
            } else {
                break;
            }
        }
        let identifier_string: String = identifier.into_iter().collect();
        match identifier_string.as_str() {
            "let" => Token::Let(start),
            "var" => Token::Var(start),
            "return" => Token::Return,
            _ => Token::Identifier(identifier_string, Span::new(start, self.input.location())),
        }
    }

    fn lex_arrow_minus_or_numeric(&mut self) -> Token {
        let first_character = self.input.read();
        if first_character == '-' && self.input.peek() == '>' {
            self.input.read();
            return Token::Arrow;
        }
        if first_character == '-' && !digit(self.input.peek()) {
            return Token::Minus;
        }
        let mut number = vec![first_character];
        loop {
            let character = self.input.peek();
            if digit(character) {
                self.input.read();
                number.push(character);
            } else {
                break;
            }
        }
        let number_string: String = number.into_iter().collect();
        return Token::Integer(number_string.parse().unwrap());
    }

    fn consume_more_newline_terminals(&mut self) {
        loop {
            self.consume_space_and_comments();
            if self.input.peek() == '\n' {
                self.input.read();
                continue;
            } else {
                break;
            }
        }
    }

    fn consume_space_and_comments(&mut self) {
        loop {
            let character = self.input.peek();
            if character == ' ' || character == '\t' {
                self.input.read();
            } else if self.input.read_if_match(&['/', '/']) {
                self.input.read_until('\n');
            } else if self.input.read_if_match(&['/', '*']) {
                loop {
                    if self.input.read_if_match(&['*', '/']) {
                        break;
                    } else {
                        self.input.read();
                    }
                }
            } else {
                // Not any kind of comment.
                break;
            }
        }
    }
}

fn identifier_head(character: char) -> bool {
    alphabetical(character)
}

fn identifier_tail(character: char) -> bool {
    alphabetical(character)
}

fn numeric_head(character: char) -> bool {
    digit(character) || (character == '-')
}

fn alphabetical(character: char) -> bool {
    (character >= 'a' && character <= 'z') || (character >= 'A' && character <= 'Z')
}

fn digit(character: char) -> bool {
    (character >= '0' && character <= '9')
}

#[cfg(test)]
mod tests {
    use super::{Location, Span, StringStream, Token, TokenStream};

    fn make_token_stream(input: &str) -> TokenStream {
        let string_stream = StringStream::new(input);
        TokenStream::new(string_stream)
    }

    fn parse(input: &str) -> Vec<Token> {
        let mut token_stream = make_token_stream(input);
        let mut tokens = vec![];
        loop {
            let token = token_stream.read();
            tokens.push(token.clone());
            if token == Token::EOF {
                break;
            }
        }
        return tokens;
    }

    #[test]
    fn it_parses_identifier() {
        assert_eq!(
            parse("foo"),
            vec![
                Token::Identifier(
                    "foo".to_string(),
                    Span::new(Location::new(0, 1, 1), Location::new(3, 1, 4)),
                ),
                Token::EOF,
            ]
        );
    }

    #[test]
    fn it_parses_keywords() {
        assert_eq!(
            parse("let"),
            vec![Token::Let(Location::new(0, 1, 1)), Token::EOF],
        );
        assert_eq!(parse("return"), vec![Token::Return, Token::EOF]);
        assert_eq!(
            parse("var"),
            vec![Token::Var(Location::new(0, 1, 1)), Token::EOF],
        );
    }

    #[test]
    fn it_parses_arrows() {
        assert_eq!(
            parse("foo() -> {}"),
            vec![
                Token::Identifier(
                    "foo".to_string(),
                    Span::new(Location::new(0, 1, 1), Location::new(3, 1, 4)),
                ),
                Token::ParenthesesLeft,
                Token::ParenthesesRight,
                Token::Arrow,
                Token::BraceLeft,
                Token::BraceRight,
                Token::EOF,
            ],
        );
    }

    #[test]
    fn it_parse_comments_and_terminals() {
        assert_eq!(
            parse("foo /* Comment */ bar"),
            vec![
                Token::Identifier(
                    "foo".to_string(),
                    Span::new(Location::new(0, 1, 1), Location::new(3, 1, 4)),
                ),
                Token::Identifier(
                    "bar".to_string(),
                    Span::new(Location::new(18, 1, 19), Location::new(21, 1, 22)),
                ),
                Token::EOF,
            ]
        );

        assert_eq!(
            parse("foo // Comment \n bar"),
            vec![
                Token::Identifier(
                    "foo".to_string(),
                    Span::new(Location::new(0, 1, 1), Location::new(3, 1, 4)),
                ),
                Token::Terminal('\n'),
                Token::Identifier(
                    "bar".to_string(),
                    Span::new(Location::new(17, 2, 2), Location::new(20, 2, 5)),
                ),
                Token::EOF,
            ]
        );

        assert_eq!(
            parse(
                "foo
            // Comment about the call
            // Another comment about the call
            bar()"
            ),
            vec![
                Token::Identifier(
                    "foo".to_string(),
                    Span::new(Location::new(0, 1, 1), Location::new(3, 1, 4)),
                ),
                Token::Terminal('\n'),
                Token::Identifier(
                    "bar".to_string(),
                    Span::new(Location::new(100, 4, 13), Location::new(103, 4, 16)),
                ),
                Token::ParenthesesLeft,
                Token::ParenthesesRight,
                Token::EOF,
            ]
        );
    }

    #[test]
    fn it_parses_integers() {
        assert_eq!(parse("1"), vec![Token::Integer(1), Token::EOF]);
        assert_eq!(parse("-1"), vec![Token::Integer(-1), Token::EOF]);
        assert_eq!(
            parse("- 1"),
            vec![Token::Minus, Token::Integer(1), Token::EOF]
        );
        assert_eq!(
            parse("1+2"),
            vec![
                Token::Integer(1),
                Token::Plus,
                Token::Integer(2),
                Token::EOF,
            ]
        );
        assert_eq!(
            parse("1 + 2"),
            vec![
                Token::Integer(1),
                Token::Plus,
                Token::Integer(2),
                Token::EOF,
            ]
        );
    }

    #[test]
    fn it_parses_blocks() {
        assert_eq!(
            parse("{ 1; }"),
            vec![
                Token::BraceLeft,
                Token::Integer(1),
                Token::Terminal(';'),
                Token::BraceRight,
                Token::EOF,
            ]
        );
    }

    #[test]
    fn test_read_if() {
        let input = "a";
        // Check that `read` works correctly.
        let actual = Token::Identifier(
            "a".to_string(),
            Span::new(Location::new(0, 1, 1), Location::new(1, 1, 2)),
        );
        assert_eq!(make_token_stream(input).read(), actual,);
        // Now check that the contents of the expected token don't equal the
        // actual token.
        let expected = Token::Identifier(
            "".to_string(),
            Span::new(Location::unknown(), Location::unknown()),
        );
        assert_ne!(make_token_stream(input).read(), expected);
        // But `read_if` should still return the actual token since they're
        // both `Identifier`s.
        assert_eq!(make_token_stream("a").read_if(expected), Some(actual));
        // And finally if we look for something that isn't an identifier it
        // should return `None`.
        assert_eq!(make_token_stream("a").read_if(Token::ParenthesesLeft), None);
    }
}
<commit_msg>Add more tests for lexing of integers<commit_after>use super::location::{Location, Span};

#[derive(Clone)]
struct StringStream {
    input: Vec<char>,
    index: usize,
    line: i32,
    column: i32,
}

impl StringStream {
    fn new(input: &str) -> Self {
        Self {
            input: input.chars().collect(),
            index: 0,
            line: 1,
            column: 1,
        }
    }

    fn read(&mut self) -> char {
        let character = self
            .input
            .get(self.index)
            .map(|value| *value)
            .unwrap_or('\0');
        self.index += 1;
        if character == '\n' {
            self.line += 1;
            self.column = 1;
        } else {
            self.column += 1;
        }
        character
    }

    fn peek(&self) -> char {
        self.input
            .get(self.index)
            .map(|value| *value)
            .unwrap_or('\0')
    }

    fn read_until(&mut self, target: char) {
        while self.peek() != target {
            let character = self.read();
            assert!(character != '\0');
        }
    }

    fn read_if_match(&mut self, target: &[char]) -> bool {
        // Bounds-check first!
        if (self.index + target.len()) > self.input.len() {
            return false;
        }
        let substr = &self.input[self.index..(self.index + target.len())];
        if substr != target {
            return false;
        }
        // Consume if they match.
        for _ in target {
            self.read();
        }
        true
    }

    fn location(&self) -> Location {
        Location::new(self.index as u32, self.line, self.column)
    }
}

#[derive(Clone, Debug, PartialEq)]
pub enum Token {
    Arrow,
    BraceLeft,
    BraceRight,
    Comma,
    Dot,
    EOF,
    Equals,
    Identifier(String, Span),
    Integer(i64),
    Let(Location),
    Minus,
    ParenthesesLeft,
    ParenthesesRight,
    Plus,
    Return,
    Star,
    Terminal(char),
    Var(Location),
}

impl Token {
    pub fn location(&self) -> Option<Location> {
        match self {
            Token::Let(location) | Token::Var(location) => Some(location.clone()),
            Token::Identifier(_, location) => Some(location.start.clone()),
            _ => None,
        }
    }
}

impl Token {
    pub fn newline(self) -> bool {
        match self {
            Token::Terminal(character) => character == '\n',
            _ => false,
        }
    }
}

#[derive(Clone)]
pub struct TokenStream {
    input: StringStream,
    peeking: bool,
    next_token: Option<Token>,
}

impl TokenStream {
    pub fn from_string(input: String) -> Self {
        Self::new(StringStream::new(input.as_str()))
    }

    fn new(input: StringStream) -> Self {
        TokenStream {
            input: input,
            peeking: false,
            next_token: None,
        }
    }

    pub fn backtrack(&mut self, other: &TokenStream) {
        self.input = other.input.clone();
        self.peeking = other.peeking;
        self.next_token = other.next_token.clone();
    }

    pub fn peek(&mut self) -> Token {
        if !self.peeking {
            self.peeking = true;
            self.next_token = Some(self.lex());
        }
        self.next_token.clone().unwrap()
    }

    /// Peeks, and if the peeked token is the same type as the expected then
    /// read that type and return the read (not expected) token.
    pub fn read_if(&mut self, expected: Token) -> Option<Token> {
        use std::mem::discriminant;

        let got = self.peek();
        if discriminant(&expected) == discriminant(&got) {
            Some(self.read())
        } else {
            None
        }
    }

    pub fn read(&mut self) -> Token {
        let token = self.peek();
        self.peeking = false;
        return token;
    }

    fn lex(&mut self) -> Token {
        self.consume_space_and_comments();

        loop {
            let location = self.input.location();
            let character = self.input.peek();

            if identifier_head(character) {
                return self.lex_identifier(location);
            } else if numeric_head(character) {
                return self.lex_arrow_minus_or_numeric();
            } else {
                self.input.read();
                match character {
                    '\0' => return Token::EOF,
                    '{' => return Token::BraceLeft,
                    '}' => return Token::BraceRight,
                    '(' => return Token::ParenthesesLeft,
                    ')' => return Token::ParenthesesRight,
                    ',' => return Token::Comma,
                    '.' => return Token::Dot,
                    '=' => return Token::Equals,
                    '+' => return Token::Plus,
                    '*' => return Token::Star,
                    ';' => return Token::Terminal(character),
                    '\n' => {
                        self.consume_more_newline_terminals();
                        return Token::Terminal(character);
                    }
                    _ => panic!("Unexpected character: {}", character),
                }
            }
        }
    }

    fn lex_identifier(&mut self, start: Location) -> Token {
        let mut identifier = vec![self.input.read()];
        loop {
            let character = self.input.peek();
            if identifier_tail(character) {
                self.input.read();
                identifier.push(character);
            } else {
                break;
            }
        }
        let identifier_string: String = identifier.into_iter().collect();
        match identifier_string.as_str() {
            "let" => Token::Let(start),
            "var" => Token::Var(start),
            "return" => Token::Return,
            _ => Token::Identifier(identifier_string, Span::new(start, self.input.location())),
        }
    }

    fn lex_arrow_minus_or_numeric(&mut self) -> Token {
        let first_character = self.input.read();
        if first_character == '-' && self.input.peek() == '>' {
            self.input.read();
            return Token::Arrow;
        }
        if first_character == '-' && !digit(self.input.peek()) {
            return Token::Minus;
        }
        let mut number = vec![first_character];
        loop {
            let character = self.input.peek();
            if digit(character) {
                self.input.read();
                number.push(character);
            } else {
                break;
            }
        }
        let number_string: String = number.into_iter().collect();
        return Token::Integer(number_string.parse().unwrap());
    }

    fn consume_more_newline_terminals(&mut self) {
        loop {
            self.consume_space_and_comments();
            if self.input.peek() == '\n' {
                self.input.read();
                continue;
            } else {
                break;
            }
        }
    }

    fn consume_space_and_comments(&mut self) {
        loop {
            let character = self.input.peek();
            if character == ' ' || character == '\t' {
                self.input.read();
            } else if self.input.read_if_match(&['/', '/']) {
                self.input.read_until('\n');
            } else if self.input.read_if_match(&['/', '*']) {
                loop {
                    if self.input.read_if_match(&['*', '/']) {
                        break;
                    } else {
                        self.input.read();
                    }
                }
            } else {
                // Not any kind of comment.
                break;
            }
        }
    }
}

fn identifier_head(character: char) -> bool {
    alphabetical(character)
}

fn identifier_tail(character: char) -> bool {
    alphabetical(character)
}

fn numeric_head(character: char) -> bool {
    digit(character) || (character == '-')
}

fn alphabetical(character: char) -> bool {
    (character >= 'a' && character <= 'z') || (character >= 'A' && character <= 'Z')
}

fn digit(character: char) -> bool {
    (character >= '0' && character <= '9')
}

#[cfg(test)]
mod tests {
    use super::{Location, Span, StringStream, Token, TokenStream};

    fn make_token_stream(input: &str) -> TokenStream {
        let string_stream = StringStream::new(input);
        TokenStream::new(string_stream)
    }

    fn parse(input: &str) -> Vec<Token> {
        let mut token_stream = make_token_stream(input);
        let mut tokens = vec![];
        loop {
            let token = token_stream.read();
            tokens.push(token.clone());
            if token == Token::EOF {
                break;
            }
        }
        return tokens;
    }

    #[test]
    fn it_parses_identifier() {
        assert_eq!(
            parse("foo"),
            vec![
                Token::Identifier(
                    "foo".to_string(),
                    Span::new(Location::new(0, 1, 1), Location::new(3, 1, 4)),
                ),
                Token::EOF,
            ]
        );
    }

    #[test]
    fn it_parses_keywords() {
        assert_eq!(
            parse("let"),
            vec![Token::Let(Location::new(0, 1, 1)), Token::EOF],
        );
        assert_eq!(parse("return"), vec![Token::Return, Token::EOF]);
        assert_eq!(
            parse("var"),
            vec![Token::Var(Location::new(0, 1, 1)), Token::EOF],
        );
    }

    #[test]
    fn it_parses_arrows() {
        assert_eq!(
            parse("foo() -> {}"),
            vec![
                Token::Identifier(
                    "foo".to_string(),
                    Span::new(Location::new(0, 1, 1), Location::new(3, 1, 4)),
                ),
                Token::ParenthesesLeft,
                Token::ParenthesesRight,
                Token::Arrow,
                Token::BraceLeft,
                Token::BraceRight,
                Token::EOF,
            ],
        );
    }

    #[test]
    fn it_parses_comments_and_terminals() {
        assert_eq!(
            parse("foo /* Comment */ bar"),
            vec![
                Token::Identifier(
                    "foo".to_string(),
                    Span::new(Location::new(0, 1, 1), Location::new(3, 1, 4)),
                ),
                Token::Identifier(
                    "bar".to_string(),
                    Span::new(Location::new(18, 1, 19), Location::new(21, 1, 22)),
                ),
                Token::EOF,
            ]
        );

        assert_eq!(
            parse("foo // Comment \n bar"),
            vec![
                Token::Identifier(
                    "foo".to_string(),
                    Span::new(Location::new(0, 1, 1), Location::new(3, 1, 4)),
                ),
                Token::Terminal('\n'),
                Token::Identifier(
                    "bar".to_string(),
                    Span::new(Location::new(17, 2, 2), Location::new(20, 2, 5)),
                ),
                Token::EOF,
            ]
        );

        assert_eq!(
            parse(
                "foo
            // Comment about the call
            // Another comment about the call
            bar()"
            ),
            vec![
                Token::Identifier(
                    "foo".to_string(),
                    Span::new(Location::new(0, 1, 1), Location::new(3, 1, 4)),
                ),
                Token::Terminal('\n'),
                Token::Identifier(
                    "bar".to_string(),
                    Span::new(Location::new(100, 4, 13), Location::new(103, 4, 16)),
                ),
                Token::ParenthesesLeft,
                Token::ParenthesesRight,
                Token::EOF,
            ]
        );
    }

    #[test]
    fn it_parses_integers() {
        assert_eq!(parse("1"), vec![Token::Integer(1), Token::EOF]);
        assert_eq!(parse("-1"), vec![Token::Integer(-1), Token::EOF]);
        assert_eq!(
            parse("- 1"),
            vec![Token::Minus, Token::Integer(1), Token::EOF],
        );
        assert_eq!(
            parse("- -2"),
            vec![Token::Minus, Token::Integer(-2), Token::EOF],
        );
        assert_eq!(
            parse("1+2"),
            vec![
                Token::Integer(1),
                Token::Plus,
                Token::Integer(2),
                Token::EOF,
            ],
        );
        assert_eq!(
            parse("1 + 2"),
            vec![
                Token::Integer(1),
                Token::Plus,
                Token::Integer(2),
                Token::EOF,
            ],
        );
        assert_eq!(
            parse("1 + -2"),
            vec![
                Token::Integer(1),
                Token::Plus,
                Token::Integer(-2),
                Token::EOF,
            ],
        );
    }

    #[test]
    fn it_parses_blocks() {
        assert_eq!(
            parse("{ 1; }"),
            vec![
                Token::BraceLeft,
                Token::Integer(1),
                Token::Terminal(';'),
                Token::BraceRight,
                Token::EOF,
            ]
        );
    }

    #[test]
    fn test_read_if() {
        let input = "a";
        // Check that `read` works correctly.
        let actual = Token::Identifier(
            "a".to_string(),
            Span::new(Location::new(0, 1, 1), Location::new(1, 1, 2)),
        );
        assert_eq!(make_token_stream(input).read(), actual,);
        // Now check that the contents of the expected token don't equal the
        // actual token.
        let expected = Token::Identifier(
            "".to_string(),
            Span::new(Location::unknown(), Location::unknown()),
        );
        assert_ne!(make_token_stream(input).read(), expected);
        // But `read_if` should still return the actual token since they're
        // both `Identifier`s.
        assert_eq!(make_token_stream("a").read_if(expected), Some(actual));
        // And finally if we look for something that isn't an identifier it
        // should return `None`.
        assert_eq!(make_token_stream("a").read_if(Token::ParenthesesLeft), None);
    }
}
<|endoftext|>