<commit_before>/// The tokenizer module contains all of the tools used to process
/// text in `tantivy`.

use std::borrow::{Borrow, BorrowMut};
use tokenizer::TokenStreamChain;

/// Token
pub struct Token {
    /// Offset (byte index) of the first character of the token.
    /// Offsets shall not be modified by token filters.
    pub offset_from: usize,
    /// Offset (byte index) of the last character of the token + 1.
    /// The text that generated the token should be obtained by
    /// &text[token.offset_from..token.offset_to]
    pub offset_to: usize,
    /// Position, expressed in number of tokens.
    pub position: usize,
    /// Actual text content of the token.
    pub text: String,
}

impl Default for Token {
    fn default() -> Token {
        Token {
            offset_from: 0,
            offset_to: 0,
            position: usize::max_value(),
            text: String::new(),
        }
    }
}

/// `Tokenizer` are in charge of splitting text into a stream of token
/// before indexing.
///
/// See the [module documentation](./index.html) for more detail.
///
/// # Warning
///
/// This API may change to use associated types.
pub trait Tokenizer<'a>: Sized + Clone {
    /// Type associated to the resulting tokenstream tokenstream.
    type TokenStreamImpl: TokenStream;

    /// Creates a token stream for a given `str`.
    fn token_stream(&self, text: &'a str) -> Self::TokenStreamImpl;

    /// Appends a token filter to the current tokenizer.
    ///
    /// The method consumes the current `TokenStream` and returns a
    /// new one.
    ///
    /// # Example
    ///
    /// ```rust
    /// # extern crate tantivy;
    ///
    /// use tantivy::tokenizer::*;
    ///
    /// # fn main() {
    /// let en_stem = SimpleTokenizer
    ///     .filter(RemoveLongFilter::limit(40))
    ///     .filter(LowerCaser)
    ///     .filter(Stemmer::new());
    /// # }
    /// ```
    ///
    fn filter<NewFilter>(self, new_filter: NewFilter) -> ChainTokenizer<NewFilter, Self>
    where
        NewFilter: TokenFilter<<Self as Tokenizer<'a>>::TokenStreamImpl>,
    {
        ChainTokenizer {
            head: new_filter,
            tail: self,
        }
    }
}

pub trait BoxedTokenizer: Send + Sync {
    fn token_stream<'a>(&self, text: &'a str) -> Box<TokenStream + 'a>;
    fn token_stream_texts<'b>(&self, texts: &'b [&'b str]) -> Box<TokenStream + 'b>;
    fn boxed_clone(&self) -> Box<BoxedTokenizer>;
}

#[derive(Clone)]
struct BoxableTokenizer(A)
where
    A: for<'a> Tokenizer<'a> + Send + Sync;

impl BoxedTokenizer for BoxableTokenizer
where
    A: 'static + Send + Sync + for<'a> Tokenizer<'a>,
{
    fn token_stream<'a>(&self, text: &'a str) -> Box<TokenStream + 'a> {
        box self.0.token_stream(text)
    }

    fn token_stream_texts<'b>(&self, texts: &'b [&'b str]) -> Box<TokenStream + 'b> {
        assert!(texts.len() > 0);
        if texts.len() == 1 {
            box self.0.token_stream(texts[0])
        } else {
            let mut offsets = vec![];
            let mut total_offset = 0;
            for &text in texts {
                offsets.push(total_offset);
                total_offset += text.len();
            }
            let token_streams: Vec<_> =
                texts.iter().map(|text| self.0.token_stream(text)).collect();
            box TokenStreamChain::new(offsets, token_streams)
        }
    }

    fn boxed_clone(&self) -> Box<BoxedTokenizer> {
        box self.clone()
    }
}

pub fn box_tokenizer(a: A) -> Box<BoxedTokenizer>
where
    A: 'static + Send + Sync + for<'a> Tokenizer<'a>,
{
    box BoxableTokenizer(a)
}

impl<'b> TokenStream for Box<TokenStream + 'b> {
    fn advance(&mut self) -> bool {
        let token_stream: &mut TokenStream = self.borrow_mut();
        token_stream.advance()
    }

    fn token(&self) -> &Token {
        let token_stream: &TokenStream = self.borrow();
        token_stream.token()
    }

    fn token_mut(&mut self) -> &mut Token {
        let token_stream: &mut TokenStream = self.borrow_mut();
        token_stream.token_mut()
    }
}

/// `TokenStream` is the result of the tokenization.
///
/// It consists consumable stream of `Token`s.
///
/// # Example
///
/// ```
/// extern crate tantivy;
/// use tantivy::tokenizer::*;
///
/// # fn main() {
/// let mut tokenizer = SimpleTokenizer
///        .filter(RemoveLongFilter::limit(40))
///        .filter(LowerCaser);
/// let mut token_stream = tokenizer.token_stream("Hello, happy tax payer");
/// {
///     let token = token_stream.next().unwrap();
///     assert_eq!(&token.text, "hello");
///     assert_eq!(token.offset_from, 0);
///     assert_eq!(token.offset_to, 5);
///     assert_eq!(token.position, 0);
/// }
/// {
///     let token = token_stream.next().unwrap();
///     assert_eq!(&token.text, "happy");
///     assert_eq!(token.offset_from, 7);
///     assert_eq!(token.offset_to, 12);
///     assert_eq!(token.position, 1);
/// }
/// # }
/// ```
///
pub trait TokenStream {
    fn advance(&mut self) -> bool;

    fn token(&self) -> &Token;

    fn token_mut(&mut self) -> &mut Token;

    fn next(&mut self) -> Option<&Token> {
        if self.advance() {
            Some(self.token())
        } else {
            None
        }
    }

    fn process(&mut self, sink: &mut FnMut(&Token)) -> u32 {
        let mut num_tokens_pushed = 0u32;
        while self.advance() {
            sink(self.token());
            num_tokens_pushed += 1u32;
        }
        num_tokens_pushed
    }
}

#[derive(Clone)]
pub struct ChainTokenizer<HeadTokenFilterFactory, TailTokenizer> {
    head: HeadTokenFilterFactory,
    tail: TailTokenizer,
}

impl<'a, HeadTokenFilterFactory, TailTokenizer> Tokenizer<'a>
    for ChainTokenizer<HeadTokenFilterFactory, TailTokenizer>
where
    HeadTokenFilterFactory: TokenFilter<TailTokenizer::TokenStreamImpl>,
    TailTokenizer: Tokenizer<'a>,
{
    type TokenStreamImpl = HeadTokenFilterFactory::ResultTokenStream;

    fn token_stream(&self, text: &'a str) -> Self::TokenStreamImpl {
        let tail_token_stream = self.tail.token_stream(text);
        self.head.transform(tail_token_stream)
    }
}

pub trait TokenFilter<TailTokenStream: TokenStream>: Clone {
    type ResultTokenStream: TokenStream;

    /// Wraps a token stream and returns the modified one.
    fn transform(&self, token_stream: TailTokenStream) -> Self::ResultTokenStream;
}
<commit_msg>fixing doctest<commit_after>/// The tokenizer module contains all of the tools used to process
/// text in `tantivy`.

use std::borrow::{Borrow, BorrowMut};
use tokenizer::TokenStreamChain;

/// Token
pub struct Token {
    /// Offset (byte index) of the first character of the token.
    /// Offsets shall not be modified by token filters.
    pub offset_from: usize,
    /// Offset (byte index) of the last character of the token + 1.
    /// The text that generated the token should be obtained by
    /// &text[token.offset_from..token.offset_to]
    pub offset_to: usize,
    /// Position, expressed in number of tokens.
    pub position: usize,
    /// Actual text content of the token.
    pub text: String,
}

impl Default for Token {
    fn default() -> Token {
        Token {
            offset_from: 0,
            offset_to: 0,
            position: usize::max_value(),
            text: String::new(),
        }
    }
}

/// `Tokenizer` are in charge of splitting text into a stream of token
/// before indexing.
///
/// See the [module documentation](./index.html) for more detail.
///
/// # Warning
///
/// This API may change to use associated types.
pub trait Tokenizer<'a>: Sized + Clone {
    /// Type associated to the resulting tokenstream tokenstream.
    type TokenStreamImpl: TokenStream;

    /// Creates a token stream for a given `str`.
    fn token_stream(&self, text: &'a str) -> Self::TokenStreamImpl;

    /// Appends a token filter to the current tokenizer.
    ///
    /// The method consumes the current `TokenStream` and returns a
    /// new one.
    ///
    /// # Example
    ///
    /// ```rust
    /// # extern crate tantivy;
    ///
    /// use tantivy::tokenizer::*;
    ///
    /// # fn main() {
    /// let en_stem = SimpleTokenizer
    ///     .filter(RemoveLongFilter::limit(40))
    ///     .filter(LowerCaser)
    ///     .filter(Stemmer::new());
    /// # }
    /// ```
    ///
    fn filter<NewFilter>(self, new_filter: NewFilter) -> ChainTokenizer<NewFilter, Self>
    where
        NewFilter: TokenFilter<<Self as Tokenizer<'a>>::TokenStreamImpl>,
    {
        ChainTokenizer {
            head: new_filter,
            tail: self,
        }
    }
}

pub trait BoxedTokenizer: Send + Sync {
    fn token_stream<'a>(&self, text: &'a str) -> Box<TokenStream + 'a>;
    fn token_stream_texts<'b>(&self, texts: &'b [&'b str]) -> Box<TokenStream + 'b>;
    fn boxed_clone(&self) -> Box<BoxedTokenizer>;
}

#[derive(Clone)]
struct BoxableTokenizer(A)
where
    A: for<'a> Tokenizer<'a> + Send + Sync;

impl BoxedTokenizer for BoxableTokenizer
where
    A: 'static + Send + Sync + for<'a> Tokenizer<'a>,
{
    fn token_stream<'a>(&self, text: &'a str) -> Box<TokenStream + 'a> {
        box self.0.token_stream(text)
    }

    fn token_stream_texts<'b>(&self, texts: &'b [&'b str]) -> Box<TokenStream + 'b> {
        assert!(texts.len() > 0);
        if texts.len() == 1 {
            box self.0.token_stream(texts[0])
        } else {
            let mut offsets = vec![];
            let mut total_offset = 0;
            for &text in texts {
                offsets.push(total_offset);
                total_offset += text.len();
            }
            let token_streams: Vec<_> =
                texts.iter().map(|text| self.0.token_stream(text)).collect();
            box TokenStreamChain::new(offsets, token_streams)
        }
    }

    fn boxed_clone(&self) -> Box<BoxedTokenizer> {
        box self.clone()
    }
}

pub fn box_tokenizer(a: A) -> Box<BoxedTokenizer>
where
    A: 'static + Send + Sync + for<'a> Tokenizer<'a>,
{
    box BoxableTokenizer(a)
}

impl<'b> TokenStream for Box<TokenStream + 'b> {
    fn advance(&mut self) -> bool {
        let token_stream: &mut TokenStream = self.borrow_mut();
        token_stream.advance()
    }

    fn token(&self) -> &Token {
        let token_stream: &TokenStream = self.borrow();
        token_stream.token()
    }

    fn token_mut(&mut self) -> &mut Token {
        let token_stream: &mut TokenStream = self.borrow_mut();
        token_stream.token_mut()
    }
}

/// `TokenStream` is the result of the tokenization.
///
/// It consists consumable stream of `Token`s.
///
/// # Example
///
/// ```
/// extern crate tantivy;
/// use tantivy::tokenizer::*;
///
/// # fn main() {
/// let tokenizer = SimpleTokenizer
///        .filter(RemoveLongFilter::limit(40))
///        .filter(LowerCaser);
/// let mut token_stream = tokenizer.token_stream("Hello, happy tax payer");
/// {
///     let token = token_stream.next().unwrap();
///     assert_eq!(&token.text, "hello");
///     assert_eq!(token.offset_from, 0);
///     assert_eq!(token.offset_to, 5);
///     assert_eq!(token.position, 0);
/// }
/// {
///     let token = token_stream.next().unwrap();
///     assert_eq!(&token.text, "happy");
///     assert_eq!(token.offset_from, 7);
///     assert_eq!(token.offset_to, 12);
///     assert_eq!(token.position, 1);
/// }
/// # }
/// ```
///
pub trait TokenStream {
    fn advance(&mut self) -> bool;

    fn token(&self) -> &Token;

    fn token_mut(&mut self) -> &mut Token;

    fn next(&mut self) -> Option<&Token> {
        if self.advance() {
            Some(self.token())
        } else {
            None
        }
    }

    fn process(&mut self, sink: &mut FnMut(&Token)) -> u32 {
        let mut num_tokens_pushed = 0u32;
        while self.advance() {
            sink(self.token());
            num_tokens_pushed += 1u32;
        }
        num_tokens_pushed
    }
}

#[derive(Clone)]
pub struct ChainTokenizer<HeadTokenFilterFactory, TailTokenizer> {
    head: HeadTokenFilterFactory,
    tail: TailTokenizer,
}

impl<'a, HeadTokenFilterFactory, TailTokenizer> Tokenizer<'a>
    for ChainTokenizer<HeadTokenFilterFactory, TailTokenizer>
where
    HeadTokenFilterFactory: TokenFilter<TailTokenizer::TokenStreamImpl>,
    TailTokenizer: Tokenizer<'a>,
{
    type TokenStreamImpl = HeadTokenFilterFactory::ResultTokenStream;

    fn token_stream(&self, text: &'a str) -> Self::TokenStreamImpl {
        let tail_token_stream = self.tail.token_stream(text);
        self.head.transform(tail_token_stream)
    }
}

pub trait TokenFilter<TailTokenStream: TokenStream>: Clone {
    type ResultTokenStream: TokenStream;

    /// Wraps a token stream and returns the modified one.
    fn transform(&self, token_stream: TailTokenStream) -> Self::ResultTokenStream;
}
<|endoftext|>


<commit_before>use std::{
    fs::File,
    path::{Path, PathBuf},
    sync::{Arc, Mutex},
};

use tempfile::TempDir;
use uucore::error::{UResult, USimpleError};

use crate::SortError;

/// A wrapper around TempDir that may only exist once in a process.
///
/// `TmpDirWrapper` handles the allocation of new temporary files in this temporary directory and
/// deleting the whole directory when `SIGINT` is received. Creating a second `TmpDirWrapper` will
/// fail because `ctrlc::set_handler()` fails when there's already a handler.
/// The directory is only created once the first file is requested.
pub struct TmpDirWrapper {
    temp_dir: Option<TempDir>,
    parent_path: PathBuf,
    size: usize,
    lock: Arc<Mutex<()>>,
}

impl TmpDirWrapper {
    pub fn new(path: PathBuf) -> Self {
        Self {
            parent_path: path,
            size: 0,
            temp_dir: None,
            lock: Default::default(),
        }
    }

    fn init_tmp_dir(&mut self) -> UResult<()> {
        assert!(self.temp_dir.is_none());
        assert_eq!(self.size, 0);
        self.temp_dir = Some(
            tempfile::Builder::new()
                .prefix("uutils_sort")
                .tempdir_in(&self.parent_path)
                .map_err(|_| SortError::TmpDirCreationFailed)?,
        );

        let path = self.temp_dir.as_ref().unwrap().path().to_owned();
        let lock = self.lock.clone();
        ctrlc::set_handler(move || {
            // Take the lock so that `next_file_path` returns no new file path.
            let _lock = lock.lock().unwrap();
            if let Err(e) = remove_tmp_dir(&path) {
                show_error!("failed to delete temporary directory: {}", e);
            }
            std::process::exit(2)
        })
        .map_err(|e| USimpleError::new(2, format!("failed to set up signal handler: {}", e)))
    }

    pub fn next_file(&mut self) -> UResult<(File, PathBuf)> {
        if self.temp_dir.is_none() {
            self.init_tmp_dir()?;
        }

        let _lock = self.lock.lock().unwrap();
        let file_name = self.size.to_string();
        self.size += 1;
        let path = self.temp_dir.as_ref().unwrap().path().join(file_name);
        Ok((
            File::create(&path).map_err(|error| SortError::OpenTmpFileFailed { error })?,
            path,
        ))
    }

    pub fn wait_if_signal(&self) {
        let _ = self.lock.lock().unwrap();
    }
}

/// Remove the directory at `path` by deleting its child files and then itself.
/// Errors while deleting child files are ignored.
fn remove_tmp_dir(path: &Path) -> std::io::Result<()> {
    if let Ok(read_dir) = std::fs::read_dir(&path) {
        for file in read_dir.flatten() {
            // if we fail to delete the file here it was probably deleted by another thread
            // in the meantime, but that's ok.
            let _ = std::fs::remove_file(file.path());
        }
    }
    std::fs::remove_dir(path)
}
<commit_msg>sort: rename lock binding variable<commit_after>use std::{
    fs::File,
    path::{Path, PathBuf},
    sync::{Arc, Mutex},
};

use tempfile::TempDir;
use uucore::error::{UResult, USimpleError};

use crate::SortError;

/// A wrapper around TempDir that may only exist once in a process.
///
/// `TmpDirWrapper` handles the allocation of new temporary files in this temporary directory and
/// deleting the whole directory when `SIGINT` is received. Creating a second `TmpDirWrapper` will
/// fail because `ctrlc::set_handler()` fails when there's already a handler.
/// The directory is only created once the first file is requested.
pub struct TmpDirWrapper {
    temp_dir: Option<TempDir>,
    parent_path: PathBuf,
    size: usize,
    lock: Arc<Mutex<()>>,
}

impl TmpDirWrapper {
    pub fn new(path: PathBuf) -> Self {
        Self {
            parent_path: path,
            size: 0,
            temp_dir: None,
            lock: Default::default(),
        }
    }

    fn init_tmp_dir(&mut self) -> UResult<()> {
        assert!(self.temp_dir.is_none());
        assert_eq!(self.size, 0);
        self.temp_dir = Some(
            tempfile::Builder::new()
                .prefix("uutils_sort")
                .tempdir_in(&self.parent_path)
                .map_err(|_| SortError::TmpDirCreationFailed)?,
        );

        let path = self.temp_dir.as_ref().unwrap().path().to_owned();
        let lock = self.lock.clone();
        ctrlc::set_handler(move || {
            // Take the lock so that `next_file_path` returns no new file path.
            let _lock = lock.lock().unwrap();
            if let Err(e) = remove_tmp_dir(&path) {
                show_error!("failed to delete temporary directory: {}", e);
            }
            std::process::exit(2)
        })
        .map_err(|e| USimpleError::new(2, format!("failed to set up signal handler: {}", e)))
    }

    pub fn next_file(&mut self) -> UResult<(File, PathBuf)> {
        if self.temp_dir.is_none() {
            self.init_tmp_dir()?;
        }

        let _lock = self.lock.lock().unwrap();
        let file_name = self.size.to_string();
        self.size += 1;
        let path = self.temp_dir.as_ref().unwrap().path().join(file_name);
        Ok((
            File::create(&path).map_err(|error| SortError::OpenTmpFileFailed { error })?,
            path,
        ))
    }

    pub fn wait_if_signal(&self) {
        let _lock = self.lock.lock().unwrap();
    }
}

/// Remove the directory at `path` by deleting its child files and then itself.
/// Errors while deleting child files are ignored.
fn remove_tmp_dir(path: &Path) -> std::io::Result<()> {
    if let Ok(read_dir) = std::fs::read_dir(&path) {
        for file in read_dir.flatten() {
            // if we fail to delete the file here it was probably deleted by another thread
            // in the meantime, but that's ok.
            let _ = std::fs::remove_file(file.path());
        }
    }
    std::fs::remove_dir(path)
}
<|endoftext|>


<commit_before>//! REPL (Read–eval–print loop) executable module provides interactive
//! execution of Tokamak language.

extern crate env_logger;
extern crate libc;
extern crate llvm;
extern crate llvm_sys;
#[macro_use]
extern crate log;
extern crate parser;
extern crate rl_sys; // libreadline

extern crate common;
extern crate exec;
extern crate plan;

use std::collections::HashMap;
use std::io;
use llvm::{Builder, Function, JitCompiler, Module, ValueRef, Verifier};
use rl_sys::readline;

use common::page::{Page, ROWBATCH_SIZE};
use common::plugin::{FuncRegistry, PluginManager};
use common::types::{HasType, Ty};
use common::session::Session;
use exec::{NamedSchema, ColumnarRowPrinter, RowPrinter};
use exec::processor::{MapCompiler, ExprCompiler, to_llvm_ty};
use plan::expr::Expr;
use parser::lexer;
use parser::parser as p;

// Logical Components
// Repl - Main compoenent for REPL
// SymbolTable - HashMap, keeping functions and variables in top-level module
// IncrementalCompiler - Imitate incremental execution with some tricky way
// ValuePrinter - Print the value contents.

pub struct Repl<'a> {
  // for system
  jit: &'a JitCompiler,
  module: Module,
  sess: &'a Session,
  plugin_mgr: &'a PluginManager<'a>,

  // in/out descriptors
  out: &'a mut io::Write, // can be stdout or anything else,

  // for compile internal
  compiler: IncrementalCompiler<'a>,
}

#[derive(Eq, Hash, PartialEq)]
pub enum SymbolKind {
  // Function
  Func,
  // Immutable value
  ImmVal,
  // Mutable value
  MutVal,
}

pub struct Symbol;

impl<'a> Repl<'a> {
  pub fn new(sess: &'a Session,
             plugin_mgr: &'a PluginManager<'a>,
             jit: &'a JitCompiler,
             out: &'a mut io::Write)
             -> Repl<'a> {
    Repl {
      jit: jit,
      module: Module::new(jit.context(), "root"),
      sess: sess,
      plugin_mgr: plugin_mgr,

      out: out,

      compiler: IncrementalCompiler {
        jit: jit,
        fn_reg: plugin_mgr.fn_registry(),
        sess: sess,
        sym_tb: HashMap::new(),
      },
    }
  }

  /// Execute a statement or expression (public for testing)
  pub fn execute(stmts: &str) -> Result<(), String> {
    Ok(())
  }

  /// Execute a meta command (public for testing)
  pub fn run_metacmd(cmd: &str) -> Result<(), String> {
    Ok(())
  }

  /// Loop for read and eval
  pub fn eval_loop(&mut self) {

    let mut tokens = Vec::new();
    let mut ast = Vec::new();

    loop {
      match readline::readline("\x1b[33mtkm> \x1b[0m") {
        Ok(Some(line)) => {
          tokens.extend(lexer::tokenize(&line));
          let parsed = p::parse(&tokens[..], &ast[..]);

          self.jit.add_module(&self.module);

          match parsed {
            Ok(r) => {
              match r.1.len() {
                0 => {
                  match MapCompiler::compile(self.jit,
                          &self.module,
                          self.plugin_mgr.fn_registry(),
                          self.sess,
                          &NamedSchema::new(&[], &[]),
                          &[&r.0[0]]) {
                    Ok((llvm_func, map)) => {
                      /*
                      debug!("{:?}", r.0[0]);
                      if log_enabled!(::log::LogLevel::Debug) {
                        f.dump();
                      }
                      Verifier::verify_func(f).unwrap_or_else(|err_msg| {
                        if !log_enabled!(::log::LogLevel::Debug) {
                          f.dump();
                        }
                        panic!("{}", err_msg);
                      });*/

                      let in_page = Page::empty_page(0);
                      let mut out_page = Page::new(&[r.0[0].ty()], None);
                      let sellist: [usize; ROWBATCH_SIZE] = unsafe { ::std::mem::uninitialized() };
                      map(&in_page, &mut out_page, sellist.as_ptr(), ROWBATCH_SIZE);
                      out_page.set_value_count(1);

                      let mut buf = String::new();
                      ColumnarRowPrinter::write(&[r.0[0].ty()], &out_page, &mut buf);
                      println!("{}", buf);

                      self.jit.remove_module(&self.module);
                      llvm::delete_func(&llvm_func);
                      ast.clear();
                      tokens.clear();
                    }
                    Err(msg) => {
                      panic!("error");
                      ast.clear();
                      tokens.clear();
                    }
                  }
                }
                _ => {}
              }
            }
            Err(msg) => println!("{}", msg),
          }
        }
        Ok(None) => break,
        Err(e) => println!("{}", e),
      };
      // add_history(line);
    }
  }
}

/// Evaluator: Expr -> String

/// ValuePrint: Value -> String

/// IncCompiler: Expr -> Value
pub struct IncrementalCompiler<'a> {
  pub jit: &'a JitCompiler,
  pub sess: &'a Session,
  pub fn_reg: &'a FuncRegistry,
  pub sym_tb: HashMap<(SymbolKind, String), Symbol>,
}

impl<'a> IncrementalCompiler<'a> {
  fn create_fn_proto(&self, bld: &Builder, ret_ty: &Ty) -> Function {
    let jit = self.jit;
    jit.create_func_prototype("processor",
                              to_llvm_ty(jit, ret_ty),
                              &[],
                              Some(bld))
  }

  fn compile(&self, expr: &Expr) -> Result<Function, String> {
    let bld = self.jit.new_builder();
    let func = self.create_fn_proto(&bld, expr.ty());
    let mut exprc = ExprCompiler::new(self.jit, self.fn_reg, self.sess, &bld);

    match exprc.compile2(&bld, expr) {
      Ok(value) => {
        bld.create_ret(&value);
        Ok(func)
      }
      Err(_) => Err("".to_string()),
    }
  }
}

// Execute the completed AST, then
// * return a value if expression
// * return an empty value if statement
// * return error if any error is included

// TODO - Error should include span and error message.

pub fn main() {
  env_logger::init().unwrap();

  // Output stream for repl
  let mut stdout = &mut io::stdout();

  // Initialize system context
  let plugin_mgr = &PluginManager::new();
  let jit = &JitCompiler::from_bc("../common/target/ir/common.bc").ok().unwrap();
  assert!(jit.get_ty("struct.Chunk").is_some());
  assert!(jit.get_ty("struct.Page").is_some());
  let sess = &Session;

  let mut repl = Repl::new(sess, plugin_mgr, jit, stdout);
  repl.eval_loop();
}
<commit_msg>Removed obsolte struct and implements.<commit_after>//! REPL (Read–eval–print loop) executable module provides interactive
//! execution of Tokamak language.

extern crate env_logger;
extern crate libc;
extern crate llvm;
extern crate llvm_sys;
#[macro_use]
extern crate log;
extern crate parser;
extern crate rl_sys; // libreadline

extern crate common;
extern crate exec;
extern crate plan;

use std::collections::HashMap;
use std::io;
use llvm::{Builder, Function, JitCompiler, Module, ValueRef, Verifier};
use rl_sys::readline;

use common::page::{Page, ROWBATCH_SIZE};
use common::plugin::{FuncRegistry, PluginManager};
use common::types::{HasType, Ty};
use common::session::Session;
use exec::{NamedSchema, ColumnarRowPrinter, RowPrinter};
use exec::processor::{MapCompiler, ExprCompiler, to_llvm_ty};
use plan::expr::Expr;
use parser::lexer;
use parser::parser as p;

// Logical Components
// Repl - Main compoenent for REPL
// SymbolTable - HashMap, keeping functions and variables in top-level module
// IncrementalCompiler - Imitate incremental execution with some tricky way
// ValuePrinter - Print the value contents.

pub struct Repl<'a> {
  // for system
  jit: &'a JitCompiler,
  module: Module,
  sess: &'a Session,
  plugin_mgr: &'a PluginManager<'a>,

  // in/out descriptors
  out: &'a mut io::Write, // can be stdout or anything else,
}

#[derive(Eq, Hash, PartialEq)]
pub enum SymbolKind {
  // Function
  Func,
  // Immutable value
  ImmVal,
  // Mutable value
  MutVal,
}

pub struct Symbol;

impl<'a> Repl<'a> {
  pub fn new(sess: &'a Session,
             plugin_mgr: &'a PluginManager<'a>,
             jit: &'a JitCompiler,
             out: &'a mut io::Write)
             -> Repl<'a> {
    Repl {
      jit: jit,
      module: Module::new(jit.context(), "root"),
      sess: sess,
      plugin_mgr: plugin_mgr,
      out: out
    }
  }

  /// Execute a statement or expression (public for testing)
  pub fn execute(stmts: &str) -> Result<(), String> {
    Ok(())
  }

  /// Execute a meta command (public for testing)
  pub fn run_metacmd(cmd: &str) -> Result<(), String> {
    Ok(())
  }

  /// Loop for read and eval
  pub fn eval_loop(&mut self) {

    let mut tokens = Vec::new();
    let mut ast = Vec::new();

    loop {
      match readline::readline("\x1b[33mtkm> \x1b[0m") {
        Ok(Some(line)) => {
          tokens.extend(lexer::tokenize(&line));
          let parsed = p::parse(&tokens[..], &ast[..]);

          self.jit.add_module(&self.module);

          match parsed {
            Ok((exprs, remain_tokens)) => {
              match remain_tokens.len() {
                0 => {
                  match MapCompiler::compile(self.jit,
                          &self.module,
                          self.plugin_mgr.fn_registry(),
                          self.sess,
                          &NamedSchema::new(&[], &[]),
                          &[&exprs[0]]) {
                    Ok((llvm_func, map)) => {
                      let result_ty = exprs[0].ty();

                      /*
                      debug!("{:?}", r.0[0]);
                      if log_enabled!(::log::LogLevel::Debug) {
                        f.dump();
                      }
                      Verifier::verify_func(f).unwrap_or_else(|err_msg| {
                        if !log_enabled!(::log::LogLevel::Debug) {
                          f.dump();
                        }
                        panic!("{}", err_msg);
                      });*/

                      let in_page = Page::empty_page(0);
                      let mut out_page = Page::new(&[result_ty], None);
                      let sellist: [usize; ROWBATCH_SIZE] = unsafe { ::std::mem::uninitialized() };
                      map(&in_page, &mut out_page, sellist.as_ptr(), ROWBATCH_SIZE);
                      out_page.set_value_count(1);

                      ColumnarRowPrinter::write(&[result_ty], &out_page, self.out);
                      self.out.write("\n".as_bytes()).ok().unwrap();
                      self.out.flush().ok().unwrap();

                      self.jit.remove_module(&self.module);
                      llvm::delete_func(&llvm_func);
                      ast.clear();
                      tokens.clear();
                    }
                    Err(msg) => {
                      panic!("error");
                      ast.clear();
                      tokens.clear();
                    }
                  }
                }
                _ => {}
              }
            }
            Err(msg) => println!("{}", msg),
          }
        }
        Ok(None) => break,
        Err(e) => println!("{}", e),
      };
      // add_history(line);
    }
  }
}

/// Evaluator: Expr -> String
/// ValuePrint: Value -> String
/// IncCompiler: Expr -> Value

// Execute the completed AST, then
// * return a value if expression
// * return an empty value if statement
// * return error if any error is included

// TODO - Error should include span and error message.

pub fn main() {
  env_logger::init().unwrap();

  // Output stream for repl
  let mut stdout = &mut io::stdout();

  // Initialize system context
  let plugin_mgr = &PluginManager::new();
  let jit = &JitCompiler::from_bc("../common/target/ir/common.bc").ok().unwrap();
  assert!(jit.get_ty("struct.Chunk").is_some());
  assert!(jit.get_ty("struct.Page").is_some());
  let sess = &Session;

  let mut repl = Repl::new(sess, plugin_mgr, jit, stdout);
  repl.eval_loop();
}
<|endoftext|>


<commit_before>// TODO: Remove
#![allow(dead_code)]

use std::fmt::{Display, Formatter, Error};
use base::code::{BytePos, Span};
use std::vec::Vec;
use std::default::Default;

macro_rules! java_enum { (
    $name:ident { $( $variant:ident => $java_word:expr, )* }
) => {
    #[derive(Copy, Clone, PartialEq, Eq, Debug, Hash)]
    pub enum $name {
        $( $variant, )*
    }

    impl $name {
        pub fn as_java_string(&self) -> &str {
            match *self {
                $( $name::$variant => $java_word , )*
            }
        }
    }

    impl Display for $name {
        fn fmt(&self, f: &mut Formatter) -> Result<(), Error> {
            self.as_java_string().fmt(f)
        }
    }
}}

// ============================================================================
// Definition of types that are common in AST nodes
// ============================================================================
#[derive(Debug, Clone)]
pub struct Ident {
    pub name: String,
    pub span: Span,
}

#[derive(Debug, Clone)]
pub struct Path {
    pub segments: Vec<Ident>,
}

// pub struct PathSegment {

// }

// ============================================================================
// Top-Down AST definition starting with the goal symbol
// ============================================================================
/// A Java compilation unit. This is the goal symbol for the syntactic grammar.
#[derive(Debug, Clone)]
pub struct CompilationUnit {
    pub package: Option<Path>,
    pub imports: Vec<Import>,
    pub types: Vec<Type>,
}

#[derive(Debug, Clone)]
pub enum Type {
    NormalClass(Class),
    Enum(()),
    NormalInterface(()),
}

#[derive(Debug, Clone)]
pub enum Item {
    Import(Import),
    Class(Box<Class>),
    Method(Box<Method>),
}

impl Default for Ident {
    fn default() -> Self {
        Ident {
            name: "".into(),
            span: Span { lo: BytePos(0), hi: BytePos(0) },
        }
    }
}

/// A import declaration
#[derive(Debug, Clone)]
pub enum Import {
    /// e.g. `import IO.AlgoTools;`
    SingleType(Path),
    /// called "type-import-on-demand" in specs -- e.g. `import IO.*;`
    TypeOnDemand(Path),
    SingleStatic(Path),
    StaticOnDemand(Path),
}

#[derive(Debug, Copy, Clone, PartialEq, Eq)]
pub enum Visibility {
    Public,
    Protected,
    Package,
    Private,
}

#[derive(Debug, Clone)]
pub struct Class {
    pub name: Ident,
    pub vis: Visibility,
    pub methods: Vec<Method>,
    pub fields: Vec<Field>,
}

#[derive(Debug, Clone)]
pub struct Name {
    // for qualified names
    pub path: Vec<Ident>,
    pub last: Option<Ident>,
}

java_enum! (Modifier {
    Public => "public",
    Protected => "protected",
    Private => "private",
    Abstract => "abstract",
    Static => "static",
    Final => "final",
    Synchronized => "synchronized",
    Native => "native",
    Strictfp => "strictfp",
    Transient => "transient",
    Volatile => "volatile",
});

#[derive(Debug, Clone)]
pub struct Field {
    pub vis: Visibility,
    pub static_: bool,
    pub final_: bool,
    pub ty: String,
    pub name: Ident,
    // pub init ...
}

#[derive(Debug, Clone)]
pub struct Method {
    pub vis: Visibility,
    pub name: Ident,
    pub ret_ty: Ident,
    pub static_: bool,
    pub final_: bool,
    pub params: Vec<FormalParameter>,
}

#[derive(Debug, Clone)]
pub struct FormalParameter {
    pub ty: Ident,
    pub name: Ident,
    pub dims: usize,
    pub final_: bool,
}
<commit_msg>Add custom `Debug` impl for `Ident` type to improve readability<commit_after>// TODO: Remove
#![allow(dead_code)]

use std::fmt::{Display, Formatter, Error};
use base::code::{BytePos, Span};
use std::vec::Vec;
use std::default::Default;
use std::fmt;

macro_rules! java_enum { (
    $name:ident { $( $variant:ident => $java_word:expr, )* }
) => {
    #[derive(Copy, Clone, PartialEq, Eq, Debug, Hash)]
    pub enum $name {
        $( $variant, )*
    }

    impl $name {
        pub fn as_java_string(&self) -> &str {
            match *self {
                $( $name::$variant => $java_word , )*
            }
        }
    }

    impl Display for $name {
        fn fmt(&self, f: &mut Formatter) -> Result<(), Error> {
            self.as_java_string().fmt(f)
        }
    }
}}

// ============================================================================
// Definition of types that are common in AST nodes
// ============================================================================
#[derive(Clone)]
pub struct Ident {
    pub name: String,
    pub span: Span,
}

// custom `Debug` impl to shorten debug output and improve readability
impl fmt::Debug for Ident {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        write!(f, r#"Ident("{}" @ ({}, {}))"#,
            self.name, self.span.lo.0, self.span.hi.0)
    }
}

#[derive(Debug, Clone)]
pub struct Path {
    pub segments: Vec<Ident>,
}

// pub struct PathSegment {

// }

// ============================================================================
// Top-Down AST definition starting with the goal symbol
// ============================================================================
/// A Java compilation unit. This is the goal symbol for the syntactic grammar.
#[derive(Debug, Clone)]
pub struct CompilationUnit {
    pub package: Option<Path>,
    pub imports: Vec<Import>,
    pub types: Vec<Type>,
}

#[derive(Debug, Clone)]
pub enum Type {
    NormalClass(Class),
    Enum(()),
    NormalInterface(()),
}

#[derive(Debug, Clone)]
pub enum Item {
    Import(Import),
    Class(Box<Class>),
    Method(Box<Method>),
}

impl Default for Ident {
    fn default() -> Self {
        Ident {
            name: "".into(),
            span: Span { lo: BytePos(0), hi: BytePos(0) },
        }
    }
}

/// A import declaration
#[derive(Debug, Clone)]
pub enum Import {
    /// e.g. `import IO.AlgoTools;`
    SingleType(Path),
    /// called "type-import-on-demand" in specs -- e.g. `import IO.*;`
    TypeOnDemand(Path),
    SingleStatic(Path),
    StaticOnDemand(Path),
}

#[derive(Debug, Copy, Clone, PartialEq, Eq)]
pub enum Visibility {
    Public,
    Protected,
    Package,
    Private,
}

#[derive(Debug, Clone)]
pub struct Class {
    pub name: Ident,
    pub vis: Visibility,
    pub methods: Vec<Method>,
    pub fields: Vec<Field>,
}

#[derive(Debug, Clone)]
pub struct Name {
    // for qualified names
    pub path: Vec<Ident>,
    pub last: Option<Ident>,
}

java_enum! (Modifier {
    Public => "public",
    Protected => "protected",
    Private => "private",
    Abstract => "abstract",
    Static => "static",
    Final => "final",
    Synchronized => "synchronized",
    Native => "native",
    Strictfp => "strictfp",
    Transient => "transient",
    Volatile => "volatile",
});

#[derive(Debug, Clone)]
pub struct Field {
    pub vis: Visibility,
    pub static_: bool,
    pub final_: bool,
    pub ty: String,
    pub name: Ident,
    // pub init ...
}

#[derive(Debug, Clone)]
pub struct Method {
    pub vis: Visibility,
    pub name: Ident,
    pub ret_ty: Ident,
    pub static_: bool,
    pub final_: bool,
    pub params: Vec<FormalParameter>,
}

#[derive(Debug, Clone)]
pub struct FormalParameter {
    pub ty: Ident,
    pub name: Ident,
    pub dims: usize,
    pub final_: bool,
}
<|endoftext|>


<commit_before>//! Abstraction around the object writing crate

use std::convert::{TryFrom, TryInto};

use rustc_data_structures::fx::FxHashMap;
use rustc_session::Session;

use cranelift_codegen::isa::TargetIsa;
use cranelift_module::FuncId;
use cranelift_object::{ObjectBuilder, ObjectModule, ObjectProduct};

use object::write::*;
use object::{RelocationEncoding, SectionKind, SymbolFlags};

use gimli::SectionId;

use crate::debuginfo::{DebugReloc, DebugRelocName};

pub(crate) trait WriteMetadata {
    fn add_rustc_section(&mut self, symbol_name: String, data: Vec<u8>);
}

impl WriteMetadata for object::write::Object {
    fn add_rustc_section(&mut self, symbol_name: String, data: Vec<u8>) {
        let segment = self.segment_name(object::write::StandardSegment::Data).to_vec();
        let section_id = self.add_section(segment, b".rustc".to_vec(), object::SectionKind::Data);
        let offset = self.append_section_data(section_id, &data, 1);
        // For MachO and probably PE this is necessary to prevent the linker from throwing away the
        // .rustc section. For ELF this isn't necessary, but it also doesn't harm.
        self.add_symbol(object::write::Symbol {
            name: symbol_name.into_bytes(),
            value: offset,
            size: data.len() as u64,
            kind: object::SymbolKind::Data,
            scope: object::SymbolScope::Dynamic,
            weak: false,
            section: SymbolSection::Section(section_id),
            flags: SymbolFlags::None,
        });
    }
}

pub(crate) trait WriteDebugInfo {
    type SectionId: Copy;

    fn add_debug_section(&mut self, name: SectionId, data: Vec<u8>) -> Self::SectionId;
    fn add_debug_reloc(
        &mut self,
        section_map: &FxHashMap<SectionId, Self::SectionId>,
        from: &Self::SectionId,
        reloc: &DebugReloc,
    );
}

impl WriteDebugInfo for ObjectProduct {
    type SectionId = (object::write::SectionId, object::write::SymbolId);

    fn add_debug_section(
        &mut self,
        id: SectionId,
        data: Vec<u8>,
    ) -> (object::write::SectionId, object::write::SymbolId) {
        let name = if self.object.format() == object::BinaryFormat::MachO {
            id.name().replace('.', "__") // machO expects __debug_info instead of .debug_info
        } else {
            id.name().to_string()
        }
        .into_bytes();

        let segment = self.object.segment_name(StandardSegment::Debug).to_vec();
        // FIXME use SHT_X86_64_UNWIND for .eh_frame
        let section_id = self.object.add_section(
            segment,
            name,
            if id == SectionId::EhFrame { SectionKind::ReadOnlyData } else { SectionKind::Debug },
        );
        self.object
            .section_mut(section_id)
            .set_data(data, if id == SectionId::EhFrame { 8 } else { 1 });
        let symbol_id = self.object.section_symbol(section_id);
        (section_id, symbol_id)
    }

    fn add_debug_reloc(
        &mut self,
        section_map: &FxHashMap<SectionId, Self::SectionId>,
        from: &Self::SectionId,
        reloc: &DebugReloc,
    ) {
        let (symbol, symbol_offset) = match reloc.name {
            DebugRelocName::Section(id) => (section_map.get(&id).unwrap().1, 0),
            DebugRelocName::Symbol(id) => {
                let symbol_id = self.function_symbol(FuncId::from_u32(id.try_into().unwrap()));
                self.object
                    .symbol_section_and_offset(symbol_id)
                    .expect("Debug reloc for undef sym???")
            }
        };
        self.object
            .add_relocation(
                from.0,
                Relocation {
                    offset: u64::from(reloc.offset),
                    symbol,
                    kind: reloc.kind,
                    encoding: RelocationEncoding::Generic,
                    size: reloc.size * 8,
                    addend: i64::try_from(symbol_offset).unwrap() + reloc.addend,
                },
            )
            .unwrap();
    }
}

pub(crate) fn with_object(sess: &Session, name: &str, f: impl FnOnce(&mut Object)) -> Vec<u8> {
    let triple = crate::target_triple(sess);

    let binary_format = match triple.binary_format {
        target_lexicon::BinaryFormat::Elf => object::BinaryFormat::Elf,
        target_lexicon::BinaryFormat::Coff => object::BinaryFormat::Coff,
        target_lexicon::BinaryFormat::Macho => object::BinaryFormat::MachO,
        binary_format => sess.fatal(&format!("binary format {} is unsupported", binary_format)),
    };
    let architecture = match triple.architecture {
        target_lexicon::Architecture::X86_32(_) => object::Architecture::I386,
        target_lexicon::Architecture::X86_64 => object::Architecture::X86_64,
        target_lexicon::Architecture::Arm(_) => object::Architecture::Arm,
        target_lexicon::Architecture::Aarch64(_) => object::Architecture::Aarch64,
        architecture => {
            sess.fatal(&format!("target architecture {:?} is unsupported", architecture,))
        }
    };
    let endian = match triple.endianness().unwrap() {
        target_lexicon::Endianness::Little => object::Endianness::Little,
        target_lexicon::Endianness::Big => object::Endianness::Big,
    };

    let mut metadata_object = object::write::Object::new(binary_format, architecture, endian);
    metadata_object.add_file_symbol(name.as_bytes().to_vec());
    f(&mut metadata_object);
    metadata_object.write().unwrap()
}

pub(crate) fn make_module(sess: &Session, isa: Box<dyn TargetIsa>, name: String) -> ObjectModule {
    let mut builder =
        ObjectBuilder::new(isa, name + ".o", cranelift_module::default_libcall_names()).unwrap();
    // Unlike cg_llvm, cg_clif defaults to disabling -Zfunction-sections. For cg_llvm binary size
    // is important, while cg_clif cares more about compilation times. Enabling -Zfunction-sections
    // can easily double the amount of time necessary to perform linking.
    builder.per_function_section(sess.opts.debugging_opts.function_sections.unwrap_or(false));
    ObjectModule::new(builder)
}
<commit_msg>Add more architectures to with_object<commit_after>//! Abstraction around the object writing crate

use std::convert::{TryFrom, TryInto};

use rustc_data_structures::fx::FxHashMap;
use rustc_session::Session;

use cranelift_codegen::isa::TargetIsa;
use cranelift_module::FuncId;
use cranelift_object::{ObjectBuilder, ObjectModule, ObjectProduct};

use object::write::*;
use object::{RelocationEncoding, SectionKind, SymbolFlags};

use gimli::SectionId;

use crate::debuginfo::{DebugReloc, DebugRelocName};

pub(crate) trait WriteMetadata {
    fn add_rustc_section(&mut self, symbol_name: String, data: Vec<u8>);
}

impl WriteMetadata for object::write::Object {
    fn add_rustc_section(&mut self, symbol_name: String, data: Vec<u8>) {
        let segment = self.segment_name(object::write::StandardSegment::Data).to_vec();
        let section_id = self.add_section(segment, b".rustc".to_vec(), object::SectionKind::Data);
        let offset = self.append_section_data(section_id, &data, 1);
        // For MachO and probably PE this is necessary to prevent the linker from throwing away the
        // .rustc section. For ELF this isn't necessary, but it also doesn't harm.
        self.add_symbol(object::write::Symbol {
            name: symbol_name.into_bytes(),
            value: offset,
            size: data.len() as u64,
            kind: object::SymbolKind::Data,
            scope: object::SymbolScope::Dynamic,
            weak: false,
            section: SymbolSection::Section(section_id),
            flags: SymbolFlags::None,
        });
    }
}

pub(crate) trait WriteDebugInfo {
    type SectionId: Copy;

    fn add_debug_section(&mut self, name: SectionId, data: Vec<u8>) -> Self::SectionId;
    fn add_debug_reloc(
        &mut self,
        section_map: &FxHashMap<SectionId, Self::SectionId>,
        from: &Self::SectionId,
        reloc: &DebugReloc,
    );
}

impl WriteDebugInfo for ObjectProduct {
    type SectionId = (object::write::SectionId, object::write::SymbolId);

    fn add_debug_section(
        &mut self,
        id: SectionId,
        data: Vec<u8>,
    ) -> (object::write::SectionId, object::write::SymbolId) {
        let name = if self.object.format() == object::BinaryFormat::MachO {
            id.name().replace('.', "__") // machO expects __debug_info instead of .debug_info
        } else {
            id.name().to_string()
        }
        .into_bytes();

        let segment = self.object.segment_name(StandardSegment::Debug).to_vec();
        // FIXME use SHT_X86_64_UNWIND for .eh_frame
        let section_id = self.object.add_section(
            segment,
            name,
            if id == SectionId::EhFrame { SectionKind::ReadOnlyData } else { SectionKind::Debug },
        );
        self.object
            .section_mut(section_id)
            .set_data(data, if id == SectionId::EhFrame { 8 } else { 1 });
        let symbol_id = self.object.section_symbol(section_id);
        (section_id, symbol_id)
    }

    fn add_debug_reloc(
        &mut self,
        section_map: &FxHashMap<SectionId, Self::SectionId>,
        from: &Self::SectionId,
        reloc: &DebugReloc,
    ) {
        let (symbol, symbol_offset) = match reloc.name {
            DebugRelocName::Section(id) => (section_map.get(&id).unwrap().1, 0),
            DebugRelocName::Symbol(id) => {
                let symbol_id = self.function_symbol(FuncId::from_u32(id.try_into().unwrap()));
                self.object
                    .symbol_section_and_offset(symbol_id)
                    .expect("Debug reloc for undef sym???")
            }
        };
        self.object
            .add_relocation(
                from.0,
                Relocation {
                    offset: u64::from(reloc.offset),
                    symbol,
                    kind: reloc.kind,
                    encoding: RelocationEncoding::Generic,
                    size: reloc.size * 8,
                    addend: i64::try_from(symbol_offset).unwrap() + reloc.addend,
                },
            )
            .unwrap();
    }
}

pub(crate) fn with_object(sess: &Session, name: &str, f: impl FnOnce(&mut Object)) -> Vec<u8> {
    let triple = crate::target_triple(sess);

    let binary_format = match triple.binary_format {
        target_lexicon::BinaryFormat::Elf => object::BinaryFormat::Elf,
        target_lexicon::BinaryFormat::Coff => object::BinaryFormat::Coff,
        target_lexicon::BinaryFormat::Macho => object::BinaryFormat::MachO,
        binary_format => sess.fatal(&format!("binary format {} is unsupported", binary_format)),
    };
    let architecture = match triple.architecture {
        target_lexicon::Architecture::Aarch64(_) => object::Architecture::Aarch64,
        target_lexicon::Architecture::Arm(_) => object::Architecture::Arm,
        target_lexicon::Architecture::Avr => object::Architecture::Avr,
        target_lexicon::Architecture::Hexagon => object::Architecture::Hexagon,
        target_lexicon::Architecture::Mips32(_) => object::Architecture::Mips,
        target_lexicon::Architecture::Mips64(_) => object::Architecture::Mips64,
        target_lexicon::Architecture::Msp430 => object::Architecture::Msp430,
        target_lexicon::Architecture::Powerpc => object::Architecture::PowerPc,
        target_lexicon::Architecture::Powerpc64 => object::Architecture::PowerPc64,
        target_lexicon::Architecture::Powerpc64le => todo!(),
        target_lexicon::Architecture::Riscv32(_) => object::Architecture::Riscv32,
        target_lexicon::Architecture::Riscv64(_) => object::Architecture::Riscv64,
        target_lexicon::Architecture::S390x => object::Architecture::S390x,
        target_lexicon::Architecture::Sparc64 => object::Architecture::Sparc64,
        target_lexicon::Architecture::Sparcv9 => object::Architecture::Sparc64,
        target_lexicon::Architecture::X86_32(_) => object::Architecture::I386,
        target_lexicon::Architecture::X86_64 => object::Architecture::X86_64,
        architecture => {
            sess.fatal(&format!("target architecture {:?} is unsupported", architecture,))
        }
    };
    let endian = match triple.endianness().unwrap() {
        target_lexicon::Endianness::Little => object::Endianness::Little,
        target_lexicon::Endianness::Big => object::Endianness::Big,
    };

    let mut metadata_object = object::write::Object::new(binary_format, architecture, endian);
    metadata_object.add_file_symbol(name.as_bytes().to_vec());
    f(&mut metadata_object);
    metadata_object.write().unwrap()
}

pub(crate) fn make_module(sess: &Session, isa: Box<dyn TargetIsa>, name: String) -> ObjectModule {
    let mut builder =
        ObjectBuilder::new(isa, name + ".o", cranelift_module::default_libcall_names()).unwrap();
    // Unlike cg_llvm, cg_clif defaults to disabling -Zfunction-sections. For cg_llvm binary size
    // is important, while cg_clif cares more about compilation times. Enabling -Zfunction-sections
    // can easily double the amount of time necessary to perform linking.
    builder.per_function_section(sess.opts.debugging_opts.function_sections.unwrap_or(false));
    ObjectModule::new(builder)
}
<|endoftext|>


<commit_before>//! The Brainfuck interpreter executable.
//!
//! ```
//! USAGE:
//!     bfi [FLAGS] [OPTIONS] [--] [FILE]...
//!
//! FLAGS:
//!         --ast          Interpret the unoptimized AST
//!         --byte         Compile AST to bytecode
//!     -h, --help         Prints help information
//!         --jit          JIT to native x64 (default)
//!         --llvm         JIT using LLVM
//!         --peep         Interpret the peephole-optimized AST
//!         --rle          Interpret the run-length encoded the AST
//!     -u, --unchecked    Omit memory bounds checks in JIT
//!     -V, --version      Prints version information
//!
//! OPTIONS:
//!     -e, --expr <CODE>...    BF code to execute
//!     -s, --size <SIZE>       Memory size in bytes (default 30,000)
//!
//! ARGS:
//!     <FILE>...    The source file(s) to interpret
//! ```
//!
//! See [the library crate documentation](../bf/index.html) for more.

extern crate bf;
extern crate clap;

use std::io::Read;
use std::fs::File;
use std::process::exit;

use clap::{Arg, App};

use bf::ast;
use bf::traits::*;

const VERSION: &str = "0.4.6";

#[derive(Debug, Clone)]
struct Options {
    program_text:  Vec<u8>,
    memory_size:   Option<usize>,
    compiler_pass: Pass,
    unchecked:     bool,
}

#[derive(Debug, Clone, Copy)]
enum Pass {
    Ast,
    Rle,
    Bytecode,
    Peephole,
    #[cfg(feature = "jit")]
    Jit,
    #[cfg(feature = "llvm")]
    Llvm,
}

fn main() {
    let options = get_options();

    let program = parse(&options);

    match options.compiler_pass {
        Pass::Ast => {
            interpret(&*program, &options);
        }

        Pass::Rle => {
            let program = program.rle_compile();
            interpret(&*program, &options);
        }

        Pass::Peephole => {
            let program = program.peephole_compile();
            interpret(&*program, &options);
        }

        Pass::Bytecode => {
            let program = program.bytecode_compile();
            interpret(&*program, &options);
        }

        #[cfg(feature = "jit")]
        Pass::Jit => {
            let program = program.jit_compile(!options.unchecked);
            interpret(&program, &options);
        }

        #[cfg(feature = "llvm")]
        Pass::Llvm => {
            program.llvm_run(options.memory_size)
                .unwrap_or_else(|e| error_exit(3, &format!("runtime error: {}.", e)));
        }
    }
}

fn parse(options: &Options) -> Box<ast::Program> {
    ast::parse_program(&options.program_text)
        .unwrap_or_else(|e| error_exit(2, &format!("syntax error: {}.", e)))
}

fn interpret<P: Interpretable + ?Sized>(program: &P, options: &Options) {
    program.interpret_stdin(options.memory_size)
        .unwrap_or_else(|e| error_exit(3, &format!("runtime error: {}.", e)))
}

#[cfg(feature = "jit")]
const DEFAULT_PASS: Pass = Pass::Jit;

#[cfg(not(feature = "jit"))]
const DEFAULT_PASS: Pass = Pass::Peephole;

fn get_options() -> Options {
    let mut result = Options {
        program_text:  Vec::new(),
        memory_size:   None,
        compiler_pass: DEFAULT_PASS,
        unchecked:     false,
    };

    let matches = build_clap_app().get_matches();

    if let Some(size) = matches.value_of("size") {
        let size = size.parse()
            .unwrap_or_else(|e|
                error_exit(1, &format!("error: could not parse memory size: {}.", e)));
        if size == 0 {
            error_exit(1, "error: memory size must be at least 1.");
        }
        result.memory_size = Some(size);
    }

    if matches.is_present("jit") {
        #[cfg(feature = "jit")]
        let _ = result.compiler_pass = Pass::Jit;
    } else if matches.is_present("llvm") {
        #[cfg(feature = "llvm")]
        let _ = result.compiler_pass = Pass::Llvm;
    } else if matches.is_present("byte") {
        result.compiler_pass = Pass::Bytecode;
    } else if matches.is_present("peep") {
        result.compiler_pass = Pass::Peephole;
    } else if matches.is_present("rle") {
        result.compiler_pass = Pass::Rle;
    } else if matches.is_present("ast") {
        result.compiler_pass = Pass::Ast;
    }

    if matches.is_present("unchecked") {
        result.unchecked = true;
    }

    if let Some(exprs) = matches.values_of("expr") {
        for e in exprs {
            result.program_text.extend(e.as_bytes());
        }
    } else if let Some(files) = matches.values_of("FILE") {
        for f in files {
            let mut file = File::open(f)
                .unwrap_or_else(|e| error_exit(1, &format!("{}: ‘{}’.", e, f)));
            file.read_to_end(&mut result.program_text)
                .unwrap_or_else(|e| error_exit(1, &format!("{}: ‘{}’.", e, f)));
        }
    } else {
        error_exit(1, "error: no program given.");
    }

    result
}

fn build_clap_app() -> App<'static, 'static> {
    let app = App::new("bfi")
        .version(VERSION)
        .author("Jesse A. Tov <jesse.tov@gmail.com>")
        .about("A Brainfuck interpreter")
        .arg(Arg::with_name("expr")
            .short("e")
            .long("expr")
            .value_name("CODE")
            .help("BF code to execute")
            .multiple(true)
            .takes_value(true)
            .conflicts_with("FILE"))
        .arg(Arg::with_name("FILE")
            .help("The source file(s) to interpret")
            .multiple(true)
            .conflicts_with("expr")
            .index(1))
        .arg(Arg::with_name("size")
            .short("s")
            .long("size")
            .value_name("SIZE")
            .help("Memory size in bytes (default 30,000)")
            .takes_value(true))
        .arg(Arg::with_name("ast")
            .long("ast")
            .help("Interpret the unoptimized AST")
            .conflicts_with_all(&["rle", "peep", "byte", "jit", "llvm"]))
        .arg(Arg::with_name("rle")
            .long("rle")
            .help("Interpret the run-length encoded the AST")
            .conflicts_with_all(&["ast", "peep", "byte", "jit", "llvm"]))
        .arg(Arg::with_name("peep")
            .long("peep")
            .help(
                if cfg!(feature = "jit") {
                    "Interpret the peephole-optimized AST"
                } else {
                    "Interpret the peephole-optimized AST (default)"
                })
            .conflicts_with_all(&["ast", "rle", "byte", "jit", "llvm"]))
        .arg(Arg::with_name("byte")
            .long("byte")
            .help("Compile AST to bytecode")
            .conflicts_with_all(&["ast", "rle", "peep", "jit", "llvm"]));

    #[cfg(feature = "llvm")]
    let app = app
        .arg(Arg::with_name("llvm")
            .long("llvm")
            .help("JIT using LLVM")
            .conflicts_with_all(&["ast", "rle", "peep", "byte", "jit"]));

    #[cfg(feature = "jit")]
    let app = app
        .arg(Arg::with_name("jit")
            .long("jit")
            .help("JIT to native x64 (default)")
            .conflicts_with_all(&["ast", "rle", "peep", "byte", "llvm"]));

    #[cfg(any(feature = "jit"))]
    let app = app
        .arg(Arg::with_name("unchecked")
            .short("u")
            .long("unchecked")
            .help("Omit memory bounds checks in JIT")
            .conflicts_with_all(&["ast", "rle", "peep", "byte", "llvm"]));

    app
}

fn error_exit(code: i32, msg: &str) -> ! {
    eprintln!("bfi: {}", msg);
    exit(code)
}

<commit_msg>bfi gets version from cargo now.<commit_after>//! The Brainfuck interpreter executable.
//!
//! ```
//! USAGE:
//!     bfi [FLAGS] [OPTIONS] [--] [FILE]...
//!
//! FLAGS:
//!         --ast          Interpret the unoptimized AST
//!         --byte         Compile AST to bytecode
//!     -h, --help         Prints help information
//!         --jit          JIT to native x64 (default)
//!         --llvm         JIT using LLVM
//!         --peep         Interpret the peephole-optimized AST
//!         --rle          Interpret the run-length encoded the AST
//!     -u, --unchecked    Omit memory bounds checks in JIT
//!     -V, --version      Prints version information
//!
//! OPTIONS:
//!     -e, --expr <CODE>...    BF code to execute
//!     -s, --size <SIZE>       Memory size in bytes (default 30,000)
//!
//! ARGS:
//!     <FILE>...    The source file(s) to interpret
//! ```
//!
//! See [the library crate documentation](../bf/index.html) for more.

extern crate bf;

#[macro_use]
extern crate clap;

use std::io::Read;
use std::fs::File;
use std::process::exit;

use clap::{Arg, App};

use bf::ast;
use bf::traits::*;

#[derive(Debug, Clone)]
struct Options {
    program_text:  Vec<u8>,
    memory_size:   Option<usize>,
    compiler_pass: Pass,
    unchecked:     bool,
}

#[derive(Debug, Clone, Copy)]
enum Pass {
    Ast,
    Rle,
    Bytecode,
    Peephole,
    #[cfg(feature = "jit")]
    Jit,
    #[cfg(feature = "llvm")]
    Llvm,
}

fn main() {
    let options = get_options();

    let program = parse(&options);

    match options.compiler_pass {
        Pass::Ast => {
            interpret(&*program, &options);
        }

        Pass::Rle => {
            let program = program.rle_compile();
            interpret(&*program, &options);
        }

        Pass::Peephole => {
            let program = program.peephole_compile();
            interpret(&*program, &options);
        }

        Pass::Bytecode => {
            let program = program.bytecode_compile();
            interpret(&*program, &options);
        }

        #[cfg(feature = "jit")]
        Pass::Jit => {
            let program = program.jit_compile(!options.unchecked);
            interpret(&program, &options);
        }

        #[cfg(feature = "llvm")]
        Pass::Llvm => {
            program.llvm_run(options.memory_size)
                .unwrap_or_else(|e| error_exit(3, &format!("runtime error: {}.", e)));
        }
    }
}

fn parse(options: &Options) -> Box<ast::Program> {
    ast::parse_program(&options.program_text)
        .unwrap_or_else(|e| error_exit(2, &format!("syntax error: {}.", e)))
}

fn interpret<P: Interpretable + ?Sized>(program: &P, options: &Options) {
    program.interpret_stdin(options.memory_size)
        .unwrap_or_else(|e| error_exit(3, &format!("runtime error: {}.", e)))
}

#[cfg(feature = "jit")]
const DEFAULT_PASS: Pass = Pass::Jit;

#[cfg(not(feature = "jit"))]
const DEFAULT_PASS: Pass = Pass::Peephole;

fn get_options() -> Options {
    let mut result = Options {
        program_text:  Vec::new(),
        memory_size:   None,
        compiler_pass: DEFAULT_PASS,
        unchecked:     false,
    };

    let matches = build_clap_app().get_matches();

    if let Some(size) = matches.value_of("size") {
        let size = size.parse()
            .unwrap_or_else(|e|
                error_exit(1, &format!("error: could not parse memory size: {}.", e)));
        if size == 0 {
            error_exit(1, "error: memory size must be at least 1.");
        }
        result.memory_size = Some(size);
    }

    if matches.is_present("jit") {
        #[cfg(feature = "jit")]
        let _ = result.compiler_pass = Pass::Jit;
    } else if matches.is_present("llvm") {
        #[cfg(feature = "llvm")]
        let _ = result.compiler_pass = Pass::Llvm;
    } else if matches.is_present("byte") {
        result.compiler_pass = Pass::Bytecode;
    } else if matches.is_present("peep") {
        result.compiler_pass = Pass::Peephole;
    } else if matches.is_present("rle") {
        result.compiler_pass = Pass::Rle;
    } else if matches.is_present("ast") {
        result.compiler_pass = Pass::Ast;
    }

    if matches.is_present("unchecked") {
        result.unchecked = true;
    }

    if let Some(exprs) = matches.values_of("expr") {
        for e in exprs {
            result.program_text.extend(e.as_bytes());
        }
    } else if let Some(files) = matches.values_of("FILE") {
        for f in files {
            let mut file = File::open(f)
                .unwrap_or_else(|e| error_exit(1, &format!("{}: ‘{}’.", e, f)));
            file.read_to_end(&mut result.program_text)
                .unwrap_or_else(|e| error_exit(1, &format!("{}: ‘{}’.", e, f)));
        }
    } else {
        error_exit(1, "error: no program given.");
    }

    result
}

fn build_clap_app() -> App<'static, 'static> {
    let app = App::new("bfi")
        .version(crate_version!())
        .author("Jesse A. Tov <jesse.tov@gmail.com>")
        .about("A Brainfuck interpreter")
        .arg(Arg::with_name("expr")
            .short("e")
            .long("expr")
            .value_name("CODE")
            .help("BF code to execute")
            .multiple(true)
            .takes_value(true)
            .conflicts_with("FILE"))
        .arg(Arg::with_name("FILE")
            .help("The source file(s) to interpret")
            .multiple(true)
            .conflicts_with("expr")
            .index(1))
        .arg(Arg::with_name("size")
            .short("s")
            .long("size")
            .value_name("SIZE")
            .help("Memory size in bytes (default 30,000)")
            .takes_value(true))
        .arg(Arg::with_name("ast")
            .long("ast")
            .help("Interpret the unoptimized AST")
            .conflicts_with_all(&["rle", "peep", "byte", "jit", "llvm"]))
        .arg(Arg::with_name("rle")
            .long("rle")
            .help("Interpret the run-length encoded the AST")
            .conflicts_with_all(&["ast", "peep", "byte", "jit", "llvm"]))
        .arg(Arg::with_name("peep")
            .long("peep")
            .help(
                if cfg!(feature = "jit") {
                    "Interpret the peephole-optimized AST"
                } else {
                    "Interpret the peephole-optimized AST (default)"
                })
            .conflicts_with_all(&["ast", "rle", "byte", "jit", "llvm"]))
        .arg(Arg::with_name("byte")
            .long("byte")
            .help("Compile AST to bytecode")
            .conflicts_with_all(&["ast", "rle", "peep", "jit", "llvm"]));

    #[cfg(feature = "llvm")]
    let app = app
        .arg(Arg::with_name("llvm")
            .long("llvm")
            .help("JIT using LLVM")
            .conflicts_with_all(&["ast", "rle", "peep", "byte", "jit"]));

    #[cfg(feature = "jit")]
    let app = app
        .arg(Arg::with_name("jit")
            .long("jit")
            .help("JIT to native x64 (default)")
            .conflicts_with_all(&["ast", "rle", "peep", "byte", "llvm"]));

    #[cfg(any(feature = "jit"))]
    let app = app
        .arg(Arg::with_name("unchecked")
            .short("u")
            .long("unchecked")
            .help("Omit memory bounds checks in JIT")
            .conflicts_with_all(&["ast", "rle", "peep", "byte", "llvm"]));

    app
}

fn error_exit(code: i32, msg: &str) -> ! {
    eprintln!("bfi: {}", msg);
    exit(code)
}

<|endoftext|>


<commit_before>use std::io::Cursor;

use byteorder::BigEndian;

pub fn encode<'a, I>(iter: I) -> Encode<I>
where
    I: Iterator<Item = &'a [u8]>,
{
    Encode {
        iter: iter,
        cur: Cursor::new(vec!()),
    }
}

pub fn decode<R>(buf: R) -> Decode<R>
where
    R: std::io::Read,
{
    Decode {
        buf: buf,
    }
}

pub struct Encode<I> {
    iter: I,
    cur: Cursor<Vec<u8>>,
}

impl<'a, I> Encode<I>
where
    I: Iterator<Item = &'a [u8]>,
{
    fn is_empty(&self) -> bool {
        self.cur.position() == self.cur.get_ref().len() as u64
    }

    fn pull(&mut self) -> std::io::Result<()> {
        use std::io::Write as _;
        use byteorder::WriteBytesExt as _;

        let data = match self.iter.next() {
            Some(data) => data,
            None => return Ok(()),
        };

        self.cur.get_mut().clear();
        self.cur.set_position(0);

        self.cur.write_u64::<BigEndian>(data.len() as u64)?;
        self.cur.write_all(data)?;
        self.cur.set_position(0);

        Ok(())
    }
}

impl<'a, I> std::io::Read for Encode<I>
where
    I: Iterator<Item = &'a [u8]>,
{
    fn read(&mut self, buf: &mut [u8]) -> std::io::Result<usize> {
        if self.is_empty() {
            self.pull()?;
        }

        self.cur.read(buf)
    }
}

pub struct Decode<R> {
    buf: R,
}

impl<R: std::io::Read> Decode<R> {

    fn read_len(&mut self) -> std::io::Result<Option<usize>> {
        use byteorder::ReadBytesExt as _;

        let mut buf = [0; 8];
        match self.buf.read(&mut buf[..])? {
            8 => (),
            0 => return Ok(None),
            _ => return Err(SizeTagError.into()),
        }

        let len = (&buf[..]).read_u64::<BigEndian>()? as usize;
        Ok(Some(len))
    }
}

impl<R: std::io::Read> Iterator for Decode<R> {

    type Item = std::io::Result<Vec<u8>>;

    fn next(&mut self) -> Option<std::io::Result<Vec<u8>>> {
        let len = match self.read_len() {
            Ok(Some(len)) => len,
            Ok(None) => return None,
            Err(error) => return Some(Err(error)),
        };

        let mut buf = vec!(0; len);
        match self.buf.read_exact(&mut buf[..]) {
            Ok(()) => (),
            Err(error) => return Some(Err(error)),
        }

        Some(Ok(buf))
    }
}

#[derive(Debug)]
struct SizeTagError;

impl std::fmt::Display for SizeTagError {

    fn fmt(&self, fmt: &mut std::fmt::Formatter) -> std::fmt::Result {
        write!(fmt, "incorrect length of the size tag")
    }
}

impl std::error::Error for SizeTagError {
}

impl From<SizeTagError> for std::io::Error {

    fn from(error: SizeTagError) -> std::io::Error {
        std::io::Error::new(std::io::ErrorKind::InvalidData, error)
    }
}
<commit_msg>Write tests for the chunked format utilities.<commit_after>use std::io::Cursor;

use byteorder::BigEndian;

pub fn encode<'a, I>(iter: I) -> Encode<I>
where
    I: Iterator<Item = &'a [u8]>,
{
    Encode {
        iter: iter,
        cur: Cursor::new(vec!()),
    }
}

pub fn decode<R>(buf: R) -> Decode<R>
where
    R: std::io::Read,
{
    Decode {
        buf: buf,
    }
}

pub struct Encode<I> {
    iter: I,
    cur: Cursor<Vec<u8>>,
}

impl<'a, I> Encode<I>
where
    I: Iterator<Item = &'a [u8]>,
{
    fn is_empty(&self) -> bool {
        self.cur.position() == self.cur.get_ref().len() as u64
    }

    fn pull(&mut self) -> std::io::Result<()> {
        use std::io::Write as _;
        use byteorder::WriteBytesExt as _;

        let data = match self.iter.next() {
            Some(data) => data,
            None => return Ok(()),
        };

        self.cur.get_mut().clear();
        self.cur.set_position(0);

        self.cur.write_u64::<BigEndian>(data.len() as u64)?;
        self.cur.write_all(data)?;
        self.cur.set_position(0);

        Ok(())
    }
}

impl<'a, I> std::io::Read for Encode<I>
where
    I: Iterator<Item = &'a [u8]>,
{
    fn read(&mut self, buf: &mut [u8]) -> std::io::Result<usize> {
        if self.is_empty() {
            self.pull()?;
        }

        self.cur.read(buf)
    }
}

pub struct Decode<R> {
    buf: R,
}

impl<R: std::io::Read> Decode<R> {

    fn read_len(&mut self) -> std::io::Result<Option<usize>> {
        use byteorder::ReadBytesExt as _;

        let mut buf = [0; 8];
        match self.buf.read(&mut buf[..])? {
            8 => (),
            0 => return Ok(None),
            _ => return Err(SizeTagError.into()),
        }

        let len = (&buf[..]).read_u64::<BigEndian>()? as usize;
        Ok(Some(len))
    }
}

impl<R: std::io::Read> Iterator for Decode<R> {

    type Item = std::io::Result<Vec<u8>>;

    fn next(&mut self) -> Option<std::io::Result<Vec<u8>>> {
        let len = match self.read_len() {
            Ok(Some(len)) => len,
            Ok(None) => return None,
            Err(error) => return Some(Err(error)),
        };

        let mut buf = vec!(0; len);
        match self.buf.read_exact(&mut buf[..]) {
            Ok(()) => (),
            Err(error) => return Some(Err(error)),
        }

        Some(Ok(buf))
    }
}

#[derive(Debug)]
struct SizeTagError;

impl std::fmt::Display for SizeTagError {

    fn fmt(&self, fmt: &mut std::fmt::Formatter) -> std::fmt::Result {
        write!(fmt, "incorrect length of the size tag")
    }
}

impl std::error::Error for SizeTagError {
}

impl From<SizeTagError> for std::io::Error {

    fn from(error: SizeTagError) -> std::io::Error {
        std::io::Error::new(std::io::ErrorKind::InvalidData, error)
    }
}

#[cfg(test)]
pub mod tests {

    use std::io::Read as _;

    use super::*;

    #[test]
    pub fn test_encode_empty() {
        let mut stream = encode(std::iter::empty());

        let mut output = vec!();
        stream.read_to_end(&mut output).unwrap();

        assert!(output.is_empty());
    }

    #[test]
    pub fn test_encode_single_chunk() {
        let mut stream = encode(vec!(&b"foo"[..]).into_iter());

        let mut output = vec!();
        stream.read_to_end(&mut output).unwrap();

        assert_eq!(output, b"\x00\x00\x00\x00\x00\x00\x00\x03foo");
    }

    #[test]
    pub fn test_encode_single_empty_chunk() {
        let mut stream = encode(vec!(&b""[..]).into_iter());

        let mut output = vec!();
        stream.read_to_end(&mut output).unwrap();

        assert_eq!(output, b"\x00\x00\x00\x00\x00\x00\x00\x00");
    }

    #[test]
    pub fn test_encode_multiple_empty_chunks() {
        let mut stream = encode(vec!(&b""[..], &b""[..]).into_iter());

        let mut output = vec!();
        stream.read_to_end(&mut output).unwrap();

        assert_eq!(&output[..8], b"\x00\x00\x00\x00\x00\x00\x00\x00");
        assert_eq!(&output[8..], b"\x00\x00\x00\x00\x00\x00\x00\x00");
    }

    #[test]
    pub fn test_decode_empty() {
        let mut iter = decode(&b""[..]);
        assert!(iter.next().is_none());
    }

    #[test]
    pub fn test_decode_single_chunk() {
        let mut iter = decode(&b"\x00\x00\x00\x00\x00\x00\x00\x03foo"[..])
            .map(Result::unwrap);

        assert_eq!(iter.next(), Some(b"foo".to_vec()));
        assert_eq!(iter.next(), None);
    }

    #[test]
    pub fn test_decode_empty_chunk() {
        let mut iter = decode(&b"\x00\x00\x00\x00\x00\x00\x00\x00"[..])
            .map(Result::unwrap);

        assert_eq!(iter.next(), Some(b"".to_vec()));
        assert_eq!(iter.next(), None);
    }

    #[test]
    pub fn test_decode_incorrect_size_tag() {
        let mut iter = decode(&b"\x12\x34\x56"[..]);

        let error = iter.next().unwrap().unwrap_err();
        assert_eq!(error.kind(), std::io::ErrorKind::InvalidData);
    }

    #[test]
    pub fn test_decode_short_input() {
        let mut iter = decode(&b"\x00\x00\x00\x00\x00\x00\x00\x42foo"[..]);

        let error = iter.next().unwrap().unwrap_err();
        assert_eq!(error.kind(), std::io::ErrorKind::UnexpectedEof);
    }

    #[test]
    pub fn test_encode_and_decode_multiple_chunks() {
        let chunks = vec! {
            &b"foo"[..],
            &b"bar"[..],
            &b"baz"[..],
        };

        let mut iter = decode(encode(chunks.into_iter()))
            .map(Result::unwrap);

        assert_eq!(iter.next(), Some(b"foo".to_vec()));
        assert_eq!(iter.next(), Some(b"bar".to_vec()));
        assert_eq!(iter.next(), Some(b"baz".to_vec()));
        assert_eq!(iter.next(), None);
    }

    #[test]
    pub fn test_encode_and_decode_multiple_empty_chunks() {
        let chunks = vec!(&b""[..], &b""[..], &b""[..]);

        let mut iter = decode(encode(chunks.into_iter()))
            .map(Result::unwrap);

        assert_eq!(iter.next(), Some(b"".to_vec()));
        assert_eq!(iter.next(), Some(b"".to_vec()));
        assert_eq!(iter.next(), Some(b"".to_vec()));
        assert_eq!(iter.next(), None);
    }
}
<|endoftext|>


<commit_before>use gl;
use glutin;
use native::NativeTaskBuilder;
use std::mem;
use std::sync::atomic::{AtomicUint, Relaxed};
use std::sync::{Arc, Mutex};
use std::task::TaskBuilder;

enum Message {
    EndFrame,
    Execute(proc(&gl::Gl, &mut GLState, (u8, u8), &ExtensionsList):Send),
}

pub struct Context {
    commands: Mutex<Sender<Message>>,
    events: Mutex<Receiver<glutin::Event>>,

    /// Dimensions of the frame buffer.
    dimensions: Arc<(AtomicUint, AtomicUint)>,
}

/// Represents the current OpenGL state.
/// The current state is passed to each function and can be freely updated.
pub struct GLState {
    /// Whether GL_BLEND is enabled
    pub enabled_blend: bool,

    /// Whether GL_CULL_FACE is enabled
    pub enabled_cull_face: bool,

    /// Whether GL_DEPTH_TEST is enabled
    pub enabled_depth_test: bool,

    /// Whether GL_DITHER is enabled
    pub enabled_dither: bool,

    /// Whether GL_POLYGON_OFFSET_FILL is enabled
    pub enabled_polygon_offset_fill: bool,

    /// Whether GL_SAMPLE_ALPHA_TO_COVERAGE is enabled
    pub enabled_sample_alpha_to_coverage: bool,

    /// Whether GL_SAMPLE_COVERAGE is enabled
    pub enabled_sample_coverage: bool,

    /// Whether GL_SCISSOR_TEST is enabled
    pub enabled_scissor_test: bool,

    /// Whether GL_STENCIL_TEST is enabled
    pub enabled_stencil_test: bool,

    // The latest value passed to `glUseProgram`.
    pub program: gl::types::GLuint,

    // The latest value passed to `glClearColor`.
    pub clear_color: (gl::types::GLclampf, gl::types::GLclampf,
                      gl::types::GLclampf, gl::types::GLclampf),

    // The latest value passed to `glClearDepthf`.
    pub clear_depth: gl::types::GLclampf,

    // The latest value passed to `glClearStencil`.
    pub clear_stencil: gl::types::GLint,

    /// The latest buffer bound to `GL_ARRAY_BUFFER`.
    pub array_buffer_binding: Option<gl::types::GLuint>,

    /// The latest buffer bound to `GL_ELEMENT_ARRAY_BUFFER`.
    pub element_array_buffer_binding: Option<gl::types::GLuint>,

    /// The latest framebuffer bound to `GL_READ_FRAMEBUFFER`.
    pub read_framebuffer: Option<gl::types::GLuint>,

    /// The latest framebuffer bound to `GL_DRAW_FRAMEBUFFER`.
    pub draw_framebuffer: Option<gl::types::GLuint>,

    /// The latest values passed to `glBlendFunc`.
    pub blend_func: (gl::types::GLenum, gl::types::GLenum),

    /// The latest value passed to `glDepthFunc`.
    pub depth_func: gl::types::GLenum,

    /// The latest values passed to `glViewport`.
    pub viewport: (gl::types::GLint, gl::types::GLint, gl::types::GLsizei, gl::types::GLsizei),

    /// The latest value passed to `glLineWidth`.
    pub line_width: gl::types::GLfloat,

    /// The latest value passed to `glCullFace`.
    pub cull_face: gl::types::GLenum,
}

impl GLState {
    /// Builds the `GLState` corresponding to the default GL values.
    fn new_defaults(viewport: (gl::types::GLint, gl::types::GLint, gl::types::GLsizei, gl::types::GLsizei)) -> GLState {
        GLState {
            enabled_blend: false,
            enabled_cull_face: false,
            enabled_depth_test: false,
            enabled_dither: false,
            enabled_polygon_offset_fill: false,
            enabled_sample_alpha_to_coverage: false,
            enabled_sample_coverage: false,
            enabled_scissor_test: false,
            enabled_stencil_test: false,

            program: 0,
            clear_color: (0.0, 0.0, 0.0, 0.0),
            clear_depth: 1.0,
            clear_stencil: 0,
            array_buffer_binding: None,
            element_array_buffer_binding: None,
            read_framebuffer: None,
            draw_framebuffer: None,
            depth_func: gl::LESS,
            blend_func: (0, 0),     // no default specified
            viewport: viewport,
            line_width: 1.0,
            cull_face: gl::BACK,
        }
    }
}

/// Contains data about the list of extensions
pub struct ExtensionsList {
    /// GL_EXT_direct_state_access
    pub gl_ext_direct_state_access: bool,
    /// GL_EXT_framebuffer_object
    pub gl_ext_framebuffer_object: bool,
}

impl Context {
    pub fn new_from_window(window: glutin::Window) -> Context {
        let (tx_events, rx_events) = channel();
        let (tx_commands, rx_commands) = channel();

        let dimensions = Arc::new((AtomicUint::new(800), AtomicUint::new(600)));

        let context = Context {
            commands: Mutex::new(tx_commands),
            events: Mutex::new(rx_events),
            dimensions: dimensions.clone(),
        };

        TaskBuilder::new().native().spawn(proc() {
            unsafe { window.make_current(); }

            let gl = gl::Gl::load_with(|symbol| window.get_proc_address(symbol));

            // building the GLState and modifying to GL state to match it
            let mut gl_state = {
                let viewport = {
                    let dim = window.get_inner_size().unwrap();
                    dimensions.0.store(dim.0, Relaxed);
                    dimensions.1.store(dim.1, Relaxed);
                    (0, 0, dim.0 as gl::types::GLsizei, dim.1 as gl::types::GLsizei)
                };

                GLState::new_defaults(viewport)
            };

            // getting the GL version and extensions
            let version = get_gl_version(&gl);
            let extensions = get_extensions(&gl);

            // main loop
            'main: loop {
                // processing commands
                loop {
                    match rx_commands.recv_opt() {
                        Ok(EndFrame) => break,
                        Ok(Execute(cmd)) => cmd(&gl, &mut gl_state, version, &extensions),
                        Err(_) => break 'main
                    }
                }

                // this is necessary on Windows 8, or nothing is being displayed
                gl.Flush();

                // swapping
                window.swap_buffers();

                // getting events
                for event in window.poll_events() {
                    // calling `glViewport` if the window has been resized
                    if let &glutin::Resized(width, height) = &event {
                        dimensions.0.store(width, Relaxed);
                        dimensions.1.store(height, Relaxed);

                        if gl_state.viewport != (0, 0, width as gl::types::GLsizei,
                                                 height as gl::types::GLsizei)
                        {
                            gl.Viewport(0, 0, width as gl::types::GLsizei,
                                height as gl::types::GLsizei);
                            gl_state.viewport = (0, 0, width as gl::types::GLsizei,
                                height as gl::types::GLsizei);
                        }
                    }

                    // sending the event outside
                    if tx_events.send_opt(event.clone()).is_err() {
                        break 'main;
                    }
                }
            }
        });

        context
    }

    pub fn new_from_headless(window: glutin::HeadlessContext) -> Context {
        let (_, rx_events) = channel();
        let (tx_commands, rx_commands) = channel();

        // TODO: fixme
        let dimensions = Arc::new((AtomicUint::new(800), AtomicUint::new(600)));

        let context = Context {
            commands: Mutex::new(tx_commands),
            events: Mutex::new(rx_events),
            dimensions: dimensions,
        };

        TaskBuilder::new().native().spawn(proc() {
            unsafe { window.make_current(); }

            let gl = gl::Gl::load_with(|symbol| window.get_proc_address(symbol));

            // building the GLState, version, and extensions
            let mut gl_state = GLState::new_defaults((0, 0, 0, 0));    // FIXME:
            let version = get_gl_version(&gl);
            let extensions = get_extensions(&gl);

            loop {
                match rx_commands.recv_opt() {
                    Ok(Execute(cmd)) => cmd(&gl, &mut gl_state, version, &extensions),
                    Ok(EndFrame) => (),     // ignoring buffer swapping
                    Err(_) => break
                }
            }
        });

        context
    }

    pub fn get_framebuffer_dimensions(&self) -> (uint, uint) {
        (
            self.dimensions.0.load(Relaxed),
            self.dimensions.1.load(Relaxed),
        )
    }

    pub fn exec(&self, f: proc(&gl::Gl, &mut GLState, (u8, u8), &ExtensionsList): Send) {
        self.commands.lock().send(Execute(f));
    }

    pub fn swap_buffers(&self) {
        self.commands.lock().send(EndFrame);
    }

    pub fn recv(&self) -> Vec<glutin::Event> {
        let events = self.events.lock();

        let mut result = Vec::new();
        loop {
            match events.try_recv() {
                Ok(ev) => result.push(ev),
                Err(_) => break
            }
        }
        result
    }
}

fn get_gl_version(gl: &gl::Gl) -> (u8, u8) {
    use std::c_str::CString;

    unsafe {
        let version = gl.GetString(gl::VERSION);
        let version = CString::new(version as *const i8, false);
        let version = version.as_str().expect("OpenGL version contains non-utf8 characters");

        let version = version.words().next().expect("glGetString(GL_VERSION) returned an empty \
                                                     string");

        let mut iter = version.split(|c: char| c == '.');
        let major = iter.next().unwrap();
        let minor = iter.next().expect("glGetString(GL_VERSION) did not return a correct version");

        (
            from_str(major).expect("failed to parse GL major version"),
            from_str(minor).expect("failed to parse GL minor version")
        )
    }
}

fn get_extensions_strings(gl: &gl::Gl) -> Vec<String> {
    use std::c_str::CString;

    unsafe {
        let list = gl.GetString(gl::EXTENSIONS);

        if list.is_null() {
            let mut num_extensions = 0;
            gl.GetIntegerv(gl::NUM_EXTENSIONS, &mut num_extensions);

            range(0, num_extensions).map(|num| {
                let ext = gl.GetStringi(gl::EXTENSIONS, num as gl::types::GLuint);
                let ext = CString::new(ext as *const i8, false);
                ext.as_str().expect("OpenGL extension contains non-utf8 characters").to_string()
            }).collect()

        } else {
            let list = CString::new(list as *const i8, false);
            let list = list.as_str()
                .expect("List of OpenGL extensions contains non-utf8 characters");
            list.words().map(|e| e.to_string()).collect()
        }
    }
}

fn get_extensions(gl: &gl::Gl) -> ExtensionsList {
    let strings = get_extensions_strings(gl);

    let mut extensions = ExtensionsList {
        gl_ext_direct_state_access: false,
        gl_ext_framebuffer_object: false,
    };

    for extension in strings.into_iter() {
        match extension.as_slice() {
            "GL_EXT_direct_state_access" => extensions.gl_ext_direct_state_access = true,
            "GL_EXT_framebuffer_object" => extensions.gl_ext_framebuffer_object = true,
            _ => ()
        }
    }

    extensions
}
<commit_msg>Now passing a `GlVersion` instead of `(u8, u8)`<commit_after>use gl;
use glutin;
use native::NativeTaskBuilder;
use std::mem;
use std::sync::atomic::{AtomicUint, Relaxed};
use std::sync::{Arc, Mutex};
use std::task::TaskBuilder;

enum Message {
    EndFrame,
    Execute(proc(&gl::Gl, &mut GLState, &GlVersion, &ExtensionsList):Send),
}

pub struct Context {
    commands: Mutex<Sender<Message>>,
    events: Mutex<Receiver<glutin::Event>>,

    /// Dimensions of the frame buffer.
    dimensions: Arc<(AtomicUint, AtomicUint)>,
}

/// Represents the current OpenGL state.
/// The current state is passed to each function and can be freely updated.
pub struct GLState {
    /// Whether GL_BLEND is enabled
    pub enabled_blend: bool,

    /// Whether GL_CULL_FACE is enabled
    pub enabled_cull_face: bool,

    /// Whether GL_DEPTH_TEST is enabled
    pub enabled_depth_test: bool,

    /// Whether GL_DITHER is enabled
    pub enabled_dither: bool,

    /// Whether GL_POLYGON_OFFSET_FILL is enabled
    pub enabled_polygon_offset_fill: bool,

    /// Whether GL_SAMPLE_ALPHA_TO_COVERAGE is enabled
    pub enabled_sample_alpha_to_coverage: bool,

    /// Whether GL_SAMPLE_COVERAGE is enabled
    pub enabled_sample_coverage: bool,

    /// Whether GL_SCISSOR_TEST is enabled
    pub enabled_scissor_test: bool,

    /// Whether GL_STENCIL_TEST is enabled
    pub enabled_stencil_test: bool,

    // The latest value passed to `glUseProgram`.
    pub program: gl::types::GLuint,

    // The latest value passed to `glClearColor`.
    pub clear_color: (gl::types::GLclampf, gl::types::GLclampf,
                      gl::types::GLclampf, gl::types::GLclampf),

    // The latest value passed to `glClearDepthf`.
    pub clear_depth: gl::types::GLclampf,

    // The latest value passed to `glClearStencil`.
    pub clear_stencil: gl::types::GLint,

    /// The latest buffer bound to `GL_ARRAY_BUFFER`.
    pub array_buffer_binding: Option<gl::types::GLuint>,

    /// The latest buffer bound to `GL_ELEMENT_ARRAY_BUFFER`.
    pub element_array_buffer_binding: Option<gl::types::GLuint>,

    /// The latest framebuffer bound to `GL_READ_FRAMEBUFFER`.
    pub read_framebuffer: Option<gl::types::GLuint>,

    /// The latest framebuffer bound to `GL_DRAW_FRAMEBUFFER`.
    pub draw_framebuffer: Option<gl::types::GLuint>,

    /// The latest values passed to `glBlendFunc`.
    pub blend_func: (gl::types::GLenum, gl::types::GLenum),

    /// The latest value passed to `glDepthFunc`.
    pub depth_func: gl::types::GLenum,

    /// The latest values passed to `glViewport`.
    pub viewport: (gl::types::GLint, gl::types::GLint, gl::types::GLsizei, gl::types::GLsizei),

    /// The latest value passed to `glLineWidth`.
    pub line_width: gl::types::GLfloat,

    /// The latest value passed to `glCullFace`.
    pub cull_face: gl::types::GLenum,
}

impl GLState {
    /// Builds the `GLState` corresponding to the default GL values.
    fn new_defaults(viewport: (gl::types::GLint, gl::types::GLint, gl::types::GLsizei, gl::types::GLsizei)) -> GLState {
        GLState {
            enabled_blend: false,
            enabled_cull_face: false,
            enabled_depth_test: false,
            enabled_dither: false,
            enabled_polygon_offset_fill: false,
            enabled_sample_alpha_to_coverage: false,
            enabled_sample_coverage: false,
            enabled_scissor_test: false,
            enabled_stencil_test: false,

            program: 0,
            clear_color: (0.0, 0.0, 0.0, 0.0),
            clear_depth: 1.0,
            clear_stencil: 0,
            array_buffer_binding: None,
            element_array_buffer_binding: None,
            read_framebuffer: None,
            draw_framebuffer: None,
            depth_func: gl::LESS,
            blend_func: (0, 0),     // no default specified
            viewport: viewport,
            line_width: 1.0,
            cull_face: gl::BACK,
        }
    }
}

/// Describes an OpenGL version.
#[deriving(Show, Clone, PartialEq, Eq)]
pub struct GlVersion(pub u8, pub u8);

impl PartialOrd for GlVersion {
    fn partial_cmp(&self, other: &GlVersion) -> Option<Ordering> {
        Some(self.cmp(other))
    }
}

impl Ord for GlVersion {
    fn cmp(&self, other: &GlVersion) -> Ordering {
        match self.0.cmp(&other.0) {
            Equal => self.1.cmp(&other.1),
            a => a
        }
    }
}

/// Contains data about the list of extensions
pub struct ExtensionsList {
    /// GL_EXT_direct_state_access
    pub gl_ext_direct_state_access: bool,
    /// GL_EXT_framebuffer_object
    pub gl_ext_framebuffer_object: bool,
}

impl Context {
    pub fn new_from_window(window: glutin::Window) -> Context {
        let (tx_events, rx_events) = channel();
        let (tx_commands, rx_commands) = channel();

        let dimensions = Arc::new((AtomicUint::new(800), AtomicUint::new(600)));

        let context = Context {
            commands: Mutex::new(tx_commands),
            events: Mutex::new(rx_events),
            dimensions: dimensions.clone(),
        };

        TaskBuilder::new().native().spawn(proc() {
            unsafe { window.make_current(); }

            let gl = gl::Gl::load_with(|symbol| window.get_proc_address(symbol));

            // building the GLState and modifying to GL state to match it
            let mut gl_state = {
                let viewport = {
                    let dim = window.get_inner_size().unwrap();
                    dimensions.0.store(dim.0, Relaxed);
                    dimensions.1.store(dim.1, Relaxed);
                    (0, 0, dim.0 as gl::types::GLsizei, dim.1 as gl::types::GLsizei)
                };

                GLState::new_defaults(viewport)
            };

            // getting the GL version and extensions
            let version = get_gl_version(&gl);
            let extensions = get_extensions(&gl);

            // main loop
            'main: loop {
                // processing commands
                loop {
                    match rx_commands.recv_opt() {
                        Ok(EndFrame) => break,
                        Ok(Execute(cmd)) => cmd(&gl, &mut gl_state, &version, &extensions),
                        Err(_) => break 'main
                    }
                }

                // this is necessary on Windows 8, or nothing is being displayed
                gl.Flush();

                // swapping
                window.swap_buffers();

                // getting events
                for event in window.poll_events() {
                    // calling `glViewport` if the window has been resized
                    if let &glutin::Resized(width, height) = &event {
                        dimensions.0.store(width, Relaxed);
                        dimensions.1.store(height, Relaxed);

                        if gl_state.viewport != (0, 0, width as gl::types::GLsizei,
                                                 height as gl::types::GLsizei)
                        {
                            gl.Viewport(0, 0, width as gl::types::GLsizei,
                                height as gl::types::GLsizei);
                            gl_state.viewport = (0, 0, width as gl::types::GLsizei,
                                height as gl::types::GLsizei);
                        }
                    }

                    // sending the event outside
                    if tx_events.send_opt(event.clone()).is_err() {
                        break 'main;
                    }
                }
            }
        });

        context
    }

    pub fn new_from_headless(window: glutin::HeadlessContext) -> Context {
        let (_, rx_events) = channel();
        let (tx_commands, rx_commands) = channel();

        // TODO: fixme
        let dimensions = Arc::new((AtomicUint::new(800), AtomicUint::new(600)));

        let context = Context {
            commands: Mutex::new(tx_commands),
            events: Mutex::new(rx_events),
            dimensions: dimensions,
        };

        TaskBuilder::new().native().spawn(proc() {
            unsafe { window.make_current(); }

            let gl = gl::Gl::load_with(|symbol| window.get_proc_address(symbol));

            // building the GLState, version, and extensions
            let mut gl_state = GLState::new_defaults((0, 0, 0, 0));    // FIXME:
            let version = get_gl_version(&gl);
            let extensions = get_extensions(&gl);

            loop {
                match rx_commands.recv_opt() {
                    Ok(Execute(cmd)) => cmd(&gl, &mut gl_state, &version, &extensions),
                    Ok(EndFrame) => (),     // ignoring buffer swapping
                    Err(_) => break
                }
            }
        });

        context
    }

    pub fn get_framebuffer_dimensions(&self) -> (uint, uint) {
        (
            self.dimensions.0.load(Relaxed),
            self.dimensions.1.load(Relaxed),
        )
    }

    pub fn exec(&self, f: proc(&gl::Gl, &mut GLState, &GlVersion, &ExtensionsList): Send) {
        self.commands.lock().send(Execute(f));
    }

    pub fn swap_buffers(&self) {
        self.commands.lock().send(EndFrame);
    }

    pub fn recv(&self) -> Vec<glutin::Event> {
        let events = self.events.lock();

        let mut result = Vec::new();
        loop {
            match events.try_recv() {
                Ok(ev) => result.push(ev),
                Err(_) => break
            }
        }
        result
    }
}

fn get_gl_version(gl: &gl::Gl) -> GlVersion {
    use std::c_str::CString;

    unsafe {
        let version = gl.GetString(gl::VERSION);
        let version = CString::new(version as *const i8, false);
        let version = version.as_str().expect("OpenGL version contains non-utf8 characters");

        let version = version.words().next().expect("glGetString(GL_VERSION) returned an empty \
                                                     string");

        let mut iter = version.split(|c: char| c == '.');
        let major = iter.next().unwrap();
        let minor = iter.next().expect("glGetString(GL_VERSION) did not return a correct version");

        GlVersion(
            from_str(major).expect("failed to parse GL major version"),
            from_str(minor).expect("failed to parse GL minor version"),
        )
    }
}

fn get_extensions_strings(gl: &gl::Gl) -> Vec<String> {
    use std::c_str::CString;

    unsafe {
        let list = gl.GetString(gl::EXTENSIONS);

        if list.is_null() {
            let mut num_extensions = 0;
            gl.GetIntegerv(gl::NUM_EXTENSIONS, &mut num_extensions);

            range(0, num_extensions).map(|num| {
                let ext = gl.GetStringi(gl::EXTENSIONS, num as gl::types::GLuint);
                let ext = CString::new(ext as *const i8, false);
                ext.as_str().expect("OpenGL extension contains non-utf8 characters").to_string()
            }).collect()

        } else {
            let list = CString::new(list as *const i8, false);
            let list = list.as_str()
                .expect("List of OpenGL extensions contains non-utf8 characters");
            list.words().map(|e| e.to_string()).collect()
        }
    }
}

fn get_extensions(gl: &gl::Gl) -> ExtensionsList {
    let strings = get_extensions_strings(gl);

    let mut extensions = ExtensionsList {
        gl_ext_direct_state_access: false,
        gl_ext_framebuffer_object: false,
    };

    for extension in strings.into_iter() {
        match extension.as_slice() {
            "GL_EXT_direct_state_access" => extensions.gl_ext_direct_state_access = true,
            "GL_EXT_framebuffer_object" => extensions.gl_ext_framebuffer_object = true,
            _ => ()
        }
    }

    extensions
}
<|endoftext|>