<commit_before>// Copyright 2015-2016 Deyan Ginev. See the LICENSE
// file at the top-level directory of this distribution.
//
// Licensed under the MIT license <LICENSE-MIT or http://opensource.org/licenses/MIT>.
// This file may not be copied, modified, or distributed
// except according to those terms.

//! Import a new corpus into the framework

extern crate glob;
extern crate Archive;

use glob::glob;
// use regex::Regex;
use Archive::*;
use std::path::Path;
use std::path::PathBuf;
use std::fs;
use std::env;
use std::io::Error;
use backend::{Backend};
use data::{Task, TaskStatus, Corpus};

/// Struct for performing corpus imports into CorTeX
pub struct Importer {
  /// a `Corpus` to be imported, containing all relevant metadata
  pub corpus : Corpus,
  /// a `Backend` on which to persist the import into the Task store
  pub backend : Backend,
  /// the current working directory, to resolve relative paths
  pub cwd : PathBuf
}
impl Default for Importer {
  fn default() -> Importer {
    let default_backend = Backend::default();
    Importer {
      corpus : default_backend.add(
        Corpus {
          id: None,
          path : ".".to_string(),
          name : "default".to_string(),
          complex : false }).unwrap(),
      backend : default_backend,
      cwd : Importer::cwd()
    }
  }
}

impl Importer {
  /// Convenience method for (recklessly?) obtaining the current working dir
  pub fn cwd() -> PathBuf {
    env::current_dir().unwrap()
  }
  /// Top-level method for unpacking an arxiv-toplogy corpus from its tar-ed form
  fn unpack(&self) -> Result<(),Error> {
    try!(self.unpack_arxiv_top());
    try!(self.unpack_arxiv_months());
    Ok(())
  }
  fn unpack_extend(&self) -> Result<(),Error> {
    try!(self.unpack_extend_arxiv_top());
    // We can reuse the monthly unpack, as it deletes all unpacked document archives
    // In other words, it always acts as a conservative extension
    try!(self.unpack_arxiv_months());
    Ok(())
  }

  /// Unpack the top-level tar files from an arxiv-topology corpus
  fn unpack_arxiv_top(&self) -> Result<(),Error> {
    println!("-- Starting top-level unpack process");
    let path_str = self.corpus.path.clone();
    let tars_path = path_str.to_string() + "/*.tar";
    for entry in glob(&tars_path).unwrap() {
      match entry {
        Ok(path) => {
          // let base_name = path.file_stem().unwrap().to_str().unwrap();
          // If we wanted fine-grained control, we could infer the dir name:
          // let arxiv_name_re = Regex::new(r"arXiv_src_(\d+)_").unwrap();
          // let captures = arxiv_name_re.captures(base_name).unwrap();
          // let unpack_dirname = match captures.at(1) {
          //   Some(month) => month,
          //   None => base_name
          // };
          // --- but not for now

          // Let's open the tar file and unpack it:
          let archive_reader = Reader::new().unwrap()
            .support_filter_all()
            .support_format_all()
            .open_filename(path.to_str().unwrap(), 10240).unwrap();
          loop {
            match archive_reader.next_header() {
              Ok(e) => {
                let full_extract_path = path_str.to_string() + &e.pathname();
                match fs::metadata(full_extract_path.clone()) {
                  Ok(_) => println!("File {:?} exists, won't unpack.", e.pathname()),
                  Err(_) => {
                    println!("To unpack: {:?}", full_extract_path);
                    match e.extract_to(&full_extract_path, Vec::new()) {
                      Ok(_) => {},
                      _ => {
                        println!("Failed to extract {:?}", full_extract_path);
                      }
                    }
                  }
                }
              },
              Err(_) => { break }
            }
          }
        },
        Err(e) => {println!("Failed tar glob: {:?}", e)}
      }
    }
    Ok(())
  }
  /// Top-level extension unpacking for arxiv-topology corpora
  fn unpack_extend_arxiv_top(&self) -> Result<(),Error> {
    // What I am trying to figure out right now is how to avoid a monstrous amount of code duplication
    // I don't want to copy-paste all of unpack_arxiv_top in here ...
    println!("-- Starting top-level unpack-extend process");
    let path_str = self.corpus.path.clone();
    let tars_path = path_str.to_string() + "/*.tar";
    for entry in glob(&tars_path).unwrap() {
      match entry {
        Ok(path) => {
          // Let's open the tar file and unpack it:
          let archive_reader = Reader::new().unwrap()
            .support_filter_all()
            .support_format_all()
            .open_filename(path.to_str().unwrap(), 10240).unwrap();
          loop {
            match archive_reader.next_header() {
              Ok(e) => {
                let full_extract_path = path_str.to_string() + &e.pathname();
                match fs::metadata(full_extract_path.clone()) {
                  Ok(_) => {},//println!("File {:?} exists, won't unpack.", e.pathname()),
                  Err(_) => {
                    // Archive entries end in .gz, let's try that as well, to check if the directory is there
                    let dir_extract_path = &full_extract_path[0..full_extract_path.len()-3];
                    match fs::metadata(dir_extract_path) {
                      Ok(_) => {},//println!("Directory for {:?} already exists, won't unpack.", e.pathname()),
                      Err(_) => {
                        println!("To unpack: {:?}", full_extract_path);
                        match e.extract_to(&full_extract_path, Vec::new()) {
                          Ok(_) => {},
                          _ => {
                            println!("Failed to extract {:?}", full_extract_path);
                          }
                        }
                      }
                    }
                  }
                }
              },
              Err(_) => { break }
            }
          }
        },
        Err(e) => {println!("Failed tar glob: {:?}", e)}
      }
    }
    Ok(())
  }

  /// Unpack the monthly sub-archives of an arxiv-topology corpus, into the CorTeX organization
  fn unpack_arxiv_months(&self) -> Result<(),Error> {
    println!("-- Starting to unpack monthly .gz archives");
    let path_str = self.corpus.path.clone();
    let gzs_path = path_str.to_string() + "/*/*.gz";
    for entry in glob(&gzs_path).unwrap() {
      match entry {
        Ok(path) => {
          let entry_path = path.to_str().unwrap();
          let entry_dir = path.parent().unwrap().to_str().unwrap();
          let base_name = path.file_stem().unwrap().to_str().unwrap();
          let entry_cp_dir = entry_dir.to_string() + "/" + base_name;
          fs::create_dir_all(entry_cp_dir.clone()).unwrap_or_else( |reason| {
            println!("Failed to mkdir -p {:?} because: {:?}", entry_cp_dir.clone(), reason.kind());
          });
          // Careful here, some of arXiv's .gz files are really plain-text TeX files (surprise!!!)
          let archive_reader_new = Reader::new().unwrap()
            .support_filter_all()
            .support_format_all()
            .open_filename(entry_path, 10240);
          // We'll write out a ZIP file for each entry
          let full_extract_path = entry_cp_dir.to_string() + "/" + base_name + ".zip";
          let mut archive_writer_new = Writer::new().unwrap()
            //.add_filter(ArchiveFilter::Lzip)
            .set_compression(ArchiveFilter::None)
            .set_format(ArchiveFormat::Zip);
          archive_writer_new.open_filename(&full_extract_path.clone()).unwrap();

          match archive_reader_new {
            Err(_) => {
              let raw_reader_new = Reader::new().unwrap()
                .support_filter_all()
                .support_format_raw()
                .open_filename(entry_path, 10240);
              match raw_reader_new {
                Ok(raw_reader) => {
                  match raw_reader.next_header() {
                    Ok(_) => {
                      let tex_target = base_name.to_string() + ".tex";
                      // In a "raw" read, we don't know the data size in advance. So we bite the bullet and
                      // read the usually tiny tex file in memory, obtaining a size estimate
                      let mut raw_data = Vec::new();
                      loop {
                        let chunk_data = raw_reader.read_data(10240);
                        match chunk_data {
                          Ok(chunk) => raw_data.extend(chunk.into_iter()),
                          Err(_) => {break}
                        };
                      }
                      match archive_writer_new.write_header_new(&tex_target,raw_data.len() as i64) {
                        Ok(_) => {},
                        Err(e) => {
                          println!("Couldn't write header: {:?}", e);
                          break;
                        }
                      }
                      match archive_writer_new.write_data(raw_data) {
                        Ok(_) => {},
                        Err(e) => println!("Failed to write data to {:?} because {:?}", tex_target.clone(), e)
                      };
                    },
                    Err(_) => println!("No content in archive: {:?}", entry_path)
                  }
                },
                Err(_) => println!("Unrecognizeable archive: {:?}", entry_path)
              }
            },
            Ok(archive_reader) => {
              loop {
                match archive_reader.next_header() {
                  Ok(e) => {
                    match archive_writer_new.write_header(e) {
                      Ok(_) => {},
                      _ => {} // TODO: If we need to print an error message, we can do so later.
                    };
                    loop {
                      let entry_data = archive_reader.read_data(10240);
                      match entry_data {
                        Ok(chunk) => { archive_writer_new.write_data(chunk).unwrap(); },
                        Err(_) => { break; }
                      };
                    }
                  },
                  Err(_) => { break; }
                }
              }
            }
          }
          // Done with this .gz , remove it:
          match fs::remove_file(path.clone()) {
            Ok(_) => {},
            Err(e) => println!("Can't remove source .gz: {:?}", e)
          };
        },
        Err(e) => println!("Failed gz glob: {:?}", e)
      }
    }
    Ok(())
  }

  /// Given a CorTeX-topology corpus, walk the file system and import it into the Task store
  pub fn walk_import<'walk>(&self) -> Result<(),Error> {
    println!("-- Starting import walk");
    let import_extension = if self.corpus.complex { "zip" } else { "tex" };
    let mut walk_q : Vec<PathBuf> = vec![Path::new(&self.corpus.path).to_owned()];
    let mut import_q : Vec<Task> = Vec::new();
    let mut import_counter = 0;
    while walk_q.len() > 0 {
      let current_path = walk_q.pop().unwrap();
      // println!("-- current path {:?}", current_path);
      let current_metadata = try!(fs::metadata(current_path.clone()));
      if current_metadata.is_dir() { // Ignore files
        // First, test if we just found an entry:
        let current_local_dir = current_path.file_name().unwrap();
        let current_entry = current_local_dir.to_str().unwrap().to_string() + "." + import_extension;
        let current_entry_path = current_path.to_str().unwrap().to_string() + "/" + &current_entry;
        match fs::metadata(current_entry_path.clone()) {
          Ok(_) => {
            // Found the expected file, import this entry:
            // println!("Found entry: {:?}", current_entry_path);
            import_counter += 1;
            import_q.push(self.new_task(current_entry_path));
            if import_q.len() >= 1000 {
              // Flush the import queue to backend:
              println!("Checkpoint backend writer: job {:?}", import_q.len());
              self.backend.mark_imported(&import_q).unwrap(); // TODO: Proper Error-handling
              import_q.clear();
            }
          },
          Err(_) => {
            //  No such entry found, traversing into the directory:
            for subentry in try!(fs::read_dir(current_path.clone())) {
              let subentry = try!(subentry);
              walk_q.push(subentry.path());
            }
          }
        }
      }
    }
    if !import_q.is_empty() {
      println!("Checkpoint backend writer: job {:?}", import_q.len());
      self.backend.mark_imported(&import_q).unwrap();} // TODO: Proper Error-handling
    println!("--- Imported {:?} entries.", import_counter);
    Ok(())
  }

  /// Create a new NoProblem task for the "import" service and the Importer-specified corpus
  pub fn new_task(&self, entry : String) -> Task {
    let abs_entry : String = if Path::new(&entry).is_relative() {
      let mut new_abs = self.cwd.clone();
      new_abs.push(&entry);
      new_abs.to_str().unwrap().to_string()
    } else {
      entry.clone()
    };

    Task {id: None, entry : abs_entry, status : TaskStatus::NoProblem.raw(), corpusid : self.corpus.id.unwrap(), serviceid: 2}
  }
  /// Top-level import driver, performs an optional unpack, and then an import into the Task store
  pub fn process(&self) -> Result<(),Error> {
    // println!("Greetings from the import processor");
    if self.corpus.complex { // Complex setup has an unpack step:
      try!(self.unpack());
    }
    // Walk the directory tree and import the files in the Task store:
    try!(self.walk_import());

    Ok(())
  }

  /// Top-level corpus extension, performs a check for newly added documents and extracts+adds them to the existing corpus tasks
  pub fn extend_corpus(&self) -> Result<(), Error> {
    if self.corpus.complex { // Complex setup has an unpack step:
      try!(self.unpack_extend());
    }
    // Use the regular walk_import, at the cost of more database work,
    // the "Backend::mark_imported" ORM method allows us to insert only if new
    try!(self.walk_import());
    Ok(())
  }
}

<commit_msg>record the current job, rather than the queue size, for debug msgs<commit_after>// Copyright 2015-2016 Deyan Ginev. See the LICENSE
// file at the top-level directory of this distribution.
//
// Licensed under the MIT license <LICENSE-MIT or http://opensource.org/licenses/MIT>.
// This file may not be copied, modified, or distributed
// except according to those terms.

//! Import a new corpus into the framework

extern crate glob;
extern crate Archive;

use glob::glob;
// use regex::Regex;
use Archive::*;
use std::path::Path;
use std::path::PathBuf;
use std::fs;
use std::env;
use std::io::Error;
use backend::{Backend};
use data::{Task, TaskStatus, Corpus};

/// Struct for performing corpus imports into CorTeX
pub struct Importer {
  /// a `Corpus` to be imported, containing all relevant metadata
  pub corpus : Corpus,
  /// a `Backend` on which to persist the import into the Task store
  pub backend : Backend,
  /// the current working directory, to resolve relative paths
  pub cwd : PathBuf
}
impl Default for Importer {
  fn default() -> Importer {
    let default_backend = Backend::default();
    Importer {
      corpus : default_backend.add(
        Corpus {
          id: None,
          path : ".".to_string(),
          name : "default".to_string(),
          complex : false }).unwrap(),
      backend : default_backend,
      cwd : Importer::cwd()
    }
  }
}

impl Importer {
  /// Convenience method for (recklessly?) obtaining the current working dir
  pub fn cwd() -> PathBuf {
    env::current_dir().unwrap()
  }
  /// Top-level method for unpacking an arxiv-toplogy corpus from its tar-ed form
  fn unpack(&self) -> Result<(),Error> {
    try!(self.unpack_arxiv_top());
    try!(self.unpack_arxiv_months());
    Ok(())
  }
  fn unpack_extend(&self) -> Result<(),Error> {
    try!(self.unpack_extend_arxiv_top());
    // We can reuse the monthly unpack, as it deletes all unpacked document archives
    // In other words, it always acts as a conservative extension
    try!(self.unpack_arxiv_months());
    Ok(())
  }

  /// Unpack the top-level tar files from an arxiv-topology corpus
  fn unpack_arxiv_top(&self) -> Result<(),Error> {
    println!("-- Starting top-level unpack process");
    let path_str = self.corpus.path.clone();
    let tars_path = path_str.to_string() + "/*.tar";
    for entry in glob(&tars_path).unwrap() {
      match entry {
        Ok(path) => {
          // let base_name = path.file_stem().unwrap().to_str().unwrap();
          // If we wanted fine-grained control, we could infer the dir name:
          // let arxiv_name_re = Regex::new(r"arXiv_src_(\d+)_").unwrap();
          // let captures = arxiv_name_re.captures(base_name).unwrap();
          // let unpack_dirname = match captures.at(1) {
          //   Some(month) => month,
          //   None => base_name
          // };
          // --- but not for now

          // Let's open the tar file and unpack it:
          let archive_reader = Reader::new().unwrap()
            .support_filter_all()
            .support_format_all()
            .open_filename(path.to_str().unwrap(), 10240).unwrap();
          loop {
            match archive_reader.next_header() {
              Ok(e) => {
                let full_extract_path = path_str.to_string() + &e.pathname();
                match fs::metadata(full_extract_path.clone()) {
                  Ok(_) => println!("File {:?} exists, won't unpack.", e.pathname()),
                  Err(_) => {
                    println!("To unpack: {:?}", full_extract_path);
                    match e.extract_to(&full_extract_path, Vec::new()) {
                      Ok(_) => {},
                      _ => {
                        println!("Failed to extract {:?}", full_extract_path);
                      }
                    }
                  }
                }
              },
              Err(_) => { break }
            }
          }
        },
        Err(e) => {println!("Failed tar glob: {:?}", e)}
      }
    }
    Ok(())
  }
  /// Top-level extension unpacking for arxiv-topology corpora
  fn unpack_extend_arxiv_top(&self) -> Result<(),Error> {
    // What I am trying to figure out right now is how to avoid a monstrous amount of code duplication
    // I don't want to copy-paste all of unpack_arxiv_top in here ...
    println!("-- Starting top-level unpack-extend process");
    let path_str = self.corpus.path.clone();
    let tars_path = path_str.to_string() + "/*.tar";
    for entry in glob(&tars_path).unwrap() {
      match entry {
        Ok(path) => {
          // Let's open the tar file and unpack it:
          let archive_reader = Reader::new().unwrap()
            .support_filter_all()
            .support_format_all()
            .open_filename(path.to_str().unwrap(), 10240).unwrap();
          loop {
            match archive_reader.next_header() {
              Ok(e) => {
                let full_extract_path = path_str.to_string() + &e.pathname();
                match fs::metadata(full_extract_path.clone()) {
                  Ok(_) => {},//println!("File {:?} exists, won't unpack.", e.pathname()),
                  Err(_) => {
                    // Archive entries end in .gz, let's try that as well, to check if the directory is there
                    let dir_extract_path = &full_extract_path[0..full_extract_path.len()-3];
                    match fs::metadata(dir_extract_path) {
                      Ok(_) => {},//println!("Directory for {:?} already exists, won't unpack.", e.pathname()),
                      Err(_) => {
                        println!("To unpack: {:?}", full_extract_path);
                        match e.extract_to(&full_extract_path, Vec::new()) {
                          Ok(_) => {},
                          _ => {
                            println!("Failed to extract {:?}", full_extract_path);
                          }
                        }
                      }
                    }
                  }
                }
              },
              Err(_) => { break }
            }
          }
        },
        Err(e) => {println!("Failed tar glob: {:?}", e)}
      }
    }
    Ok(())
  }

  /// Unpack the monthly sub-archives of an arxiv-topology corpus, into the CorTeX organization
  fn unpack_arxiv_months(&self) -> Result<(),Error> {
    println!("-- Starting to unpack monthly .gz archives");
    let path_str = self.corpus.path.clone();
    let gzs_path = path_str.to_string() + "/*/*.gz";
    for entry in glob(&gzs_path).unwrap() {
      match entry {
        Ok(path) => {
          let entry_path = path.to_str().unwrap();
          let entry_dir = path.parent().unwrap().to_str().unwrap();
          let base_name = path.file_stem().unwrap().to_str().unwrap();
          let entry_cp_dir = entry_dir.to_string() + "/" + base_name;
          fs::create_dir_all(entry_cp_dir.clone()).unwrap_or_else( |reason| {
            println!("Failed to mkdir -p {:?} because: {:?}", entry_cp_dir.clone(), reason.kind());
          });
          // Careful here, some of arXiv's .gz files are really plain-text TeX files (surprise!!!)
          let archive_reader_new = Reader::new().unwrap()
            .support_filter_all()
            .support_format_all()
            .open_filename(entry_path, 10240);
          // We'll write out a ZIP file for each entry
          let full_extract_path = entry_cp_dir.to_string() + "/" + base_name + ".zip";
          let mut archive_writer_new = Writer::new().unwrap()
            //.add_filter(ArchiveFilter::Lzip)
            .set_compression(ArchiveFilter::None)
            .set_format(ArchiveFormat::Zip);
          archive_writer_new.open_filename(&full_extract_path.clone()).unwrap();

          match archive_reader_new {
            Err(_) => {
              let raw_reader_new = Reader::new().unwrap()
                .support_filter_all()
                .support_format_raw()
                .open_filename(entry_path, 10240);
              match raw_reader_new {
                Ok(raw_reader) => {
                  match raw_reader.next_header() {
                    Ok(_) => {
                      let tex_target = base_name.to_string() + ".tex";
                      // In a "raw" read, we don't know the data size in advance. So we bite the bullet and
                      // read the usually tiny tex file in memory, obtaining a size estimate
                      let mut raw_data = Vec::new();
                      loop {
                        let chunk_data = raw_reader.read_data(10240);
                        match chunk_data {
                          Ok(chunk) => raw_data.extend(chunk.into_iter()),
                          Err(_) => {break}
                        };
                      }
                      match archive_writer_new.write_header_new(&tex_target,raw_data.len() as i64) {
                        Ok(_) => {},
                        Err(e) => {
                          println!("Couldn't write header: {:?}", e);
                          break;
                        }
                      }
                      match archive_writer_new.write_data(raw_data) {
                        Ok(_) => {},
                        Err(e) => println!("Failed to write data to {:?} because {:?}", tex_target.clone(), e)
                      };
                    },
                    Err(_) => println!("No content in archive: {:?}", entry_path)
                  }
                },
                Err(_) => println!("Unrecognizeable archive: {:?}", entry_path)
              }
            },
            Ok(archive_reader) => {
              loop {
                match archive_reader.next_header() {
                  Ok(e) => {
                    match archive_writer_new.write_header(e) {
                      Ok(_) => {},
                      _ => {} // TODO: If we need to print an error message, we can do so later.
                    };
                    loop {
                      let entry_data = archive_reader.read_data(10240);
                      match entry_data {
                        Ok(chunk) => { archive_writer_new.write_data(chunk).unwrap(); },
                        Err(_) => { break; }
                      };
                    }
                  },
                  Err(_) => { break; }
                }
              }
            }
          }
          // Done with this .gz , remove it:
          match fs::remove_file(path.clone()) {
            Ok(_) => {},
            Err(e) => println!("Can't remove source .gz: {:?}", e)
          };
        },
        Err(e) => println!("Failed gz glob: {:?}", e)
      }
    }
    Ok(())
  }

  /// Given a CorTeX-topology corpus, walk the file system and import it into the Task store
  pub fn walk_import<'walk>(&self) -> Result<(),Error> {
    println!("-- Starting import walk");
    let import_extension = if self.corpus.complex { "zip" } else { "tex" };
    let mut walk_q : Vec<PathBuf> = vec![Path::new(&self.corpus.path).to_owned()];
    let mut import_q : Vec<Task> = Vec::new();
    let mut import_counter = 0;
    while walk_q.len() > 0 {
      let current_path = walk_q.pop().unwrap();
      // println!("-- current path {:?}", current_path);
      let current_metadata = try!(fs::metadata(current_path.clone()));
      if current_metadata.is_dir() { // Ignore files
        // First, test if we just found an entry:
        let current_local_dir = current_path.file_name().unwrap();
        let current_entry = current_local_dir.to_str().unwrap().to_string() + "." + import_extension;
        let current_entry_path = current_path.to_str().unwrap().to_string() + "/" + &current_entry;
        match fs::metadata(current_entry_path.clone()) {
          Ok(_) => {
            // Found the expected file, import this entry:
            // println!("Found entry: {:?}", current_entry_path);
            import_counter += 1;
            import_q.push(self.new_task(current_entry_path));
            if import_q.len() >= 1000 {
              // Flush the import queue to backend:
              println!("Checkpoint backend writer: job {:?}", import_counter);
              self.backend.mark_imported(&import_q).unwrap(); // TODO: Proper Error-handling
              import_q.clear();
            }
          },
          Err(_) => {
            //  No such entry found, traversing into the directory:
            for subentry in try!(fs::read_dir(current_path.clone())) {
              let subentry = try!(subentry);
              walk_q.push(subentry.path());
            }
          }
        }
      }
    }
    if !import_q.is_empty() {
      println!("Checkpoint backend writer: job {:?}", import_q.len());
      self.backend.mark_imported(&import_q).unwrap();} // TODO: Proper Error-handling
    println!("--- Imported {:?} entries.", import_counter);
    Ok(())
  }

  /// Create a new NoProblem task for the "import" service and the Importer-specified corpus
  pub fn new_task(&self, entry : String) -> Task {
    let abs_entry : String = if Path::new(&entry).is_relative() {
      let mut new_abs = self.cwd.clone();
      new_abs.push(&entry);
      new_abs.to_str().unwrap().to_string()
    } else {
      entry.clone()
    };

    Task {id: None, entry : abs_entry, status : TaskStatus::NoProblem.raw(), corpusid : self.corpus.id.unwrap(), serviceid: 2}
  }
  /// Top-level import driver, performs an optional unpack, and then an import into the Task store
  pub fn process(&self) -> Result<(),Error> {
    // println!("Greetings from the import processor");
    if self.corpus.complex { // Complex setup has an unpack step:
      try!(self.unpack());
    }
    // Walk the directory tree and import the files in the Task store:
    try!(self.walk_import());

    Ok(())
  }

  /// Top-level corpus extension, performs a check for newly added documents and extracts+adds them to the existing corpus tasks
  pub fn extend_corpus(&self) -> Result<(), Error> {
    if self.corpus.complex { // Complex setup has an unpack step:
      try!(self.unpack_extend());
    }
    // Use the regular walk_import, at the cost of more database work,
    // the "Backend::mark_imported" ORM method allows us to insert only if new
    try!(self.walk_import());
    Ok(())
  }
}

<|endoftext|>


<commit_before>use super::*;
use pact_matching::models::*;
use std::str::FromStr;
use std::collections::hash_map::HashMap;
use std::io::Read;
use hyper::client::Client;
use hyper::client::response::Response as HyperResponse;
use hyper::error::Error as HyperError;
use hyper::method::Method;
use hyper::header::{Headers, ContentType};
use hyper::mime::{Mime, TopLevel, SubLevel};

pub fn join_paths(base: &String, path: String) -> String {
    let mut full_path = s!(base.trim_right_matches("/"));
    full_path.push('/');
    full_path.push_str(path.trim_left_matches("/"));
    full_path
}

fn setup_headers(headers: &Option<HashMap<String, String>>) -> Headers {
    let mut hyper_headers = Headers::new();
    if headers.is_some() {
        let headers = headers.clone().unwrap();
        for (k, v) in headers.clone() {
            hyper_headers.set_raw(k.clone(), vec![v.bytes().collect()]);
        }

        if !headers.keys().any(|k| k.to_lowercase() == "content-type") {
            hyper_headers.set(ContentType(Mime(TopLevel::Application, SubLevel::Json,
                vec![])));
        }
    }
    hyper_headers
}

fn make_request(base_url: &String, request: &Request, client: &Client) -> Result<HyperResponse, HyperError> {
    match Method::from_str(&request.method) {
        Ok(method) => {
            let mut url = join_paths(base_url, request.path.clone());
            if request.query.is_some() {
                url.push('?');
                url.push_str(&build_query_string(request.query.clone().unwrap()));
            }
            debug!("Making request to '{}'", url);
            let hyper_request = client.request(method, &url)
                .headers(setup_headers(&request.headers.clone()));
            match request.body {
                OptionalBody::Present(ref s) => hyper_request.body(s.as_str()),
                OptionalBody::Null => {
                    if request.content_type() == "application/json" {
                        hyper_request.body("null")
                    } else {
                        hyper_request
                    }
                },
                _ => hyper_request
            }.send()
        },
        Err(err) => Err(err)
    }
}

fn extract_headers(headers: &Headers) -> Option<HashMap<String, String>> {
    if headers.len() > 0 {
        Some(headers.iter().map(|h| (s!(h.name()), h.value_string()) ).collect())
    } else {
        None
    }
}

pub fn extract_body(response: &mut HyperResponse) -> OptionalBody {
    let mut buffer = String::new();
    match response.read_to_string(&mut buffer) {
        Ok(size) => if size > 0 {
                OptionalBody::Present(buffer)
            } else {
                OptionalBody::Empty
            },
        Err(err) => {
            warn!("Failed to read request body: {}", err);
            OptionalBody::Missing
        }
    }
}

fn hyper_response_to_pact_response(response: &mut HyperResponse) -> Response {
    Response {
        status: response.status.to_u16(),
        headers: extract_headers(&response.headers),
        body: extract_body(response),
        matching_rules: None
    }
}

pub fn make_provider_request(provider: &ProviderInfo, request: &Request) -> Result<Response, HyperError> {
    debug!("Sending {:?} to provider", request);
    let client = Client::new();
    match make_request(&format!("{}://{}:{}{}", provider.protocol, provider.host, provider.port,
        provider.path), request, &client) {
        Ok(ref mut response) => {
            debug!("Received response: {:?}", response);
            Ok(hyper_response_to_pact_response(response))
        },
        Err(err) => {
            debug!("Request failed: {}", err);
            Err(err)
        }
    }
}

pub fn make_state_change_request(provider: &ProviderInfo, request: &Request) -> Result<(), String> {
    debug!("Sending {:?} to state change handler", request);
    let client = Client::new();
    match make_request(&provider.state_change_url.clone().unwrap(), request, &client) {
        Ok(ref mut response) => {
            debug!("Received response: {:?}", response);
            if response.status.is_success() {
                Ok(())
            } else {
                debug!("Request failed: {}", response.status);
                Err(format!("State change request failed: {}", response.status))
            }
        },
        Err(err) => {
            debug!("Request failed: {}", err);
            Err(format!("State change request failed: {}", err))
        }
    }
}

#[cfg(test)]
mod tests {
    use expectest::prelude::*;
    use super::join_paths;

    #[test]
    fn join_paths_test() {
        expect!(join_paths(&s!(""), s!(""))).to(be_equal_to(s!("/")));
        expect!(join_paths(&s!("/"), s!(""))).to(be_equal_to(s!("/")));
        expect!(join_paths(&s!(""), s!("/"))).to(be_equal_to(s!("/")));
        expect!(join_paths(&s!("/"), s!("/"))).to(be_equal_to(s!("/")));
        expect!(join_paths(&s!("/a/b"), s!("/c/d"))).to(be_equal_to(s!("/a/b/c/d")));
    }

}
<commit_msg>update the verifier to support the new V3 format matchers<commit_after>use super::*;
use pact_matching::models::*;
use pact_matching::models::matchingrules::*;
use std::str::FromStr;
use std::collections::hash_map::HashMap;
use std::io::Read;
use hyper::client::Client;
use hyper::client::response::Response as HyperResponse;
use hyper::error::Error as HyperError;
use hyper::method::Method;
use hyper::header::{Headers, ContentType};
use hyper::mime::{Mime, TopLevel, SubLevel};

pub fn join_paths(base: &String, path: String) -> String {
    let mut full_path = s!(base.trim_right_matches("/"));
    full_path.push('/');
    full_path.push_str(path.trim_left_matches("/"));
    full_path
}

fn setup_headers(headers: &Option<HashMap<String, String>>) -> Headers {
    let mut hyper_headers = Headers::new();
    if headers.is_some() {
        let headers = headers.clone().unwrap();
        for (k, v) in headers.clone() {
            hyper_headers.set_raw(k.clone(), vec![v.bytes().collect()]);
        }

        if !headers.keys().any(|k| k.to_lowercase() == "content-type") {
            hyper_headers.set(ContentType(Mime(TopLevel::Application, SubLevel::Json,
                vec![])));
        }
    }
    hyper_headers
}

fn make_request(base_url: &String, request: &Request, client: &Client) -> Result<HyperResponse, HyperError> {
    match Method::from_str(&request.method) {
        Ok(method) => {
            let mut url = join_paths(base_url, request.path.clone());
            if request.query.is_some() {
                url.push('?');
                url.push_str(&build_query_string(request.query.clone().unwrap()));
            }
            debug!("Making request to '{}'", url);
            let hyper_request = client.request(method, &url)
                .headers(setup_headers(&request.headers.clone()));
            match request.body {
                OptionalBody::Present(ref s) => hyper_request.body(s.as_str()),
                OptionalBody::Null => {
                    if request.content_type() == "application/json" {
                        hyper_request.body("null")
                    } else {
                        hyper_request
                    }
                },
                _ => hyper_request
            }.send()
        },
        Err(err) => Err(err)
    }
}

fn extract_headers(headers: &Headers) -> Option<HashMap<String, String>> {
    if headers.len() > 0 {
        Some(headers.iter().map(|h| (s!(h.name()), h.value_string()) ).collect())
    } else {
        None
    }
}

pub fn extract_body(response: &mut HyperResponse) -> OptionalBody {
    let mut buffer = String::new();
    match response.read_to_string(&mut buffer) {
        Ok(size) => if size > 0 {
                OptionalBody::Present(buffer)
            } else {
                OptionalBody::Empty
            },
        Err(err) => {
            warn!("Failed to read request body: {}", err);
            OptionalBody::Missing
        }
    }
}

fn hyper_response_to_pact_response(response: &mut HyperResponse) -> Response {
    Response {
        status: response.status.to_u16(),
        headers: extract_headers(&response.headers),
        body: extract_body(response),
        matching_rules: MatchingRules::default()
    }
}

pub fn make_provider_request(provider: &ProviderInfo, request: &Request) -> Result<Response, HyperError> {
    debug!("Sending {:?} to provider", request);
    let client = Client::new();
    match make_request(&format!("{}://{}:{}{}", provider.protocol, provider.host, provider.port,
        provider.path), request, &client) {
        Ok(ref mut response) => {
            debug!("Received response: {:?}", response);
            Ok(hyper_response_to_pact_response(response))
        },
        Err(err) => {
            debug!("Request failed: {}", err);
            Err(err)
        }
    }
}

pub fn make_state_change_request(provider: &ProviderInfo, request: &Request) -> Result<(), String> {
    debug!("Sending {:?} to state change handler", request);
    let client = Client::new();
    match make_request(&provider.state_change_url.clone().unwrap(), request, &client) {
        Ok(ref mut response) => {
            debug!("Received response: {:?}", response);
            if response.status.is_success() {
                Ok(())
            } else {
                debug!("Request failed: {}", response.status);
                Err(format!("State change request failed: {}", response.status))
            }
        },
        Err(err) => {
            debug!("Request failed: {}", err);
            Err(format!("State change request failed: {}", err))
        }
    }
}

#[cfg(test)]
mod tests {
    use expectest::prelude::*;
    use super::join_paths;

    #[test]
    fn join_paths_test() {
        expect!(join_paths(&s!(""), s!(""))).to(be_equal_to(s!("/")));
        expect!(join_paths(&s!("/"), s!(""))).to(be_equal_to(s!("/")));
        expect!(join_paths(&s!(""), s!("/"))).to(be_equal_to(s!("/")));
        expect!(join_paths(&s!("/"), s!("/"))).to(be_equal_to(s!("/")));
        expect!(join_paths(&s!("/a/b"), s!("/c/d"))).to(be_equal_to(s!("/a/b/c/d")));
    }

}
<|endoftext|>


<commit_before>#![allow(non_camel_case_types)]
#![allow(dead_code)]

#[derive(Copy,Clone,PartialEq,PartialOrd)]
pub enum PartitionType {
    PARTITION_NONE,
    PARTITION_HORZ,
    PARTITION_VERT,
    PARTITION_SPLIT,
    PARTITION_INVALID
}

pub const BLOCK_SIZES_ALL: usize = 19;

#[derive(Copy,Clone,PartialEq,PartialOrd)]
pub enum BlockSize {
    BLOCK_4X4,
    BLOCK_4X8,
    BLOCK_8X4,
    BLOCK_8X8,
    BLOCK_8X16,
    BLOCK_16X8,
    BLOCK_16X16,
    BLOCK_16X32,
    BLOCK_32X16,
    BLOCK_32X32,
    BLOCK_32X64,
    BLOCK_64X32,
    BLOCK_64X64,
    BLOCK_4X16,
    BLOCK_16X4,
    BLOCK_8X32,
    BLOCK_32X8,
    BLOCK_16X64,
    BLOCK_64X16,
    BLOCK_INVALID
}

pub const TX_SIZES: usize = 4;
pub const TX_SIZES_ALL: usize = 14;

#[derive(Copy,Clone,PartialEq,PartialOrd)]
pub enum TxSize {
    TX_4X4,
    TX_8X8,
    TX_16X16,
    TX_32X32,
    TX_4X8,
    TX_8X4,
    TX_8X16,
    TX_16X8,
    TX_16X32,
    TX_32X16,
    TX_4X16,
    TX_16X4,
    TX_8X32,
    TX_32X8,
}

impl TxSize {
    pub fn width(self) -> usize {
        1<<tx_size_wide_log2[self as usize]
    }
    pub fn height(self) -> usize {
        1<<tx_size_high_log2[self as usize]
    }
    pub fn width_mi(self) -> usize {
        (1<<tx_size_wide_log2[self as usize])>>2
    }
    pub fn height_mi(self) -> usize {
        (1<<tx_size_high_log2[self as usize])>>2
    }
}

pub const TX_TYPES: usize = 16;

#[derive(Copy,Clone)]
#[repr(C)]
pub enum TxType {
    DCT_DCT = 0,    // DCT  in both horizontal and vertical
    ADST_DCT = 1,   // ADST in vertical, DCT in horizontal
    DCT_ADST = 2,   // DCT  in vertical, ADST in horizontal
    ADST_ADST = 3,  // ADST in both directions
    FLIPADST_DCT = 4,
    DCT_FLIPADST = 5,
    FLIPADST_FLIPADST = 6,
    ADST_FLIPADST = 7,
    FLIPADST_ADST = 8,
    IDTX = 9,
    V_DCT = 10,
    H_DCT = 11,
    V_ADST = 12,
    H_ADST = 13,
    V_FLIPADST = 14,
    H_FLIPADST = 15,
}

#[derive(Copy, Clone, Debug, PartialEq, PartialOrd)]
pub enum PredictionMode {
    DC_PRED,    // Average of above and left pixels
    V_PRED,     // Vertical
    H_PRED,     // Horizontal
    D45_PRED,   // Directional 45  deg = round(arctan(1/1) * 180/pi)
    D135_PRED,  // Directional 135 deg = 180 - 45
    D117_PRED,  // Directional 117 deg = 180 - 63
    D153_PRED,  // Directional 153 deg = 180 - 27
    D207_PRED,  // Directional 207 deg = 180 + 27
    D63_PRED,   // Directional 63  deg = round(arctan(2/1) * 180/pi)
    SMOOTH_PRED,  // Combination of horizontal and vertical interpolation
    SMOOTH_V_PRED,
    SMOOTH_H_PRED,
    TM_PRED,        // True-motion
    NEARESTMV,
    NEARMV,
    ZEROMV,
    NEWMV,
    // Compound ref compound modes
    NEAREST_NEARESTMV,
    NEAR_NEARMV,
    NEAREST_NEWMV,
    NEW_NEARESTMV,
    NEAR_NEWMV,
    NEW_NEARMV,
    ZERO_ZEROMV,
    NEW_NEWMV,
}

use plane::*;
use predict::*;
use context::*;

fn setup_left(left: &mut [u16], rec: &PlaneMutSlice, height: usize) {
    let left_slice = rec.go_left(1);
    for i in 0..height {
        left[i] = left_slice.p(0, i);
    }
}

impl PredictionMode {
    pub fn predict<'a>(&self, dst: &'a mut PlaneMutSlice<'a>, tx_size: TxSize) {
        let mut above = [127u16; 64];
        let mut left = [129u16; 64];
        let stride = dst.plane.cfg.stride;
        let x = dst.x;
        let y = dst.y;

        if (self == &PredictionMode::V_PRED ||
            self == &PredictionMode::DC_PRED) && y != 0 {
            above[..tx_size.width()].copy_from_slice(&dst.go_up(1).as_slice()[..tx_size.width()]);
        }

        if (self == &PredictionMode::H_PRED ||
            self == &PredictionMode::DC_PRED) && x != 0 {
            setup_left(&mut left, dst, tx_size.height());
        }

        let slice = dst.as_mut_slice();

        match *self {
            PredictionMode::DC_PRED => {
                match (x, y) {
                    (0, 0) =>
                        match tx_size {
                            TxSize::TX_4X4 => Block4x4::pred_dc_128(slice, stride),
                            TxSize::TX_8X8 => Block8x8::pred_dc_128(slice, stride),
                            _ => panic!("unimplemented prediction size")
                        },
                    (_, 0) =>
                        match tx_size {
                            TxSize::TX_4X4 => Block4x4::pred_dc_left(slice, stride, &above[..4], &left[..4]),
                            TxSize::TX_8X8 => Block8x8::pred_dc_left(slice, stride, &above[..8], &left[..8]),
                            _ => panic!("unimplemented prediction size")
                        },
                    (0, _) =>
                        match tx_size {
                            TxSize::TX_4X4 => Block4x4::pred_dc_top(slice, stride, &above[..4], &left[..4]),
                            TxSize::TX_8X8 => Block8x8::pred_dc_top(slice, stride, &above[..8], &left[..8]),
                            _ => panic!("unimplemented prediction size")
                        },
                    _ =>
                        match tx_size {
                            TxSize::TX_4X4 => Block4x4::pred_dc(slice, stride, &above[..4], &left[..4]),
                            TxSize::TX_8X8 => Block8x8::pred_dc(slice, stride, &above[..8], &left[..8]),
                            _ => panic!("unimplemented prediction size")
                        },
                }
            },
            PredictionMode::H_PRED => {
                match tx_size {
                    TxSize::TX_4X4 => Block4x4::pred_h(slice, stride, &left[..4]),
                    TxSize::TX_8X8 => Block8x8::pred_h(slice, stride, &left[..8]),
                    _ => panic!("unimplemented prediction size")
                }
            },
            PredictionMode::V_PRED => {
                match tx_size {
                    TxSize::TX_4X4 => Block4x4::pred_v(slice, stride, &above[..4]),
                    TxSize::TX_8X8 => Block8x8::pred_v(slice, stride, &above[..8]),
                    _ => panic!("unimplemented prediction size")
                }
            },
            _ => unimplemented!(),
        }
    }
}

#[derive(Copy,Clone)]
pub enum TxSetType {
    // DCT only
    EXT_TX_SET_DCTONLY = 0,
    // DCT + Identity only
    EXT_TX_SET_DCT_IDTX = 1,
    // Discrete Trig transforms w/o flip (4) + Identity (1)
    EXT_TX_SET_DTT4_IDTX = 2,
    // Discrete Trig transforms w/o flip (4) + Identity (1) + 1D Hor/vert DCT (2)
    EXT_TX_SET_DTT4_IDTX_1DDCT = 3,
    // Discrete Trig transforms w/ flip (9) + Identity (1) + 1D Hor/Ver DCT (2)
    EXT_TX_SET_DTT9_IDTX_1DDCT = 4,
    // Discrete Trig transforms w/ flip (9) + Identity (1) + 1D Hor/Ver (6)
    EXT_TX_SET_ALL16 = 5,
    EXT_TX_SET_TYPES
}

pub fn get_subsize(bsize: BlockSize , partition: PartitionType) -> BlockSize {
    subsize_lookup[partition as usize][bsize as usize]
}
<commit_msg>Bubble up the Intra trait<commit_after>#![allow(non_camel_case_types)]
#![allow(dead_code)]

#[derive(Copy,Clone,PartialEq,PartialOrd)]
pub enum PartitionType {
    PARTITION_NONE,
    PARTITION_HORZ,
    PARTITION_VERT,
    PARTITION_SPLIT,
    PARTITION_INVALID
}

pub const BLOCK_SIZES_ALL: usize = 19;

#[derive(Copy,Clone,PartialEq,PartialOrd)]
pub enum BlockSize {
    BLOCK_4X4,
    BLOCK_4X8,
    BLOCK_8X4,
    BLOCK_8X8,
    BLOCK_8X16,
    BLOCK_16X8,
    BLOCK_16X16,
    BLOCK_16X32,
    BLOCK_32X16,
    BLOCK_32X32,
    BLOCK_32X64,
    BLOCK_64X32,
    BLOCK_64X64,
    BLOCK_4X16,
    BLOCK_16X4,
    BLOCK_8X32,
    BLOCK_32X8,
    BLOCK_16X64,
    BLOCK_64X16,
    BLOCK_INVALID
}

pub const TX_SIZES: usize = 4;
pub const TX_SIZES_ALL: usize = 14;

#[derive(Copy,Clone,PartialEq,PartialOrd)]
pub enum TxSize {
    TX_4X4,
    TX_8X8,
    TX_16X16,
    TX_32X32,
    TX_4X8,
    TX_8X4,
    TX_8X16,
    TX_16X8,
    TX_16X32,
    TX_32X16,
    TX_4X16,
    TX_16X4,
    TX_8X32,
    TX_32X8,
}

impl TxSize {
    pub fn width(self) -> usize {
        1<<tx_size_wide_log2[self as usize]
    }
    pub fn height(self) -> usize {
        1<<tx_size_high_log2[self as usize]
    }
    pub fn width_mi(self) -> usize {
        (1<<tx_size_wide_log2[self as usize])>>2
    }
    pub fn height_mi(self) -> usize {
        (1<<tx_size_high_log2[self as usize])>>2
    }
}

pub const TX_TYPES: usize = 16;

#[derive(Copy,Clone)]
#[repr(C)]
pub enum TxType {
    DCT_DCT = 0,    // DCT  in both horizontal and vertical
    ADST_DCT = 1,   // ADST in vertical, DCT in horizontal
    DCT_ADST = 2,   // DCT  in vertical, ADST in horizontal
    ADST_ADST = 3,  // ADST in both directions
    FLIPADST_DCT = 4,
    DCT_FLIPADST = 5,
    FLIPADST_FLIPADST = 6,
    ADST_FLIPADST = 7,
    FLIPADST_ADST = 8,
    IDTX = 9,
    V_DCT = 10,
    H_DCT = 11,
    V_ADST = 12,
    H_ADST = 13,
    V_FLIPADST = 14,
    H_FLIPADST = 15,
}

#[derive(Copy, Clone, Debug, PartialEq, PartialOrd)]
pub enum PredictionMode {
    DC_PRED,    // Average of above and left pixels
    V_PRED,     // Vertical
    H_PRED,     // Horizontal
    D45_PRED,   // Directional 45  deg = round(arctan(1/1) * 180/pi)
    D135_PRED,  // Directional 135 deg = 180 - 45
    D117_PRED,  // Directional 117 deg = 180 - 63
    D153_PRED,  // Directional 153 deg = 180 - 27
    D207_PRED,  // Directional 207 deg = 180 + 27
    D63_PRED,   // Directional 63  deg = round(arctan(2/1) * 180/pi)
    SMOOTH_PRED,  // Combination of horizontal and vertical interpolation
    SMOOTH_V_PRED,
    SMOOTH_H_PRED,
    TM_PRED,        // True-motion
    NEARESTMV,
    NEARMV,
    ZEROMV,
    NEWMV,
    // Compound ref compound modes
    NEAREST_NEARESTMV,
    NEAR_NEARMV,
    NEAREST_NEWMV,
    NEW_NEARESTMV,
    NEAR_NEWMV,
    NEW_NEARMV,
    ZERO_ZEROMV,
    NEW_NEWMV,
}

use plane::*;
use predict::*;
use context::*;

impl PredictionMode {
    pub fn predict<'a>(&self, dst: &'a mut PlaneMutSlice<'a>, tx_size: TxSize) {
        match tx_size {
            TxSize::TX_4X4 => self.predict_inner::<Block4x4>(dst),
            TxSize::TX_8X8 => self.predict_inner::<Block8x8>(dst),
            _ => unimplemented!()
        }
    }

    #[inline(always)]
    fn predict_inner<'a, B: Intra>(&self, dst: &'a mut PlaneMutSlice<'a>) {
        let above = &mut [127u16; 64][..B::W];
        let left = &mut [129u16; 64][..B::H];
        let stride = dst.plane.cfg.stride;
        let x = dst.x;
        let y = dst.y;

        if (self == &PredictionMode::V_PRED ||
            self == &PredictionMode::DC_PRED) && y != 0 {
            above.copy_from_slice(&dst.go_up(1).as_slice()[..B::W]);
        }

        if (self == &PredictionMode::H_PRED ||
            self == &PredictionMode::DC_PRED) && x != 0 {
            let left_slice = dst.go_left(1);
            for i in 0..B::H {
                left[i] = left_slice.p(0, i);
            }
        }

        let slice = dst.as_mut_slice();

        match *self {
            PredictionMode::DC_PRED => {
                match (x, y) {
                    (0, 0) => B::pred_dc_128(slice, stride),
                    (_, 0) => B::pred_dc_left(slice, stride, above, left),
                    (0, _) => B::pred_dc_top(slice, stride, above, left),
                    _ => B::pred_dc(slice, stride, above, left),
                }
            },
            PredictionMode::H_PRED => B::pred_h(slice, stride, left),
            PredictionMode::V_PRED => B::pred_v(slice, stride, above),
            _ => unimplemented!(),
        }
    }
}

#[derive(Copy,Clone)]
pub enum TxSetType {
    // DCT only
    EXT_TX_SET_DCTONLY = 0,
    // DCT + Identity only
    EXT_TX_SET_DCT_IDTX = 1,
    // Discrete Trig transforms w/o flip (4) + Identity (1)
    EXT_TX_SET_DTT4_IDTX = 2,
    // Discrete Trig transforms w/o flip (4) + Identity (1) + 1D Hor/vert DCT (2)
    EXT_TX_SET_DTT4_IDTX_1DDCT = 3,
    // Discrete Trig transforms w/ flip (9) + Identity (1) + 1D Hor/Ver DCT (2)
    EXT_TX_SET_DTT9_IDTX_1DDCT = 4,
    // Discrete Trig transforms w/ flip (9) + Identity (1) + 1D Hor/Ver (6)
    EXT_TX_SET_ALL16 = 5,
    EXT_TX_SET_TYPES
}

pub fn get_subsize(bsize: BlockSize , partition: PartitionType) -> BlockSize {
    subsize_lookup[partition as usize][bsize as usize]
}
<|endoftext|>


<commit_before>extern crate syscall;

use syscall::scheme::Scheme;

use std::{env, fs,thread};
use std::io::{stderr, Write};
use std::os::unix::process::CommandExt;
use std::path::Path;
use std::process::{self, Command};

use self::chroot::ChrootScheme;

mod chroot;

fn usage() -> ! {
    write!(stderr(), "contain root cmd args..\n").unwrap();
    process::exit(1);
}

fn enter(root: &Path, cmd: &str, args: &[String]) {
    let names = [
        "display",
        "pty",
        "rand",
        "tcp",
        "udp"
    ];

    let mut name_ptrs = Vec::new();
    for name in names.iter() {
        name_ptrs.push([name.as_ptr() as usize, name.len()]);
    }

    let new_ns = syscall::mkns(&name_ptrs).unwrap();

    let root_canon = fs::canonicalize(root).unwrap();
    let root_thread = thread::spawn(move || {
        syscall::setrens(-1isize as usize, new_ns).unwrap();
        let scheme_fd = syscall::open(":file", syscall::O_CREAT | syscall::O_RDWR | syscall::O_CLOEXEC).unwrap();
        syscall::setrens(-1isize as usize, syscall::getns().unwrap()).unwrap();

        let chroot_scheme = ChrootScheme::new(root_canon);
        loop {
            let mut packet = syscall::Packet::default();
            if syscall::read(scheme_fd, &mut packet).unwrap() == 0 {
                break;
            }
            chroot_scheme.handle(&mut packet);
            syscall::write(scheme_fd, &packet).unwrap();
        }

        let _ = syscall::close(scheme_fd);
    });

    let pid = unsafe { syscall::clone(0).unwrap() };
    if pid == 0 {
        syscall::setrens(new_ns, new_ns).unwrap();

        println!("Container {}: enter: {}", new_ns, cmd);

        let mut command = Command::new(&cmd);
        for arg in args {
            command.arg(&arg);
        }
        command.current_dir("/");

        let err = command.exec();

        panic!("contain: failed to launch {}: {}", cmd, err);
    } else {
        let mut status = 0;
        syscall::waitpid(pid, &mut status, 0).unwrap();

        loop {
            let mut c_status = 0;
            let c_pid = syscall::waitpid(0, &mut c_status, syscall::WNOHANG).unwrap();
            if c_pid == 0 {
                break;
            } else {
                println!("Container zombie {}: {:X}", c_pid, c_status);
            }
        }

        println!("Container {}: exit: {:X}", new_ns, status);
    }
}

pub fn main() {
    let mut args = env::args().skip(1);

    if let Some(root) = args.next() {
        let cmd = args.next().unwrap_or(env::var("SHELL").unwrap_or("sh".to_string()));
        let args: Vec<String> = args.collect();
        enter(Path::new(&root), &cmd, &args);
    } else {
        usage();
    }
}
<commit_msg>Do not include display scheme<commit_after>extern crate syscall;

use syscall::scheme::Scheme;

use std::{env, fs,thread};
use std::io::{stderr, Write};
use std::os::unix::process::CommandExt;
use std::path::Path;
use std::process::{self, Command};

use self::chroot::ChrootScheme;

mod chroot;

fn usage() -> ! {
    write!(stderr(), "contain root cmd args..\n").unwrap();
    process::exit(1);
}

fn enter(root: &Path, cmd: &str, args: &[String]) {
    let names = [
        "pty",
        "rand",
        "tcp",
        "udp"
    ];

    let mut name_ptrs = Vec::new();
    for name in names.iter() {
        name_ptrs.push([name.as_ptr() as usize, name.len()]);
    }

    let new_ns = syscall::mkns(&name_ptrs).unwrap();

    let root_canon = fs::canonicalize(root).unwrap();
    let root_thread = thread::spawn(move || {
        syscall::setrens(-1isize as usize, new_ns).unwrap();
        let scheme_fd = syscall::open(":file", syscall::O_CREAT | syscall::O_RDWR | syscall::O_CLOEXEC).unwrap();
        syscall::setrens(-1isize as usize, syscall::getns().unwrap()).unwrap();

        let chroot_scheme = ChrootScheme::new(root_canon);
        loop {
            let mut packet = syscall::Packet::default();
            if syscall::read(scheme_fd, &mut packet).unwrap() == 0 {
                break;
            }
            chroot_scheme.handle(&mut packet);
            syscall::write(scheme_fd, &packet).unwrap();
        }

        let _ = syscall::close(scheme_fd);
    });

    let pid = unsafe { syscall::clone(0).unwrap() };
    if pid == 0 {
        syscall::setrens(new_ns, new_ns).unwrap();

        println!("Container {}: enter: {}", new_ns, cmd);

        let mut command = Command::new(&cmd);
        for arg in args {
            command.arg(&arg);
        }
        command.current_dir("/");

        let err = command.exec();

        panic!("contain: failed to launch {}: {}", cmd, err);
    } else {
        let mut status = 0;
        syscall::waitpid(pid, &mut status, 0).unwrap();

        loop {
            let mut c_status = 0;
            let c_pid = syscall::waitpid(0, &mut c_status, syscall::WNOHANG).unwrap();
            if c_pid == 0 {
                break;
            } else {
                println!("Container zombie {}: {:X}", c_pid, c_status);
            }
        }

        println!("Container {}: exit: {:X}", new_ns, status);
    }
}

pub fn main() {
    let mut args = env::args().skip(1);

    if let Some(root) = args.next() {
        let cmd = args.next().unwrap_or(env::var("SHELL").unwrap_or("sh".to_string()));
        let args: Vec<String> = args.collect();
        enter(Path::new(&root), &cmd, &args);
    } else {
        usage();
    }
}
<|endoftext|>


<commit_before>/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */

use WindowWrapper;
use image::ColorType;
use image::png::PNGEncoder;
use std::fs::File;
use std::path::Path;
use std::sync::mpsc::{channel, Sender};
use webrender_traits::*;
use wrench::{Wrench, WrenchThing};
use yaml_frame_reader::YamlFrameReader;

pub fn save_flipped<P: AsRef<Path>>(path: P, orig_pixels: &[u8], size: DeviceUintSize) {
    // flip image vertically (texture is upside down)
    let mut data = orig_pixels.to_owned();
    let stride = size.width as usize * 4;
    for y in 0..size.height as usize {
        let dst_start = y * stride;
        let src_start = (size.height as usize - y - 1) * stride;
        let src_slice = &orig_pixels[src_start .. src_start + stride];
        (&mut data[dst_start .. dst_start + stride]).clone_from_slice(&src_slice[..stride]);
    }

    let encoder = PNGEncoder::new(File::create(path).unwrap());
    encoder.encode(&data,
                   size.width,
                   size.height,
                   ColorType::RGBA(8))
           .expect("Unable to encode PNG!");
}

pub fn png(wrench: &mut Wrench,
           window: &mut WindowWrapper,
           mut reader: YamlFrameReader)
{
    // setup a notifier so we can wait for frames to be finished
    struct Notifier {
        tx: Sender<()>,
    };
    impl RenderNotifier for Notifier {
        fn new_frame_ready(&mut self) {
            self.tx.send(()).unwrap();
        }
        fn new_scroll_frame_ready(&mut self, _composite_needed: bool) {}
    }
    let (tx, rx) = channel();
    wrench.renderer.set_render_notifier(Box::new(Notifier { tx: tx }));
    reader.do_frame(wrench);

    // wait for the frame
    rx.recv().unwrap();
    wrench.render();

    let size = window.get_inner_size_pixels();
    let data = wrench.renderer.read_pixels_rgba8(DeviceUintRect::new(DeviceUintPoint::zero(),
                                                                     DeviceUintSize::new(size.0, size.1)));

    let mut out_path = reader.yaml_path().clone();
    out_path.set_extension("png");

    let device_size = DeviceUintSize::new(size.0, size.1);
    save_flipped(out_path, &data, device_size);
}
<commit_msg>Factor out DeviceSize construction<commit_after>/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */

use WindowWrapper;
use image::ColorType;
use image::png::PNGEncoder;
use std::fs::File;
use std::path::Path;
use std::sync::mpsc::{channel, Sender};
use webrender_traits::*;
use wrench::{Wrench, WrenchThing};
use yaml_frame_reader::YamlFrameReader;

pub fn save_flipped<P: AsRef<Path>>(path: P, orig_pixels: &[u8], size: DeviceUintSize) {
    // flip image vertically (texture is upside down)
    let mut data = orig_pixels.to_owned();
    let stride = size.width as usize * 4;
    for y in 0..size.height as usize {
        let dst_start = y * stride;
        let src_start = (size.height as usize - y - 1) * stride;
        let src_slice = &orig_pixels[src_start .. src_start + stride];
        (&mut data[dst_start .. dst_start + stride]).clone_from_slice(&src_slice[..stride]);
    }

    let encoder = PNGEncoder::new(File::create(path).unwrap());
    encoder.encode(&data,
                   size.width,
                   size.height,
                   ColorType::RGBA(8))
           .expect("Unable to encode PNG!");
}

pub fn png(wrench: &mut Wrench,
           window: &mut WindowWrapper,
           mut reader: YamlFrameReader)
{
    // setup a notifier so we can wait for frames to be finished
    struct Notifier {
        tx: Sender<()>,
    };
    impl RenderNotifier for Notifier {
        fn new_frame_ready(&mut self) {
            self.tx.send(()).unwrap();
        }
        fn new_scroll_frame_ready(&mut self, _composite_needed: bool) {}
    }
    let (tx, rx) = channel();
    wrench.renderer.set_render_notifier(Box::new(Notifier { tx: tx }));
    reader.do_frame(wrench);

    // wait for the frame
    rx.recv().unwrap();
    wrench.render();

    let size = window.get_inner_size_pixels();
    let device_size = DeviceUintSize::new(size.0, size.1);
    let data = wrench.renderer.read_pixels_rgba8(DeviceUintRect::new(DeviceUintPoint::zero(),
                                                                     device_size));

    let mut out_path = reader.yaml_path().clone();
    out_path.set_extension("png");

    save_flipped(out_path, &data, device_size);
}
<|endoftext|>


<commit_before>extern crate env_logger;
extern crate lapin_futures as lapin;
#[macro_use]
extern crate log;
extern crate futures;
extern crate tokio;

use futures::future::Future;
use futures::Stream;
use tokio::io::{AsyncRead, AsyncWrite};
use tokio::net::TcpStream;
use lapin::types::FieldTable;
use lapin::client::{Client, ConnectionOptions};
use lapin::channel::{BasicConsumeOptions, BasicProperties, BasicPublishOptions, ConfirmSelectOptions, QueueDeclareOptions};

fn create_consumer<T: AsyncRead + AsyncWrite + Sync + Send + 'static>(client: &Client<T>, n: u8) -> Box<Future<Item = (), Error = ()> + Send + 'static> {
    info!("will create consumer {}", n);

    let queue = format!("test-queue-{}", n);

    Box::new(
        client.create_confirm_channel(ConfirmSelectOptions::default()).and_then(move |channel| {
            channel.queue_declare(&queue, &QueueDeclareOptions::default(), &FieldTable::new()).map(move |queue| (channel, queue))
        }).and_then(move |(channel, queue)| {
            info!("creating consumer {}", n);
            channel.basic_consume(&queue, "", &BasicConsumeOptions::default(), &FieldTable::new()).map(move |stream| (channel, stream))
        }).and_then(move |(channel, stream)| {
            info!("got stream for consumer {}", n);
            stream.for_each(move |message| {
                println!("consumer '{}' got '{}'", n, std::str::from_utf8(&message.data).unwrap());
                channel.basic_ack(message.delivery_tag)
            })
        }).map(|_| ()).map_err(move |err| eprintln!("got error in consumer '{}': {:?}", n, err))
    )
}

fn main() {
    env_logger::init();

    let addr = "127.0.0.1:5672".parse().unwrap();

    tokio::run(
        TcpStream::connect(&addr).and_then(|stream| {
            Client::connect(stream, &ConnectionOptions {
                frame_max: 65535,
                ..Default::default()
            })
        }).and_then(|(client, heartbeat)| {
            tokio::spawn(heartbeat.map_err(|e| eprintln!("heartbeat error: {:?}", e)));

            for n in 1..9 {
                tokio::spawn(create_consumer(&client, n));
            }

            client.create_confirm_channel(ConfirmSelectOptions::default()).and_then(move |channel| {
                futures::stream::iter_ok((1..9).flat_map(|c| {
                    (1..6).map(move |m| (c, m))
                })).for_each(move |(c, m)| {
                    let queue   = format!("test-queue-{}", c);
                    let message = format!("message {} for consumer {}", m, c);
                    let channel = channel.clone();

                    info!("will publish {}", message);

                    channel.queue_declare(&queue, &QueueDeclareOptions::default(), &FieldTable::new()).and_then(move |_| {
                        channel.basic_publish("", &queue, message.as_str().as_bytes(), &BasicPublishOptions::default(), BasicProperties::default()).map(move |confirmation| {
                            println!("got confirmation (consumer {}, message {}): {:?}", c, m, confirmation);
                        })
                    })
                })
            })
        }).map(|_| ()).map_err(|err| eprintln!("error: {:?}", err))
    );
}
<commit_msg>futures: make consumers example configurable<commit_after>extern crate env_logger;
extern crate lapin_futures as lapin;
#[macro_use]
extern crate log;
extern crate futures;
extern crate tokio;

use futures::future::Future;
use futures::Stream;
use tokio::io::{AsyncRead, AsyncWrite};
use tokio::net::TcpStream;
use lapin::types::FieldTable;
use lapin::client::{Client, ConnectionOptions};
use lapin::channel::{BasicConsumeOptions, BasicProperties, BasicPublishOptions, ConfirmSelectOptions, QueueDeclareOptions};

const N_CONSUMERS : u8 = 8;
const N_MESSAGES  : u8 = 5;

fn create_consumer<T: AsyncRead + AsyncWrite + Sync + Send + 'static>(client: &Client<T>, n: u8) -> Box<Future<Item = (), Error = ()> + Send + 'static> {
    info!("will create consumer {}", n);

    let queue = format!("test-queue-{}", n);

    Box::new(
        client.create_confirm_channel(ConfirmSelectOptions::default()).and_then(move |channel| {
            channel.queue_declare(&queue, &QueueDeclareOptions::default(), &FieldTable::new()).map(move |queue| (channel, queue))
        }).and_then(move |(channel, queue)| {
            info!("creating consumer {}", n);
            channel.basic_consume(&queue, "", &BasicConsumeOptions::default(), &FieldTable::new()).map(move |stream| (channel, stream))
        }).and_then(move |(channel, stream)| {
            info!("got stream for consumer {}", n);
            stream.for_each(move |message| {
                println!("consumer '{}' got '{}'", n, std::str::from_utf8(&message.data).unwrap());
                channel.basic_ack(message.delivery_tag)
            })
        }).map(|_| ()).map_err(move |err| eprintln!("got error in consumer '{}': {:?}", n, err))
    )
}

fn main() {
    env_logger::init();

    let addr = "127.0.0.1:5672".parse().unwrap();

    tokio::run(
        TcpStream::connect(&addr).and_then(|stream| {
            Client::connect(stream, &ConnectionOptions {
                frame_max: 65535,
                ..Default::default()
            })
        }).and_then(|(client, heartbeat)| {
            tokio::spawn(heartbeat.map_err(|e| eprintln!("heartbeat error: {:?}", e)));

            for n in 0..N_CONSUMERS {
                tokio::spawn(create_consumer(&client, n));
            }

            client.create_confirm_channel(ConfirmSelectOptions::default()).and_then(move |channel| {
                futures::stream::iter_ok((0..N_CONSUMERS).flat_map(|c| {
                    (0..N_MESSAGES).map(move |m| (c, m))
                })).for_each(move |(c, m)| {
                    let queue   = format!("test-queue-{}", c);
                    let message = format!("message {} for consumer {}", m, c);
                    let channel = channel.clone();

                    info!("will publish {}", message);

                    channel.queue_declare(&queue, &QueueDeclareOptions::default(), &FieldTable::new()).and_then(move |_| {
                        channel.basic_publish("", &queue, message.as_str().as_bytes(), &BasicPublishOptions::default(), BasicProperties::default()).map(move |confirmation| {
                            println!("got confirmation (consumer {}, message {}): {:?}", c, m, confirmation);
                        })
                    })
                })
            })
        }).map(|_| ()).map_err(|err| eprintln!("error: {:?}", err))
    );
}
<|endoftext|>


<commit_before>use crate::{
    protocol::*,
    types::*,
    types::generation::{GenSize, Length},
};

/// Enum representing an AMQP channel
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum AMQPChannel {
    /// The Global (id 0) AMQP channel used for creating other channels and for heartbeat
    Global,
    /// A regular AMQP channel
    Id(ShortUInt),
}

impl AMQPChannel {
    /// Get the channel id
    pub fn get_id(self) -> ShortUInt {
        match self {
            AMQPChannel::Global => 0,
            AMQPChannel::Id(id) => id,
        }
    }
}

impl From<ShortUInt> for AMQPChannel {
    fn from(id: ShortUInt) -> AMQPChannel {
        match id {
            0  => AMQPChannel::Global,
            id => AMQPChannel::Id(id),
        }
    }
}

/// The type of AMQP Frame
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum AMQPFrameType {
    /// Call a method
    Method,
    /// Content header
    Header,
    /// Content body
    Body,
    /// Heartbeat frame
    Heartbeat
}

/// The different possible frames
#[derive(Clone, Debug, PartialEq)]
pub enum AMQPFrame {
    /// Protocol header frame
    ProtocolHeader,
    /// Method call
    Method(ShortUInt, AMQPClass),
    /// Content header
    Header(ShortUInt, ShortUInt, Box<AMQPContentHeader>),
    /// Content body
    Body(ShortUInt, Vec<u8>),
    /// Heartbeat frame
    Heartbeat(ShortUInt)
}

/// Raw AMQP Frame
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub struct AMQPRawFrame<'a> {
    /// The type of frame
    pub frame_type: AMQPFrameType,
    /// The id this frame was received on
    pub channel_id: ShortUInt,
    /// The size of the frame
    pub size:       LongUInt,
    /// The paylaod of the frame
    pub payload:    &'a [u8],
}

/// Contente header
#[derive(Clone, Debug, PartialEq)]
pub struct AMQPContentHeader {
    /// The class of content
    pub class_id:   ShortUInt,
    /// The weight of the content
    pub weight:     ShortUInt,
    /// The size of the content's body
    pub body_size:  LongLongUInt,
    /// The AMQP properties associated with the content
    pub properties: basic::AMQPProperties,
}

impl GenSize for AMQPFrame {
    fn get_gen_size(&self) -> usize {
        match self {
            AMQPFrame::ProtocolHeader                       => 4 + metadata::NAME.get_gen_size(),
            AMQPFrame::Heartbeat(_)                         => 8,
            AMQPFrame::Method(channel_id, method)           => 2 + Length.get_gen_size() + channel_id.get_gen_size() + method.get_gen_size(),
            AMQPFrame::Header(channel_id, class_id, header) => 2 + Length.get_gen_size() + channel_id.get_gen_size() + class_id.get_gen_size() + 2 + header.body_size.get_gen_size() + header.properties.get_gen_size(),
            AMQPFrame::Body(channel_id, data)               => 2 + channel_id.get_gen_size() + 4 + data.get_gen_size(),
        }
    }
}
<commit_msg>protocol: add a test<commit_after>use crate::{
    protocol::*,
    types::*,
    types::generation::{GenSize, Length},
};

/// Enum representing an AMQP channel
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum AMQPChannel {
    /// The Global (id 0) AMQP channel used for creating other channels and for heartbeat
    Global,
    /// A regular AMQP channel
    Id(ShortUInt),
}

impl AMQPChannel {
    /// Get the channel id
    pub fn get_id(self) -> ShortUInt {
        match self {
            AMQPChannel::Global => 0,
            AMQPChannel::Id(id) => id,
        }
    }
}

impl From<ShortUInt> for AMQPChannel {
    fn from(id: ShortUInt) -> AMQPChannel {
        match id {
            0  => AMQPChannel::Global,
            id => AMQPChannel::Id(id),
        }
    }
}

/// The type of AMQP Frame
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum AMQPFrameType {
    /// Call a method
    Method,
    /// Content header
    Header,
    /// Content body
    Body,
    /// Heartbeat frame
    Heartbeat
}

/// The different possible frames
#[derive(Clone, Debug, PartialEq)]
pub enum AMQPFrame {
    /// Protocol header frame
    ProtocolHeader,
    /// Method call
    Method(ShortUInt, AMQPClass),
    /// Content header
    Header(ShortUInt, ShortUInt, Box<AMQPContentHeader>),
    /// Content body
    Body(ShortUInt, Vec<u8>),
    /// Heartbeat frame
    Heartbeat(ShortUInt)
}

/// Raw AMQP Frame
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub struct AMQPRawFrame<'a> {
    /// The type of frame
    pub frame_type: AMQPFrameType,
    /// The id this frame was received on
    pub channel_id: ShortUInt,
    /// The size of the frame
    pub size:       LongUInt,
    /// The paylaod of the frame
    pub payload:    &'a [u8],
}

/// Contente header
#[derive(Clone, Debug, PartialEq)]
pub struct AMQPContentHeader {
    /// The class of content
    pub class_id:   ShortUInt,
    /// The weight of the content
    pub weight:     ShortUInt,
    /// The size of the content's body
    pub body_size:  LongLongUInt,
    /// The AMQP properties associated with the content
    pub properties: basic::AMQPProperties,
}

impl GenSize for AMQPFrame {
    fn get_gen_size(&self) -> usize {
        match self {
            AMQPFrame::ProtocolHeader                       => 4 + metadata::NAME.get_gen_size(),
            AMQPFrame::Heartbeat(_)                         => 8,
            AMQPFrame::Method(channel_id, method)           => 2 + Length.get_gen_size() + channel_id.get_gen_size() + method.get_gen_size(),
            AMQPFrame::Header(channel_id, class_id, header) => 2 + Length.get_gen_size() + channel_id.get_gen_size() + class_id.get_gen_size() + 2 + header.body_size.get_gen_size() + header.properties.get_gen_size(),
            AMQPFrame::Body(channel_id, data)               => 2 + channel_id.get_gen_size() + 4 + data.get_gen_size(),
        }
    }
}

#[test]
fn blah() {
    let mut client_properties = FieldTable::default();
    client_properties.insert("product".into(), AMQPValue::LongString("lapin-async".into()));
    client_properties.insert("version".into(), AMQPValue::LongString("0.19.0-alpha3".into()));
    client_properties.insert("platform".into(), AMQPValue::LongString("rust".into()));

    let mut capabilities = FieldTable::default();
    capabilities.insert("publisher_confirms".into(), AMQPValue::Boolean(true));
    capabilities.insert("exchange_exchange_bindings".into(), AMQPValue::Boolean(true));
    capabilities.insert("basic.nack".into(), AMQPValue::Boolean(true));
    capabilities.insert("consumer_cancel_notify".into(), AMQPValue::Boolean(true));
    capabilities.insert("connection.blocked".into(), AMQPValue::Boolean(true));
    capabilities.insert("authentication_failure_close".into(), AMQPValue::Boolean(true));

    client_properties.insert("capabilities".into(), AMQPValue::FieldTable(capabilities));

    let method = AMQPFrame::Method(0, AMQPClass::Connection(connection::AMQPMethod::StartOk(connection::StartOk { client_properties, mechanism: ShortString::from("PLAIN"), response: LongString::from("\u{0}guest\u{0}guest"), locale: ShortString::from("en_US") })));

    assert_eq!(method.get_gen_size(), 270);
}
<|endoftext|>


<commit_before>use std::collections::HashMap;
use std::collections::HashSet;
use std::io::Read;

use crate::common::Lines;
use crate::Solution;

struct Tile {
    grid: Vec<Vec<bool>>,
    id: u64,
    sides: [Vec<bool>; 4],
}

fn read_input(input: &mut dyn Read) -> HashMap<u64, Tile> {
    let mut tiles = HashMap::new();
    let mut lines = Lines::new(input);

    while let Some(line) = lines.next() {
        let id = line[5..(line.len() - 1)].parse().unwrap();
        drop(line);

        let mut grid: Vec<Vec<_>> = Vec::new();

        while let Some(line) = lines.next() {
            if line.is_empty() {
                break;
            }

            grid.push(line.chars().map(|c| c == '#').collect());
        }

        let sides = [
            grid[0].clone(),
            grid.iter().map(|v| *v.last().unwrap()).collect(),
            grid.last().cloned().unwrap(),
            grid.iter().map(|v| v[0]).collect(),
        ];

        let tile = Tile { grid, sides, id };

        tiles.insert(id, tile);
    }

    tiles
}

fn compute_matching<'a>(tiles: impl IntoIterator<Item = &'a Tile>) -> HashMap<Vec<bool>, Vec<u64>> {
    let mut matching: HashMap<Vec<_>, Vec<u64>> = HashMap::new();

    for tile in tiles {
        for side in &tile.sides {
            let neighbours = matching.entry(side.clone()).or_default();

            if !neighbours.contains(&tile.id) {
                neighbours.push(tile.id);
            }

            let mut rev = side.clone();
            rev.reverse();

            let neighbours = matching.entry(rev).or_default();

            if !neighbours.contains(&tile.id) {
                neighbours.push(tile.id);
            }
        }
    }

    // Check if my tile assumption is violated
    debug_assert!(matching.values().map(Vec::len).max().unwrap_or(0) <= 2);

    matching
}

fn compute_neighbours<'a>(
    cliques: impl IntoIterator<Item = &'a Vec<u64>>,
) -> HashMap<u64, Vec<u64>> {
    let mut neighbours: HashMap<u64, Vec<u64>> = HashMap::new();

    for clique in cliques {
        for &a in clique {
            let related = neighbours.entry(a).or_default();

            for &b in clique {
                if a != b && !related.contains(&b) {
                    related.push(b);
                }
            }
        }
    }

    neighbours
}

fn rev_eq(a: &[bool], b: &[bool]) -> bool {
    a.iter().zip(b.iter().rev()).all(|(&a, &b)| a == b)
}

#[derive(Default)]
pub struct Day20;

impl Solution for Day20 {
    fn part1(&mut self, input: &mut dyn Read) -> String {
        let tiles = read_input(input);
        let matching = compute_matching(tiles.values());
        let neighbours = compute_neighbours(matching.values());

        neighbours
            .into_iter()
            .filter_map(|(i, n)| if n.len() == 2 { Some(i) } else { None })
            .fold(1, |a, b| a * b)
            .to_string()
    }
}

#[cfg(test)]
mod tests {
    use crate::test_implementation;

    use super::*;

    const SAMPLE: &[u8] = include_bytes!("../samples/20.txt");

    #[test]
    fn sample_part1() {
        test_implementation!(Day20, 1, SAMPLE, 20899048083289u64);
    }
}
<commit_msg>Finally a correct implementation<commit_after>use std::collections::HashMap;
use std::collections::HashSet;
use std::io::Read;

use regex::bytes::Regex;

use crate::common::Lines;
use crate::Solution;

#[derive(Clone, Debug)]
struct Tile {
    grid: Vec<Vec<bool>>,
    id: u64,
    // NESW order
    sides: [Vec<bool>; 4],
    rotations: u8,
    flipped: bool,
}

impl Tile {
    /// Rotate the slice once to the left. Affects
    fn rotate(&mut self) {
        self.rotations = (self.rotations + 1) % 4;
        self.sides.rotate_right(1);
        self.sides[0].reverse();
        self.sides[2].reverse();
    }

    /// Horizontally flip the tile
    fn flip(&mut self) {
        debug_assert!(!self.flipped);
        self.flipped = !self.flipped;
        self.sides[0].reverse();
        self.sides[2].reverse();
        self.sides.swap(1, 3);
    }

    fn len(&self) -> usize {
        self.grid.len() - 2
    }
}

fn read_input(input: &mut dyn Read) -> HashMap<u64, Tile> {
    let mut tiles = HashMap::new();
    let mut lines = Lines::new(input);

    while let Some(line) = lines.next() {
        let id = line[5..(line.len() - 1)].parse().unwrap();
        drop(line);

        let mut grid: Vec<Vec<_>> = Vec::new();

        while let Some(line) = lines.next() {
            if line.is_empty() {
                break;
            }

            grid.push(line.chars().map(|c| c == '#').collect());
        }

        let sides = [
            grid[0].clone(),
            grid.iter().map(|v| *v.last().unwrap()).collect(),
            grid.last().cloned().unwrap(),
            grid.iter().map(|v| v[0]).collect(),
        ];

        let tile = Tile {
            grid,
            sides,
            id,
            rotations: 0,
            flipped: false,
        };

        tiles.insert(id, tile);
    }

    tiles
}

fn compute_matching<'a>(tiles: impl IntoIterator<Item = &'a Tile>) -> HashMap<Vec<bool>, Vec<u64>> {
    let mut matching: HashMap<Vec<_>, Vec<u64>> = HashMap::new();

    for tile in tiles {
        for side in &tile.sides {
            let neighbours = matching.entry(side.clone()).or_default();

            if !neighbours.contains(&tile.id) {
                neighbours.push(tile.id);
            }

            let mut rev = side.clone();
            rev.reverse();

            let neighbours = matching.entry(rev).or_default();

            if !neighbours.contains(&tile.id) {
                neighbours.push(tile.id);
            }
        }
    }

    // Check if my tile assumption is violated
    debug_assert!(matching.values().map(Vec::len).max().unwrap_or(0) <= 2);

    matching
}

fn compute_neighbours<'a>(
    cliques: impl IntoIterator<Item = &'a Vec<u64>>,
) -> HashMap<u64, Vec<u64>> {
    let mut neighbours: HashMap<u64, Vec<u64>> = HashMap::new();

    for clique in cliques {
        for &a in clique {
            let related = neighbours.entry(a).or_default();

            for &b in clique {
                if a != b && !related.contains(&b) {
                    related.push(b);
                }
            }
        }
    }

    neighbours
}

fn rev_eq(a: &[bool], b: &[bool]) -> bool {
    a.iter().zip(b.iter().rev()).all(|(&a, &b)| a == b)
}

fn complete_row(
    mut first: u64,
    neighbours: &HashMap<u64, Vec<u64>>,
    tiles: &mut HashMap<u64, Tile>,
    used_tiles: &mut HashSet<u64>,
) -> Vec<u64> {
    let mut row = vec![first];
    used_tiles.insert(first);

    loop {
        let last = first;
        let mut next = None;

        for n in &neighbours[&last] {
            if used_tiles.contains(n) {
                continue;
            }

            for _ in 0..4 {
                if &tiles[&last].sides[1] == &tiles[n].sides[3] {
                    break;
                } else if rev_eq(&tiles[&last].sides[1], &tiles[n].sides[3]) {
                    let tile = tiles.get_mut(n).unwrap();

                    // Vertical flip == horizontal flip + 180
                    tile.rotate();
                    tile.rotate();
                    tile.flip();
                    break;
                } else {
                    tiles.get_mut(&n).unwrap().rotate();
                }
            }

            if &tiles[&last].sides[1] == &tiles[n].sides[3] {
                // This tile matches, add it
                next = Some(*n);
                break;
            }
        }

        if let Some(next) = next {
            row.push(next);
            used_tiles.insert(next);
            first = next;
        } else {
            return row;
        }
    }
}

fn compute_image(
    neighbours: &HashMap<u64, Vec<u64>>,
    matching: HashMap<Vec<bool>, Vec<u64>>,
    tiles: &mut HashMap<u64, Tile>,
) -> Vec<Vec<u64>> {
    let mut corner = Vec::new();

    for (&id, connected) in neighbours {
        if connected.len() == 2 && !corner.contains(&id) {
            corner.push(id);
        }
    }

    // Randomly put down the first corner
    let mut used_tiles = HashSet::new();

    let corner_tile = tiles.get_mut(&corner[0]).unwrap();

    // Rotate it until it fits
    while matching[&corner_tile.sides[2]].len() != 2 {
        corner_tile.rotate();
    }

    if matching[&corner_tile.sides[1]].len() != 2 {
        corner_tile.flip();
    }

    debug_assert_eq!(matching[&corner_tile.sides[1]].len(), 2);
    debug_assert_eq!(matching[&corner_tile.sides[2]].len(), 2);

    let rotation = corner_tile.rotations;
    let sides = corner_tile.sides.clone();

    let mut rows = vec![complete_row(corner[0], &neighbours, tiles, &mut used_tiles)];

    debug_assert_eq!(rotation, tiles[&rows[0][0]].rotations);
    debug_assert_eq!(sides, tiles[&rows[0][0]].sides);

    while used_tiles.len() < tiles.len() {
        let prev = rows.last().unwrap()[0];

        // Should be just one tile that can go there.
        let next = neighbours[&prev]
            .iter()
            .filter(|&n| !used_tiles.contains(n))
            .next()
            .unwrap();

        for _ in 0..4 {
            if &tiles[&prev].sides[2] == &tiles[next].sides[0] {
                break;
            } else if rev_eq(&tiles[&prev].sides[2], &tiles[next].sides[0]) {
                tiles.get_mut(next).unwrap().flip();
                break;
            } else {
                tiles.get_mut(next).unwrap().rotate();
            }
        }

        debug_assert_eq!(&tiles[&prev].sides[2], &tiles[next].sides[0]);

        rows.push(complete_row(*next, &neighbours, tiles, &mut used_tiles));
    }

    rows
}

fn combine_tiles(rows: &[Vec<u64>], tiles: &mut HashMap<u64, Tile>) -> Vec<Vec<u8>> {
    // Fix orientation
    for tile in tiles.values_mut() {
        let to_rotate = tile.rotations;

        for _ in 0..to_rotate {
            tile.grid = rotate(&tile.grid);
        }

        if tile.flipped {
            reverse(&mut tile.grid);
        }

        tile.rotations = 0;
        tile.flipped = false;
    }

    rows.iter()
        .flat_map(|row| {
            let len = tiles[row.first().unwrap()].len();

            let mut rows = vec![Vec::new(); len];

            for id in row {
                let tile = &tiles[id];
                for (r, line) in tile.grid.iter().skip(1).take(len).enumerate() {
                    rows[r].extend(
                        line.iter()
                            .skip(1)
                            .take(len)
                            .map(|&b| if b { b'#' } else { b'.' }),
                    );
                }
            }

            rows
        })
        .collect()
}

fn monster_regex() -> [Regex; 3] {
    [
        Regex::new(&"                  # ".replace(' ', ".")).unwrap(),
        Regex::new(&"#    ##    ##    ###".replace(' ', ".")).unwrap(),
        Regex::new(&" #  #  #  #  #  #   ".replace(' ', ".")).unwrap(),
    ]
}

fn rotate<T: Copy>(image: &[Vec<T>]) -> Vec<Vec<T>> {
    let mut new = vec![Vec::new(); image[0].len()];

    for (c, target) in new.iter_mut().enumerate() {
        target.extend(image.iter().rev().map(|r| r[c]));
    }

    new
}

fn reverse<T>(image: &mut [Vec<T>]) {
    for row in image {
        row.reverse();
    }
}

fn replace_monster(image: &mut [Vec<u8>]) {
    let searchers = monster_regex();

    for i in 1..(image.len() - 1) {
        let mut start = 0;

        while let Some(found) = searchers[1].find_at(&image[i], start) {
            start = found.start() + 1;
            let range = found.range();

            if searchers[2].is_match(&image[i + 1][range.clone()])
                && searchers[0].is_match(&image[i - 1][range.clone()])
            {
                image[(i - 1)..=(i + 1)]
                    .iter_mut()
                    .zip(&searchers)
                    .for_each(|(line, expr)| {
                        line[range.clone()]
                            .iter_mut()
                            .zip(expr.as_str().as_bytes().iter())
                            .for_each(|(b, &r)| {
                                if *b == r {
                                    *b = b'O';
                                }
                            })
                    });
            }
        }
    }
}

#[derive(Default)]
pub struct Day20;

impl Solution for Day20 {
    fn part1(&mut self, input: &mut dyn Read) -> String {
        let tiles = read_input(input);
        let matching = compute_matching(tiles.values());
        let neighbours = compute_neighbours(matching.values());

        neighbours
            .into_iter()
            .filter_map(|(i, n)| if n.len() == 2 { Some(i) } else { None })
            .fold(1, |a, b| a * b)
            .to_string()
    }

    fn part2(&mut self, input: &mut dyn Read) -> String {
        let mut tiles = read_input(input);
        let matching = compute_matching(tiles.values());
        let neighbours = compute_neighbours(matching.values());

        let rows = compute_image(&neighbours, matching, &mut tiles);

        let mut image = combine_tiles(&rows, &mut tiles);

        for _ in 0..4 {
            replace_monster(&mut image);
            image = rotate(&image);
        }

        reverse(&mut image);

        for _ in 0..4 {
            replace_monster(&mut image);
            image = rotate(&image);
        }

        image
            .iter()
            .map(|b| bytecount::count(&b, b'#'))
            .sum::<usize>()
            .to_string()
    }
}

#[cfg(test)]
mod tests {
    use crate::test_implementation;

    use super::*;

    const SAMPLE: &[u8] = include_bytes!("../samples/20.txt");

    #[test]
    fn sample_part1() {
        test_implementation!(Day20, 1, SAMPLE, 20899048083289u64);
    }

    #[test]
    fn sample_part2() {
        test_implementation!(Day20, 2, SAMPLE, 273);
    }

    #[test]
    fn test_tile_rotate() {
        let mut tile = Tile {
            grid: Vec::new(),
            id: 1,
            sides: [
                vec![false, true],
                vec![false, false, true],
                vec![false, true],
                vec![true, true, false],
            ],
            rotations: 0,
            flipped: false,
        };

        tile.rotate();

        assert_eq!(
            tile.sides,
            [
                vec![false, true, true],
                vec![false, true],
                vec![true, false, false],
                vec![false, true],
            ]
        );
    }

    #[test]
    fn test_tile_flip() {
        let mut tile = Tile {
            grid: Vec::new(),
            id: 1,
            sides: [
                vec![false, true],
                vec![false, false, true],
                vec![false, true],
                vec![true, true, false],
            ],
            rotations: 0,
            flipped: false,
        };

        tile.flip();

        assert_eq!(
            tile.sides,
            [
                vec![true, false],
                vec![true, true, false],
                vec![true, false],
                vec![false, false, true],
            ]
        );
    }

    #[test]
    fn test_rotate() {
        let sample: Vec<Vec<u8>> = vec![
            b"###".as_ref().to_owned(),
            b"  #".as_ref().to_owned(),
            b"# #".as_ref().to_owned(),
        ];

        let rotated = rotate(&sample);

        let correct: Vec<Vec<u8>> = vec![
            b"# #".as_ref().to_owned(),
            b"  #".as_ref().to_owned(),
            b"###".as_ref().to_owned(),
        ];

        assert_eq!(correct, rotated);
    }
}
<|endoftext|>


<commit_before>use std::collections::HashMap;
use std::io::Read;

use crate::common::from_lines;
use crate::Solution;

const MOD_BASE: u32 = 20201227;
const SUBJECT_NUMBER: u32 = 7;

fn loop_count(public_key: u32) -> u64 {
    discrete_log(SUBJECT_NUMBER, public_key, MOD_BASE).unwrap() as u64
}

// Implementation of the baby-step giant-step algorithm
//
// Based on:https://en.wikipedia.org/wiki/Baby-step_giant-step#C++_algorithm_(C++17)
fn discrete_log(g: u32, h: u32, mod_base: u32) -> Option<u32> {
    let m = (mod_base as f64).sqrt().ceil() as u32;
    let mut table = HashMap::new();
    let mut e: u32 = 1;

    for i in 0..m {
        table.insert(e, i);
        e = ((e as u64 * g as u64) % mod_base as u64) as u32;
    }

    let factor = mod_exp(g as u64, (mod_base - m - 1) as u64, mod_base as u64);
    e = h;

    for i in 0..m {
        if let Some(&val) = table.get(&e) {
            return Some(i * m + val);
        }

        e = ((e as u64 * factor) % mod_base as u64) as u32;
    }

    None
}

#[inline]
fn mod_exp(base: u64, mut power: u64, mod_base: u64) -> u64 {
    let mut result = 1;
    let mut cur = base;

    while power > 0 {
        if power % 2 == 1 {
            result *= cur;
            result %= mod_base;
        }

        cur *= cur;
        cur %= mod_base;

        power /= 2;
    }

    result
}

#[derive(Default)]
pub struct Day25;

impl Solution for Day25 {
    fn part1(&mut self, input: &mut dyn Read) -> String {
        let nums: Vec<_> = from_lines(input);

        let exponent: u64 = nums.into_iter().map(loop_count).product();
        let result = mod_exp(SUBJECT_NUMBER as u64, exponent, MOD_BASE as u64);

        result.to_string()
    }

    fn part2(&mut self, _input: &mut dyn Read) -> String {
        "Part 2 is free!".to_string()
    }
}

#[cfg(test)]
mod tests {
    use crate::test_implementation;

    use super::*;

    const SAMPLE: &[u8] = include_bytes!("../samples/25.txt");

    #[test]
    fn test_loop_count() {
        assert_eq!(8, loop_count(5764801));
        assert_eq!(11, loop_count(17807724));
    }

    #[test]
    fn sample_part1() {
        test_implementation(Day25, 1, SAMPLE, 14897079);
    }
}
<commit_msg>Don't compute unnecessary info.<commit_after>use std::collections::HashMap;
use std::io::Read;

use crate::common::from_lines;
use crate::Solution;

const MOD_BASE: u32 = 20201227;
const SUBJECT_NUMBER: u32 = 7;

fn loop_count(public_key: u32) -> u64 {
    discrete_log(SUBJECT_NUMBER, public_key, MOD_BASE).unwrap() as u64
}

// Implementation of the baby-step giant-step algorithm
//
// Based on:https://en.wikipedia.org/wiki/Baby-step_giant-step#C++_algorithm_(C++17)
fn discrete_log(g: u32, h: u32, mod_base: u32) -> Option<u32> {
    let m = (mod_base as f64).sqrt().ceil() as u32;
    let mut table = HashMap::with_capacity(m as usize);
    let mut e: u32 = 1;

    for i in 0..m {
        table.insert(e, i);
        e = ((e as u64 * g as u64) % mod_base as u64) as u32;
    }

    let factor = mod_exp(g as u64, (mod_base - m - 1) as u64, mod_base as u64);
    e = h;

    for i in 0..m {
        if let Some(&val) = table.get(&e) {
            return Some(i * m + val);
        }

        e = ((e as u64 * factor) % mod_base as u64) as u32;
    }

    None
}

#[inline]
fn mod_exp(base: u64, mut power: u64, mod_base: u64) -> u64 {
    let mut result = 1;
    let mut cur = base;

    while power > 0 {
        if power % 2 == 1 {
            result *= cur;
            result %= mod_base;
        }

        cur *= cur;
        cur %= mod_base;

        power /= 2;
    }

    result
}

#[derive(Default)]
pub struct Day25;

impl Solution for Day25 {
    fn part1(&mut self, input: &mut dyn Read) -> String {
        let nums: Vec<_> = from_lines(input);

        let key_exponent = loop_count(nums[0]);

        mod_exp(nums[1] as u64, key_exponent, MOD_BASE as u64).to_string()
    }

    fn part2(&mut self, _input: &mut dyn Read) -> String {
        "Part 2 is free!".to_string()
    }
}

#[cfg(test)]
mod tests {
    use crate::test_implementation;

    use super::*;

    const SAMPLE: &[u8] = include_bytes!("../samples/25.txt");

    #[test]
    fn test_loop_count() {
        assert_eq!(8, loop_count(5764801));
        assert_eq!(11, loop_count(17807724));
    }

    #[test]
    fn sample_part1() {
        test_implementation(Day25, 1, SAMPLE, 14897079);
    }
}
<|endoftext|>


<commit_before><commit_msg>Inherit Debug trait for FloatType.<commit_after><|endoftext|>


<commit_before>/*
 * A commandline based four function calculator.
 */
use std::io::{self, Write};

fn main() {
    println!("A simple four function calculator");
    println!("Type \"quit\" or \"exit\" to quit.");
    println!();

    // process input in a loop indefinitely.
    loop {
        emit_prompt();

        let mut exp = String::new();

        io::stdin().read_line(&mut exp)
            .expect("Failed to read line");
        let exp = exp.trim();

        // allow a graceful exit from the program.
        if exp == "quit" { // '==' can do string comparison
            break;
        } else if exp == "exit" {
            break;
        }

        println!("expression: {}", exp);
    } // end loop
} // end main

fn emit_prompt() {
    print!("> ");
    io::stdout().flush().unwrap();
} // end emit_prompt
<commit_msg>everything seems to work. finished?<commit_after>/*
 * A commandline based four function calculator.
 */
use std::io::{self, Write};

fn main() {
    println!("A simple four function calculator");
    println!();
    println!("Type \"quit\" or \"exit\" to quit.");
    println!("Operators are: +, -, *, and /");
    println!();

    // process input in a loop indefinitely.
    loop {
        emit_prompt();

        let mut exp = String::new();

        io::stdin().read_line(&mut exp)
            .expect("Failed to read line");
        let exp = exp.trim();

        // allow a graceful exit from the program.
        if exp == "quit" { // '==' can do string comparison
            break;
        } else if exp == "exit" {
            break;
        }

        println!("expression: {}", exp);

        if exp.contains('+') {
            //println!("Addition!");
            let v: Vec<&str> = exp.split('+').collect();

            let (x, y) = (v[0].parse::<i64>().unwrap(),
                        v[1].parse::<i64>().unwrap());

            println!("Answer: {}", add(x, y));
        } else if exp.contains('-') {
            //println!("Subtraction");
            let v: Vec<&str> = exp.split('-').collect();

            let (x, y) = (v[0].parse::<i64>().unwrap(),
                        v[1].parse::<i64>().unwrap());

            println!("Answer: {}", sub(x, y));
        } else if exp.contains('*') {
            //println!("Multiplication");
            let v: Vec<&str> = exp.split('*').collect();

            let (x, y) = (v[0].parse::<i64>().unwrap(),
                        v[1].parse::<i64>().unwrap());

            println!("Answer: {}", mul(x, y));
        } else if exp.contains('/') {
            //println!("Division");
            let v: Vec<&str> = exp.split('/').collect();

            let (x, y) = (v[0].parse::<i64>().unwrap(),
                        v[1].parse::<i64>().unwrap());

            println!("Answer: {}", div(x, y));
        } else {
            println!("Error! Try again.");
        }
    } // end loop
} // end main

fn emit_prompt() {
    print!("> ");
    io::stdout().flush().unwrap();
} // end emit_prompt

fn add(x: i64, y: i64) -> i64 {
    x+y
}

fn sub(x: i64, y: i64) -> i64 {
    x-y
}

fn mul(x: i64, y: i64) -> i64 {
    x*y
}

fn div(x: i64, y: i64) -> i64 {
    if y == 0 {
        println!("Division by zero! Answer undefined!");
        0
    } else {
        x/y
    }
}
<|endoftext|>


<commit_before><commit_msg>Expand FromSql impl from just Option<T><commit_after><|endoftext|>