<commit_before>// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

extern crate devicemapper;
extern crate env_logger;
extern crate libstratis;
extern crate log;
extern crate serde_json;

mod util;

use std::cmp;
use std::fs::OpenOptions;
use std::path::{Path, PathBuf};

use serde_json::{Value, from_reader};

use self::devicemapper::{Bytes, Sectors};

use libstratis::engine::IEC;
use libstratis::engine::strat_engine::device::wipe_sectors;

use util::logger::init_logger;
use util::blockdev_tests::test_blockdevmgr_used;
use util::blockdev_tests::test_force_flag_dirty;
use util::blockdev_tests::test_force_flag_stratis;
use util::blockdev_tests::test_pool_blockdevs;
use util::dm_tests::test_thinpool_device;
use util::dm_tests::test_linear_device;
use util::filesystem_tests::test_xfs_expand;
use util::pool_tests::test_filesystem_rename;
use util::pool_tests::test_thinpool_expand;
use util::pool_tests::test_thinpool_thindev_destroy;
use util::setup_tests::test_basic_metadata;
use util::setup_tests::test_initialize;
use util::setup_tests::test_pool_rename;
use util::setup_tests::test_pool_setup;
use util::setup_tests::test_setup;
use util::simple_tests::test_empty_pool;
use util::simple_tests::test_teardown;

pub struct RealTestDev {
    path: PathBuf,
}

impl RealTestDev {
    /// Construct a new test device
    pub fn new(path: &str) -> RealTestDev {
        wipe_sectors(path, Sectors(0), Bytes(IEC::Mi).sectors()).unwrap();
        RealTestDev { path: PathBuf::from(path) }
    }

    fn as_path(&self) -> &Path {
        &self.path
    }
}

impl Drop for RealTestDev {
    fn drop(&mut self) {
        wipe_sectors(&self.path, Sectors(0), Bytes(IEC::Mi).sectors()).unwrap();
    }
}

enum DeviceLimits {
    Exactly(usize),
    AtLeast(usize),
    #[allow(dead_code)]
    Range(usize, usize), // inclusive
}

/// Return one or more lists of device nodes to use, based upon the
/// constraints. Returns None if constraints can't be met.
fn get_devices(limits: DeviceLimits) -> Option<Vec<Vec<String>>> {

    let file = OpenOptions::new()
        .read(true)
        .open("tests/test_config.json")
        .unwrap();
    let config: Value = from_reader(&file).unwrap();
    let devpaths = config
        .get("ok_to_destroy_dev_array_key")
        .unwrap()
        .as_array()
        .unwrap();

    // Convert enum to [lower, Option<upper>) values
    let (lower, maybe_upper) = match limits {
        DeviceLimits::Exactly(num) => (num, Some(num + 1)),
        DeviceLimits::AtLeast(num) => (num, None),
        DeviceLimits::Range(lower, upper) => {
            assert!(lower < upper);
            (lower, Some(upper + 1))
        }
    };

    // Check these values against available blockdevs
    let avail = devpaths.len();
    if lower > avail {
        return None;
    }
    let maybe_upper = {
        if lower == avail {
            None
        } else {
            match maybe_upper {
                None => Some(avail),
                Some(upper) => {
                    if lower + 1 == upper {
                        None
                    } else {
                        Some(cmp::min(upper - 1, avail))
                    }
                }
            }
        }
    };

    let low_paths: Vec<String> = devpaths
        .iter()
        .take(lower)
        .map(|x| x.as_str().unwrap().to_owned())
        .collect();

    if let Some(upper) = maybe_upper {
        let high_paths: Vec<String> = devpaths
            .iter()
            .take(upper)
            .map(|x| x.as_str().unwrap().to_owned())
            .collect();
        Some(vec![low_paths, high_paths])
    } else {
        Some(vec![low_paths])
    }
}

/// Run test on real devices, using given constraints. Constraints may result
/// in multiple invocations of the test, with differing numbers of block
/// devices.
fn test_with_spec<F>(limits: DeviceLimits, test: F) -> ()
    where F: Fn(&[&Path]) -> ()
{
    init_logger();
    let runs = get_devices(limits).unwrap();
    for run_paths in runs {
        let devices: Vec<_> = run_paths.iter().map(|x| RealTestDev::new(x)).collect();
        test(&devices.iter().map(|x| x.as_path()).collect::<Vec<_>>());
    }
}

#[test]
pub fn real_test_force_flag_stratis() {
    test_with_spec(DeviceLimits::AtLeast(1), test_force_flag_stratis);
}

#[test]
pub fn real_test_linear_device() {
    test_with_spec(DeviceLimits::AtLeast(1), test_linear_device);
}

#[test]
pub fn real_test_thinpool_device() {
    test_with_spec(DeviceLimits::AtLeast(1), test_thinpool_device);
}

#[test]
pub fn real_test_thinpool_expand() {
    test_with_spec(DeviceLimits::AtLeast(1), test_thinpool_expand);
}

#[test]
pub fn real_test_thinpool_thindev_destroy() {
    test_with_spec(DeviceLimits::AtLeast(1), test_thinpool_thindev_destroy);
}

#[test]
pub fn real_test_pool_blockdevs() {
    test_with_spec(DeviceLimits::AtLeast(1), test_pool_blockdevs);
}

#[test]
pub fn real_test_force_flag_dirty() {
    test_with_spec(DeviceLimits::AtLeast(1), test_force_flag_dirty);
}

#[test]
pub fn real_test_teardown() {
    test_with_spec(DeviceLimits::AtLeast(1), test_teardown);
}

#[test]
pub fn real_test_initialize() {
    test_with_spec(DeviceLimits::AtLeast(2), test_initialize);
}

#[test]
pub fn real_test_empty_pool() {
    test_with_spec(DeviceLimits::Exactly(0), test_empty_pool)
}

#[test]
pub fn real_test_basic_metadata() {
    test_with_spec(DeviceLimits::AtLeast(2), test_basic_metadata);
}

#[test]
pub fn real_test_setup() {
    test_with_spec(DeviceLimits::AtLeast(2), test_setup);
}

#[test]
pub fn real_test_pool_rename() {
    test_with_spec(DeviceLimits::AtLeast(1), test_pool_rename);
}

#[test]
pub fn real_test_blockdevmgr_used() {
    test_with_spec(DeviceLimits::AtLeast(1), test_blockdevmgr_used);
}

#[test]
pub fn real_test_filesystem_rename() {
    test_with_spec(DeviceLimits::AtLeast(1), test_filesystem_rename);
}

#[test]
pub fn real_test_pool_setup() {
    test_with_spec(DeviceLimits::AtLeast(1), test_pool_setup);
}

#[test]
pub fn real_test_xfs_expand() {
    test_with_spec(DeviceLimits::AtLeast(1), test_xfs_expand);
}
<commit_msg>Have RealTestDev::new() take a &Path<commit_after>// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

extern crate devicemapper;
extern crate env_logger;
extern crate libstratis;
extern crate log;
extern crate serde_json;

mod util;

use std::cmp;
use std::fs::OpenOptions;
use std::path::{Path, PathBuf};

use serde_json::{Value, from_reader};

use self::devicemapper::{Bytes, Sectors};

use libstratis::engine::IEC;
use libstratis::engine::strat_engine::device::wipe_sectors;

use util::logger::init_logger;
use util::blockdev_tests::test_blockdevmgr_used;
use util::blockdev_tests::test_force_flag_dirty;
use util::blockdev_tests::test_force_flag_stratis;
use util::blockdev_tests::test_pool_blockdevs;
use util::dm_tests::test_thinpool_device;
use util::dm_tests::test_linear_device;
use util::filesystem_tests::test_xfs_expand;
use util::pool_tests::test_filesystem_rename;
use util::pool_tests::test_thinpool_expand;
use util::pool_tests::test_thinpool_thindev_destroy;
use util::setup_tests::test_basic_metadata;
use util::setup_tests::test_initialize;
use util::setup_tests::test_pool_rename;
use util::setup_tests::test_pool_setup;
use util::setup_tests::test_setup;
use util::simple_tests::test_empty_pool;
use util::simple_tests::test_teardown;

pub struct RealTestDev {
    path: PathBuf,
}

impl RealTestDev {
    /// Construct a new test device for the given path.
    /// Wipe initial MiB to clear metadata.
    pub fn new(path: &Path) -> RealTestDev {
        wipe_sectors(path, Sectors(0), Bytes(IEC::Mi).sectors()).unwrap();
        RealTestDev { path: PathBuf::from(path) }
    }

    fn as_path(&self) -> &Path {
        &self.path
    }
}

impl Drop for RealTestDev {
    fn drop(&mut self) {
        wipe_sectors(&self.path, Sectors(0), Bytes(IEC::Mi).sectors()).unwrap();
    }
}

enum DeviceLimits {
    Exactly(usize),
    AtLeast(usize),
    #[allow(dead_code)]
    Range(usize, usize), // inclusive
}

/// Return one or more lists of device nodes to use, based upon the
/// constraints. Returns None if constraints can't be met.
fn get_devices(limits: DeviceLimits) -> Option<Vec<Vec<String>>> {

    let file = OpenOptions::new()
        .read(true)
        .open("tests/test_config.json")
        .unwrap();
    let config: Value = from_reader(&file).unwrap();
    let devpaths = config
        .get("ok_to_destroy_dev_array_key")
        .unwrap()
        .as_array()
        .unwrap();

    // Convert enum to [lower, Option<upper>) values
    let (lower, maybe_upper) = match limits {
        DeviceLimits::Exactly(num) => (num, Some(num + 1)),
        DeviceLimits::AtLeast(num) => (num, None),
        DeviceLimits::Range(lower, upper) => {
            assert!(lower < upper);
            (lower, Some(upper + 1))
        }
    };

    // Check these values against available blockdevs
    let avail = devpaths.len();
    if lower > avail {
        return None;
    }
    let maybe_upper = {
        if lower == avail {
            None
        } else {
            match maybe_upper {
                None => Some(avail),
                Some(upper) => {
                    if lower + 1 == upper {
                        None
                    } else {
                        Some(cmp::min(upper - 1, avail))
                    }
                }
            }
        }
    };

    let low_paths: Vec<String> = devpaths
        .iter()
        .take(lower)
        .map(|x| x.as_str().unwrap().to_owned())
        .collect();

    if let Some(upper) = maybe_upper {
        let high_paths: Vec<String> = devpaths
            .iter()
            .take(upper)
            .map(|x| x.as_str().unwrap().to_owned())
            .collect();
        Some(vec![low_paths, high_paths])
    } else {
        Some(vec![low_paths])
    }
}

/// Run test on real devices, using given constraints. Constraints may result
/// in multiple invocations of the test, with differing numbers of block
/// devices.
fn test_with_spec<F>(limits: DeviceLimits, test: F) -> ()
    where F: Fn(&[&Path]) -> ()
{
    init_logger();
    let runs = get_devices(limits).unwrap();
    for run_paths in runs {
        let devices: Vec<_> = run_paths
            .iter()
            .map(|x| RealTestDev::new(Path::new(x)))
            .collect();
        test(&devices.iter().map(|x| x.as_path()).collect::<Vec<_>>());
    }
}

#[test]
pub fn real_test_force_flag_stratis() {
    test_with_spec(DeviceLimits::AtLeast(1), test_force_flag_stratis);
}

#[test]
pub fn real_test_linear_device() {
    test_with_spec(DeviceLimits::AtLeast(1), test_linear_device);
}

#[test]
pub fn real_test_thinpool_device() {
    test_with_spec(DeviceLimits::AtLeast(1), test_thinpool_device);
}

#[test]
pub fn real_test_thinpool_expand() {
    test_with_spec(DeviceLimits::AtLeast(1), test_thinpool_expand);
}

#[test]
pub fn real_test_thinpool_thindev_destroy() {
    test_with_spec(DeviceLimits::AtLeast(1), test_thinpool_thindev_destroy);
}

#[test]
pub fn real_test_pool_blockdevs() {
    test_with_spec(DeviceLimits::AtLeast(1), test_pool_blockdevs);
}

#[test]
pub fn real_test_force_flag_dirty() {
    test_with_spec(DeviceLimits::AtLeast(1), test_force_flag_dirty);
}

#[test]
pub fn real_test_teardown() {
    test_with_spec(DeviceLimits::AtLeast(1), test_teardown);
}

#[test]
pub fn real_test_initialize() {
    test_with_spec(DeviceLimits::AtLeast(2), test_initialize);
}

#[test]
pub fn real_test_empty_pool() {
    test_with_spec(DeviceLimits::Exactly(0), test_empty_pool)
}

#[test]
pub fn real_test_basic_metadata() {
    test_with_spec(DeviceLimits::AtLeast(2), test_basic_metadata);
}

#[test]
pub fn real_test_setup() {
    test_with_spec(DeviceLimits::AtLeast(2), test_setup);
}

#[test]
pub fn real_test_pool_rename() {
    test_with_spec(DeviceLimits::AtLeast(1), test_pool_rename);
}

#[test]
pub fn real_test_blockdevmgr_used() {
    test_with_spec(DeviceLimits::AtLeast(1), test_blockdevmgr_used);
}

#[test]
pub fn real_test_filesystem_rename() {
    test_with_spec(DeviceLimits::AtLeast(1), test_filesystem_rename);
}

#[test]
pub fn real_test_pool_setup() {
    test_with_spec(DeviceLimits::AtLeast(1), test_pool_setup);
}

#[test]
pub fn real_test_xfs_expand() {
    test_with_spec(DeviceLimits::AtLeast(1), test_xfs_expand);
}
<|endoftext|>


<commit_before>// Copyright 2015-2018 Deyan Ginev. See the LICENSE
// file at the top-level directory of this distribution.
//
// Licensed under the MIT license <LICENSE-MIT or http://opensource.org/licenses/MIT>.
// This file may not be copied, modified, or distributed
// except according to those terms.

//! Import a new corpus into the framework
use glob::glob;
// use regex::Regex;
use crate::backend::Backend;
use crate::helpers::TaskStatus;
use crate::models::{Corpus, NewCorpus, NewTask};
use std::env;
use std::fs;
use std::io::Error;
use std::path::Path;
use std::path::PathBuf;
use Archive::*;

const BUFFER_SIZE: usize = 10_240;

/// Struct for performing corpus imports into `CorTeX`
pub struct Importer {
  /// a `Corpus` to be imported, containing all relevant metadata
  pub corpus: Corpus,
  /// a `Backend` on which to persist the import into the Task store
  pub backend: Backend,
  /// the current working directory, to resolve relative paths
  pub cwd: PathBuf,
}
impl Default for Importer {
  fn default() -> Importer {
    let default_backend = Backend::default();
    let name = "default";
    default_backend
      .add(&NewCorpus {
        path: ".".to_string(),
        name: name.to_string(),
        complex: false,
        description: String::new(),
      })
      .expect("Failed to create new corpus.");
    let registered_corpus = Corpus::find_by_name(name, &default_backend.connection).unwrap();

    Importer {
      corpus: registered_corpus,
      backend: default_backend,
      cwd: Importer::cwd(),
    }
  }
}

impl Importer {
  /// Convenience method for (recklessly?) obtaining the current working dir
  pub fn cwd() -> PathBuf { env::current_dir().unwrap() }
  /// Top-level method for unpacking an arxiv-toplogy corpus from its tar-ed form
  fn unpack(&self) -> Result<(), Error> {
    r#try!(self.unpack_arxiv_top());
    r#try!(self.unpack_arxiv_months());
    Ok(())
  }
  fn unpack_extend(&self) -> Result<(), Error> {
    r#try!(self.unpack_extend_arxiv_top());
    // We can reuse the monthly unpack, as it deletes all unpacked document archives
    // In other words, it always acts as a conservative extension
    r#try!(self.unpack_arxiv_months());
    Ok(())
  }

  /// Unpack the top-level tar files from an arxiv-topology corpus
  fn unpack_arxiv_top(&self) -> Result<(), Error> {
    println!("-- Starting top-level unpack process");
    let path_str = self.corpus.path.clone();
    let tars_path = path_str.to_string() + "/*.tar";
    for entry in glob(&tars_path).unwrap() {
      match entry {
        Ok(path) => {
          // let base_name = path.file_stem().unwrap().to_str().unwrap();
          // If we wanted fine-grained control, we could infer the dir name:
          // let arxiv_name_re = Regex::new(r"arXiv_src_(\d+)_").unwrap();
          // let captures = arxiv_name_re.captures(base_name).unwrap();
          // let unpack_dirname = match captures.at(1) {
          //   Some(month) => month,
          //   None => base_name
          // };
          // --- but not for now

          // Let's open the tar file and unpack it:
          let archive_reader = Reader::new()
            .unwrap()
            .support_filter_all()
            .support_format_all()
            .open_filename(path.to_str().unwrap(), BUFFER_SIZE)
            .unwrap();
          while let Ok(e) = archive_reader.next_header() {
            if e.pathname().ends_with(".pdf") {
              continue;
            }
            let full_extract_path = path_str.to_string() + &e.pathname();
            match fs::metadata(full_extract_path.clone()) {
              Ok(_) => println!("File {:?} exists, won't unpack.", e.pathname()),
              Err(_) => {
                println!("To unpack: {:?}", full_extract_path);
                match e.extract_to(&full_extract_path, Vec::new()) {
                  Ok(_) => {},
                  _ => {
                    println!("Failed to extract {:?}", full_extract_path);
                  },
                }
              },
            }
          }
        },
        Err(e) => println!("Failed tar glob: {:?}", e),
      }
    }
    Ok(())
  }
  /// Top-level extension unpacking for arxiv-topology corpora
  fn unpack_extend_arxiv_top(&self) -> Result<(), Error> {
    println!("-- Starting top-level unpack-extend process");
    let path_str = self.corpus.path.clone();
    let tars_path = path_str.to_string() + "/*.tar";
    for entry in glob(&tars_path).unwrap() {
      match entry {
        Ok(path) => {
          // Let's open the tar file and unpack it:
          let archive_reader = Reader::new()
            .unwrap()
            .support_filter_all()
            .support_format_all()
            .open_filename(path.to_str().unwrap(), BUFFER_SIZE)
            .unwrap();
          while let Ok(e) = archive_reader.next_header() {
            if e.pathname().ends_with(".pdf") {
              continue;
            }
            let full_extract_path = path_str.to_string() + &e.pathname();
            match fs::metadata(full_extract_path.clone()) {
              Ok(_) => {}//println!("File {:?} exists, won't unpack.", e.pathname()),
              Err(_) => {
                // Archive entries end in .gz, let's try that as well, to check if the directory is there
                let dir_extract_path = &full_extract_path[0..full_extract_path.len() - 3];
                match fs::metadata(dir_extract_path) {
                  Ok(_) => {}//println!("Directory for {:?} already exists, won't unpack.", e.pathname()),
                  Err(_) => {
                    // println!("To unpack: {:?}", full_extract_path);
                    match e.extract_to(&full_extract_path, Vec::new()) {
                      Ok(_) => {}
                      _ => {
                        println!("Failed to extract {:?}", full_extract_path);
                      }
                    }
                  }
                }
              }
            }
          }
        },
        Err(e) => println!("Failed tar glob: {:?}", e),
      }
    }
    Ok(())
  }

  /// Unpack the monthly sub-archives of an arxiv-topology corpus, into the CorTeX organization
  fn unpack_arxiv_months(&self) -> Result<(), Error> {
    println!("-- Starting to unpack monthly .gz archives");
    let path_str = self.corpus.path.clone();
    let gzs_path = path_str.to_string() + "/*/*.gz";
    for entry in glob(&gzs_path).unwrap() {
      match entry {
        Ok(path) => {
          let entry_path = path.to_str().unwrap();
          let entry_dir = path.parent().unwrap().to_str().unwrap();
          let base_name = path.file_stem().unwrap().to_str().unwrap();
          let default_tex_target = base_name.to_string() + ".tex";
          let entry_cp_dir = entry_dir.to_string() + "/" + base_name;
          fs::create_dir_all(entry_cp_dir.clone()).unwrap_or_else(|reason| {
            println!(
              "Failed to mkdir -p {:?} because: {:?}",
              entry_cp_dir.clone(),
              reason.kind()
            );
          });
          // We'll write out a ZIP file for each entry
          let full_extract_path = entry_cp_dir.to_string() + "/" + base_name + ".zip";
          let mut archive_writer_new = Writer::new()
            .unwrap()
            //.add_filter(ArchiveFilter::Lzip)
            // .set_compression(ArchiveFilter::None)
            .set_format(ArchiveFormat::Zip);
          archive_writer_new
            .open_filename(&full_extract_path.clone())
            .unwrap();

          // Careful here, some of arXiv's .gz files are really plain-text TeX files (surprise!!!)
          let mut raw_read_needed = false;
          match Reader::new()
            .unwrap()
            .support_filter_all()
            .support_format_all()
            .open_filename(entry_path, BUFFER_SIZE)
          {
            Err(_) => raw_read_needed = true,
            Ok(archive_reader) => {
              let mut file_count = 0;
              while let Ok(e) = archive_reader.next_header() {
                file_count += 1;
                match archive_writer_new.write_header(e) {
                  Ok(_) => {}, // TODO: If we need to print an error message, we can do so later.
                  Err(e2) => println!("Header write failed: {:?}", e2),
                };
                while let Ok(chunk) = archive_reader.read_data(BUFFER_SIZE) {
                  archive_writer_new.write_data(chunk).unwrap();
                }
              }
              if file_count == 0 {
                // Special case (bug? in libarchive crate), single file in .gz
                raw_read_needed = true;
              }
            },
          }

          if raw_read_needed {
            let raw_reader_new = Reader::new()
              .unwrap()
              .support_filter_all()
              .support_format_raw()
              .open_filename(entry_path, BUFFER_SIZE);
            match raw_reader_new {
              Ok(raw_reader) => match raw_reader.next_header() {
                Ok(_) => {
                  single_file_transfer(&default_tex_target, &raw_reader, &mut archive_writer_new);
                },
                Err(_) => println!("No content in archive: {:?}", entry_path),
              },
              Err(_) => println!("Unrecognizeable archive: {:?}", entry_path),
            }
          }
          // Done with this .gz , remove it:
          match fs::remove_file(path.clone()) {
            Ok(_) => {},
            Err(e) => println!("Can't remove source .gz: {:?}", e),
          };
        },
        Err(e) => println!("Failed gz glob: {:?}", e),
      }
    }
    Ok(())
  }

  /// Given a CorTeX-topology corpus, walk the file system and import it into the Task store
  pub fn walk_import(&self) -> Result<usize, Error> {
    println!("-- Starting import walk");
    let import_extension = if self.corpus.complex { "zip" } else { "tex" };
    let mut walk_q: Vec<PathBuf> = vec![Path::new(&self.corpus.path).to_owned()];
    let mut import_q: Vec<NewTask> = Vec::new();
    let mut import_counter = 0;
    while !walk_q.is_empty() {
      let current_path = walk_q.pop().unwrap();
      let current_metadata = r#try!(fs::metadata(current_path.clone()));
      if current_metadata.is_dir() {
        println!("-- current path {:?}", current_path);
        // First, test if we just found an entry:
        let current_local_dir = current_path.file_name().unwrap();
        let current_entry =
          current_local_dir.to_str().unwrap().to_string() + "." + import_extension;
        let current_entry_path = current_path.to_str().unwrap().to_string() + "/" + &current_entry;
        match fs::metadata(&current_entry_path) {
          Ok(_) => {
            // Found the expected file, import this entry:
            println!("Found entry: {:?}", current_entry_path);
            import_counter += 1;
            import_q.push(self.new_task(&current_entry_path));
            if import_q.len() >= 1000 {
              // Flush the import queue to backend:
              println!("Checkpoint backend writer: job {:?}", import_counter);
              self.backend.mark_imported(&import_q).unwrap(); // TODO: Proper Error-handling
              import_q.clear();
            }
          },
          Err(_) => {
            // No such entry found, traversing into the directory:
            for subentry in r#try!(fs::read_dir(current_path.clone())) {
              let subentry = r#try!(subentry);
              walk_q.push(subentry.path());
            }
          },
        }
      }
    }
    if !import_q.is_empty() {
      println!("Checkpoint backend writer: job {:?}", import_q.len());
      self.backend.mark_imported(&import_q).unwrap();
    } // TODO: Proper Error-handling
    println!("--- Imported {:?} entries.", import_counter);
    Ok(import_counter)
  }

  /// Create a new NoProblem task for the "import" service and the Importer-specified corpus
  pub fn new_task(&self, entry: &str) -> NewTask {
    let abs_entry: String = if Path::new(&entry).is_relative() {
      let mut new_abs = self.cwd.clone();
      new_abs.push(entry);
      new_abs.to_str().unwrap().to_string()
    } else {
      entry.to_string()
    };

    NewTask {
      entry: abs_entry,
      status: TaskStatus::NoProblem.raw(),
      corpus_id: self.corpus.id,
      service_id: 2,
    }
  }
  /// Top-level import driver, performs an optional unpack, and then an import into the Task store
  pub fn process(&self) -> Result<(), Error> {
    // println!("Greetings from the import processor");
    if self.corpus.complex {
      // Complex setup has an unpack step:
      r#try!(self.unpack());
    }
    // Walk the directory tree and import the files in the Task store:
    r#try!(self.walk_import());

    Ok(())
  }

  /// Top-level corpus extension, performs a check for newly added documents and extracts+adds
  /// them to the existing corpus tasks
  pub fn extend_corpus(&self) -> Result<(), Error> {
    if self.corpus.complex {
      // Complex setup has an unpack step:
      self.unpack_extend()?;
    }
    // Use the regular walk_import, at the cost of more database work,
    // the "Backend::mark_imported" ORM method allows us to insert only if new
    self.walk_import()?;
    // TODO: Should we mark new runs in the metadata, on extending the corpus, for all services?
    // Probably not?
    Ok(())
  }
}

/// Transfer the data contained within `Reader` to a `Writer`, assuming it was a single file
pub fn single_file_transfer(tex_target: &str, reader: &Reader, writer: &mut Writer) {
  // In a "raw" read, we don't know the data size in advance. So we bite the
  // bullet and read the usually tiny tex file in memory,
  // obtaining a size estimate
  let mut raw_data = Vec::new();
  while let Ok(chunk) = reader.read_data(BUFFER_SIZE) {
    raw_data.extend(chunk.into_iter());
  }
  let mut ok_header = false;
  match writer.write_header_new(&tex_target, raw_data.len() as i64) {
    Ok(_) => {
      ok_header = true;
    },
    Err(e) => {
      println!("Couldn't write header: {:?}", e);
    },
  }
  if ok_header {
    match writer.write_data(raw_data) {
      Ok(_) => {},
      Err(e) => println!("Failed to write data to {:?} because {:?}", tex_target, e),
    };
  }
}
<commit_msg>mark a historical run, when extending corpora<commit_after>// Copyright 2015-2018 Deyan Ginev. See the LICENSE
// file at the top-level directory of this distribution.
//
// Licensed under the MIT license <LICENSE-MIT or http://opensource.org/licenses/MIT>.
// This file may not be copied, modified, or distributed
// except according to those terms.

//! Import a new corpus into the framework
use glob::glob;
// use regex::Regex;
use crate::backend::{Backend, RerunOptions};
use crate::helpers::TaskStatus;
use crate::models::{Corpus, NewCorpus, NewTask};
use std::env;
use std::fs;
use std::error::Error;
use std::path::Path;
use std::path::PathBuf;
use Archive::*;

const BUFFER_SIZE: usize = 10_240;

/// Struct for performing corpus imports into `CorTeX`
pub struct Importer {
  /// a `Corpus` to be imported, containing all relevant metadata
  pub corpus: Corpus,
  /// a `Backend` on which to persist the import into the Task store
  pub backend: Backend,
  /// the current working directory, to resolve relative paths
  pub cwd: PathBuf,
}
impl Default for Importer {
  fn default() -> Importer {
    let default_backend = Backend::default();
    let name = "default";
    default_backend
      .add(&NewCorpus {
        path: ".".to_string(),
        name: name.to_string(),
        complex: false,
        description: String::new(),
      })
      .expect("Failed to create new corpus.");
    let registered_corpus = Corpus::find_by_name(name, &default_backend.connection).unwrap();

    Importer {
      corpus: registered_corpus,
      backend: default_backend,
      cwd: Importer::cwd(),
    }
  }
}

impl Importer {
  /// Convenience method for (recklessly?) obtaining the current working dir
  pub fn cwd() -> PathBuf { env::current_dir().unwrap() }
  /// Top-level method for unpacking an arxiv-toplogy corpus from its tar-ed form
  fn unpack(&self) -> Result<(), Box<Error>> {
    self.unpack_arxiv_top()?;
    self.unpack_arxiv_months()?;
    Ok(())
  }
  fn unpack_extend(&self) -> Result<(), Box<Error>> {
    self.unpack_extend_arxiv_top()?;
    // We can reuse the monthly unpack, as it deletes all unpacked document archives
    // In other words, it always acts as a conservative extension
    self.unpack_arxiv_months()?;
    Ok(())
  }

  /// Unpack the top-level tar files from an arxiv-topology corpus
  fn unpack_arxiv_top(&self) -> Result<(), Box<Error>> {
    println!("-- Starting top-level unpack process");
    let path_str = self.corpus.path.clone();
    let tars_path = path_str.to_string() + "/*.tar";
    for entry in glob(&tars_path).unwrap() {
      match entry {
        Ok(path) => {
          // let base_name = path.file_stem().unwrap().to_str().unwrap();
          // If we wanted fine-grained control, we could infer the dir name:
          // let arxiv_name_re = Regex::new(r"arXiv_src_(\d+)_").unwrap();
          // let captures = arxiv_name_re.captures(base_name).unwrap();
          // let unpack_dirname = match captures.at(1) {
          //   Some(month) => month,
          //   None => base_name
          // };
          // --- but not for now

          // Let's open the tar file and unpack it:
          let archive_reader = Reader::new()
            .unwrap()
            .support_filter_all()
            .support_format_all()
            .open_filename(path.to_str().unwrap(), BUFFER_SIZE)
            .unwrap();
          while let Ok(e) = archive_reader.next_header() {
            if e.pathname().ends_with(".pdf") {
              continue;
            }
            let full_extract_path = path_str.to_string() + &e.pathname();
            match fs::metadata(full_extract_path.clone()) {
              Ok(_) => println!("File {:?} exists, won't unpack.", e.pathname()),
              Err(_) => {
                println!("To unpack: {:?}", full_extract_path);
                match e.extract_to(&full_extract_path, Vec::new()) {
                  Ok(_) => {},
                  _ => {
                    println!("Failed to extract {:?}", full_extract_path);
                  },
                }
              },
            }
          }
        },
        Err(e) => println!("Failed tar glob: {:?}", e),
      }
    }
    Ok(())
  }
  /// Top-level extension unpacking for arxiv-topology corpora
  fn unpack_extend_arxiv_top(&self) -> Result<(), Box<Error>> {
    println!("-- Starting top-level unpack-extend process");
    let path_str = self.corpus.path.clone();
    let tars_path = path_str.to_string() + "/*.tar";
    for entry in glob(&tars_path).unwrap() {
      match entry {
        Ok(path) => {
          // Let's open the tar file and unpack it:
          let archive_reader = Reader::new()
            .unwrap()
            .support_filter_all()
            .support_format_all()
            .open_filename(path.to_str().unwrap(), BUFFER_SIZE)
            .unwrap();
          while let Ok(e) = archive_reader.next_header() {
            if e.pathname().ends_with(".pdf") {
              continue;
            }
            let full_extract_path = path_str.to_string() + &e.pathname();
            match fs::metadata(full_extract_path.clone()) {
              Ok(_) => {}//println!("File {:?} exists, won't unpack.", e.pathname()),
              Err(_) => {
                // Archive entries end in .gz, let's try that as well, to check if the directory is there
                let dir_extract_path = &full_extract_path[0..full_extract_path.len() - 3];
                match fs::metadata(dir_extract_path) {
                  Ok(_) => {}//println!("Directory for {:?} already exists, won't unpack.", e.pathname()),
                  Err(_) => {
                    // println!("To unpack: {:?}", full_extract_path);
                    match e.extract_to(&full_extract_path, Vec::new()) {
                      Ok(_) => {}
                      _ => {
                        println!("Failed to extract {:?}", full_extract_path);
                      }
                    }
                  }
                }
              }
            }
          }
        },
        Err(e) => println!("Failed tar glob: {:?}", e),
      }
    }
    Ok(())
  }

  /// Unpack the monthly sub-archives of an arxiv-topology corpus, into the CorTeX organization
  fn unpack_arxiv_months(&self) -> Result<(), Box<Error>> {
    println!("-- Starting to unpack monthly .gz archives");
    let path_str = self.corpus.path.clone();
    let gzs_path = path_str.to_string() + "/*/*.gz";
    for entry in glob(&gzs_path).unwrap() {
      match entry {
        Ok(path) => {
          let entry_path = path.to_str().unwrap();
          let entry_dir = path.parent().unwrap().to_str().unwrap();
          let base_name = path.file_stem().unwrap().to_str().unwrap();
          let default_tex_target = base_name.to_string() + ".tex";
          let entry_cp_dir = entry_dir.to_string() + "/" + base_name;
          fs::create_dir_all(entry_cp_dir.clone()).unwrap_or_else(|reason| {
            println!(
              "Failed to mkdir -p {:?} because: {:?}",
              entry_cp_dir.clone(),
              reason.kind()
            );
          });
          // We'll write out a ZIP file for each entry
          let full_extract_path = entry_cp_dir.to_string() + "/" + base_name + ".zip";
          let mut archive_writer_new = Writer::new()
            .unwrap()
            //.add_filter(ArchiveFilter::Lzip)
            // .set_compression(ArchiveFilter::None)
            .set_format(ArchiveFormat::Zip);
          archive_writer_new
            .open_filename(&full_extract_path.clone())
            .unwrap();

          // Careful here, some of arXiv's .gz files are really plain-text TeX files (surprise!!!)
          let mut raw_read_needed = false;
          match Reader::new()
            .unwrap()
            .support_filter_all()
            .support_format_all()
            .open_filename(entry_path, BUFFER_SIZE)
          {
            Err(_) => raw_read_needed = true,
            Ok(archive_reader) => {
              let mut file_count = 0;
              while let Ok(e) = archive_reader.next_header() {
                file_count += 1;
                match archive_writer_new.write_header(e) {
                  Ok(_) => {}, // TODO: If we need to print an error message, we can do so later.
                  Err(e2) => println!("Header write failed: {:?}", e2),
                };
                while let Ok(chunk) = archive_reader.read_data(BUFFER_SIZE) {
                  archive_writer_new.write_data(chunk).unwrap();
                }
              }
              if file_count == 0 {
                // Special case (bug? in libarchive crate), single file in .gz
                raw_read_needed = true;
              }
            },
          }

          if raw_read_needed {
            let raw_reader_new = Reader::new()
              .unwrap()
              .support_filter_all()
              .support_format_raw()
              .open_filename(entry_path, BUFFER_SIZE);
            match raw_reader_new {
              Ok(raw_reader) => match raw_reader.next_header() {
                Ok(_) => {
                  single_file_transfer(&default_tex_target, &raw_reader, &mut archive_writer_new);
                },
                Err(_) => println!("No content in archive: {:?}", entry_path),
              },
              Err(_) => println!("Unrecognizeable archive: {:?}", entry_path),
            }
          }
          // Done with this .gz , remove it:
          match fs::remove_file(path.clone()) {
            Ok(_) => {},
            Err(e) => println!("Can't remove source .gz: {:?}", e),
          };
        },
        Err(e) => println!("Failed gz glob: {:?}", e),
      }
    }
    Ok(())
  }

  /// Given a CorTeX-topology corpus, walk the file system and import it into the Task store
  pub fn walk_import(&self) -> Result<usize, Box<Error>> {
    println!("-- Starting import walk");
    let import_extension = if self.corpus.complex { "zip" } else { "tex" };
    let mut walk_q: Vec<PathBuf> = vec![Path::new(&self.corpus.path).to_owned()];
    let mut import_q: Vec<NewTask> = Vec::new();
    let mut import_counter = 0;
    while !walk_q.is_empty() {
      let current_path = walk_q.pop().unwrap();
      let current_metadata = r#try!(fs::metadata(current_path.clone()));
      if current_metadata.is_dir() {
        println!("-- current path {:?}", current_path);
        // First, test if we just found an entry:
        let current_local_dir = current_path.file_name().unwrap();
        let current_entry =
          current_local_dir.to_str().unwrap().to_string() + "." + import_extension;
        let current_entry_path = current_path.to_str().unwrap().to_string() + "/" + &current_entry;
        match fs::metadata(&current_entry_path) {
          Ok(_) => {
            // Found the expected file, import this entry:
            println!("Found entry: {:?}", current_entry_path);
            import_counter += 1;
            import_q.push(self.new_task(&current_entry_path));
            if import_q.len() >= 1000 {
              // Flush the import queue to backend:
              println!("Checkpoint backend writer: job {:?}", import_counter);
              self.backend.mark_imported(&import_q).unwrap(); // TODO: Proper Error-handling
              import_q.clear();
            }
          },
          Err(_) => {
            // No such entry found, traversing into the directory:
            for subentry in r#try!(fs::read_dir(current_path.clone())) {
              let subentry = r#try!(subentry);
              walk_q.push(subentry.path());
            }
          },
        }
      }
    }
    if !import_q.is_empty() {
      println!("Checkpoint backend writer: job {:?}", import_q.len());
      self.backend.mark_imported(&import_q).unwrap();
    } // TODO: Proper Error-handling
    println!("--- Imported {:?} entries.", import_counter);
    Ok(import_counter)
  }

  /// Create a new NoProblem task for the "import" service and the Importer-specified corpus
  pub fn new_task(&self, entry: &str) -> NewTask {
    let abs_entry: String = if Path::new(&entry).is_relative() {
      let mut new_abs = self.cwd.clone();
      new_abs.push(entry);
      new_abs.to_str().unwrap().to_string()
    } else {
      entry.to_string()
    };

    NewTask {
      entry: abs_entry,
      status: TaskStatus::NoProblem.raw(),
      corpus_id: self.corpus.id,
      service_id: 2,
    }
  }
  /// Top-level import driver, performs an optional unpack, and then an import into the Task store
  pub fn process(&self) -> Result<(), Box<Error>> {
    // println!("Greetings from the import processor");
    if self.corpus.complex {
      // Complex setup has an unpack step:
      r#try!(self.unpack());
    }
    // Walk the directory tree and import the files in the Task store:
    r#try!(self.walk_import());

    Ok(())
  }

  /// Top-level corpus extension, performs a check for newly added documents and extracts+adds
  /// them to the existing corpus tasks
  pub fn extend_corpus(&self) -> Result<(), Box<Error>> {
    if self.corpus.complex {
      // Complex setup has an unpack step:
      self.unpack_extend()?;
    }
    // Before we import, mark any current runs as completed.
    for service in self.corpus.select_services(&self.backend.connection).unwrap_or_default().iter() {
      self.backend.mark_rerun(RerunOptions {
        corpus: &self.corpus,
        service,
        severity_opt: None,
        category_opt: None,
        what_opt: None,
        owner_opt: Some("cli-admin".to_string()), // command line interface only?
        description_opt: Some("extending corpus with more entries".to_string())
      })?;
    }
    // Use the regular walk_import, at the cost of more database work,
    // the "Backend::mark_imported" ORM method allows us to insert only if new
    self.walk_import()?;
    Ok(())
  }
}

/// Transfer the data contained within `Reader` to a `Writer`, assuming it was a single file
pub fn single_file_transfer(tex_target: &str, reader: &Reader, writer: &mut Writer) {
  // In a "raw" read, we don't know the data size in advance. So we bite the
  // bullet and read the usually tiny tex file in memory,
  // obtaining a size estimate
  let mut raw_data = Vec::new();
  while let Ok(chunk) = reader.read_data(BUFFER_SIZE) {
    raw_data.extend(chunk.into_iter());
  }
  let mut ok_header = false;
  match writer.write_header_new(&tex_target, raw_data.len() as i64) {
    Ok(_) => {
      ok_header = true;
    },
    Err(e) => {
      println!("Couldn't write header: {:?}", e);
    },
  }
  if ok_header {
    match writer.write_data(raw_data) {
      Ok(_) => {},
      Err(e) => println!("Failed to write data to {:?} because {:?}", tex_target, e),
    };
  }
}
<|endoftext|>


<commit_before>use std::{fs, io, path};

use syntax::print::pprust;
use syntax::{ast, source_map, visit};

use tree::{Tree, Visibility};

pub struct Config {
    pub include_orphans: bool,
    pub ignored_files: Vec<path::PathBuf>,
}

pub struct Builder<'a> {
    tree: Tree,
    path: Vec<String>,
    config: Config,
    source_map: &'a source_map::SourceMap,
}

impl<'a> Builder<'a> {
    pub fn new(config: Config, crate_name: String, source_map: &'a source_map::SourceMap) -> Self {
        let tree = Tree::new_crate(crate_name);
        Builder {
            tree,
            path: vec![],
            config,
            source_map,
        }
    }

    pub fn tree(&self) -> &Tree {
        &self.tree
    }

    fn find_orphan_candidates(
        path: &[String],
        ignored_paths: &[path::PathBuf],
    ) -> Result<Vec<String>, io::Error> {
        let mut dir_path = "./src/".to_string();
        for name in path.iter() {
            dir_path.push_str(name);
            dir_path.push_str("/");
        }
        fn is_mod(entry: &fs::DirEntry) -> bool {
            let path = entry.path();
            if path.is_file() {
                if path.extension().map_or(false, |ext| ext == "rs") {
                    return file_name(entry)
                        .map_or(false, |s| (s != "mod") && (s != "lib") && (s != "main"));
                }
            } else if path.join("mod.rs").exists() {
                return true;
            }
            false
        }
        fn file_name(entry: &fs::DirEntry) -> Option<String> {
            entry
                .path()
                .file_stem()
                .and_then(|s| s.to_str().map(|s| s.to_string()))
        }
        Ok(try!(fs::read_dir(&dir_path))
            .filter_map(|e| e.ok())
            .filter(is_mod)
            .filter(|e| !ignored_paths.contains(&e.path()))
            .filter_map(|e| file_name(&e))
            .collect::<Vec<_>>())
    }

    fn sanitize_condition(condition: String) -> String {
        let words: Vec<&str> = condition.split_whitespace().collect();
        words.join(" ")
    }
}

impl<'a> visit::Visitor<'a> for Builder<'a> {
    fn visit_item(&mut self, item: &'a ast::Item) {
        let condition = item
            .attrs
            .iter()
            .find(|attr| attr.check_name("cfg"))
            .map(|attr| {
                self.source_map
                    .span_to_snippet(attr.span)
                    .map(Builder::sanitize_condition)
                    .unwrap_or_else(|_| "".to_string())
            });

        match item.node {
            ast::ItemKind::Mod(_) => {
                let name = item.ident.to_string();
                {
                    let tree = self.tree.subtree_at_path(&self.path).unwrap();
                    let visibility = if let ast::VisibilityKind::Public = item.vis.node {
                        Visibility::Public
                    } else {
                        Visibility::Private
                    };
                    tree.insert(Tree::new_module(name.clone(), visibility, condition));
                }
                self.path.push(name);
                visit::walk_item(self, item);
                let _ = self.path.pop();
            }
            ast::ItemKind::Use(ref use_tree) => {
                {
                    let tree = self.tree.subtree_at_path(&self.path).unwrap();
                    let visibility = if let ast::VisibilityKind::Public = item.vis.node {
                        Visibility::Public
                    } else {
                        Visibility::Private
                    };

                    add_use_tree(tree, visibility, "", use_tree);
                }

                visit::walk_item(self, item);
            }
            _ => {
                visit::walk_item(self, item);
            }
        }
    }

    fn visit_mod(
        &mut self,
        m: &'a ast::Mod,
        _s: source_map::Span,
        _attrs: &[ast::Attribute],
        _n: ast::NodeId,
    ) {
        visit::walk_mod(self, m);
        if !self.config.include_orphans {
            return;
        }
        let path = &self.path;
        if let Ok(candidates) = Builder::find_orphan_candidates(path, &self.config.ignored_files) {
            let tree = self.tree.subtree_at_path(path).unwrap();
            let names: Vec<_> = tree.subtree_names();
            for candidate in candidates {
                if !names.contains(&candidate) {
                    tree.insert(Tree::new_orphan(candidate.clone()));
                }
            }
        }
    }

    fn visit_mac(&mut self, mac: &ast::Mac) {
        visit::walk_mac(self, mac);
    }
}

fn add_use_tree(tree: &mut Tree, visibility: Visibility, prefix: &str, use_tree: &ast::UseTree) {
    let prefix = if prefix.is_empty() {
        String::new()
    } else {
        prefix.to_string()
    };

    match use_tree.kind {
        ast::UseTreeKind::Simple(_, ..) => {
            let path = pprust::path_to_string(&use_tree.prefix);
            if path != "self" {
                tree.insert_use((visibility, prefix + &path));
            } else {
                let mut prefix = prefix;
                let new_len = prefix.len() - 2;
                prefix.truncate(new_len);
                tree.insert_use((visibility, prefix));
            }
        }
        ast::UseTreeKind::Glob => {
            let mut use_path = if use_tree.prefix.segments.is_empty() {
                prefix
            } else {
                prefix + &pprust::path_to_string(&use_tree.prefix) + "::"
            };
            use_path += "*";
            tree.insert_use((visibility, use_path));
        }
        ast::UseTreeKind::Nested(ref items) => {
            let mut prefix = if use_tree.prefix.segments.is_empty() {
                prefix
            } else {
                prefix + &pprust::path_to_string(&use_tree.prefix) + "::"
            };
            for (sub_tree, _) in items {
                add_use_tree(tree, visibility, &prefix, sub_tree);
            }
        }
    }
}
<commit_msg>Remove unnecessary closure (clippy)<commit_after>use std::{fs, io, path, string::ToString};

use syntax::print::pprust;
use syntax::{ast, source_map, visit};

use tree::{Tree, Visibility};

pub struct Config {
    pub include_orphans: bool,
    pub ignored_files: Vec<path::PathBuf>,
}

pub struct Builder<'a> {
    tree: Tree,
    path: Vec<String>,
    config: Config,
    source_map: &'a source_map::SourceMap,
}

impl<'a> Builder<'a> {
    pub fn new(config: Config, crate_name: String, source_map: &'a source_map::SourceMap) -> Self {
        let tree = Tree::new_crate(crate_name);
        Builder {
            tree,
            path: vec![],
            config,
            source_map,
        }
    }

    pub fn tree(&self) -> &Tree {
        &self.tree
    }

    fn find_orphan_candidates(
        path: &[String],
        ignored_paths: &[path::PathBuf],
    ) -> Result<Vec<String>, io::Error> {
        let mut dir_path = "./src/".to_string();
        for name in path.iter() {
            dir_path.push_str(name);
            dir_path.push_str("/");
        }
        fn is_mod(entry: &fs::DirEntry) -> bool {
            let path = entry.path();
            if path.is_file() {
                if path.extension().map_or(false, |ext| ext == "rs") {
                    return file_name(entry)
                        .map_or(false, |s| (s != "mod") && (s != "lib") && (s != "main"));
                }
            } else if path.join("mod.rs").exists() {
                return true;
            }
            false
        }
        fn file_name(entry: &fs::DirEntry) -> Option<String> {
            entry
                .path()
                .file_stem()
                .and_then(|s| s.to_str())
                .map(ToString::to_string)
        }
        Ok(try!(fs::read_dir(&dir_path))
            .filter_map(|e| e.ok())
            .filter(is_mod)
            .filter(|e| !ignored_paths.contains(&e.path()))
            .filter_map(|e| file_name(&e))
            .collect::<Vec<_>>())
    }

    fn sanitize_condition(condition: String) -> String {
        let words: Vec<&str> = condition.split_whitespace().collect();
        words.join(" ")
    }
}

impl<'a> visit::Visitor<'a> for Builder<'a> {
    fn visit_item(&mut self, item: &'a ast::Item) {
        let condition = item
            .attrs
            .iter()
            .find(|attr| attr.check_name("cfg"))
            .map(|attr| {
                self.source_map
                    .span_to_snippet(attr.span)
                    .map(Builder::sanitize_condition)
                    .unwrap_or_else(|_| "".to_string())
            });

        match item.node {
            ast::ItemKind::Mod(_) => {
                let name = item.ident.to_string();
                {
                    let tree = self.tree.subtree_at_path(&self.path).unwrap();
                    let visibility = if let ast::VisibilityKind::Public = item.vis.node {
                        Visibility::Public
                    } else {
                        Visibility::Private
                    };
                    tree.insert(Tree::new_module(name.clone(), visibility, condition));
                }
                self.path.push(name);
                visit::walk_item(self, item);
                let _ = self.path.pop();
            }
            ast::ItemKind::Use(ref use_tree) => {
                {
                    let tree = self.tree.subtree_at_path(&self.path).unwrap();
                    let visibility = if let ast::VisibilityKind::Public = item.vis.node {
                        Visibility::Public
                    } else {
                        Visibility::Private
                    };

                    add_use_tree(tree, visibility, "", use_tree);
                }

                visit::walk_item(self, item);
            }
            _ => {
                visit::walk_item(self, item);
            }
        }
    }

    fn visit_mod(
        &mut self,
        m: &'a ast::Mod,
        _s: source_map::Span,
        _attrs: &[ast::Attribute],
        _n: ast::NodeId,
    ) {
        visit::walk_mod(self, m);
        if !self.config.include_orphans {
            return;
        }
        let path = &self.path;
        if let Ok(candidates) = Builder::find_orphan_candidates(path, &self.config.ignored_files) {
            let tree = self.tree.subtree_at_path(path).unwrap();
            let names: Vec<_> = tree.subtree_names();
            for candidate in candidates {
                if !names.contains(&candidate) {
                    tree.insert(Tree::new_orphan(candidate.clone()));
                }
            }
        }
    }

    fn visit_mac(&mut self, mac: &ast::Mac) {
        visit::walk_mac(self, mac);
    }
}

fn add_use_tree(tree: &mut Tree, visibility: Visibility, prefix: &str, use_tree: &ast::UseTree) {
    let prefix = if prefix.is_empty() {
        String::new()
    } else {
        prefix.to_string()
    };

    match use_tree.kind {
        ast::UseTreeKind::Simple(_, ..) => {
            let path = pprust::path_to_string(&use_tree.prefix);
            if path != "self" {
                tree.insert_use((visibility, prefix + &path));
            } else {
                let mut prefix = prefix;
                let new_len = prefix.len() - 2;
                prefix.truncate(new_len);
                tree.insert_use((visibility, prefix));
            }
        }
        ast::UseTreeKind::Glob => {
            let mut use_path = if use_tree.prefix.segments.is_empty() {
                prefix
            } else {
                prefix + &pprust::path_to_string(&use_tree.prefix) + "::"
            };
            use_path += "*";
            tree.insert_use((visibility, use_path));
        }
        ast::UseTreeKind::Nested(ref items) => {
            let mut prefix = if use_tree.prefix.segments.is_empty() {
                prefix
            } else {
                prefix + &pprust::path_to_string(&use_tree.prefix) + "::"
            };
            for (sub_tree, _) in items {
                add_use_tree(tree, visibility, &prefix, sub_tree);
            }
        }
    }
}
<|endoftext|>


<commit_before>// Copyright (c) 2016-2018 Fabian Schuiki

//! A hardware description language compiler.

extern crate clap;
extern crate llhd;
extern crate moore;
extern crate sha1;
#[macro_use]
extern crate log;

use clap::{App, Arg, ArgMatches};
use moore::common::score::NodeRef;
use moore::errors::*;
use moore::name::Name;
use moore::score::{ScoreBoard, ScoreContext};
use moore::*;
use std::{path::Path, str::FromStr};

#[derive(Debug)]
enum Language {
    Verilog,
    SystemVerilog,
    Vhdl,
}

fn main() {
    let matches = App::new(env!("CARGO_PKG_NAME"))
        .version(clap::crate_version!())
        .author(clap::crate_authors!())
        .about(clap::crate_description!())
        .arg(
            Arg::with_name("trace_scoreboard")
                .long("trace-scoreboard")
                .global(true),
        )
        .arg(
            Arg::with_name("verbosity")
                .short("v")
                .multiple(true)
                .help("Increase message verbosity"),
        )
        .arg(
            Arg::with_name("timestamp")
                .short("t")
                .help("Prepend log lines with a timestamp")
                .takes_value(true)
                .possible_values(&["none", "sec", "ms", "ns"]),
        )
        .arg(
            Arg::with_name("verbosity-opts")
                .short("V")
                .help("Sets verbosity settings")
                .takes_value(true)
                .multiple(true)
                .number_of_values(1)
                .possible_values(&["types", "expr-types", "type-contexts", "typeck", "names"])
                .global(true),
        )
        .arg(
            Arg::with_name("inc")
                .short("I")
                .value_name("DIR")
                .help("Add a search path for SystemVerilog includes")
                .multiple(true)
                .takes_value(true)
                .number_of_values(1),
        )
        .arg(
            Arg::with_name("dump_ast")
                .long("dump-ast")
                .help("Dump the parsed abstract syntax tree"),
        )
        .arg(
            Arg::with_name("emit_pkgs")
                .long("emit-pkgs")
                .help("Dump VHDL packages for debugging"),
        )
        .arg(
            Arg::with_name("lib")
                .short("l")
                .long("lib")
                .value_name("LIB")
                .help("Name of the library to compile into")
                .takes_value(true)
                .number_of_values(1),
        )
        .arg(
            Arg::with_name("elaborate")
                .short("e")
                .long("elaborate")
                .value_name("ENTITY")
                .help("Elaborate an entity or module")
                .multiple(true)
                .takes_value(true)
                .number_of_values(1),
        )
        .arg(
            Arg::with_name("INPUT")
                .help("The input files to compile")
                .multiple(true)
                .required(true),
        )
        .get_matches();

    // Configure the logger.
    let verbose = matches.occurrences_of("verbosity") as usize;
    let quiet = !matches.is_present("verbosity");
    let ts = matches
        .value_of("timestamp")
        .map(|v| {
            stderrlog::Timestamp::from_str(v).unwrap_or_else(|_| {
                clap::Error {
                    message: "invalid value for 'timestamp'".into(),
                    kind: clap::ErrorKind::InvalidValue,
                    info: None,
                }
                .exit()
            })
        })
        .unwrap_or(stderrlog::Timestamp::Off);

    stderrlog::new()
        .module("moore")
        .module("moore_common")
        .module("moore_svlog")
        .module("moore_svlog_syntax")
        .module("moore_vhdl")
        .module("moore_vhdl_syntax")
        .quiet(quiet)
        .verbosity(verbose)
        .timestamp(ts)
        .init()
        .unwrap();

    // Configure the session.
    let mut session = Session::new();
    session.opts.trace_scoreboard = matches.is_present("trace_scoreboard");
    for v in matches
        .values_of("verbosity-opts")
        .into_iter()
        .flat_map(|v| v)
    {
        session.opts.verbosity |= match v {
            "types" => Verbosity::TYPES,
            "expr-types" => Verbosity::EXPR_TYPES,
            "type-contexts" => Verbosity::TYPE_CONTEXTS,
            "typeck" => Verbosity::TYPECK,
            "names" => Verbosity::NAMES,
            _ => unreachable!(),
        };
    }

    // Invoke the compiler.
    score(&session, &matches);
}

fn score(sess: &Session, matches: &ArgMatches) {
    use crate::name::get_name_table;

    // Prepare a list of include paths.
    let include_paths: Vec<_> = match matches.values_of("inc") {
        Some(args) => args.map(|x| std::path::Path::new(x)).collect(),
        None => Vec::new(),
    };

    // Establish into which library the entities will be compiled. Later on this
    // should be made configurable per entity.
    let lib = get_name_table().intern(matches.value_of("lib").unwrap_or("work"), true);

    // Parse the input files.
    let mut failed = false;
    let mut asts = Vec::new();
    for filename in matches.values_of("INPUT").unwrap() {
        // Detect the file type.
        let language = match Path::new(&filename).extension().and_then(|s| s.to_str()) {
            Some("sv") | Some("svh") => Language::SystemVerilog,
            Some("v") => Language::Verilog,
            Some("vhd") | Some("vhdl") => Language::Vhdl,
            Some(_) => panic!("Unrecognized extension of file '{}'", filename),
            None => panic!("Unable to determine language of file '{}'", filename),
        };

        // Add the file to the source manager.
        let sm = source::get_source_manager();
        let source = match sm.open(&filename) {
            Some(s) => s,
            None => panic!("Unable to open input file '{}'", filename),
        };

        // Parse the file.
        match language {
            Language::SystemVerilog | Language::Verilog => {
                let preproc = svlog::preproc::Preprocessor::new(source, &include_paths);
                let lexer = svlog::lexer::Lexer::new(preproc);
                match svlog::parser::parse(lexer) {
                    Ok(x) => asts.push(score::Ast::Svlog(x)),
                    Err(()) => failed = true,
                }
            }
            Language::Vhdl => match vhdl::syntax::parse(source) {
                Ok(x) => asts.push(score::Ast::Vhdl(x)),
                Err(()) => failed = true,
            },
        }
    }
    if failed || sess.failed() {
        std::process::exit(1);
    }

    // Dump the AST if so requested.
    if matches.is_present("dump_ast") {
        println!("{:#?}", asts);
    }

    if matches.is_present("emit_pkgs") {
        vhdl::debug::emit_pkgs(
            sess,
            asts.iter()
                .flat_map(|ast| match *ast {
                    score::Ast::Vhdl(ref x) => x.iter(),
                    _ => [].iter(),
                })
                .collect(),
        );
    }

    // Create the scoreboard and add the initial map of libraries.
    let arenas = score::Arenas::new();
    let sb = ScoreBoard::new(&arenas);
    let vhdl_sb = vhdl::score::ScoreBoard::new(&arenas.vhdl);
    let svlog_arenas = svlog::GlobalArenas::default();
    let svlog_sb = svlog::GlobalContext::new(&sess, &svlog_arenas);

    // Elaborate the requested entities or modules.
    if let Some(names) = matches.values_of("elaborate") {
        let vhdl_phases = vhdl::lazy::LazyPhaseTable::new(&vhdl_sb);
        let ctx = ScoreContext {
            sess: sess,
            sb: &sb,
            vhdl: &vhdl_sb,
            vhdl_phases: &vhdl_phases,
            svlog: &svlog_sb,
        };
        let lib_id = ctx.add_library(lib, &asts);
        debug!("lib_id = {:?}", lib_id);
        debug!("{:?}", sb);
        for name in names {
            match elaborate_name(&ctx, lib_id, name) {
                Ok(_) => (),
                Err(_) => failed = true,
            };
        }
        if sess.failed() {
            failed = true;
        }
    }
    if failed || sess.failed() {
        std::process::exit(1);
    }

    // Extract the populated LLHD modules from the scoreboards and link them
    // together.
    let vhdl_module = vhdl_sb.llmod.into_inner();

    // Emit the module.
    llhd::assembly::write(&mut std::io::stdout().lock(), &vhdl_module);

    if sess.failed() {
        std::process::exit(1);
    }
}

/// Resolve an entity/module specificaiton of the form `[lib.]entity[.arch]` for
/// elaboration.
fn elaborate_name(ctx: &ScoreContext, lib_id: score::LibRef, input_name: &str) -> Result<(), ()> {
    let (lib, name, arch) = parse_elaborate_name(input_name)?;
    debug!(
        "parsed `{}` into (lib: {:?}, name: {:?}, arch: {:?})",
        input_name, lib, name, arch
    );

    // Resolve the library name if one was provided.
    let lib = {
        if let Some(lib) = lib {
            let rid = ctx.sb.root;
            let defs = ctx.defs(score::ScopeRef::Root(rid))?;
            match defs.get(&lib) {
                Some(&score::Def::Lib(d)) => d,
                _ => {
                    let mut d = DiagBuilder2::error(format!("Library `{}` does not exist", lib))
                        .add_note("The following libraries do exist:");
                    let mut names: Vec<_> = defs.iter().map(|(&k, _)| k).collect();
                    names.sort(); // sorts by name ID, roughly equivalent to order of declaration
                    for name in names {
                        d = d.add_note(format!("- {}", name));
                    }
                    ctx.sess.emit(d);
                    return Err(());
                }
            }
        } else {
            lib_id
        }
    };
    debug!("using library {:?}", lib);

    // Resolve the entity name.
    // TODO: Make sure that the thing we resolve to actually is a VHDL entity or
    // a SystemVerilog module. Right we happily accept packages as well.
    #[derive(Debug)]
    enum Elaborate {
        VhdlEntity(vhdl::score::EntityRef, vhdl::score::ArchRef),
        VhdlPkg(vhdl::score::PkgDeclRef),
        Svlog(NodeId), // TODO: handle svlog case
    };
    let defs = ctx.defs(lib.into())?;
    let elab = match defs.get(&name) {
        Some(&score::Def::Vhdl(vhdl::score::Def::Entity(entity))) => {
            let archs = ctx
                .vhdl()
                .archs(vhdl::score::LibRef::new(lib.into()))?
                .by_entity
                .get(&entity)
                .unwrap();
            let arch_ref = if let Some(arch) = arch {
                match archs.by_name.get(&arch) {
                    Some(&id) => id,
                    None => {
                        ctx.sess.emit(DiagBuilder2::error(format!(
                            "`{}` is not an architecture of entity `{}`",
                            arch, name
                        )));
                        return Err(());
                    }
                }
            } else {
                match archs.ordered.last() {
                    Some(&id) => id,
                    None => {
                        ctx.sess.emit(DiagBuilder2::error(format!(
                            "Entity `{}` has no architecture defined",
                            name
                        )));
                        return Err(());
                    }
                }
            };
            Elaborate::VhdlEntity(entity, arch_ref)
        }
        Some(&score::Def::Vhdl(vhdl::score::Def::Pkg(p))) => Elaborate::VhdlPkg(p),
        Some(&score::Def::Svlog(e)) => Elaborate::Svlog(e),
        _ => {
            let mut d = DiagBuilder2::error(format!("Item `{}` does not exist", name))
                .add_note("The following items are defined:");
            let mut names: Vec<_> = defs.iter().map(|(&k, _)| k).collect();
            names.sort(); // sorts by name ID, roughly equivalent to order of declaration
            for name in names {
                d = d.add_note(format!("- {}", name));
            }
            ctx.sess.emit(d);
            return Err(());
        }
    };
    debug!("elaborating {:?}", elab);

    // Generate the LLHD definition for whatever we're elaborating.
    match elab {
        Elaborate::VhdlEntity(_entity, arch) => {
            // let decl = ctx.vhdl.lldecl(arch);
            // println!("Architecture declared as {:?}", decl);
            let def = ctx.vhdl().lldef(arch)?;
            eprintln!("Architecture declared as {:?}", def);
        }
        Elaborate::VhdlPkg(pkg) => {
            use moore::vhdl::typeck::{Typeck, TypeckContext};
            let sbc = ctx.vhdl();
            let tyc = TypeckContext::new(&sbc);
            tyc.typeck(pkg);
            // use moore::vhdl::codegen::Codegen;
            // ctx.vhdl().codegen(pkg, &mut ())?;
        }
        Elaborate::Svlog(m) => {
            let mut cg = svlog::CodeGenerator::new(ctx.svlog);
            cg.emit_module(m)?;
            let code = cg.finalize();
            llhd::assembly::write(&mut std::io::stdout().lock(), &code);
        }
    }
    Ok(())
}

/// Parse an entity name of the form `(first\.)?second((arch))?` for
/// elaboration.
fn parse_elaborate_name<S: AsRef<str>>(name: S) -> Result<(Option<Name>, Name, Option<Name>), ()> {
    use self::name::get_name_table;
    let name = name.as_ref();
    let nt = get_name_table();

    // Isolate the first name.
    let x: &[_] = &['.', '('];
    let (first, rest) = {
        if let Some(pos) = name.find(x) {
            let (a, b) = name.split_at(pos);
            (a, Some(b))
        } else {
            (name, None)
        }
    };
    let first = nt.intern(first, true);

    // Isolate the second name.
    let (second, rest) = {
        if let Some(rest) = rest {
            if rest.starts_with('.') {
                let rest = &rest[1..];
                if let Some(pos) = rest.find('(') {
                    let (a, b) = rest.split_at(pos);
                    (Some(a), Some(b))
                } else {
                    (Some(rest), None)
                }
            } else {
                (None, Some(rest))
            }
        } else {
            (None, None)
        }
    };
    let second = second.map(|s| nt.intern(s, true));

    // Isolate the architecture name.
    let third = {
        if let Some(rest) = rest {
            if rest.starts_with('(') && rest.ends_with(')') {
                Some(&rest[1..rest.len() - 1])
            } else {
                None
            }
        } else {
            None
        }
    };
    let third = third.map(|t| nt.intern(t, true));

    // Return the names in the appropriate order.
    let (lib, ent) = {
        if let Some(second) = second {
            (Some(first), second)
        } else {
            (None, first)
        }
    };

    Ok((lib, ent, third))
}
<commit_msg>moore: Optionally output pre-processor artifact<commit_after>// Copyright (c) 2016-2018 Fabian Schuiki

//! A hardware description language compiler.

extern crate clap;
extern crate llhd;
extern crate moore;
extern crate sha1;
#[macro_use]
extern crate log;

use clap::{App, Arg, ArgMatches};
use moore::common::score::NodeRef;
use moore::errors::*;
use moore::name::Name;
use moore::score::{ScoreBoard, ScoreContext};
use moore::*;
use std::{path::Path, str::FromStr};

#[derive(Debug)]
enum Language {
    Verilog,
    SystemVerilog,
    Vhdl,
}

fn main() {
    let matches = App::new(env!("CARGO_PKG_NAME"))
        .version(clap::crate_version!())
        .author(clap::crate_authors!())
        .about(clap::crate_description!())
        .arg(
            Arg::with_name("trace_scoreboard")
                .long("trace-scoreboard")
                .global(true),
        )
        .arg(
            Arg::with_name("verbosity")
                .short("v")
                .multiple(true)
                .help("Increase message verbosity"),
        )
        .arg(
            Arg::with_name("timestamp")
                .short("t")
                .help("Prepend log lines with a timestamp")
                .takes_value(true)
                .possible_values(&["none", "sec", "ms", "ns"]),
        )
        .arg(
            Arg::with_name("verbosity-opts")
                .short("V")
                .help("Sets verbosity settings")
                .takes_value(true)
                .multiple(true)
                .number_of_values(1)
                .possible_values(&["types", "expr-types", "type-contexts", "typeck", "names"])
                .global(true),
        )
        .arg(
            Arg::with_name("inc")
                .short("I")
                .value_name("DIR")
                .help("Add a search path for SystemVerilog includes")
                .multiple(true)
                .takes_value(true)
                .number_of_values(1),
        )
        .arg(
            Arg::with_name("preproc")
                .short("E")
                .help("Write preprocessed input files to stdout"),
        )
        .arg(
            Arg::with_name("dump_ast")
                .long("dump-ast")
                .help("Dump the parsed abstract syntax tree"),
        )
        .arg(
            Arg::with_name("emit_pkgs")
                .long("emit-pkgs")
                .help("Dump VHDL packages for debugging"),
        )
        .arg(
            Arg::with_name("lib")
                .short("l")
                .long("lib")
                .value_name("LIB")
                .help("Name of the library to compile into")
                .takes_value(true)
                .number_of_values(1),
        )
        .arg(
            Arg::with_name("elaborate")
                .short("e")
                .long("elaborate")
                .value_name("ENTITY")
                .help("Elaborate an entity or module")
                .multiple(true)
                .takes_value(true)
                .number_of_values(1),
        )
        .arg(
            Arg::with_name("INPUT")
                .help("The input files to compile")
                .multiple(true)
                .required(true),
        )
        .get_matches();

    // Configure the logger.
    let verbose = matches.occurrences_of("verbosity") as usize;
    let quiet = !matches.is_present("verbosity");
    let ts = matches
        .value_of("timestamp")
        .map(|v| {
            stderrlog::Timestamp::from_str(v).unwrap_or_else(|_| {
                clap::Error {
                    message: "invalid value for 'timestamp'".into(),
                    kind: clap::ErrorKind::InvalidValue,
                    info: None,
                }
                .exit()
            })
        })
        .unwrap_or(stderrlog::Timestamp::Off);

    stderrlog::new()
        .module("moore")
        .module("moore_common")
        .module("moore_svlog")
        .module("moore_svlog_syntax")
        .module("moore_vhdl")
        .module("moore_vhdl_syntax")
        .quiet(quiet)
        .verbosity(verbose)
        .timestamp(ts)
        .init()
        .unwrap();

    // Configure the session.
    let mut session = Session::new();
    session.opts.trace_scoreboard = matches.is_present("trace_scoreboard");
    for v in matches
        .values_of("verbosity-opts")
        .into_iter()
        .flat_map(|v| v)
    {
        session.opts.verbosity |= match v {
            "types" => Verbosity::TYPES,
            "expr-types" => Verbosity::EXPR_TYPES,
            "type-contexts" => Verbosity::TYPE_CONTEXTS,
            "typeck" => Verbosity::TYPECK,
            "names" => Verbosity::NAMES,
            _ => unreachable!(),
        };
    }

    // Invoke the compiler.
    score(&session, &matches);
}

fn score(sess: &Session, matches: &ArgMatches) {
    use crate::name::get_name_table;

    // Prepare a list of include paths.
    let include_paths: Vec<_> = match matches.values_of("inc") {
        Some(args) => args.map(|x| std::path::Path::new(x)).collect(),
        None => Vec::new(),
    };

    // Establish into which library the entities will be compiled. Later on this
    // should be made configurable per entity.
    let lib = get_name_table().intern(matches.value_of("lib").unwrap_or("work"), true);

    // Parse the input files.
    let mut failed = false;
    let mut asts = Vec::new();
    for filename in matches.values_of("INPUT").unwrap() {
        // Detect the file type.
        let language = match Path::new(&filename).extension().and_then(|s| s.to_str()) {
            Some("sv") | Some("svh") => Language::SystemVerilog,
            Some("v") => Language::Verilog,
            Some("vhd") | Some("vhdl") => Language::Vhdl,
            Some(_) => panic!("Unrecognized extension of file '{}'", filename),
            None => panic!("Unable to determine language of file '{}'", filename),
        };

        // Add the file to the source manager.
        let sm = source::get_source_manager();
        let source = match sm.open(&filename) {
            Some(s) => s,
            None => panic!("Unable to open input file '{}'", filename),
        };

        // Parse the file.
        match language {
            Language::SystemVerilog | Language::Verilog => {
                let preproc = svlog::preproc::Preprocessor::new(source, &include_paths);
                if matches.is_present("preproc") {
                    for token in preproc {
                        print!("{}", token.unwrap().1.extract());
                    }
                    return;
                }

                let lexer = svlog::lexer::Lexer::new(preproc);
                match svlog::parser::parse(lexer) {
                    Ok(x) => asts.push(score::Ast::Svlog(x)),
                    Err(()) => failed = true,
                }
            }
            Language::Vhdl => match vhdl::syntax::parse(source) {
                Ok(x) => asts.push(score::Ast::Vhdl(x)),
                Err(()) => failed = true,
            },
        }
    }
    if failed || sess.failed() {
        std::process::exit(1);
    }

    // Dump the AST if so requested.
    if matches.is_present("dump_ast") {
        println!("{:#?}", asts);
    }

    if matches.is_present("emit_pkgs") {
        vhdl::debug::emit_pkgs(
            sess,
            asts.iter()
                .flat_map(|ast| match *ast {
                    score::Ast::Vhdl(ref x) => x.iter(),
                    _ => [].iter(),
                })
                .collect(),
        );
    }

    // Create the scoreboard and add the initial map of libraries.
    let arenas = score::Arenas::new();
    let sb = ScoreBoard::new(&arenas);
    let vhdl_sb = vhdl::score::ScoreBoard::new(&arenas.vhdl);
    let svlog_arenas = svlog::GlobalArenas::default();
    let svlog_sb = svlog::GlobalContext::new(&sess, &svlog_arenas);

    // Elaborate the requested entities or modules.
    if let Some(names) = matches.values_of("elaborate") {
        let vhdl_phases = vhdl::lazy::LazyPhaseTable::new(&vhdl_sb);
        let ctx = ScoreContext {
            sess: sess,
            sb: &sb,
            vhdl: &vhdl_sb,
            vhdl_phases: &vhdl_phases,
            svlog: &svlog_sb,
        };
        let lib_id = ctx.add_library(lib, &asts);
        debug!("lib_id = {:?}", lib_id);
        debug!("{:?}", sb);
        for name in names {
            match elaborate_name(&ctx, lib_id, name) {
                Ok(_) => (),
                Err(_) => failed = true,
            };
        }
        if sess.failed() {
            failed = true;
        }
    }
    if failed || sess.failed() {
        std::process::exit(1);
    }

    // Extract the populated LLHD modules from the scoreboards and link them
    // together.
    let vhdl_module = vhdl_sb.llmod.into_inner();

    // Emit the module.
    llhd::assembly::write(&mut std::io::stdout().lock(), &vhdl_module);

    if sess.failed() {
        std::process::exit(1);
    }
}

/// Resolve an entity/module specificaiton of the form `[lib.]entity[.arch]` for
/// elaboration.
fn elaborate_name(ctx: &ScoreContext, lib_id: score::LibRef, input_name: &str) -> Result<(), ()> {
    let (lib, name, arch) = parse_elaborate_name(input_name)?;
    debug!(
        "parsed `{}` into (lib: {:?}, name: {:?}, arch: {:?})",
        input_name, lib, name, arch
    );

    // Resolve the library name if one was provided.
    let lib = {
        if let Some(lib) = lib {
            let rid = ctx.sb.root;
            let defs = ctx.defs(score::ScopeRef::Root(rid))?;
            match defs.get(&lib) {
                Some(&score::Def::Lib(d)) => d,
                _ => {
                    let mut d = DiagBuilder2::error(format!("Library `{}` does not exist", lib))
                        .add_note("The following libraries do exist:");
                    let mut names: Vec<_> = defs.iter().map(|(&k, _)| k).collect();
                    names.sort(); // sorts by name ID, roughly equivalent to order of declaration
                    for name in names {
                        d = d.add_note(format!("- {}", name));
                    }
                    ctx.sess.emit(d);
                    return Err(());
                }
            }
        } else {
            lib_id
        }
    };
    debug!("using library {:?}", lib);

    // Resolve the entity name.
    // TODO: Make sure that the thing we resolve to actually is a VHDL entity or
    // a SystemVerilog module. Right we happily accept packages as well.
    #[derive(Debug)]
    enum Elaborate {
        VhdlEntity(vhdl::score::EntityRef, vhdl::score::ArchRef),
        VhdlPkg(vhdl::score::PkgDeclRef),
        Svlog(NodeId), // TODO: handle svlog case
    };
    let defs = ctx.defs(lib.into())?;
    let elab = match defs.get(&name) {
        Some(&score::Def::Vhdl(vhdl::score::Def::Entity(entity))) => {
            let archs = ctx
                .vhdl()
                .archs(vhdl::score::LibRef::new(lib.into()))?
                .by_entity
                .get(&entity)
                .unwrap();
            let arch_ref = if let Some(arch) = arch {
                match archs.by_name.get(&arch) {
                    Some(&id) => id,
                    None => {
                        ctx.sess.emit(DiagBuilder2::error(format!(
                            "`{}` is not an architecture of entity `{}`",
                            arch, name
                        )));
                        return Err(());
                    }
                }
            } else {
                match archs.ordered.last() {
                    Some(&id) => id,
                    None => {
                        ctx.sess.emit(DiagBuilder2::error(format!(
                            "Entity `{}` has no architecture defined",
                            name
                        )));
                        return Err(());
                    }
                }
            };
            Elaborate::VhdlEntity(entity, arch_ref)
        }
        Some(&score::Def::Vhdl(vhdl::score::Def::Pkg(p))) => Elaborate::VhdlPkg(p),
        Some(&score::Def::Svlog(e)) => Elaborate::Svlog(e),
        _ => {
            let mut d = DiagBuilder2::error(format!("Item `{}` does not exist", name))
                .add_note("The following items are defined:");
            let mut names: Vec<_> = defs.iter().map(|(&k, _)| k).collect();
            names.sort(); // sorts by name ID, roughly equivalent to order of declaration
            for name in names {
                d = d.add_note(format!("- {}", name));
            }
            ctx.sess.emit(d);
            return Err(());
        }
    };
    debug!("elaborating {:?}", elab);

    // Generate the LLHD definition for whatever we're elaborating.
    match elab {
        Elaborate::VhdlEntity(_entity, arch) => {
            // let decl = ctx.vhdl.lldecl(arch);
            // println!("Architecture declared as {:?}", decl);
            let def = ctx.vhdl().lldef(arch)?;
            eprintln!("Architecture declared as {:?}", def);
        }
        Elaborate::VhdlPkg(pkg) => {
            use moore::vhdl::typeck::{Typeck, TypeckContext};
            let sbc = ctx.vhdl();
            let tyc = TypeckContext::new(&sbc);
            tyc.typeck(pkg);
            // use moore::vhdl::codegen::Codegen;
            // ctx.vhdl().codegen(pkg, &mut ())?;
        }
        Elaborate::Svlog(m) => {
            let mut cg = svlog::CodeGenerator::new(ctx.svlog);
            cg.emit_module(m)?;
            let code = cg.finalize();
            llhd::assembly::write(&mut std::io::stdout().lock(), &code);
        }
    }
    Ok(())
}

/// Parse an entity name of the form `(first\.)?second((arch))?` for
/// elaboration.
fn parse_elaborate_name<S: AsRef<str>>(name: S) -> Result<(Option<Name>, Name, Option<Name>), ()> {
    use self::name::get_name_table;
    let name = name.as_ref();
    let nt = get_name_table();

    // Isolate the first name.
    let x: &[_] = &['.', '('];
    let (first, rest) = {
        if let Some(pos) = name.find(x) {
            let (a, b) = name.split_at(pos);
            (a, Some(b))
        } else {
            (name, None)
        }
    };
    let first = nt.intern(first, true);

    // Isolate the second name.
    let (second, rest) = {
        if let Some(rest) = rest {
            if rest.starts_with('.') {
                let rest = &rest[1..];
                if let Some(pos) = rest.find('(') {
                    let (a, b) = rest.split_at(pos);
                    (Some(a), Some(b))
                } else {
                    (Some(rest), None)
                }
            } else {
                (None, Some(rest))
            }
        } else {
            (None, None)
        }
    };
    let second = second.map(|s| nt.intern(s, true));

    // Isolate the architecture name.
    let third = {
        if let Some(rest) = rest {
            if rest.starts_with('(') && rest.ends_with(')') {
                Some(&rest[1..rest.len() - 1])
            } else {
                None
            }
        } else {
            None
        }
    };
    let third = third.map(|t| nt.intern(t, true));

    // Return the names in the appropriate order.
    let (lib, ent) = {
        if let Some(second) = second {
            (Some(first), second)
        } else {
            (None, first)
        }
    };

    Ok((lib, ent, third))
}
<|endoftext|>


<commit_before>// "Tifflin" Kernel
// - By John Hodge (thePowersGang)
//
// Core/lib/lazy_static.rs
///
use prelude::*;

/// A lazily initialised value (for `static`s)
pub struct LazyStatic<T: Send+Sync>(pub ::core::cell::UnsafeCell<Option<T>>);
unsafe impl<T: Send+Sync> Sync for LazyStatic<T> {}	// Barring the unsafe "prep" call, is Sync
unsafe impl<T: Send+Sync> Send for LazyStatic<T> {}	// Sendable because inner is sendable

#[macro_export]
macro_rules! lazystatic_init {
	() => ($crate::lib::LazyStatic(::core::cell::UnsafeCell { value: ::core::option::Option::None }));
}

impl<T: Send+Sync> LazyStatic<T>
{
	/// (unsafe) Prepare the value using the passed function
	///
	/// Unsafe because it must NOT be called where a race is possible
	pub unsafe fn prep<Fcn: FnOnce()->T>(&self, fcn: Fcn) {
		let r = &mut *self.0.get();
		assert!(r.is_none(), "LazyStatic<{}> initialised multiple times", type_name!(T));
		if r.is_none() {
			*r = Some(fcn());
		}
	}
	pub unsafe fn ls_unsafe_mut(&self) -> &mut T {
		match *self.0.get()
		{
		Some(ref mut v) => v,
		None => panic!("Dereferencing LazyStatic<{}> without initialising", type_name!(T))
		}
	}
}
impl<T: Send+Sync> ::core::ops::Deref for LazyStatic<T>
{
	type Target = T;
	fn deref(&self) -> &T {
		match unsafe { (&*self.0.get()).as_ref() } {
		Some(v) => v,
		None => panic!("Dereferencing LazyStatic<{}> without initialising", type_name!(T))
		}
	}
}

<commit_msg>(minor) Comment in LazyStatic<commit_after>// "Tifflin" Kernel
// - By John Hodge (thePowersGang)
//
// Core/lib/lazy_static.rs
///
use prelude::*;

/// A lazily initialised value (for `static`s)
pub struct LazyStatic<T: Send+Sync>(pub ::core::cell::UnsafeCell<Option<T>>);
unsafe impl<T: Send+Sync> Sync for LazyStatic<T> {}	// Barring the unsafe "prep" call, is Sync
unsafe impl<T: Send+Sync> Send for LazyStatic<T> {}	// Sendable because inner is sendable

#[macro_export]
macro_rules! lazystatic_init {
	() => ($crate::lib::LazyStatic(::core::cell::UnsafeCell { value: ::core::option::Option::None }));
}

impl<T: Send+Sync> LazyStatic<T>
{
	/// (unsafe) Prepare the value using the passed function
	///
	/// Unsafe because it must NOT be called where a race is possible
	pub unsafe fn prep<Fcn: FnOnce()->T>(&self, fcn: Fcn) {
		let r = &mut *self.0.get();
		assert!(r.is_none(), "LazyStatic<{}> initialised multiple times", type_name!(T));
		if r.is_none() {
			*r = Some(fcn());
		}
	}
	/// (unsafe) Obtain a mutable reference to the interior
	pub unsafe fn ls_unsafe_mut(&self) -> &mut T {
		match *self.0.get()
		{
		Some(ref mut v) => v,
		None => panic!("Dereferencing LazyStatic<{}> without initialising", type_name!(T))
		}
	}
}
impl<T: Send+Sync> ::core::ops::Deref for LazyStatic<T>
{
	type Target = T;
	fn deref(&self) -> &T {
		match unsafe { (&*self.0.get()).as_ref() } {
		Some(v) => v,
		None => panic!("Dereferencing LazyStatic<{}> without initialising", type_name!(T))
		}
	}
}

<|endoftext|>