<commit_before>use std::collections::BTreeMap;
use rusqlite::Connection;
use dbVariableName;
use dbJob;
use dbFsFile;
use serde_yaml;
use std::collections::HashSet;

fn job_provide_list_by_job(conn: &Connection) -> Vec<dbVariableName::VariableName> {
    let mut items = Vec::<dbVariableName::VariableName>::new();
    let provide_list_prep_rc = conn.prepare(
        "SELECT DISTINCT
            VARIABLE_NAME.id,
            VARIABLE_NAME.name
            FROM JOBPROVIDE , VARIABLE_NAME
        WHERE
            VARIABLE_NAME.id = JOBPROVIDE.fk_variable
            order by VARIABLE_NAME.name",
    );
    if provide_list_prep_rc.is_err() {
        error!("SQL issue in job_provide_list");
        return items;
    }
    let mut provide_list_prep = provide_list_prep_rc.unwrap();

    let result_row = provide_list_prep.query(&[]);
    let mut rox = result_row.unwrap();

    while let Some(row_query) = rox.next() {
        let row = row_query.unwrap();
        let bill = dbVariableName::VariableName {
            id: row.get(0),
            name: row.get(1),
        };
        debug!("bill:{}", bill.id);
        items.push(bill);
    }
    return items;
}

fn job_depend_list_by_job(conn: &Connection) -> Vec<dbVariableName::VariableName> {
    let mut items = Vec::<dbVariableName::VariableName>::new();
    let provide_list_prep_rc = conn.prepare(
        "SELECT DISTINCT
            VARIABLE_NAME.id,
            VARIABLE_NAME.name
            FROM JOBDEPEND , VARIABLE_NAME
        WHERE
            VARIABLE_NAME.id = JOBDEPEND.fk_variable
            order by VARIABLE_NAME.name",
    );
    if provide_list_prep_rc.is_err() {
        error!("SQL issue in job_provide_list");
        return items;
    }
    let mut provide_list_prep = provide_list_prep_rc.unwrap();

    let result_row = provide_list_prep.query(&[]);
    let mut rox = result_row.unwrap();

    while let Some(row_query) = rox.next() {
        let row = row_query.unwrap();
        let bill = dbVariableName::VariableName {
            id: row.get(0),
            name: row.get(1),
        };
        debug!("bill:{}", bill.id);
        items.push(bill);
    }
    return items;
}

fn job_targets_list_by_job(conn: &Connection) -> Vec<dbVariableName::VariableName> {
    let mut items = Vec::<dbVariableName::VariableName>::new();

    let mut hs_depend: HashSet<i32> = HashSet::new();
    let list_dep = job_depend_list_by_job(&conn);
    for item in list_dep {
        hs_depend.insert(item.id);
    }
    for item in job_provide_list_by_job(&conn) {
        let foo = hs_depend.contains(&item.id);
        if foo == false {
            items.push(item)
        }
    }
    return items;
}

fn list_job(conn: &Connection, fk_variable_name: &i32) -> Vec<dbJob::Job> {
    let mut items = Vec::<dbJob::Job>::new();
    let provide_list_prep_rc = conn.prepare(
        "SELECT DISTINCT
            JOB.id,
            JOB.name,
            JOB.fk_file
            FROM JOB, JOBPROVIDE , VARIABLE_NAME
        WHERE
            JOB.id = JOBPROVIDE.fk_job
            AND
            VARIABLE_NAME.id = JOBPROVIDE.fk_variable
            AND
            VARIABLE_NAME.id = ?1
            order by JOB.name",
    );
    if provide_list_prep_rc.is_err() {
        error!("SQL issue in job_provide_list");
        return items;
    }
    let mut provide_list_prep = provide_list_prep_rc.unwrap();

    let result_row = provide_list_prep.query(&[fk_variable_name]);
    let mut rox = result_row.unwrap();

    while let Some(row_query) = rox.next() {
        let row = row_query.unwrap();
        let bill = dbJob::Job {
            id: row.get(0),
            name: row.get(1),
            fk_file: row.get(2),
        };
        debug!("bill:{}", bill.id);
        items.push(bill);
    }
    return items;
}

fn list_file(conn: &Connection, fk_variable_name: &i32) -> Vec<dbFsFile::FsFile> {
    let mut items = Vec::<dbFsFile::FsFile>::new();
    let provide_list_prep_rc = conn.prepare(
        "SELECT DISTINCT
            FS_FILE.id,
            FS_FILE.fk_fs_dir,
            FS_FILE.name
        FROM
            FS_FILE
        WHERE
            FS_FILE.id = ?1
            order by FS_FILE.name",
    );
    if provide_list_prep_rc.is_err() {
        error!("SQL issue in job_provide_list");
        return items;
    }
    let mut provide_list_prep = provide_list_prep_rc.unwrap();

    let result_row = provide_list_prep.query(&[fk_variable_name]);
    let mut rox = result_row.unwrap();

    while let Some(row_query) = rox.next() {
        let row = row_query.unwrap();
        let bill = dbFsFile::FsFile {
            id: row.get(0),
            fk_fs_dir: row.get(1),
            name: row.get(2),
        };
        debug!("bill:{}", bill.id);
        items.push(bill);
    }
    return items;
}

fn shared_foo(conn: &Connection, variables: &Vec<dbVariableName::VariableName>) -> String {
    let mut map = BTreeMap::new();
    for var_name in variables {
        let mut v_new = Vec::<BTreeMap<&str, String>>::new();
        let doop = list_job(&conn, &var_name.id);
        for item in doop {
            let mut map2 = BTreeMap::new();
            map2.insert("name", item.name.to_string());
            let woop = list_file(&conn, &item.fk_file);
            for op in woop {
                map2.insert("file_name", op.name.to_string());
            }
            v_new.push(map2);
        }
        map.insert(var_name.name.to_string(), v_new);
    }
    return serde_yaml::to_string(&map).unwrap();
}

pub fn process_list_provides(conn: &Connection) {
    let henry = job_provide_list_by_job(&conn);
    println!("{}", shared_foo(&conn, &henry));
}

pub fn process_list_targets(conn: &Connection) {
    let henry = job_targets_list_by_job(&conn);
    println!("{}", shared_foo(&conn, &henry));
}
<commit_msg>Apply rustfmt<commit_after>use std::collections::BTreeMap;
use rusqlite::Connection;
use dbVariableName;
use dbJob;
use dbFsFile;
use serde_yaml;
use std::collections::HashSet;

fn job_provide_list_by_job(conn: &Connection) -> Vec<dbVariableName::VariableName> {
    let mut items = Vec::<dbVariableName::VariableName>::new();
    let provide_list_prep_rc = conn.prepare(
        "SELECT DISTINCT
            VARIABLE_NAME.id,
            VARIABLE_NAME.name
            FROM JOBPROVIDE , VARIABLE_NAME
        WHERE
            VARIABLE_NAME.id = JOBPROVIDE.fk_variable
            order by VARIABLE_NAME.name",
    );
    if provide_list_prep_rc.is_err() {
        error!("SQL issue in job_provide_list");
        return items;
    }
    let mut provide_list_prep = provide_list_prep_rc.unwrap();

    let result_row = provide_list_prep.query(&[]);
    let mut rox = result_row.unwrap();

    while let Some(row_query) = rox.next() {
        let row = row_query.unwrap();
        let bill = dbVariableName::VariableName {
            id: row.get(0),
            name: row.get(1),
        };
        debug!("bill:{}", bill.id);
        items.push(bill);
    }
    return items;
}

fn job_depend_list_by_job(conn: &Connection) -> Vec<dbVariableName::VariableName> {
    let mut items = Vec::<dbVariableName::VariableName>::new();
    let provide_list_prep_rc = conn.prepare(
        "SELECT DISTINCT
            VARIABLE_NAME.id,
            VARIABLE_NAME.name
            FROM JOBDEPEND , VARIABLE_NAME
        WHERE
            VARIABLE_NAME.id = JOBDEPEND.fk_variable
            order by VARIABLE_NAME.name",
    );
    if provide_list_prep_rc.is_err() {
        error!("SQL issue in job_provide_list");
        return items;
    }
    let mut provide_list_prep = provide_list_prep_rc.unwrap();

    let result_row = provide_list_prep.query(&[]);
    let mut rox = result_row.unwrap();

    while let Some(row_query) = rox.next() {
        let row = row_query.unwrap();
        let bill = dbVariableName::VariableName {
            id: row.get(0),
            name: row.get(1),
        };
        debug!("bill:{}", bill.id);
        items.push(bill);
    }
    return items;
}

fn job_targets_list_by_job(conn: &Connection) -> Vec<dbVariableName::VariableName> {
    let mut items = Vec::<dbVariableName::VariableName>::new();

    let mut hs_depend: HashSet<i32> = HashSet::new();
    let list_dep = job_depend_list_by_job(&conn);
    for item in list_dep {
        hs_depend.insert(item.id);
    }
    for item in job_provide_list_by_job(&conn) {
        let foo = hs_depend.contains(&item.id);
        if foo == false {
            items.push(item)
        }
    }
    return items;
}

fn list_job(conn: &Connection, fk_variable_name: &i32) -> Vec<dbJob::Job> {
    let mut items = Vec::<dbJob::Job>::new();
    let provide_list_prep_rc = conn.prepare(
        "SELECT DISTINCT
            JOB.id,
            JOB.name,
            JOB.fk_file
            FROM JOB, JOBPROVIDE , VARIABLE_NAME
        WHERE
            JOB.id = JOBPROVIDE.fk_job
            AND
            VARIABLE_NAME.id = JOBPROVIDE.fk_variable
            AND
            VARIABLE_NAME.id = ?1
            order by JOB.name",
    );
    if provide_list_prep_rc.is_err() {
        error!("SQL issue in job_provide_list");
        return items;
    }
    let mut provide_list_prep = provide_list_prep_rc.unwrap();

    let result_row = provide_list_prep.query(&[fk_variable_name]);
    let mut rox = result_row.unwrap();

    while let Some(row_query) = rox.next() {
        let row = row_query.unwrap();
        let bill = dbJob::Job {
            id: row.get(0),
            name: row.get(1),
            fk_file: row.get(2),
        };
        debug!("bill:{}", bill.id);
        items.push(bill);
    }
    return items;
}

fn list_file(conn: &Connection, fk_variable_name: &i32) -> Vec<dbFsFile::FsFile> {
    let mut items = Vec::<dbFsFile::FsFile>::new();
    let provide_list_prep_rc = conn.prepare(
        "SELECT DISTINCT
            FS_FILE.id,
            FS_FILE.fk_fs_dir,
            FS_FILE.name
        FROM
            FS_FILE
        WHERE
            FS_FILE.id = ?1
            order by FS_FILE.name",
    );
    if provide_list_prep_rc.is_err() {
        error!("SQL issue in job_provide_list");
        return items;
    }
    let mut provide_list_prep = provide_list_prep_rc.unwrap();

    let result_row = provide_list_prep.query(&[fk_variable_name]);
    let mut rox = result_row.unwrap();

    while let Some(row_query) = rox.next() {
        let row = row_query.unwrap();
        let bill = dbFsFile::FsFile {
            id: row.get(0),
            fk_fs_dir: row.get(1),
            name: row.get(2),
        };
        debug!("bill:{}", bill.id);
        items.push(bill);
    }
    return items;
}

fn shared_foo(conn: &Connection, variables: &Vec<dbVariableName::VariableName>) -> String {
    let mut map = BTreeMap::new();
    for var_name in variables {
        let mut v_new = Vec::<BTreeMap<&str, String>>::new();
        let doop = list_job(&conn, &var_name.id);
        for item in doop {
            let mut map2 = BTreeMap::new();
            map2.insert("name", item.name.to_string());
            let woop = list_file(&conn, &item.fk_file);
            for op in woop {
                map2.insert("file_name", op.name.to_string());
            }
            v_new.push(map2);
        }
        map.insert(var_name.name.to_string(), v_new);
    }
    return serde_yaml::to_string(&map).unwrap();
}

pub fn process_list_provides(conn: &Connection) {
    let henry = job_provide_list_by_job(&conn);
    println!("{}", shared_foo(&conn, &henry));
}

pub fn process_list_targets(conn: &Connection) {
    let henry = job_targets_list_by_job(&conn);
    println!("{}", shared_foo(&conn, &henry));
}
<|endoftext|>


<commit_before>fn main() {
    #[cfg(target_os="openbsd")]
    println!(r"cargo:rustc-link-search=/usr/local/lib");
}
<commit_msg>Fix linking on FreeBSD<commit_after>fn main() {
    #[cfg(any(target_os="openbsd", target_os="freebsd"))]
    println!(r"cargo:rustc-link-search=/usr/local/lib");
}
<|endoftext|>


<commit_before>//! This build script detects target platforms that lack proper support for
//! atomics and sets `cfg` flags accordingly.

use std::env;
use std::process::Command;
use std::str::{self, FromStr};

#[cfg(feature = "kv_unstable")]
#[path = "src/kv/value/internal/cast/primitive.rs"]
mod primitive;

fn main() {
    let minor = match rustc_minor_version() {
        Some(minor) => minor,
        None => return,
    };

    let target = match rustc_target() {
        Some(target) => target,
        None => return,
    };

    match &target[..] {
        "thumbv6m-none-eabi"
        | "msp430-none-elf"
        | "riscv32i-unknown-none-elf"
        | "riscv32imc-unknown-none-elf" => {}

        _ => {
            println!("cargo:rustc-cfg=atomic_cas");
        }
    };

    match &target[..] {
        "msp430-none-elf" | "riscv32i-unknown-none-elf" | "riscv32imc-unknown-none-elf" => {}

        _ => {
            println!("cargo:rustc-cfg=has_atomics");
        }
    };

    // If the Rust version is at least 1.46.0 then we can use type ids at compile time
    if minor >= 47 {
        println!("cargo:rustc-cfg=const_type_id");
    }

    // Generate sorted type id lookup
    #[cfg(feature = "kv_unstable")]
    primitive::generate();

    println!("cargo:rustc-cfg=srcbuild");
    println!("cargo:rerun-if-changed=build.rs");
}

fn rustc_target() -> Option<String> {
    env::var("TARGET").ok()
}

// From the `serde` build script
fn rustc_minor_version() -> Option<u32> {
    let rustc = match env::var_os("RUSTC") {
        Some(rustc) => rustc,
        None => return None,
    };

    let output = match Command::new(rustc).arg("--version").output() {
        Ok(output) => output,
        Err(_) => return None,
    };

    let version = match str::from_utf8(&output.stdout) {
        Ok(version) => version,
        Err(_) => return None,
    };

    let mut pieces = version.split('.');
    if pieces.next() != Some("rustc 1") {
        return None;
    }

    let next = match pieces.next() {
        Some(next) => next,
        None => return None,
    };

    u32::from_str(next).ok()
}
<commit_msg>build.rs: Convert to functions<commit_after>//! This build script detects target platforms that lack proper support for
//! atomics and sets `cfg` flags accordingly.

use std::env;
use std::process::Command;
use std::str::{self, FromStr};

#[cfg(feature = "kv_unstable")]
#[path = "src/kv/value/internal/cast/primitive.rs"]
mod primitive;

fn main() {
    let minor = match rustc_minor_version() {
        Some(minor) => minor,
        None => return,
    };

    let target = match rustc_target() {
        Some(target) => target,
        None => return,
    };

    if target_has_atomic_cas(&target) {
        println!("cargo:rustc-cfg=atomic_cas");
    }

    if target_has_atomics(&target) {
        println!("cargo:rustc-cfg=has_atomics");
    }

    // If the Rust version is at least 1.46.0 then we can use type ids at compile time
    if minor >= 47 {
        println!("cargo:rustc-cfg=const_type_id");
    }

    // Generate sorted type id lookup
    #[cfg(feature = "kv_unstable")]
    primitive::generate();

    println!("cargo:rustc-cfg=srcbuild");
    println!("cargo:rerun-if-changed=build.rs");
}

fn target_has_atomic_cas(target: &str) -> bool {
    match &target[..] {
        "thumbv6m-none-eabi"
        | "msp430-none-elf"
        | "riscv32i-unknown-none-elf"
        | "riscv32imc-unknown-none-elf" => false,
        _ => true,
    }
}

fn target_has_atomics(target: &str) -> bool {
    match &target[..] {
        "msp430-none-elf" | "riscv32i-unknown-none-elf" | "riscv32imc-unknown-none-elf" => false,
        _ => true,
    }
}

fn rustc_target() -> Option<String> {
    env::var("TARGET").ok()
}

// From the `serde` build script
fn rustc_minor_version() -> Option<u32> {
    let rustc = match env::var_os("RUSTC") {
        Some(rustc) => rustc,
        None => return None,
    };

    let output = match Command::new(rustc).arg("--version").output() {
        Ok(output) => output,
        Err(_) => return None,
    };

    let version = match str::from_utf8(&output.stdout) {
        Ok(version) => version,
        Err(_) => return None,
    };

    let mut pieces = version.split('.');
    if pieces.next() != Some("rustc 1") {
        return None;
    }

    let next = match pieces.next() {
        Some(next) => next,
        None => return None,
    };

    u32::from_str(next).ok()
}
<|endoftext|>


<commit_before>extern crate git2;
extern crate rustc_version;

use git2::{Repository, DescribeOptions, DescribeFormatOptions};
use std::env;
use std::path::Path;
use std::fs::File;
use std::io::{self, Write};
use std::process::exit;
use rustc_version::version_matches;

fn get_version() -> String {
  let profile = env::var("PROFILE").unwrap();
  if profile == "release" {
    String::new()
  } else {
    let manifest_dir = env::var("CARGO_MANIFEST_DIR").unwrap();
    let repo = match Repository::open(&manifest_dir) {
      Ok(r) => r,
      Err(_) => return String::new()
    };
    let version = repo
    .describe(
      DescribeOptions::new().describe_tags().show_commit_oid_as_fallback(true)
    )
    .unwrap()
    .format(
      Some(DescribeFormatOptions::new().dirty_suffix("-dirty"))
    )
    .unwrap();
    String::from("-") + &version
  }
}

fn main() {
  if !version_matches(">= 1.8.0") {
    writeln!(&mut io::stderr(), "bins requires at least Rust 1.8.0").unwrap();
    exit(1);
  }
  let profile = env::var("PROFILE").unwrap();
  let version = get_version();
  let out_dir = env::var("OUT_DIR").unwrap();
  let dest_path = Path::new(&out_dir).join("git_short_tag.rs");
  let mut f = File::create(&dest_path).unwrap();
  f.write_all(format!("
      fn git_short_tag() -> &'static str {{
          \"{}\"
      }}
  ", version).as_bytes()).unwrap();
}
<commit_msg>fix(build): remove unused variable<commit_after>extern crate git2;
extern crate rustc_version;

use git2::{Repository, DescribeOptions, DescribeFormatOptions};
use std::env;
use std::path::Path;
use std::fs::File;
use std::io::{self, Write};
use std::process::exit;
use rustc_version::version_matches;

fn get_version() -> String {
  let profile = env::var("PROFILE").unwrap();
  if profile == "release" {
    String::new()
  } else {
    let manifest_dir = env::var("CARGO_MANIFEST_DIR").unwrap();
    let repo = match Repository::open(&manifest_dir) {
      Ok(r) => r,
      Err(_) => return String::new()
    };
    let version = repo
    .describe(
      DescribeOptions::new().describe_tags().show_commit_oid_as_fallback(true)
    )
    .unwrap()
    .format(
      Some(DescribeFormatOptions::new().dirty_suffix("-dirty"))
    )
    .unwrap();
    String::from("-") + &version
  }
}

fn main() {
  if !version_matches(">= 1.8.0") {
    writeln!(&mut io::stderr(), "bins requires at least Rust 1.8.0").unwrap();
    exit(1);
  }
  let version = get_version();
  let out_dir = env::var("OUT_DIR").unwrap();
  let dest_path = Path::new(&out_dir).join("git_short_tag.rs");
  let mut f = File::create(&dest_path).unwrap();
  f.write_all(format!("
      fn git_short_tag() -> &'static str {{
          \"{}\"
      }}
  ", version).as_bytes()).unwrap();
}
<|endoftext|>


<commit_before>#[derive(Debug)]
pub struct AttributeInfo {
    pub attribute_name_index: u16,
    pub attribute_length: u32,
    pub info: Vec<u8>,
}

#[derive(Debug)]
pub struct ExceptionEntry {
    pub start_pc: u16,
    pub end_pc: u16,
    pub handler_pc: u16,
    pub catch_type: u16
}

#[derive(Debug)]
pub struct CodeAttribute {
    pub max_stack: u16,
    pub max_locals: u16,
    pub code_length: u32,
    pub code: Vec<u8>,
    pub exception_table_length: u16,
    pub exception_table: Vec<ExceptionEntry>,
    pub attributes_count: u16,
    pub attributes: Vec<AttributeInfo>,
}

#[derive(Copy,Clone,Debug)]
#[repr(u8)]
pub enum VerificationTypeInfo {
    Top               = 0,
    Integer           = 1,
    Float             = 2,
    Double            = 3,
    Long              = 4,
    Null              = 5,
    UninitializedThis = 6,
    Object            = 7,
    Uninitialized     = 8,
}

#[derive(Debug)]
pub enum StackMapFrame {
    SameFrame { frame_type: u8 },
    SameLocals1StackItemFrame { frame_type: u8, stack: VerificationTypeInfo },
    SameLocals1StackItemFrameExtended { frame_type: u8, offset_delta: u16, stack: VerificationTypeInfo },
    ChopFrame { frame_type: u8, offset_delta: u16 },
    SameFrameExtended { frame_type: u8, offset_delta: u16 },
    AppendFrame { frame_type: u8, offset_delta: u16, locals: Vec<VerificationTypeInfo> },
    FullFrame { frame_type: u8, offset_delta: u16, number_of_locals: u16, locals: Vec<VerificationTypeInfo>, number_of_stack_items: u16, stack: Vec<VerificationTypeInfo> },
}

#[derive(Debug)]
pub struct StackMapTableAttribute {
    pub number_of_entries: u16,
    pub entries: Vec<StackMapFrame>,
}

#[derive(Debug)]
pub struct ExceptionsAttribute {
    pub exception_table_length: u16,
    pub exception_table: Vec<u16>,
}

#[derive(Debug)]
pub struct ConstantValueAttribute {
    pub constant_value_index: u16,
}
<commit_msg>Reformat types more nicely<commit_after>#[derive(Debug)]
pub struct AttributeInfo {
    pub attribute_name_index: u16,
    pub attribute_length: u32,
    pub info: Vec<u8>,
}

#[derive(Debug)]
pub struct ExceptionEntry {
    pub start_pc: u16,
    pub end_pc: u16,
    pub handler_pc: u16,
    pub catch_type: u16
}

#[derive(Debug)]
pub struct CodeAttribute {
    pub max_stack: u16,
    pub max_locals: u16,
    pub code_length: u32,
    pub code: Vec<u8>,
    pub exception_table_length: u16,
    pub exception_table: Vec<ExceptionEntry>,
    pub attributes_count: u16,
    pub attributes: Vec<AttributeInfo>,
}

#[derive(Copy,Clone,Debug)]
#[repr(u8)]
pub enum VerificationTypeInfo {
    Top               = 0,
    Integer           = 1,
    Float             = 2,
    Double            = 3,
    Long              = 4,
    Null              = 5,
    UninitializedThis = 6,
    Object            = 7,
    Uninitialized     = 8,
}

#[derive(Debug)]
pub enum StackMapFrame {
    SameFrame {
        frame_type: u8,
	},
    SameLocals1StackItemFrame {
		frame_type: u8,
		stack: VerificationTypeInfo,
	},
    SameLocals1StackItemFrameExtended {
		frame_type: u8,
		offset_delta: u16,
		stack: VerificationTypeInfo,
	},
    ChopFrame {
		frame_type: u8,
		offset_delta: u16,
	},
    SameFrameExtended {
		frame_type: u8,
		offset_delta: u16,
	},
    AppendFrame {
		frame_type: u8,
		offset_delta: u16,
		locals: Vec<VerificationTypeInfo>,
	},
    FullFrame {
		frame_type: u8,
		offset_delta: u16,
		number_of_locals: u16,
		locals: Vec<VerificationTypeInfo>,
		number_of_stack_items: u16,
		stack: Vec<VerificationTypeInfo>,
	},
}

#[derive(Debug)]
pub struct StackMapTableAttribute {
    pub number_of_entries: u16,
    pub entries: Vec<StackMapFrame>,
}

#[derive(Debug)]
pub struct ExceptionsAttribute {
    pub exception_table_length: u16,
    pub exception_table: Vec<u16>,
}

#[derive(Debug)]
pub struct ConstantValueAttribute {
    pub constant_value_index: u16,
}
<|endoftext|>


<commit_before>use serde::Deserialize;
use std::collections::HashMap;
use std::os::unix::fs::MetadataExt;
use std::os::unix::fs::PermissionsExt;
use std::path::{Path, PathBuf};
use std::sync::Arc;
use std::time::Instant;
use structopt::StructOpt;
use walkdir::WalkDir;
use watchman_client::prelude::*;

#[derive(StructOpt, Debug)]
#[structopt(about = "Audit Watchman's in-memory database with the filesystem")]
pub(crate) struct AuditCmd {
    #[structopt(name = "path", parse(from_os_str))]
    path: PathBuf,
}

query_result_type! {
    struct AuditQueryResult {
        name: NameField,
        mode: ModeAndPermissionsField,
        size: SizeField,
        mtime: MTimeField,
        oclock: ObservedClockField,
        ino: InodeNumberField,
    }
}

impl AuditCmd {
    pub(crate) async fn run(&self) -> crate::Result<()> {
        let client = Connector::new().connect().await?;
        let resolved = Arc::new(
            client
                .resolve_root(CanonicalPath::canonicalize(&self.path)?)
                .await?,
        );

        if resolved.watcher() == "eden" {
            return Err(watchman_client::Error::Generic(format!(
                "{} is an EdenFS mount - no need to audit",
                resolved.project_root().display()
            )));
        }

        let config = client.get_config(&resolved).await?;
        let mut ignore_dirs = config.ignore_dirs.unwrap_or_default().clone();

        // TODO: This list is duplicated in the Watchman query below.
        ignore_dirs.push(".hg".into());
        ignore_dirs.push(".git".into());
        ignore_dirs.push(".svn".into());

        let filesystem_state_handle = {
            let resolved = resolved.clone();
            tokio::spawn(async move {
                let mut filesystem_state: HashMap<PathBuf, std::fs::Metadata> = HashMap::new();

                let start_crawl = Instant::now();

                // Allocate outside the loop to save time.
                let resolved_path = resolved.path();

                for entry in WalkDir::new(&resolved_path)
                    .into_iter()
                    .filter_entry(|entry| {
                        // It sucks we have to do this here and in the loop body below. It would be nice if traversal and filtering were folded into the same callback.
                        let from_root = match entry.path().strip_prefix(&resolved_path) {
                            Ok(from_root) => from_root,
                            Err(_) => return true,
                        };
                        let join_storage;
                        let path_from_root = match resolved.project_relative_path() {
                            Some(relpath) => {
                                join_storage = relpath.join(&from_root);
                                join_storage.as_ref()
                            }
                            None => from_root,
                        };
                        !ignore_dirs.iter().any(|i| i == path_from_root)
                    })
                {
                    let entry = match entry {
                        Ok(entry) => entry,
                        Err(err) => {
                            eprintln!("error while traversing directory: {}", err);
                            continue;
                        }
                    };

                    let relpath = match entry.path().strip_prefix(&resolved_path) {
                        Ok(relpath) => relpath,
                        Err(err) => {
                            eprintln!(
                                "unable to form relative path from {} to {}: {}",
                                resolved_path.display(),
                                entry.path().display(),
                                err
                            );
                            continue;
                        }
                    };

                    let metadata = match entry.metadata() {
                        Ok(metadata) => metadata,
                        Err(err) => {
                            eprintln!(
                                "error fetching metadata for {}: {}",
                                entry.path().display(),
                                err
                            );
                            continue;
                        }
                    };
                    filesystem_state.insert(relpath.to_path_buf(), metadata);
                }
                // Watchman doesn't return information about the root, so remove it here.
                filesystem_state.remove(&PathBuf::new());
                eprintln!("Crawled filesystem in {:#?}", start_crawl.elapsed());

                filesystem_state
            })
        };

        let start_query = Instant::now();

        // TODO: should we ignore fresh instance results?

        use Expr::*;
        let result = client
            .query::<AuditQueryResult>(
                &resolved,
                QueryRequestCommon {
                    expression: Some(All(vec![
                        Exists,
                        Not(Box::new(Any(vec![
                            DirName(DirNameTerm {
                                path: ".git".into(),
                                depth: None,
                            }),
                            DirName(DirNameTerm {
                                path: ".hg".into(),
                                depth: None,
                            }),
                            DirName(DirNameTerm {
                                path: ".svn".into(),
                                depth: None,
                            }),
                            Name(NameTerm {
                                paths: vec![".git".into()],
                                wholename: true,
                            }),
                            Name(NameTerm {
                                paths: vec![".svn".into()],
                                wholename: true,
                            }),
                            Name(NameTerm {
                                paths: vec![".hg".into()],
                                wholename: true,
                            }),
                        ]))),
                    ])),
                    ..Default::default()
                },
            )
            .await?;

        eprintln!("Queried Watchman in {:#?}", start_query.elapsed());

        let filesystem_state = filesystem_state_handle.await.unwrap();

        let diff_start = Instant::now();

        let watchman_files = match result.files {
            Some(files) => files,
            None => {
                return Err(watchman_client::Error::Generic(
                    "No files set in result {}".into(),
                ));
            }
        };

        let mut any_differences = false;
        let mut phantoms = vec![];
        let mut missing = vec![];

        let mut watchman_state: HashMap<&Path, &AuditQueryResult> =
            HashMap::with_capacity(watchman_files.len());
        for watchman_file in &watchman_files {
            let filename = &*watchman_file.name;
            watchman_state.insert(filename, watchman_file);

            let metadata = match filesystem_state.get(filename) {
                Some(metadata) => metadata,
                None => {
                    phantoms.push(watchman_file);
                    continue;
                }
            };

            let mut diffs = Vec::new();

            if *watchman_file.mode != u64::from(metadata.permissions().mode()) {
                diffs.push(format!(
                    "watchman mode is {} vs. fs {}",
                    *watchman_file.mode,
                    metadata.permissions().mode()
                ));
            }

            if metadata.is_file() && *watchman_file.size != metadata.size() {
                diffs.push(format!(
                    "watchman size is {} vs. fs {}",
                    *watchman_file.size,
                    metadata.len()
                ));
            }

            if metadata.is_file() && *watchman_file.mtime != metadata.mtime() {
                diffs.push(format!(
                    "watchman mtime is {} vs. fs {}",
                    *watchman_file.mtime,
                    metadata.mtime()
                ));
            }

            if *watchman_file.ino != metadata.ino() {
                diffs.push(format!(
                    "watchman ino is {} vs. fs {}",
                    *watchman_file.ino,
                    metadata.ino()
                ));
            }

            if !diffs.is_empty() {
                println!(
                    "Conflicting information for {}:",
                    watchman_file.name.display()
                );
                println!("  oclock is {:#?}", *watchman_file.oclock);
                for diff in diffs {
                    println!("  {}", diff);
                }
                any_differences = true;
            }
        }

        for (path, val) in &filesystem_state {
            if !watchman_state.contains_key(&path.as_path()) {
                missing.push((path, val));
            }
        }

        phantoms.sort_by(|x, y| x.name.cmp(&y.name));
        missing.sort_by(|x, y| x.0.cmp(&y.0));

        if !phantoms.is_empty() {
            println!(
                "There are {} items reported by watchman not on the filesystem:",
                phantoms.len()
            );
            for phantom in &phantoms {
                println!("  {}", phantom.name.display());
            }
            any_differences = true;
        }

        if !missing.is_empty() {
            println!(
                "There are {} items on the filesystem not reported by watchman:",
                missing.len()
            );
            for (path, _) in &missing {
                println!("  {}", path.display());
            }
            any_differences = true;
        }

        if any_differences {
            // This is dumb, but Rust doesn't have a standard way to return
            // nonzero exit codes yet.
            std::process::exit(1);
        }

        eprintln!("Diffed in {:#?}", diff_start.elapsed());

        Ok(())
    }
}
<commit_msg>audit: use ahash<commit_after>use ahash::AHashMap;
use serde::Deserialize;
use std::os::unix::fs::MetadataExt;
use std::os::unix::fs::PermissionsExt;
use std::path::{Path, PathBuf};
use std::sync::Arc;
use std::time::Instant;
use structopt::StructOpt;
use walkdir::WalkDir;
use watchman_client::prelude::*;

#[derive(StructOpt, Debug)]
#[structopt(about = "Audit Watchman's in-memory database with the filesystem")]
pub(crate) struct AuditCmd {
    #[structopt(name = "path", parse(from_os_str))]
    path: PathBuf,
}

query_result_type! {
    struct AuditQueryResult {
        name: NameField,
        mode: ModeAndPermissionsField,
        size: SizeField,
        mtime: MTimeField,
        oclock: ObservedClockField,
        ino: InodeNumberField,
    }
}

impl AuditCmd {
    pub(crate) async fn run(&self) -> crate::Result<()> {
        let client = Connector::new().connect().await?;
        let resolved = Arc::new(
            client
                .resolve_root(CanonicalPath::canonicalize(&self.path)?)
                .await?,
        );

        if resolved.watcher() == "eden" {
            return Err(watchman_client::Error::Generic(format!(
                "{} is an EdenFS mount - no need to audit",
                resolved.project_root().display()
            )));
        }

        let config = client.get_config(&resolved).await?;
        let mut ignore_dirs = config.ignore_dirs.unwrap_or_default().clone();

        // TODO: This list is duplicated in the Watchman query below.
        ignore_dirs.push(".hg".into());
        ignore_dirs.push(".git".into());
        ignore_dirs.push(".svn".into());

        let filesystem_state_handle = {
            let resolved = resolved.clone();
            tokio::spawn(async move {
                let mut filesystem_state: AHashMap<PathBuf, std::fs::Metadata> = AHashMap::new();

                let start_crawl = Instant::now();

                // Allocate outside the loop to save time.
                let resolved_path = resolved.path();

                for entry in WalkDir::new(&resolved_path)
                    .into_iter()
                    .filter_entry(|entry| {
                        // It sucks we have to do this here and in the loop body below. It would be nice if traversal and filtering were folded into the same callback.
                        let from_root = match entry.path().strip_prefix(&resolved_path) {
                            Ok(from_root) => from_root,
                            Err(_) => return true,
                        };
                        let join_storage;
                        let path_from_root = match resolved.project_relative_path() {
                            Some(relpath) => {
                                join_storage = relpath.join(&from_root);
                                join_storage.as_ref()
                            }
                            None => from_root,
                        };
                        !ignore_dirs.iter().any(|i| i == path_from_root)
                    })
                {
                    let entry = match entry {
                        Ok(entry) => entry,
                        Err(err) => {
                            eprintln!("error while traversing directory: {}", err);
                            continue;
                        }
                    };

                    let relpath = match entry.path().strip_prefix(&resolved_path) {
                        Ok(relpath) => relpath,
                        Err(err) => {
                            eprintln!(
                                "unable to form relative path from {} to {}: {}",
                                resolved_path.display(),
                                entry.path().display(),
                                err
                            );
                            continue;
                        }
                    };

                    let metadata = match entry.metadata() {
                        Ok(metadata) => metadata,
                        Err(err) => {
                            eprintln!(
                                "error fetching metadata for {}: {}",
                                entry.path().display(),
                                err
                            );
                            continue;
                        }
                    };
                    filesystem_state.insert(relpath.to_path_buf(), metadata);
                }
                // Watchman doesn't return information about the root, so remove it here.
                filesystem_state.remove(&PathBuf::new());
                eprintln!("Crawled filesystem in {:#?}", start_crawl.elapsed());

                filesystem_state
            })
        };

        let start_query = Instant::now();

        // TODO: should we ignore fresh instance results?

        use Expr::*;
        let result = client
            .query::<AuditQueryResult>(
                &resolved,
                QueryRequestCommon {
                    expression: Some(All(vec![
                        Exists,
                        Not(Box::new(Any(vec![
                            DirName(DirNameTerm {
                                path: ".git".into(),
                                depth: None,
                            }),
                            DirName(DirNameTerm {
                                path: ".hg".into(),
                                depth: None,
                            }),
                            DirName(DirNameTerm {
                                path: ".svn".into(),
                                depth: None,
                            }),
                            Name(NameTerm {
                                paths: vec![".git".into()],
                                wholename: true,
                            }),
                            Name(NameTerm {
                                paths: vec![".svn".into()],
                                wholename: true,
                            }),
                            Name(NameTerm {
                                paths: vec![".hg".into()],
                                wholename: true,
                            }),
                        ]))),
                    ])),
                    ..Default::default()
                },
            )
            .await?;

        eprintln!("Queried Watchman in {:#?}", start_query.elapsed());

        let filesystem_state = filesystem_state_handle.await.unwrap();

        let diff_start = Instant::now();

        let watchman_files = match result.files {
            Some(files) => files,
            None => {
                return Err(watchman_client::Error::Generic(
                    "No files set in result {}".into(),
                ));
            }
        };

        let mut any_differences = false;
        let mut phantoms = vec![];
        let mut missing = vec![];

        let mut watchman_state: AHashMap<&Path, &AuditQueryResult> =
            AHashMap::with_capacity(watchman_files.len());
        for watchman_file in &watchman_files {
            let filename = &*watchman_file.name;
            watchman_state.insert(filename, watchman_file);

            let metadata = match filesystem_state.get(filename) {
                Some(metadata) => metadata,
                None => {
                    phantoms.push(watchman_file);
                    continue;
                }
            };

            let mut diffs = Vec::new();

            if *watchman_file.mode != u64::from(metadata.permissions().mode()) {
                diffs.push(format!(
                    "watchman mode is {} vs. fs {}",
                    *watchman_file.mode,
                    metadata.permissions().mode()
                ));
            }

            if metadata.is_file() && *watchman_file.size != metadata.size() {
                diffs.push(format!(
                    "watchman size is {} vs. fs {}",
                    *watchman_file.size,
                    metadata.len()
                ));
            }

            if metadata.is_file() && *watchman_file.mtime != metadata.mtime() {
                diffs.push(format!(
                    "watchman mtime is {} vs. fs {}",
                    *watchman_file.mtime,
                    metadata.mtime()
                ));
            }

            if *watchman_file.ino != metadata.ino() {
                diffs.push(format!(
                    "watchman ino is {} vs. fs {}",
                    *watchman_file.ino,
                    metadata.ino()
                ));
            }

            if !diffs.is_empty() {
                println!(
                    "Conflicting information for {}:",
                    watchman_file.name.display()
                );
                println!("  oclock is {:#?}", *watchman_file.oclock);
                for diff in diffs {
                    println!("  {}", diff);
                }
                any_differences = true;
            }
        }

        for (path, val) in &filesystem_state {
            if !watchman_state.contains_key(&path.as_path()) {
                missing.push((path, val));
            }
        }

        phantoms.sort_by(|x, y| x.name.cmp(&y.name));
        missing.sort_by(|x, y| x.0.cmp(&y.0));

        if !phantoms.is_empty() {
            println!(
                "There are {} items reported by watchman not on the filesystem:",
                phantoms.len()
            );
            for phantom in &phantoms {
                println!("  {}", phantom.name.display());
            }
            any_differences = true;
        }

        if !missing.is_empty() {
            println!(
                "There are {} items on the filesystem not reported by watchman:",
                missing.len()
            );
            for (path, _) in &missing {
                println!("  {}", path.display());
            }
            any_differences = true;
        }

        if any_differences {
            // This is dumb, but Rust doesn't have a standard way to return
            // nonzero exit codes yet.
            std::process::exit(1);
        }

        eprintln!("Diffed in {:#?}", diff_start.elapsed());

        Ok(())
    }
}
<|endoftext|>


<commit_before>#![feature(test)]
extern crate test;

use std::ops::{Mul};

extern crate nalgebra;
use self::nalgebra::{DMat, Transpose};

extern crate rblas;
use rblas::matrix::ops::{Gemm, Syrk};
use rblas::math::mat::Mat;
use rblas::attribute::Transpose as BlasTranspose;

const NSAMPLES: usize = 6 * 1000;
const NOBSERVED: usize = 10 * 10;
const NHIDDEN: usize = 20;

#[bench]
fn bench_ortho_nmf_weights_dividend_nalgebra(bencher: &mut test::Bencher) {
    let samples = DMat::<f64>::new_ones(NSAMPLES, NOBSERVED);
    let hidden = DMat::<f64>::new_ones(NHIDDEN, NOBSERVED);
    bencher.iter(|| {
        samples.clone().mul(hidden.transpose())
    });
}

#[bench]
fn bench_ortho_nmf_weights_dividend_rblas(bencher: &mut test::Bencher) {
    let samples = Mat::<f64>::fill(1., NSAMPLES, NOBSERVED);
    let hidden = Mat::<f64>::fill(1., NHIDDEN, NOBSERVED);
    bencher.iter(|| {
        let mut result = Mat::<f64>::fill(1., NSAMPLES, NHIDDEN);
        Gemm::gemm(
            &1.,
            BlasTranspose::NoTrans, &samples,
            BlasTranspose::Trans, &hidden,
            &0.,
            &mut result);
        result
    });
}

#[bench]
fn bench_ortho_nmf_weights_divisor_nalgebra(bencher: &mut test::Bencher) {
    let weights = DMat::<f64>::new_ones(NSAMPLES, NHIDDEN);
    let hidden = DMat::<f64>::new_ones(NHIDDEN, NOBSERVED);
    bencher.iter(|| {
        weights.clone().mul(&hidden).mul(hidden.transpose())
    });
}

#[bench]
fn bench_ortho_nmf_weights_divisor_rblas(bencher: &mut test::Bencher) {
    let weights = Mat::<f64>::fill(1., NSAMPLES, NHIDDEN);
    let hidden = Mat::<f64>::fill(1., NHIDDEN, NOBSERVED);
    let mut tmp = Mat::<f64>::new(NSAMPLES, NOBSERVED);
    bencher.iter(|| {
        let mut result = Mat::<f64>::fill(1., NSAMPLES, NHIDDEN);
        Gemm::gemm(
            &1.,
            BlasTranspose::NoTrans, &weights,
            BlasTranspose::NoTrans, &hidden,
            &0.,
            &mut tmp);
        Gemm::gemm(
            &1.,
            BlasTranspose::NoTrans, &tmp,
            BlasTranspose::Trans, &hidden,
            &0.,
            &mut result);
        result
    });
}
<commit_msg>more benchmarks<commit_after>#![feature(test)]
extern crate test;

use std::ops::{Mul, Add};

extern crate nalgebra;
use self::nalgebra::{DMat, Transpose};

extern crate rblas;
use rblas::matrix::ops::{Gemm};
use rblas::math::mat::Mat;
use rblas::attribute::Transpose as BlasTranspose;

const NSAMPLES: usize = 6 * 1000;
const NOBSERVED: usize = 10 * 10;
const NHIDDEN: usize = 20;

#[bench]
fn bench_ortho_nmf_weights_dividend_nalgebra(bencher: &mut test::Bencher) {
    let samples = DMat::<f64>::new_ones(NSAMPLES, NOBSERVED);
    let hidden = DMat::<f64>::new_ones(NHIDDEN, NOBSERVED);
    bencher.iter(|| {
        samples.clone().mul(hidden.transpose())
    });
}

#[bench]
fn bench_ortho_nmf_weights_dividend_rblas(bencher: &mut test::Bencher) {
    let samples = Mat::<f64>::fill(1., NSAMPLES, NOBSERVED);
    let hidden = Mat::<f64>::fill(1., NHIDDEN, NOBSERVED);
    bencher.iter(|| {
        let mut result = Mat::<f64>::fill(1., NSAMPLES, NHIDDEN);
        Gemm::gemm(
            &1.,
            BlasTranspose::NoTrans, &samples,
            BlasTranspose::Trans, &hidden,
            &0.,
            &mut result);
        result
    });
}

#[bench]
fn bench_ortho_nmf_weights_divisor_nalgebra(bencher: &mut test::Bencher) {
    let weights = DMat::<f64>::new_ones(NSAMPLES, NHIDDEN);
    let hidden = DMat::<f64>::new_ones(NHIDDEN, NOBSERVED);
    bencher.iter(|| {
        weights.clone().mul(&hidden).mul(hidden.transpose())
    });
}

#[bench]
fn bench_ortho_nmf_weights_divisor_rblas(bencher: &mut test::Bencher) {
    let weights = Mat::<f64>::fill(1., NSAMPLES, NHIDDEN);
    let hidden = Mat::<f64>::fill(1., NHIDDEN, NOBSERVED);
    let mut reconstruction = Mat::<f64>::new(NSAMPLES, NOBSERVED);
    bencher.iter(|| {
        let mut result = Mat::<f64>::fill(1., NSAMPLES, NHIDDEN);
        Gemm::gemm(
            &1.,
            BlasTranspose::NoTrans, &weights,
            BlasTranspose::NoTrans, &hidden,
            &0.,
            &mut reconstruction);
        Gemm::gemm(
            &1.,
            BlasTranspose::NoTrans, &reconstruction,
            BlasTranspose::Trans, &hidden,
            &0.,
            &mut result);
        result
    });
}

// TODO why is this so much faster than the other ones
// when it does so much more ?
// is the clone so expensive ?
#[bench]
fn bench_ortho_nmf_hidden_divisor_nalgebra(bencher: &mut test::Bencher) {
    let weights = DMat::<f64>::new_ones(NSAMPLES, NHIDDEN);
    let hidden = DMat::<f64>::new_ones(NHIDDEN, NOBSERVED);
    bencher.iter(|| {
        let gamma = DMat::<f64>::new_ones(NHIDDEN, NHIDDEN);
        weights.transpose().mul(&weights).mul(&hidden)
            .add(gamma.mul(&hidden))
    });
}
<|endoftext|>


<commit_before>use std::cell::RefCell;
use std::ops::Deref;

use finalfusion::embeddings::Embeddings as FFEmbeddings;
use finalfusion::storage::StorageWrap;
use finalfusion::vocab::{Vocab, VocabWrap};
use ndarray::{Array1, CowArray, Ix1};
use serde_derive::{Deserialize, Serialize};

use crate::Numberer;

pub enum LookupResult<'a> {
    Index(usize),
    Embedding(CowArray<'a, f32, Ix1>),
}

impl<'a> LookupResult<'a> {
    pub fn index(&self) -> Option<usize> {
        use self::LookupResult::*;

        match self {
            Index(idx) => Some(*idx),
            Embedding(_) => None,
        }
    }
}

/// The type of lookup.
pub enum LookupType {
    /// Lookup that returns indices.
    Index,

    /// Lookup that returns embeddings with the given size.
    Embedding(usize),
}

/// Trait for feature index lookup.
///
/// In dpar, the parser state is encoded as a vector of integers. The integers
/// are offsets into an embedding matrix. However, one might want to use two
/// different kinds of embeddings:
///
/// 1. Pretrained embeddings: the vector should use the offset of the datum
///    in an existing embedding matrix.
/// 2. Embeddings trained through backprop: the vector should generate an
///    offset for the datum during training.
///
/// To accommodate for these different representation, this trait is provided,
/// so that parser state vectorizers can be agnostic to where the indices come
/// from.
#[allow(clippy::len_without_is_empty)]
pub trait Lookup {
    /// Size of the table.
    fn len(&self) -> usize;

    /// Lookup a feature.
    fn lookup(&self, feature: &str) -> Option<LookupResult<'_>>;

    /// Lookup type.
    fn lookup_type(&self) -> LookupType;

    /// Null value.
    fn null(&self) -> LookupResult<'_>;

    // Unknown value.
    fn unknown(&self) -> LookupResult<'_>;
}

pub struct Embeddings {
    inner: FFEmbeddings<VocabWrap, StorageWrap>,
    null: Array1<f32>,
    unknown: Array1<f32>,
}

impl Deref for Embeddings {
    type Target = FFEmbeddings<VocabWrap, StorageWrap>;

    fn deref(&self) -> &Self::Target {
        &self.inner
    }
}

impl From<FFEmbeddings<VocabWrap, StorageWrap>> for Embeddings {
    fn from(embeddings: FFEmbeddings<VocabWrap, StorageWrap>) -> Self {
        let null = Array1::zeros(embeddings.dims());

        // Compute the average embedding.
        let mut unknown = Array1::zeros(embeddings.dims());
        for (_, embed) in &embeddings {
            unknown += &embed;
        }
        let l2norm = unknown.dot(&unknown).sqrt();
        if l2norm != 0f32 {
            unknown /= l2norm;
        }

        Embeddings {
            inner: embeddings,
            null,
            unknown,
        }
    }
}

impl Lookup for Embeddings {
    fn len(&self) -> usize {
        self.vocab().words_len()
    }

    fn lookup(&self, feature: &str) -> Option<LookupResult<'_>> {
        self.embedding(feature).map(LookupResult::Embedding)
    }

    fn lookup_type(&self) -> LookupType {
        LookupType::Embedding(self.dims())
    }

    fn null(&self) -> LookupResult<'_> {
        LookupResult::Embedding(CowArray::from(self.null.view()))
    }

    fn unknown(&self) -> LookupResult<'_> {
        LookupResult::Embedding(CowArray::from(self.unknown.view()))
    }
}

pub struct MutableLookupTable(RefCell<Numberer<String>>);

impl MutableLookupTable {
    pub fn new() -> Self {
        MutableLookupTable(RefCell::new(Numberer::new(1)))
    }
}

impl Default for MutableLookupTable {
    fn default() -> Self {
        Self::new()
    }
}

impl Lookup for MutableLookupTable {
    fn len(&self) -> usize {
        self.0.borrow().len() + 1
    }

    fn lookup(&self, feature: &str) -> Option<LookupResult<'_>> {
        let mut numberer = self.0.borrow_mut();
        Some(LookupResult::Index(numberer.add(feature.to_owned())))
    }

    fn lookup_type(&self) -> LookupType {
        LookupType::Index
    }

    fn null(&self) -> LookupResult<'_> {
        LookupResult::Index(0)
    }

    fn unknown(&self) -> LookupResult<'_> {
        LookupResult::Index(0)
    }
}

#[derive(Eq, PartialEq, Serialize, Deserialize)]
pub struct LookupTable(Numberer<String>);

impl From<MutableLookupTable> for LookupTable {
    fn from(t: MutableLookupTable) -> Self {
        LookupTable(t.0.into_inner())
    }
}

impl Lookup for LookupTable {
    fn len(&self) -> usize {
        self.0.len() + 1
    }

    fn lookup(&self, feature: &str) -> Option<LookupResult<'_>> {
        self.0.number(feature).map(LookupResult::Index)
    }

    fn lookup_type(&self) -> LookupType {
        LookupType::Index
    }

    fn null(&self) -> LookupResult<'_> {
        LookupResult::Index(0)
    }

    fn unknown(&self) -> LookupResult<'_> {
        LookupResult::Index(0)
    }
}

pub struct BoxedLookup(Option<Box<dyn Lookup>>);

impl BoxedLookup {
    /// Construct a boxed lookup from a lookup.
    pub fn new<L>(lookup: L) -> Self
    where
        L: Into<Box<dyn Lookup>>,
    {
        BoxedLookup(Some(lookup.into()))
    }

    /// Get the lookup as a reference.
    pub fn as_ref(&self) -> Option<&dyn Lookup> {
        self.0.as_ref().map(AsRef::as_ref)
    }
}

impl Default for BoxedLookup {
    fn default() -> Self {
        BoxedLookup(None)
    }
}

#[cfg(test)]
mod tests {
    use super::{Lookup, LookupTable, MutableLookupTable};

    #[test]
    fn mutable_lookup_table_test() {
        let table = MutableLookupTable::new();
        assert_eq!(table.lookup("a").unwrap().index(), Some(1));
        assert_eq!(table.lookup("b").unwrap().index(), Some(2));
        assert_eq!(table.lookup("a").unwrap().index(), Some(1));
        assert_eq!(table.null().index(), Some(0));
        assert_eq!(table.unknown().index(), Some(0));
    }

    #[test]
    fn lookup_table_test() {
        let table = MutableLookupTable::new();
        assert_eq!(table.lookup("a").unwrap().index(), Some(1));
        assert_eq!(table.lookup("b").unwrap().index(), Some(2));

        let table: LookupTable = table.into();
        assert_eq!(table.lookup("a").unwrap().index(), Some(1));
        assert_eq!(table.lookup("a").unwrap().index(), Some(1));
        assert_eq!(table.lookup("b").unwrap().index(), Some(2));
        assert!(table.lookup("c").is_none());
        assert_eq!(table.null().index(), Some(0));
        assert_eq!(table.unknown().index(), Some(0));
    }
}
<commit_msg>Compute the unknown vector from the subquantizers for quantized embeddings<commit_after>use std::cell::RefCell;
use std::ops::Deref;

use finalfusion::embeddings::Embeddings as FFEmbeddings;
use finalfusion::storage::StorageWrap;
use finalfusion::vocab::{Vocab, VocabWrap};
use ndarray::{Array1, Axis, CowArray, Ix1};
use serde_derive::{Deserialize, Serialize};

use crate::Numberer;

pub enum LookupResult<'a> {
    Index(usize),
    Embedding(CowArray<'a, f32, Ix1>),
}

impl<'a> LookupResult<'a> {
    pub fn index(&self) -> Option<usize> {
        use self::LookupResult::*;

        match self {
            Index(idx) => Some(*idx),
            Embedding(_) => None,
        }
    }
}

/// The type of lookup.
pub enum LookupType {
    /// Lookup that returns indices.
    Index,

    /// Lookup that returns embeddings with the given size.
    Embedding(usize),
}

/// Trait for feature index lookup.
///
/// In dpar, the parser state is encoded as a vector of integers. The integers
/// are offsets into an embedding matrix. However, one might want to use two
/// different kinds of embeddings:
///
/// 1. Pretrained embeddings: the vector should use the offset of the datum
///    in an existing embedding matrix.
/// 2. Embeddings trained through backprop: the vector should generate an
///    offset for the datum during training.
///
/// To accommodate for these different representation, this trait is provided,
/// so that parser state vectorizers can be agnostic to where the indices come
/// from.
#[allow(clippy::len_without_is_empty)]
pub trait Lookup {
    /// Size of the table.
    fn len(&self) -> usize;

    /// Lookup a feature.
    fn lookup(&self, feature: &str) -> Option<LookupResult<'_>>;

    /// Lookup type.
    fn lookup_type(&self) -> LookupType;

    /// Null value.
    fn null(&self) -> LookupResult<'_>;

    // Unknown value.
    fn unknown(&self) -> LookupResult<'_>;
}

pub struct Embeddings {
    inner: FFEmbeddings<VocabWrap, StorageWrap>,
    null: Array1<f32>,
    unknown: Array1<f32>,
}

impl Deref for Embeddings {
    type Target = FFEmbeddings<VocabWrap, StorageWrap>;

    fn deref(&self) -> &Self::Target {
        &self.inner
    }
}

impl From<FFEmbeddings<VocabWrap, StorageWrap>> for Embeddings {
    fn from(embeddings: FFEmbeddings<VocabWrap, StorageWrap>) -> Self {
        let null = Array1::zeros(embeddings.dims());

        let mut unknown = match embeddings.storage() {
            StorageWrap::QuantizedArray(_) | StorageWrap::MmapQuantizedArray(_) => {
                let quantizer = match embeddings.storage() {
                    StorageWrap::QuantizedArray(a) => a.quantizer(),
                    StorageWrap::MmapQuantizedArray(a) => a.quantizer(),
                    _ => unreachable!(),
                };

                // The subquantizer tensor has shape:
                //
                // n_quantizers * n_centroids * quantizer_len
                //
                // Here we average the centroids of the quantizers to obtain
                // the average subquantizers of shape:
                //
                // n_quantizers * quantizer_len
                let avg_subquantizers = quantizer.subquantizers().sum_axis(Axis(1))
                    / quantizer.subquantizers().len_of(Axis(1)) as f32;

                // Now we reshape to get the average quantizer embedding.
                let embed_len = avg_subquantizers.len();
                let avg_quantizer_embed = avg_subquantizers
                    .into_shape((embed_len,))
                    .expect("Cannot reshape averaged subquantizers into vector");

                // Finally, project the embedding into the original space.
                match quantizer.projection() {
                    Some(ref projection) => avg_quantizer_embed.dot(&projection.t()),
                    None => avg_quantizer_embed,
                }
            }
            _ => {
                let mut unknown = Array1::zeros(embeddings.dims());

                for (_, embed) in &embeddings {
                    unknown += &embed.view();
                }

                unknown
            }
        };

        let l2norm = unknown.dot(&unknown).sqrt();
        if l2norm != 0f32 {
            unknown /= l2norm;
        }

        Embeddings {
            inner: embeddings,
            null,
            unknown,
        }
    }
}

impl Lookup for Embeddings {
    fn len(&self) -> usize {
        self.vocab().words_len()
    }

    fn lookup(&self, feature: &str) -> Option<LookupResult<'_>> {
        self.embedding(feature).map(LookupResult::Embedding)
    }

    fn lookup_type(&self) -> LookupType {
        LookupType::Embedding(self.dims())
    }

    fn null(&self) -> LookupResult<'_> {
        LookupResult::Embedding(CowArray::from(self.null.view()))
    }

    fn unknown(&self) -> LookupResult<'_> {
        LookupResult::Embedding(CowArray::from(self.unknown.view()))
    }
}

pub struct MutableLookupTable(RefCell<Numberer<String>>);

impl MutableLookupTable {
    pub fn new() -> Self {
        MutableLookupTable(RefCell::new(Numberer::new(1)))
    }
}

impl Default for MutableLookupTable {
    fn default() -> Self {
        Self::new()
    }
}

impl Lookup for MutableLookupTable {
    fn len(&self) -> usize {
        self.0.borrow().len() + 1
    }

    fn lookup(&self, feature: &str) -> Option<LookupResult<'_>> {
        let mut numberer = self.0.borrow_mut();
        Some(LookupResult::Index(numberer.add(feature.to_owned())))
    }

    fn lookup_type(&self) -> LookupType {
        LookupType::Index
    }

    fn null(&self) -> LookupResult<'_> {
        LookupResult::Index(0)
    }

    fn unknown(&self) -> LookupResult<'_> {
        LookupResult::Index(0)
    }
}

#[derive(Eq, PartialEq, Serialize, Deserialize)]
pub struct LookupTable(Numberer<String>);

impl From<MutableLookupTable> for LookupTable {
    fn from(t: MutableLookupTable) -> Self {
        LookupTable(t.0.into_inner())
    }
}

impl Lookup for LookupTable {
    fn len(&self) -> usize {
        self.0.len() + 1
    }

    fn lookup(&self, feature: &str) -> Option<LookupResult<'_>> {
        self.0.number(feature).map(LookupResult::Index)
    }

    fn lookup_type(&self) -> LookupType {
        LookupType::Index
    }

    fn null(&self) -> LookupResult<'_> {
        LookupResult::Index(0)
    }

    fn unknown(&self) -> LookupResult<'_> {
        LookupResult::Index(0)
    }
}

pub struct BoxedLookup(Option<Box<dyn Lookup>>);

impl BoxedLookup {
    /// Construct a boxed lookup from a lookup.
    pub fn new<L>(lookup: L) -> Self
    where
        L: Into<Box<dyn Lookup>>,
    {
        BoxedLookup(Some(lookup.into()))
    }

    /// Get the lookup as a reference.
    pub fn as_ref(&self) -> Option<&dyn Lookup> {
        self.0.as_ref().map(AsRef::as_ref)
    }
}

impl Default for BoxedLookup {
    fn default() -> Self {
        BoxedLookup(None)
    }
}

#[cfg(test)]
mod tests {
    use super::{Lookup, LookupTable, MutableLookupTable};

    #[test]
    fn mutable_lookup_table_test() {
        let table = MutableLookupTable::new();
        assert_eq!(table.lookup("a").unwrap().index(), Some(1));
        assert_eq!(table.lookup("b").unwrap().index(), Some(2));
        assert_eq!(table.lookup("a").unwrap().index(), Some(1));
        assert_eq!(table.null().index(), Some(0));
        assert_eq!(table.unknown().index(), Some(0));
    }

    #[test]
    fn lookup_table_test() {
        let table = MutableLookupTable::new();
        assert_eq!(table.lookup("a").unwrap().index(), Some(1));
        assert_eq!(table.lookup("b").unwrap().index(), Some(2));

        let table: LookupTable = table.into();
        assert_eq!(table.lookup("a").unwrap().index(), Some(1));
        assert_eq!(table.lookup("a").unwrap().index(), Some(1));
        assert_eq!(table.lookup("b").unwrap().index(), Some(2));
        assert!(table.lookup("c").is_none());
        assert_eq!(table.null().index(), Some(0));
        assert_eq!(table.unknown().index(), Some(0));
    }
}
<|endoftext|>


<commit_before>use std::fs::File;
use std::io::ErrorKind;

fn main() {
    let f = File::open("hello.txt");

    let f = match f {
        Ok(file) => file,
        Err(ref error) if error.kind() == ErrorKind::NotFound => {
            match File::create("hello.txt") {
                Ok(fc) => fc,
                Err(e) => {
                    panic!("File creation failed: {:?}", e)
                }
            }
        },
        Err(error) => {
            panic!("Problem with opening file: {:?}", error)
        }
    };

    // Example of unwrap usage to shorten code (Ok(res) => res and Err(e) => panic!("...")
    let f_unwrap = File::open("hello.txt").unwrap();
    // Similar to unwrap but it's possible to provide panic message
    let f_expect = File::open("hello.txt").expect("Failed to open file hello.txt");
}
<commit_msg>Example of how function can propagate error up the call stack<commit_after>use std::fs::File;
use std::io::ErrorKind;

fn main() {
    let f = File::open("hello.txt");

    let f = match f {
        Ok(file) => file,
        Err(ref error) if error.kind() == ErrorKind::NotFound => {
            match File::create("hello.txt") {
                Ok(fc) => fc,
                Err(e) => {
                    panic!("File creation failed: {:?}", e)
                }
            }
        },
        Err(error) => {
            panic!("Problem with opening file: {:?}", error)
        }
    };

    // Example of unwrap usage to shorten code (Ok(res) => res and Err(e) => panic!("...")
    let f_unwrap = File::open("hello.txt").unwrap();
    // Similar to unwrap but it's possible to provide panic message
    let f_expect = File::open("hello.txt").expect("Failed to open file hello.txt");
}

fn read_username_from_file() -> Result<String, io::Error> {
    let f = File::open("hello.txt");

    let mut f = match f {
        Ok(file) => file,
        Err(e) => return Err(e),
    };

    let mut s = String::new();

    match f.read_to_string(&mut s) {
        Ok(_) => Ok(s),
        Err(e) => Err(e),
    }
}<|endoftext|>


<commit_before>// Copyright (c) 2017 The vulkano developers
// Licensed under the Apache License, Version 2.0
// <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT
// license <LICENSE-MIT or http://opensource.org/licenses/MIT>,
// at your option. All files in the project carrying such
// notice may not be copied, modified, or distributed except
// according to those terms.

//! This example contains the source code of the second part of the guide at http://vulkano.rs.
//!
//! It is not commented, as the explanations can be found in the guide itself.

#[macro_use]
extern crate vulkano;
#[macro_use]
extern crate vulkano_shader_derive;

use std::sync::Arc;
use vulkano::buffer::BufferUsage;
use vulkano::buffer::CpuAccessibleBuffer;
use vulkano::command_buffer::AutoCommandBufferBuilder;
use vulkano::command_buffer::CommandBuffer;
use vulkano::descriptor::descriptor_set::PersistentDescriptorSet;
use vulkano::device::Device;
use vulkano::device::DeviceExtensions;
use vulkano::instance::Features;
use vulkano::instance::Instance;
use vulkano::instance::InstanceExtensions;
use vulkano::instance::PhysicalDevice;
use vulkano::pipeline::ComputePipeline;
use vulkano::sync::GpuFuture;

fn main() {
    let instance = Instance::new(None, &InstanceExtensions::none(), None)
        .expect("failed to create instance");

    let physical = PhysicalDevice::enumerate(&instance).next().expect("no device available");

    let queue_family = physical.queue_families()
        .find(|&q| q.supports_graphics())
        .expect("couldn't find a graphical queue family");

    let (device, mut queues) = {
        Device::new(physical, &Features::none(), &DeviceExtensions::none(),
                    [(queue_family, 0.5)].iter().cloned()).expect("failed to create device")
    };

    let queue = queues.next().unwrap();

    // Introduction to compute operations
    let data_iter = 0 .. 65536;
    let data_buffer = CpuAccessibleBuffer::from_iter(device.clone(), BufferUsage::all(),
                                                    Some(queue.family()),
                                                    data_iter).expect("failed to create buffer");

    // Compute pipelines
    mod cs {
        #[derive(VulkanoShader)]
        #[ty = "compute"]
        #[src = "
    #version 450

    layout(local_size_x = 64, local_size_y = 1, local_size_z = 1) in;

    layout(set = 0, binding = 0) buffer Data {
        uint data[];
    } data;

    void main() {
        uint idx = gl_GlobalInvocationID.x;
        data.data[idx] *= 12;
    }"
        ]
        struct Dummy;
    }

    let shader = cs::Shader::load(device.clone())
        .expect("failed to create shader module");
    let compute_pipeline = Arc::new(ComputePipeline::new(device.clone(), &shader.main_entry_point(), &())
        .expect("failed to create compute pipeline"));

    // Descriptor sets
    let set = Arc::new(PersistentDescriptorSet::start(compute_pipeline.clone(), 0)
        .add_buffer(data_buffer.clone()).unwrap()
        .build().unwrap()
    );

    // Dispatch
    let command_buffer = AutoCommandBufferBuilder::new(device.clone(), queue.family()).unwrap()
        .dispatch([1024, 1, 1], compute_pipeline.clone(), set.clone(), ()).unwrap()
        .build().unwrap();

    let finished = command_buffer.execute(queue.clone()).unwrap();
    finished.then_signal_fence_and_flush().unwrap()
        .wait(None).unwrap();

    let content = data_buffer.read().unwrap();
    for (n, val) in content.iter().enumerate() {
        assert_eq!(*val, n as u32 * 12);
    }

    println!("Everything succeeded!");
}
<commit_msg>Use supports_compute() in compute example.<commit_after>// Copyright (c) 2017 The vulkano developers
// Licensed under the Apache License, Version 2.0
// <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT
// license <LICENSE-MIT or http://opensource.org/licenses/MIT>,
// at your option. All files in the project carrying such
// notice may not be copied, modified, or distributed except
// according to those terms.

//! This example contains the source code of the second part of the guide at http://vulkano.rs.
//!
//! It is not commented, as the explanations can be found in the guide itself.

#[macro_use]
extern crate vulkano;
#[macro_use]
extern crate vulkano_shader_derive;

use std::sync::Arc;
use vulkano::buffer::BufferUsage;
use vulkano::buffer::CpuAccessibleBuffer;
use vulkano::command_buffer::AutoCommandBufferBuilder;
use vulkano::command_buffer::CommandBuffer;
use vulkano::descriptor::descriptor_set::PersistentDescriptorSet;
use vulkano::device::Device;
use vulkano::device::DeviceExtensions;
use vulkano::instance::Features;
use vulkano::instance::Instance;
use vulkano::instance::InstanceExtensions;
use vulkano::instance::PhysicalDevice;
use vulkano::pipeline::ComputePipeline;
use vulkano::sync::GpuFuture;

fn main() {
    let instance = Instance::new(None, &InstanceExtensions::none(), None)
        .expect("failed to create instance");

    let physical = PhysicalDevice::enumerate(&instance).next().expect("no device available");

    let queue_family = physical.queue_families()
        .find(|&q| q.supports_compute())
        .expect("couldn't find a compute queue family");

    let (device, mut queues) = {
        Device::new(physical, &Features::none(), &DeviceExtensions::none(),
                    [(queue_family, 0.5)].iter().cloned()).expect("failed to create device")
    };

    let queue = queues.next().unwrap();

    // Introduction to compute operations
    let data_iter = 0 .. 65536;
    let data_buffer = CpuAccessibleBuffer::from_iter(device.clone(), BufferUsage::all(),
                                                    Some(queue.family()),
                                                    data_iter).expect("failed to create buffer");

    // Compute pipelines
    mod cs {
        #[derive(VulkanoShader)]
        #[ty = "compute"]
        #[src = "
    #version 450

    layout(local_size_x = 64, local_size_y = 1, local_size_z = 1) in;

    layout(set = 0, binding = 0) buffer Data {
        uint data[];
    } data;

    void main() {
        uint idx = gl_GlobalInvocationID.x;
        data.data[idx] *= 12;
    }"
        ]
        struct Dummy;
    }

    let shader = cs::Shader::load(device.clone())
        .expect("failed to create shader module");
    let compute_pipeline = Arc::new(ComputePipeline::new(device.clone(), &shader.main_entry_point(), &())
        .expect("failed to create compute pipeline"));

    // Descriptor sets
    let set = Arc::new(PersistentDescriptorSet::start(compute_pipeline.clone(), 0)
        .add_buffer(data_buffer.clone()).unwrap()
        .build().unwrap()
    );

    // Dispatch
    let command_buffer = AutoCommandBufferBuilder::new(device.clone(), queue.family()).unwrap()
        .dispatch([1024, 1, 1], compute_pipeline.clone(), set.clone(), ()).unwrap()
        .build().unwrap();

    let finished = command_buffer.execute(queue.clone()).unwrap();
    finished.then_signal_fence_and_flush().unwrap()
        .wait(None).unwrap();

    let content = data_buffer.read().unwrap();
    for (n, val) in content.iter().enumerate() {
        assert_eq!(*val, n as u32 * 12);
    }

    println!("Everything succeeded!");
}
<|endoftext|>


<commit_before>use prelude::*;
use sha1;

impl Uuid {
    /// Creates a UUID using a name from a namespace, based on the SHA-1 hash.
    ///
    /// A number of namespaces are available as constants in this crate:
    ///
    /// * [`NAMESPACE_DNS`]
    /// * [`NAMESPACE_OID`]
    /// * [`NAMESPACE_URL`]
    /// * [`NAMESPACE_X500`]
    ///
    /// Note that usage of this method requires the `v5` feature of this crate
    /// to be enabled.
    ///
    /// [`NAMESPACE_DNS`]: ../ns/const.NAMESPACE_DNS.html
    /// [`NAMESPACE_OID`]: ../ns/const.NAMESPACE_OID.html
    /// [`NAMESPACE_URL`]: ../ns/const.NAMESPACE_URL.html
    /// [`NAMESPACE_X500`]: ../ns/const.NAMESPACE_X500.html
    pub fn new_v5(namespace: &Uuid, name: &[u8]) -> Uuid {
        let mut hash = sha1::Sha1::new();

        hash.update(namespace.as_bytes());
        hash.update(name);

        let buffer = hash.digest().bytes();
        let mut uuid = Uuid::default();

        uuid.0.copy_from_slice(&buffer[..16]);
        uuid.set_variant(UuidVariant::RFC4122);
        uuid.set_version(UuidVersion::Sha1);

        uuid
    }
}

#[cfg(test)]
mod tests {
    use prelude::*;

    static FIXTURE: &'static [(&'static Uuid, &'static str, &'static str)] = &[
        (
            &Uuid::NAMESPACE_DNS,
            "example.org",
            "aad03681-8b63-5304-89e0-8ca8f49461b5",
        ),
        (
            &Uuid::NAMESPACE_DNS,
            "rust-lang.org",
            "c66bbb60-d62e-5f17-a399-3a0bd237c503",
        ),
        (
            &Uuid::NAMESPACE_DNS,
            "42",
            "7c411b5e-9d3f-50b5-9c28-62096e41c4ed",
        ),
        (
            &Uuid::NAMESPACE_DNS,
            "lorem ipsum",
            "97886a05-8a68-5743-ad55-56ab2d61cf7b",
        ),
        (
            &Uuid::NAMESPACE_URL,
            "example.org",
            "54a35416-963c-5dd6-a1e2-5ab7bb5bafc7",
        ),
        (
            &Uuid::NAMESPACE_URL,
            "rust-lang.org",
            "c48d927f-4122-5413-968c-598b1780e749",
        ),
        (
            &Uuid::NAMESPACE_URL,
            "42",
            "5c2b23de-4bad-58ee-a4b3-f22f3b9cfd7d",
        ),
        (
            &Uuid::NAMESPACE_URL,
            "lorem ipsum",
            "15c67689-4b85-5253-86b4-49fbb138569f",
        ),
        (
            &Uuid::NAMESPACE_OID,
            "example.org",
            "34784df9-b065-5094-92c7-00bb3da97a30",
        ),
        (
            &Uuid::NAMESPACE_OID,
            "rust-lang.org",
            "8ef61ecb-977a-5844-ab0f-c25ef9b8d5d6",
        ),
        (
            &Uuid::NAMESPACE_OID,
            "42",
            "ba293c61-ad33-57b9-9671-f3319f57d789",
        ),
        (
            &Uuid::NAMESPACE_OID,
            "lorem ipsum",
            "6485290d-f79e-5380-9e64-cb4312c7b4a6",
        ),
        (
            &Uuid::NAMESPACE_X500,
            "example.org",
            "e3635e86-f82b-5bbc-a54a-da97923e5c76",
        ),
        (
            &Uuid::NAMESPACE_X500,
            "rust-lang.org",
            "26c9c3e9-49b7-56da-8b9f-a0fb916a71a3",
        ),
        (
            &Uuid::NAMESPACE_X500,
            "42",
            "e4b88014-47c6-5fe0-a195-13710e5f6e27",
        ),
        (
            &Uuid::NAMESPACE_X500,
            "lorem ipsum",
            "b11f79a5-1e6d-57ce-a4b5-ba8531ea03d0",
        ),
    ];

    #[test]
    fn test_get_version() {
        let uuid =
            { Uuid::new_v5(&Uuid::NAMESPACE_DNS, "rust-lang.org".as_bytes()) };

        assert_eq!(uuid.get_version(), Some(UuidVersion::Sha1));
        assert_eq!(uuid.get_version_num(), 5);
    }

    #[test]
    fn test_hyphenated() {
        for &(ref ns, ref name, ref expected) in FIXTURE {
            let uuid = Uuid::new_v5(*ns, name.as_bytes());

            assert_eq!(uuid.to_hyphenated().to_string(), *expected)
        }
    }

    #[test]
    fn test_new() {
        for &(ref ns, ref name, ref u) in FIXTURE {
            let uuid = Uuid::new_v5(*ns, name.as_bytes());

            assert_eq!(uuid.get_variant(), Some(UuidVariant::RFC4122));
            assert_eq!(uuid.get_version(), Some(UuidVersion::Sha1));
            assert_eq!(Ok(uuid), u.parse());
        }
    }
}
<commit_msg>remove unnecessary scope<commit_after>use prelude::*;
use sha1;

impl Uuid {
    /// Creates a UUID using a name from a namespace, based on the SHA-1 hash.
    ///
    /// A number of namespaces are available as constants in this crate:
    ///
    /// * [`NAMESPACE_DNS`]
    /// * [`NAMESPACE_OID`]
    /// * [`NAMESPACE_URL`]
    /// * [`NAMESPACE_X500`]
    ///
    /// Note that usage of this method requires the `v5` feature of this crate
    /// to be enabled.
    ///
    /// [`NAMESPACE_DNS`]: ../ns/const.NAMESPACE_DNS.html
    /// [`NAMESPACE_OID`]: ../ns/const.NAMESPACE_OID.html
    /// [`NAMESPACE_URL`]: ../ns/const.NAMESPACE_URL.html
    /// [`NAMESPACE_X500`]: ../ns/const.NAMESPACE_X500.html
    pub fn new_v5(namespace: &Uuid, name: &[u8]) -> Uuid {
        let mut hash = sha1::Sha1::new();

        hash.update(namespace.as_bytes());
        hash.update(name);

        let buffer = hash.digest().bytes();
        let mut uuid = Uuid::default();

        uuid.0.copy_from_slice(&buffer[..16]);
        uuid.set_variant(UuidVariant::RFC4122);
        uuid.set_version(UuidVersion::Sha1);

        uuid
    }
}

#[cfg(test)]
mod tests {
    use prelude::*;

    static FIXTURE: &'static [(&'static Uuid, &'static str, &'static str)] = &[
        (
            &Uuid::NAMESPACE_DNS,
            "example.org",
            "aad03681-8b63-5304-89e0-8ca8f49461b5",
        ),
        (
            &Uuid::NAMESPACE_DNS,
            "rust-lang.org",
            "c66bbb60-d62e-5f17-a399-3a0bd237c503",
        ),
        (
            &Uuid::NAMESPACE_DNS,
            "42",
            "7c411b5e-9d3f-50b5-9c28-62096e41c4ed",
        ),
        (
            &Uuid::NAMESPACE_DNS,
            "lorem ipsum",
            "97886a05-8a68-5743-ad55-56ab2d61cf7b",
        ),
        (
            &Uuid::NAMESPACE_URL,
            "example.org",
            "54a35416-963c-5dd6-a1e2-5ab7bb5bafc7",
        ),
        (
            &Uuid::NAMESPACE_URL,
            "rust-lang.org",
            "c48d927f-4122-5413-968c-598b1780e749",
        ),
        (
            &Uuid::NAMESPACE_URL,
            "42",
            "5c2b23de-4bad-58ee-a4b3-f22f3b9cfd7d",
        ),
        (
            &Uuid::NAMESPACE_URL,
            "lorem ipsum",
            "15c67689-4b85-5253-86b4-49fbb138569f",
        ),
        (
            &Uuid::NAMESPACE_OID,
            "example.org",
            "34784df9-b065-5094-92c7-00bb3da97a30",
        ),
        (
            &Uuid::NAMESPACE_OID,
            "rust-lang.org",
            "8ef61ecb-977a-5844-ab0f-c25ef9b8d5d6",
        ),
        (
            &Uuid::NAMESPACE_OID,
            "42",
            "ba293c61-ad33-57b9-9671-f3319f57d789",
        ),
        (
            &Uuid::NAMESPACE_OID,
            "lorem ipsum",
            "6485290d-f79e-5380-9e64-cb4312c7b4a6",
        ),
        (
            &Uuid::NAMESPACE_X500,
            "example.org",
            "e3635e86-f82b-5bbc-a54a-da97923e5c76",
        ),
        (
            &Uuid::NAMESPACE_X500,
            "rust-lang.org",
            "26c9c3e9-49b7-56da-8b9f-a0fb916a71a3",
        ),
        (
            &Uuid::NAMESPACE_X500,
            "42",
            "e4b88014-47c6-5fe0-a195-13710e5f6e27",
        ),
        (
            &Uuid::NAMESPACE_X500,
            "lorem ipsum",
            "b11f79a5-1e6d-57ce-a4b5-ba8531ea03d0",
        ),
    ];

    #[test]
    fn test_get_version() {
        let uuid =
            Uuid::new_v5(&Uuid::NAMESPACE_DNS, "rust-lang.org".as_bytes());

        assert_eq!(uuid.get_version(), Some(UuidVersion::Sha1));
        assert_eq!(uuid.get_version_num(), 5);
    }

    #[test]
    fn test_hyphenated() {
        for &(ref ns, ref name, ref expected) in FIXTURE {
            let uuid = Uuid::new_v5(*ns, name.as_bytes());

            assert_eq!(uuid.to_hyphenated().to_string(), *expected)
        }
    }

    #[test]
    fn test_new() {
        for &(ref ns, ref name, ref u) in FIXTURE {
            let uuid = Uuid::new_v5(*ns, name.as_bytes());

            assert_eq!(uuid.get_variant(), Some(UuidVariant::RFC4122));
            assert_eq!(uuid.get_version(), Some(UuidVersion::Sha1));
            assert_eq!(Ok(uuid), u.parse());
        }
    }
}
<|endoftext|>


<commit_before>use std::path::PathBuf;
use std::ops::Deref;
use std::fmt::Display;
use std::process::exit;

use clap::ArgMatches;
use toml::Value;

use libimagstore::store::FileLockEntry;
use libimagstore::storeid::build_entry_path;
use libimagrt::runtime::Runtime;
use libimagutil::trace::trace_error;

pub fn retrieve(rt: &Runtime) {
    rt.cli()
        .subcommand_matches("retrieve")
        .map(|scmd| {
            scmd.value_of("id")
                .map(|id| {
                    let path = build_entry_path(rt.store(), id);
                    if path.is_err() {
                        trace_error(&path.err().unwrap());
                        exit(1);
                    }
                    let path = path.unwrap();
                    debug!("path = {:?}", path);

                    rt.store()
                        // "id" must be present, enforced via clap spec
                        .retrieve(path)
                        .map(|e| print_entry(rt, scmd, e))
                        .map_err(|e| {
                            debug!("No entry.");
                            debug!("{}", e);
                        })
                })
        });
}

fn print_entry(rt: &Runtime, scmd: &ArgMatches, e: FileLockEntry) {
    if do_print_raw(scmd) {
        debug!("Printing raw content...");
        println!("{}", e.deref().to_str());
    } else if do_filter(scmd) {
        debug!("Filtering...");
        warn!("Filtering via header specs is currently now supported.");
        warn!("Will fail now!");
        unimplemented!()
    } else {
        debug!("Printing structured...");
        let entry = e.deref();
        if do_print_header(scmd) {
            debug!("Printing header...");
            if do_print_header_as_json(rt.cli()) {
                debug!("Printing header as json...");
                warn!("Printing as JSON currently not supported.");
                warn!("Will fail now!");
                unimplemented!()
            } else {
                debug!("Printing header as TOML...");
                // We have to Value::Table() for Display
                println!("{}", Value::Table(entry.get_header().clone().into()))
            }
        }

        if do_print_content(scmd) {
            debug!("Printing content...");
            println!("{}", entry.get_content());
        }

    }
}

fn do_print_header(m: &ArgMatches) -> bool {
    m.is_present("header")
}

fn do_print_header_as_json(m: &ArgMatches) -> bool {
    m.is_present("header-json")
}

fn do_print_content(m: &ArgMatches) -> bool {
    m.is_present("content")
}

fn do_print_raw(m: &ArgMatches) -> bool {
    m.is_present("raw")
}

fn do_filter(m: &ArgMatches) -> bool {
    m.subcommand_matches("filter-header").is_some()
}

<commit_msg>Trace error on retrieve failure<commit_after>use std::path::PathBuf;
use std::ops::Deref;
use std::fmt::Display;
use std::process::exit;

use clap::ArgMatches;
use toml::Value;

use libimagstore::store::FileLockEntry;
use libimagstore::storeid::build_entry_path;
use libimagrt::runtime::Runtime;
use libimagutil::trace::trace_error;

pub fn retrieve(rt: &Runtime) {
    rt.cli()
        .subcommand_matches("retrieve")
        .map(|scmd| {
            scmd.value_of("id")
                .map(|id| {
                    let path = build_entry_path(rt.store(), id);
                    if path.is_err() {
                        trace_error(&path.err().unwrap());
                        exit(1);
                    }
                    let path = path.unwrap();
                    debug!("path = {:?}", path);

                    rt.store()
                        // "id" must be present, enforced via clap spec
                        .retrieve(path)
                        .map(|e| print_entry(rt, scmd, e))
                        .map_err(|e| {
                            debug!("No entry.");
                            debug!("{}:", e);
                            trace_error(&e);
                        })
                })
        });
}

fn print_entry(rt: &Runtime, scmd: &ArgMatches, e: FileLockEntry) {
    if do_print_raw(scmd) {
        debug!("Printing raw content...");
        println!("{}", e.deref().to_str());
    } else if do_filter(scmd) {
        debug!("Filtering...");
        warn!("Filtering via header specs is currently now supported.");
        warn!("Will fail now!");
        unimplemented!()
    } else {
        debug!("Printing structured...");
        let entry = e.deref();
        if do_print_header(scmd) {
            debug!("Printing header...");
            if do_print_header_as_json(rt.cli()) {
                debug!("Printing header as json...");
                warn!("Printing as JSON currently not supported.");
                warn!("Will fail now!");
                unimplemented!()
            } else {
                debug!("Printing header as TOML...");
                // We have to Value::Table() for Display
                println!("{}", Value::Table(entry.get_header().clone().into()))
            }
        }

        if do_print_content(scmd) {
            debug!("Printing content...");
            println!("{}", entry.get_content());
        }

    }
}

fn do_print_header(m: &ArgMatches) -> bool {
    m.is_present("header")
}

fn do_print_header_as_json(m: &ArgMatches) -> bool {
    m.is_present("header-json")
}

fn do_print_content(m: &ArgMatches) -> bool {
    m.is_present("content")
}

fn do_print_raw(m: &ArgMatches) -> bool {
    m.is_present("raw")
}

fn do_filter(m: &ArgMatches) -> bool {
    m.subcommand_matches("filter-header").is_some()
}

<|endoftext|>


<commit_before>extern mod extra;

#[allow(dead_code)]
pub mod helpers {
    use extra::getopts;
    use std::io::buffered::BufferedReader;
    use std::io::{Open, Truncate, Read, Write};
    use std::io::fs::File;
    use std::os;
    use std::run::Process;
    // Ugly, ugly, word-splitting parser for individual commands. Gets the job
    // done, though.
    pub fn split_words(words : &str) -> ~[~str] {
        let mut splits = ~[];
        let mut lastword = 0;
        let mut quoted = false;
        for i in range(0, words.len()) {
            if !quoted {
                if i == 0 {
                    if words.char_at(i) == '"' {
                        quoted = true;
                        lastword = i+1;
                    }
                }
                else {
                    if words.char_at(i) == '"' && words.char_at(i-1) != '\\' {
                        quoted = true;
                        lastword = i+1;
                    }
                    else if words.char_at(i) == ' ' && words.char_at(i-1) != '\\' {
                        let word = words.slice(lastword, i).to_owned();
                        if word != ~"" {
                            splits.push(word);
                        }
                        lastword = i+1;
                    }
                }
            }
            else {
                if words.char_at(i) == '"' {
                    splits.push(words.slice(lastword, i).to_owned());
                    lastword = i+1;
                    quoted = false;
                }
            }
        }
        if lastword != words.len() {
            splits.push(words.slice_from(lastword).to_owned());
        }
        splits.iter().map(|x| x.replace("\\n", "\n")).collect()
    }

    #[allow(dead_code)]
    pub fn input_redirect(mut process: ~Process, path: &Path) -> ~Process {
        let file = File::open_mode(path, Open, Read)
            .expect(format!("ERR: Failed opening input file"));
        let file_buffer = &mut BufferedReader::new(file);
        process.input().write(file_buffer.read_to_end());
        process
    }

    #[allow(dead_code)]
    pub fn write_output_to_file(output : ~[u8],
                            path : &Path) {
        let mut file = File::open_mode(path,
                                        Truncate,
                                        Write)
            .expect(format!("ERR: Failed opening output"));
        file.write(output);
    }

    #[allow(dead_code)]
    pub fn output_redirect(mut process : ~Process, path : &Path) -> ~Process {
        let output = process.finish_with_output();
        if output.status.success() {
            write_output_to_file(output.output, path);
        }
        process
    }

    #[allow(dead_code)]
    pub fn pipe_redirect(mut left: ~Process, mut right: ~Process) -> ~Process {
        right.input().write(left.finish_with_output().output);
        left.close_outputs();
        right
    }

    // Begin processing program arguments and initiate the parameters.
    #[allow(dead_code)]
    pub fn get_cmdline_from_args() -> Option<~str> {
        let args = os::args();

        let opts = ~[
            getopts::optopt("c")
        ];

        let matches = match getopts::getopts(args.tail(), opts) {
            Ok(m) => { m }
            Err(f) => { fail!(f.to_err_msg()) }
        };

        if matches.opt_present("c") {
            let cmd_str = match matches.opt_str("c") {
                Some(cmd_str) => {cmd_str.to_owned()},
                None          => {~""}
            };
            return Some(cmd_str);
        } else {
            return None;
        }
    }
}
<commit_msg>remove redundant `#[allow(dead_code)]`s<commit_after>extern mod extra;

#[allow(dead_code)]
pub mod helpers {
    use extra::getopts;
    use std::io::buffered::BufferedReader;
    use std::io::{Open, Truncate, Read, Write};
    use std::io::fs::File;
    use std::os;
    use std::run::Process;
    // Ugly, ugly, word-splitting parser for individual commands. Gets the job
    // done, though.
    pub fn split_words(words : &str) -> ~[~str] {
        let mut splits = ~[];
        let mut lastword = 0;
        let mut quoted = false;
        for i in range(0, words.len()) {
            if !quoted {
                if i == 0 {
                    if words.char_at(i) == '"' {
                        quoted = true;
                        lastword = i+1;
                    }
                }
                else {
                    if words.char_at(i) == '"' && words.char_at(i-1) != '\\' {
                        quoted = true;
                        lastword = i+1;
                    }
                    else if words.char_at(i) == ' ' && words.char_at(i-1) != '\\' {
                        let word = words.slice(lastword, i).to_owned();
                        if word != ~"" {
                            splits.push(word);
                        }
                        lastword = i+1;
                    }
                }
            }
            else {
                if words.char_at(i) == '"' {
                    splits.push(words.slice(lastword, i).to_owned());
                    lastword = i+1;
                    quoted = false;
                }
            }
        }
        if lastword != words.len() {
            splits.push(words.slice_from(lastword).to_owned());
        }
        splits.iter().map(|x| x.replace("\\n", "\n")).collect()
    }

    pub fn input_redirect(mut process: ~Process, path: &Path) -> ~Process {
        let file = File::open_mode(path, Open, Read)
            .expect(format!("ERR: Failed opening input file"));
        let file_buffer = &mut BufferedReader::new(file);
        process.input().write(file_buffer.read_to_end());
        process
    }

    pub fn write_output_to_file(output : ~[u8],
                            path : &Path) {
        let mut file = File::open_mode(path,
                                        Truncate,
                                        Write)
            .expect(format!("ERR: Failed opening output"));
        file.write(output);
    }

    pub fn output_redirect(mut process : ~Process, path : &Path) -> ~Process {
        let output = process.finish_with_output();
        if output.status.success() {
            write_output_to_file(output.output, path);
        }
        process
    }

    pub fn pipe_redirect(mut left: ~Process, mut right: ~Process) -> ~Process {
        right.input().write(left.finish_with_output().output);
        left.close_outputs();
        right
    }

    // Begin processing program arguments and initiate the parameters.
    pub fn get_cmdline_from_args() -> Option<~str> {
        let args = os::args();

        let opts = ~[
            getopts::optopt("c")
        ];

        let matches = match getopts::getopts(args.tail(), opts) {
            Ok(m) => { m }
            Err(f) => { fail!(f.to_err_msg()) }
        };

        if matches.opt_present("c") {
            let cmd_str = match matches.opt_str("c") {
                Some(cmd_str) => {cmd_str.to_owned()},
                None          => {~""}
            };
            return Some(cmd_str);
        } else {
            return None;
        }
    }
}
<|endoftext|>