<commit_before>use crate::account_address::AccountAddress;
use canonical_serialization::{
    CanonicalDeserialize, CanonicalDeserializer, CanonicalSerialize, CanonicalSerializer,
};
use crypto::HashValue;
use failure::prelude::*;
use hex;
#[cfg(any(test, feature = "testing"))]
use proptest_derive::Arbitrary;
use serde::{Deserialize, Serialize};
use std::{convert::TryFrom, fmt};

/// Size of an event key.
pub const EVENT_KEY_LENGTH: usize = 32;

/// A struct that represents a globally unique id for an Event stream that a user can listen to.
#[derive(
    Ord, PartialOrd, Eq, PartialEq, Hash, Debug, Default, Clone, Serialize, Deserialize, Copy,
)]
#[cfg_attr(any(test, feature = "testing"), derive(Arbitrary))]
pub struct EventKey([u8; EVENT_KEY_LENGTH]);

impl EventKey {
    /// Construct a new EventKey from a byte array slice.
    pub fn new(key: [u8; EVENT_KEY_LENGTH]) -> Self {
        EventKey(key)
    }

    /// Get the byte representation of the event key.
    pub fn as_bytes(&self) -> &[u8] {
        &self.0
    }

    /// Convert event key into a byte array.
    pub fn to_vec(&self) -> Vec<u8> {
        self.0.to_vec()
    }

    #[cfg(any(test, feature = "testing"))]
    /// Create a random event key for testing
    pub fn random() -> Self {
        EventKey::try_from(HashValue::random().to_vec().as_slice()).unwrap()
    }

    /// Create a unique handle by using an AccountAddress and a counter.
    pub fn new_from_address(addr: &AccountAddress, salt: u64) -> Self {
        let mut output_bytes = salt.to_be_bytes().to_vec();
        output_bytes.append(&mut addr.to_vec());
        EventKey(*HashValue::from_sha3_256(&output_bytes).as_ref())
    }
}

impl TryFrom<&[u8]> for EventKey {
    type Error = failure::Error;

    /// Tries to convert the provided byte array into Event Key.
    fn try_from(bytes: &[u8]) -> Result<EventKey> {
        ensure!(
            bytes.len() == EVENT_KEY_LENGTH,
            "The Address {:?} is of invalid length",
            bytes
        );
        let mut addr = [0u8; EVENT_KEY_LENGTH];
        addr.copy_from_slice(bytes);
        Ok(EventKey(addr))
    }
}

/// A Rust representation of an Event Handle Resource.
#[derive(Debug, Default, Clone, PartialEq, Eq)]
pub struct EventHandle {
    /// The associated globally unique key that is used as the key to the EventStore.
    key: EventKey,
    /// Number of events in the event stream.
    count: u64,
}

impl EventHandle {
    /// Constructs a new Event Handle
    pub fn new(key: EventKey, count: u64) -> Self {
        EventHandle { key, count }
    }

    /// Return the key to where this event is stored in EventStore.
    pub fn key(&self) -> &EventKey {
        &self.key
    }
    /// Return the counter for the handle
    pub fn count(&self) -> u64 {
        self.count
    }

    #[cfg(any(test, feature = "testing"))]
    pub fn count_mut(&mut self) -> &mut u64 {
        &mut self.count
    }

    #[cfg(any(test, feature = "testing"))]
    /// Create a random event handle for testing
    pub fn random_handle(count: u64) -> Self {
        Self {
            key: EventKey::random(),
            count,
        }
    }

    #[cfg(any(test, feature = "testing"))]
    /// Derive a unique handle by using an AccountAddress and a counter.
    pub fn new_from_address(addr: &AccountAddress, salt: u64) -> Self {
        Self {
            key: EventKey::new_from_address(addr, salt),
            count: 0,
        }
    }
}

impl fmt::LowerHex for EventKey {
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        write!(f, "{}", hex::encode(&self.0))
    }
}

impl fmt::Display for EventKey {
    fn fmt(&self, f: &mut fmt::Formatter) -> std::fmt::Result {
        // Forward to the LowerHex impl with a "0x" prepended (the # flag).
        write!(f, "{:#x}", self)
    }
}

impl CanonicalSerialize for EventKey {
    fn serialize(&self, serializer: &mut impl CanonicalSerializer) -> Result<()> {
        // We cannot use encode_raw_bytes as this structure will represent how Move Value of type
        // EventKey is serialized into. And since Move doesn't have fix length bytearray, values
        // can't be encoded in the fix length fasion.
        serializer.encode_bytes(&self.0)?;
        Ok(())
    }
}

impl CanonicalDeserialize for EventKey {
    fn deserialize(deserializer: &mut impl CanonicalDeserializer) -> Result<Self> {
        let bytes = deserializer.decode_bytes()?;
        Self::try_from(bytes.as_slice())
    }
}

impl CanonicalSerialize for EventHandle {
    fn serialize(&self, serializer: &mut impl CanonicalSerializer) -> Result<()> {
        serializer
            .encode_u64(self.count)?
            .encode_struct(&self.key)?;
        Ok(())
    }
}

impl CanonicalDeserialize for EventHandle {
    fn deserialize(deserializer: &mut impl CanonicalDeserializer) -> Result<Self> {
        let count = deserializer.decode_u64()?;
        let key = deserializer.decode_struct()?;
        Ok(EventHandle { count, key })
    }
}
<commit_msg>[types] Delete this comment because function 'encode_raw_bytes' no longer exsits.<commit_after>use crate::account_address::AccountAddress;
use canonical_serialization::{
    CanonicalDeserialize, CanonicalDeserializer, CanonicalSerialize, CanonicalSerializer,
};
use crypto::HashValue;
use failure::prelude::*;
use hex;
#[cfg(any(test, feature = "testing"))]
use proptest_derive::Arbitrary;
use serde::{Deserialize, Serialize};
use std::{convert::TryFrom, fmt};

/// Size of an event key.
pub const EVENT_KEY_LENGTH: usize = 32;

/// A struct that represents a globally unique id for an Event stream that a user can listen to.
#[derive(
    Ord, PartialOrd, Eq, PartialEq, Hash, Debug, Default, Clone, Serialize, Deserialize, Copy,
)]
#[cfg_attr(any(test, feature = "testing"), derive(Arbitrary))]
pub struct EventKey([u8; EVENT_KEY_LENGTH]);

impl EventKey {
    /// Construct a new EventKey from a byte array slice.
    pub fn new(key: [u8; EVENT_KEY_LENGTH]) -> Self {
        EventKey(key)
    }

    /// Get the byte representation of the event key.
    pub fn as_bytes(&self) -> &[u8] {
        &self.0
    }

    /// Convert event key into a byte array.
    pub fn to_vec(&self) -> Vec<u8> {
        self.0.to_vec()
    }

    #[cfg(any(test, feature = "testing"))]
    /// Create a random event key for testing
    pub fn random() -> Self {
        EventKey::try_from(HashValue::random().to_vec().as_slice()).unwrap()
    }

    /// Create a unique handle by using an AccountAddress and a counter.
    pub fn new_from_address(addr: &AccountAddress, salt: u64) -> Self {
        let mut output_bytes = salt.to_be_bytes().to_vec();
        output_bytes.append(&mut addr.to_vec());
        EventKey(*HashValue::from_sha3_256(&output_bytes).as_ref())
    }
}

impl TryFrom<&[u8]> for EventKey {
    type Error = failure::Error;

    /// Tries to convert the provided byte array into Event Key.
    fn try_from(bytes: &[u8]) -> Result<EventKey> {
        ensure!(
            bytes.len() == EVENT_KEY_LENGTH,
            "The Address {:?} is of invalid length",
            bytes
        );
        let mut addr = [0u8; EVENT_KEY_LENGTH];
        addr.copy_from_slice(bytes);
        Ok(EventKey(addr))
    }
}

/// A Rust representation of an Event Handle Resource.
#[derive(Debug, Default, Clone, PartialEq, Eq)]
pub struct EventHandle {
    /// The associated globally unique key that is used as the key to the EventStore.
    key: EventKey,
    /// Number of events in the event stream.
    count: u64,
}

impl EventHandle {
    /// Constructs a new Event Handle
    pub fn new(key: EventKey, count: u64) -> Self {
        EventHandle { key, count }
    }

    /// Return the key to where this event is stored in EventStore.
    pub fn key(&self) -> &EventKey {
        &self.key
    }
    /// Return the counter for the handle
    pub fn count(&self) -> u64 {
        self.count
    }

    #[cfg(any(test, feature = "testing"))]
    pub fn count_mut(&mut self) -> &mut u64 {
        &mut self.count
    }

    #[cfg(any(test, feature = "testing"))]
    /// Create a random event handle for testing
    pub fn random_handle(count: u64) -> Self {
        Self {
            key: EventKey::random(),
            count,
        }
    }

    #[cfg(any(test, feature = "testing"))]
    /// Derive a unique handle by using an AccountAddress and a counter.
    pub fn new_from_address(addr: &AccountAddress, salt: u64) -> Self {
        Self {
            key: EventKey::new_from_address(addr, salt),
            count: 0,
        }
    }
}

impl fmt::LowerHex for EventKey {
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        write!(f, "{}", hex::encode(&self.0))
    }
}

impl fmt::Display for EventKey {
    fn fmt(&self, f: &mut fmt::Formatter) -> std::fmt::Result {
        // Forward to the LowerHex impl with a "0x" prepended (the # flag).
        write!(f, "{:#x}", self)
    }
}

impl CanonicalSerialize for EventKey {
    fn serialize(&self, serializer: &mut impl CanonicalSerializer) -> Result<()> {
        serializer.encode_bytes(&self.0)?;
        Ok(())
    }
}

impl CanonicalDeserialize for EventKey {
    fn deserialize(deserializer: &mut impl CanonicalDeserializer) -> Result<Self> {
        let bytes = deserializer.decode_bytes()?;
        Self::try_from(bytes.as_slice())
    }
}

impl CanonicalSerialize for EventHandle {
    fn serialize(&self, serializer: &mut impl CanonicalSerializer) -> Result<()> {
        serializer
            .encode_u64(self.count)?
            .encode_struct(&self.key)?;
        Ok(())
    }
}

impl CanonicalDeserialize for EventHandle {
    fn deserialize(deserializer: &mut impl CanonicalDeserializer) -> Result<Self> {
        let count = deserializer.decode_u64()?;
        let key = deserializer.decode_struct()?;
        Ok(EventHandle { count, key })
    }
}
<|endoftext|>


<commit_before>/// The event system will have a trigger function that takes an
/// argument that decides if the event will be triggered this tick
/// or the next tick. The default should be next tick to prevent
/// inadvertent infinite loop bugs.
///
/// The event system can then schedule all events handlers that
/// do not mutate components to be run in parallel.
/// All handlers that do mutate the component list can be run
/// sequentially after that. Perhaps we can even schedule
/// those with non-overlapping mutable component lists to
/// run in parallel.

use std::collections::HashMap;
use std::any::Any;
use std::mem;
use shared_mutex::{ SharedMutex, MappedSharedMutexWriteGuard };

lazy_static! {
	pub static ref EventQueues: SharedMutex<HashMap<&'static str, &'static mut usize>> = SharedMutex::new(HashMap::new());
}

/// The event loop should trigger every n ms and execute any
/// events that are queued up. Every event has a function
/// that is called by the trigger! macro that puts the event
/// on the queue.
pub fn run_loop() {

}

pub fn event_queue_apply<T, F>(event_name: &str, action: F) -> bool
		where F: FnOnce(&mut Vec<T>) {
	let map = &mut EventQueues.write().unwrap();
	return match map.get_mut(event_name) {
		Some(queue) => {
			// :(
			let r : &mut Vec<T> = unsafe { mem::transmute(queue) };
			action(r);
			true
		},
		None => false
	}
}

pub fn set_event_queue<T>(event_name: &'static str, initial_value: T) {
	let map = &mut EventQueues.write().unwrap();
	let queue = Box::new(vec![initial_value]);

	// :(
	let v : &mut usize = unsafe { mem::transmute(queue) };
	map.insert(event_name, v);
}

///

/// Defines an event.
macro_rules! event {
	() => ()
}

/// Queues an event to be dispatched.
/// This means that the argument is put into the trigger queue for the
/// event and the event handlers will be invoked either at the next run loop.
macro_rules! trigger {
	//
	// Implementation idea: since we can't initialize events implicitly
	// in any way (not even through a rustc plugin atm) maybe we can
	// do it like lazy_static does it and initialize the event on the
	// first invocation of trigger.
	//
	// Initialisation that needs to happen:
	//   The trigger queue for this particular event needs to be initialized
	//   The run loop needs to be handed a reference to the trigger queue
	//
	() => ()
}

<commit_msg>fix missing box to make spec pass<commit_after>/// The event system will have a trigger function that takes an
/// argument that decides if the event will be triggered this tick
/// or the next tick. The default should be next tick to prevent
/// inadvertent infinite loop bugs.
///
/// The event system can then schedule all events handlers that
/// do not mutate components to be run in parallel.
/// All handlers that do mutate the component list can be run
/// sequentially after that. Perhaps we can even schedule
/// those with non-overlapping mutable component lists to
/// run in parallel.

use std::collections::HashMap;
use std::any::Any;
use std::mem;
use shared_mutex::{ SharedMutex, MappedSharedMutexWriteGuard };

lazy_static! {
	pub static ref EventQueues: SharedMutex<HashMap<&'static str, &'static mut usize>> = SharedMutex::new(HashMap::new());
}

/// The event loop should trigger every n ms and execute any
/// events that are queued up. Every event has a function
/// that is called by the trigger! macro that puts the event
/// on the queue.
pub fn run_loop() {

}

pub fn event_queue_apply<T, F>(event_name: &str, action: F) -> bool
		where F: FnOnce(&mut Vec<T>) {
	let map = &mut EventQueues.write().unwrap();
	return match map.get_mut(event_name) {
		Some(queue) => {
			// :(
			let r : &mut Box<Vec<T>> = unsafe { mem::transmute(queue) };
			action(r);
			true
		},
		None => false
	}
}

pub fn set_event_queue<T>(event_name: &'static str, initial_value: T) {
	let map = &mut EventQueues.write().unwrap();
	let queue = Box::new(vec![initial_value]);

	// :(
	let v : &mut usize = unsafe { mem::transmute(queue) };
	map.insert(event_name, v);
}

///

/// Defines an event.
macro_rules! event {
	() => ()
}

/// Queues an event to be dispatched.
/// This means that the argument is put into the trigger queue for the
/// event and the event handlers will be invoked either at the next run loop.
macro_rules! trigger {
	//
	// Implementation idea: since we can't initialize events implicitly
	// in any way (not even through a rustc plugin atm) maybe we can
	// do it like lazy_static does it and initialize the event on the
	// first invocation of trigger.
	//
	// Initialisation that needs to happen:
	//   The trigger queue for this particular event needs to be initialized
	//   The run loop needs to be handed a reference to the trigger queue
	//
	() => ()
}

<|endoftext|>


<commit_before>mod never_type;
mod coercion;
mod regression;
mod simple;
mod patterns;
mod traits;
mod method_resolution;
mod macros;

use std::fmt::Write;
use std::sync::Arc;

use hir_def::{
    body::BodySourceMap, db::DefDatabase, nameres::CrateDefMap, AssocItemId, DefWithBodyId,
    LocalModuleId, Lookup, ModuleDefId,
};
use hir_expand::InFile;
use insta::assert_snapshot;
use ra_db::{fixture::WithFixture, salsa::Database, FilePosition, SourceDatabase};
use ra_syntax::{
    algo,
    ast::{self, AstNode},
};

use crate::{db::HirDatabase, display::HirDisplay, test_db::TestDB, InferenceResult};

// These tests compare the inference results for all expressions in a file
// against snapshots of the expected results using insta. Use cargo-insta to
// update the snapshots.

fn type_at_pos(db: &TestDB, pos: FilePosition) -> String {
    let file = db.parse(pos.file_id).ok().unwrap();
    let expr = algo::find_node_at_offset::<ast::Expr>(file.syntax(), pos.offset).unwrap();

    let module = db.module_for_file(pos.file_id);
    let crate_def_map = db.crate_def_map(module.krate);
    for decl in crate_def_map[module.local_id].scope.declarations() {
        if let ModuleDefId::FunctionId(func) = decl {
            let (_body, source_map) = db.body_with_source_map(func.into());
            if let Some(expr_id) = source_map.node_expr(InFile::new(pos.file_id.into(), &expr)) {
                let infer = db.infer(func.into());
                let ty = &infer[expr_id];
                return ty.display(db).to_string();
            }
        }
    }
    panic!("Can't find expression")
}

fn type_at(content: &str) -> String {
    let (db, file_pos) = TestDB::with_position(content);
    type_at_pos(&db, file_pos)
}

fn infer(content: &str) -> String {
    let (db, file_id) = TestDB::with_single_file(content);

    let mut acc = String::new();

    let mut infer_def = |inference_result: Arc<InferenceResult>,
                         body_source_map: Arc<BodySourceMap>| {
        let mut types = Vec::new();

        for (pat, ty) in inference_result.type_of_pat.iter() {
            let syntax_ptr = match body_source_map.pat_syntax(pat) {
                Some(sp) => {
                    sp.map(|ast| ast.either(|it| it.syntax_node_ptr(), |it| it.syntax_node_ptr()))
                }
                None => continue,
            };
            types.push((syntax_ptr, ty));
        }

        for (expr, ty) in inference_result.type_of_expr.iter() {
            let syntax_ptr = match body_source_map.expr_syntax(expr) {
                Some(sp) => {
                    sp.map(|ast| ast.either(|it| it.syntax_node_ptr(), |it| it.syntax_node_ptr()))
                }
                None => continue,
            };
            types.push((syntax_ptr, ty));
        }

        // sort ranges for consistency
        types.sort_by_key(|(src_ptr, _)| {
            (src_ptr.value.range().start(), src_ptr.value.range().end())
        });
        for (src_ptr, ty) in &types {
            let node = src_ptr.value.to_node(&src_ptr.file_syntax(&db));

            let (range, text) = if let Some(self_param) = ast::SelfParam::cast(node.clone()) {
                (self_param.self_kw_token().text_range(), "self".to_string())
            } else {
                (src_ptr.value.range(), node.text().to_string().replace("\n", " "))
            };
            let macro_prefix = if src_ptr.file_id != file_id.into() { "!" } else { "" };
            write!(
                acc,
                "{}{} '{}': {}\n",
                macro_prefix,
                range,
                ellipsize(text, 15),
                ty.display(&db)
            )
            .unwrap();
        }
    };

    let module = db.module_for_file(file_id);
    let crate_def_map = db.crate_def_map(module.krate);

    let mut defs: Vec<DefWithBodyId> = Vec::new();
    visit_module(&db, &crate_def_map, module.local_id, &mut |it| defs.push(it));
    defs.sort_by_key(|def| match def {
        DefWithBodyId::FunctionId(it) => {
            it.lookup(&db).ast_id.to_node(&db).syntax().text_range().start()
        }
        DefWithBodyId::ConstId(it) => {
            it.lookup(&db).ast_id.to_node(&db).syntax().text_range().start()
        }
        DefWithBodyId::StaticId(it) => {
            it.lookup(&db).ast_id.to_node(&db).syntax().text_range().start()
        }
    });
    for def in defs {
        let (_body, source_map) = db.body_with_source_map(def);
        let infer = db.infer(def);
        infer_def(infer, source_map);
    }

    acc.truncate(acc.trim_end().len());
    acc
}

fn visit_module(
    db: &TestDB,
    crate_def_map: &CrateDefMap,
    module_id: LocalModuleId,
    cb: &mut dyn FnMut(DefWithBodyId),
) {
    for decl in crate_def_map[module_id].scope.declarations() {
        match decl {
            ModuleDefId::FunctionId(it) => cb(it.into()),
            ModuleDefId::ConstId(it) => cb(it.into()),
            ModuleDefId::StaticId(it) => cb(it.into()),
            ModuleDefId::TraitId(it) => {
                let trait_data = db.trait_data(it);
                for &(_, item) in trait_data.items.iter() {
                    match item {
                        AssocItemId::FunctionId(it) => cb(it.into()),
                        AssocItemId::ConstId(it) => cb(it.into()),
                        AssocItemId::TypeAliasId(_) => (),
                    }
                }
            }
            ModuleDefId::ModuleId(it) => visit_module(db, crate_def_map, it.local_id, cb),
            _ => (),
        }
    }
    for &impl_id in crate_def_map[module_id].impls.iter() {
        let impl_data = db.impl_data(impl_id);
        for &item in impl_data.items.iter() {
            match item {
                AssocItemId::FunctionId(it) => cb(it.into()),
                AssocItemId::ConstId(it) => cb(it.into()),
                AssocItemId::TypeAliasId(_) => (),
            }
        }
    }
}

fn ellipsize(mut text: String, max_len: usize) -> String {
    if text.len() <= max_len {
        return text;
    }
    let ellipsis = "...";
    let e_len = ellipsis.len();
    let mut prefix_len = (max_len - e_len) / 2;
    while !text.is_char_boundary(prefix_len) {
        prefix_len += 1;
    }
    let mut suffix_len = max_len - e_len - prefix_len;
    while !text.is_char_boundary(text.len() - suffix_len) {
        suffix_len += 1;
    }
    text.replace_range(prefix_len..text.len() - suffix_len, ellipsis);
    text
}

#[test]
fn typing_whitespace_inside_a_function_should_not_invalidate_types() {
    let (mut db, pos) = TestDB::with_position(
        "
        //- /lib.rs
        fn foo() -> i32 {
            <|>1 + 1
        }
    ",
    );
    {
        let events = db.log_executed(|| {
            let module = db.module_for_file(pos.file_id);
            let crate_def_map = db.crate_def_map(module.krate);
            visit_module(&db, &crate_def_map, module.local_id, &mut |def| {
                db.infer(def);
            });
        });
        assert!(format!("{:?}", events).contains("infer"))
    }

    let new_text = "
        fn foo() -> i32 {
            1
            +
            1
        }
    "
    .to_string();

    db.query_mut(ra_db::FileTextQuery).set(pos.file_id, Arc::new(new_text));

    {
        let events = db.log_executed(|| {
            let module = db.module_for_file(pos.file_id);
            let crate_def_map = db.crate_def_map(module.krate);
            visit_module(&db, &crate_def_map, module.local_id, &mut |def| {
                db.infer(def);
            });
        });
        assert!(!format!("{:?}", events).contains("infer"), "{:#?}", events)
    }
}

#[test]
fn no_such_field_diagnostics() {
    let diagnostics = TestDB::with_files(
        r"
        //- /lib.rs
        struct S { foo: i32, bar: () }
        impl S {
            fn new() -> S {
                S {
                    foo: 92,
                    baz: 62,
                }
            }
        }
        ",
    )
    .diagnostics();

    assert_snapshot!(diagnostics, @r###"
    "baz: 62": no such field
    "{\n            foo: 92,\n            baz: 62,\n        }": Missing structure fields:
    - bar
    "###
    );
}
<commit_msg>Simplify test<commit_after>mod never_type;
mod coercion;
mod regression;
mod simple;
mod patterns;
mod traits;
mod method_resolution;
mod macros;

use std::fmt::Write;
use std::sync::Arc;

use hir_def::{
    body::BodySourceMap, child_from_source::ChildFromSource, db::DefDatabase, nameres::CrateDefMap,
    AssocItemId, DefWithBodyId, LocalModuleId, Lookup, ModuleDefId,
};
use hir_expand::InFile;
use insta::assert_snapshot;
use ra_db::{fixture::WithFixture, salsa::Database, FilePosition, SourceDatabase};
use ra_syntax::{
    algo,
    ast::{self, AstNode},
};

use crate::{db::HirDatabase, display::HirDisplay, test_db::TestDB, InferenceResult};

// These tests compare the inference results for all expressions in a file
// against snapshots of the expected results using insta. Use cargo-insta to
// update the snapshots.

fn type_at_pos(db: &TestDB, pos: FilePosition) -> String {
    let file = db.parse(pos.file_id).ok().unwrap();
    let expr = algo::find_node_at_offset::<ast::Expr>(file.syntax(), pos.offset).unwrap();
    let fn_def = expr.syntax().ancestors().find_map(ast::FnDef::cast).unwrap();
    let module = db.module_for_file(pos.file_id);
    let func = module.child_from_source(db, InFile::new(pos.file_id.into(), fn_def)).unwrap();

    let (_body, source_map) = db.body_with_source_map(func.into());
    if let Some(expr_id) = source_map.node_expr(InFile::new(pos.file_id.into(), &expr)) {
        let infer = db.infer(func.into());
        let ty = &infer[expr_id];
        return ty.display(db).to_string();
    }
    panic!("Can't find expression")
}

fn type_at(content: &str) -> String {
    let (db, file_pos) = TestDB::with_position(content);
    type_at_pos(&db, file_pos)
}

fn infer(content: &str) -> String {
    let (db, file_id) = TestDB::with_single_file(content);

    let mut acc = String::new();

    let mut infer_def = |inference_result: Arc<InferenceResult>,
                         body_source_map: Arc<BodySourceMap>| {
        let mut types = Vec::new();

        for (pat, ty) in inference_result.type_of_pat.iter() {
            let syntax_ptr = match body_source_map.pat_syntax(pat) {
                Some(sp) => {
                    sp.map(|ast| ast.either(|it| it.syntax_node_ptr(), |it| it.syntax_node_ptr()))
                }
                None => continue,
            };
            types.push((syntax_ptr, ty));
        }

        for (expr, ty) in inference_result.type_of_expr.iter() {
            let syntax_ptr = match body_source_map.expr_syntax(expr) {
                Some(sp) => {
                    sp.map(|ast| ast.either(|it| it.syntax_node_ptr(), |it| it.syntax_node_ptr()))
                }
                None => continue,
            };
            types.push((syntax_ptr, ty));
        }

        // sort ranges for consistency
        types.sort_by_key(|(src_ptr, _)| {
            (src_ptr.value.range().start(), src_ptr.value.range().end())
        });
        for (src_ptr, ty) in &types {
            let node = src_ptr.value.to_node(&src_ptr.file_syntax(&db));

            let (range, text) = if let Some(self_param) = ast::SelfParam::cast(node.clone()) {
                (self_param.self_kw_token().text_range(), "self".to_string())
            } else {
                (src_ptr.value.range(), node.text().to_string().replace("\n", " "))
            };
            let macro_prefix = if src_ptr.file_id != file_id.into() { "!" } else { "" };
            write!(
                acc,
                "{}{} '{}': {}\n",
                macro_prefix,
                range,
                ellipsize(text, 15),
                ty.display(&db)
            )
            .unwrap();
        }
    };

    let module = db.module_for_file(file_id);
    let crate_def_map = db.crate_def_map(module.krate);

    let mut defs: Vec<DefWithBodyId> = Vec::new();
    visit_module(&db, &crate_def_map, module.local_id, &mut |it| defs.push(it));
    defs.sort_by_key(|def| match def {
        DefWithBodyId::FunctionId(it) => {
            it.lookup(&db).ast_id.to_node(&db).syntax().text_range().start()
        }
        DefWithBodyId::ConstId(it) => {
            it.lookup(&db).ast_id.to_node(&db).syntax().text_range().start()
        }
        DefWithBodyId::StaticId(it) => {
            it.lookup(&db).ast_id.to_node(&db).syntax().text_range().start()
        }
    });
    for def in defs {
        let (_body, source_map) = db.body_with_source_map(def);
        let infer = db.infer(def);
        infer_def(infer, source_map);
    }

    acc.truncate(acc.trim_end().len());
    acc
}

fn visit_module(
    db: &TestDB,
    crate_def_map: &CrateDefMap,
    module_id: LocalModuleId,
    cb: &mut dyn FnMut(DefWithBodyId),
) {
    for decl in crate_def_map[module_id].scope.declarations() {
        match decl {
            ModuleDefId::FunctionId(it) => cb(it.into()),
            ModuleDefId::ConstId(it) => cb(it.into()),
            ModuleDefId::StaticId(it) => cb(it.into()),
            ModuleDefId::TraitId(it) => {
                let trait_data = db.trait_data(it);
                for &(_, item) in trait_data.items.iter() {
                    match item {
                        AssocItemId::FunctionId(it) => cb(it.into()),
                        AssocItemId::ConstId(it) => cb(it.into()),
                        AssocItemId::TypeAliasId(_) => (),
                    }
                }
            }
            ModuleDefId::ModuleId(it) => visit_module(db, crate_def_map, it.local_id, cb),
            _ => (),
        }
    }
    for &impl_id in crate_def_map[module_id].impls.iter() {
        let impl_data = db.impl_data(impl_id);
        for &item in impl_data.items.iter() {
            match item {
                AssocItemId::FunctionId(it) => cb(it.into()),
                AssocItemId::ConstId(it) => cb(it.into()),
                AssocItemId::TypeAliasId(_) => (),
            }
        }
    }
}

fn ellipsize(mut text: String, max_len: usize) -> String {
    if text.len() <= max_len {
        return text;
    }
    let ellipsis = "...";
    let e_len = ellipsis.len();
    let mut prefix_len = (max_len - e_len) / 2;
    while !text.is_char_boundary(prefix_len) {
        prefix_len += 1;
    }
    let mut suffix_len = max_len - e_len - prefix_len;
    while !text.is_char_boundary(text.len() - suffix_len) {
        suffix_len += 1;
    }
    text.replace_range(prefix_len..text.len() - suffix_len, ellipsis);
    text
}

#[test]
fn typing_whitespace_inside_a_function_should_not_invalidate_types() {
    let (mut db, pos) = TestDB::with_position(
        "
        //- /lib.rs
        fn foo() -> i32 {
            <|>1 + 1
        }
    ",
    );
    {
        let events = db.log_executed(|| {
            let module = db.module_for_file(pos.file_id);
            let crate_def_map = db.crate_def_map(module.krate);
            visit_module(&db, &crate_def_map, module.local_id, &mut |def| {
                db.infer(def);
            });
        });
        assert!(format!("{:?}", events).contains("infer"))
    }

    let new_text = "
        fn foo() -> i32 {
            1
            +
            1
        }
    "
    .to_string();

    db.query_mut(ra_db::FileTextQuery).set(pos.file_id, Arc::new(new_text));

    {
        let events = db.log_executed(|| {
            let module = db.module_for_file(pos.file_id);
            let crate_def_map = db.crate_def_map(module.krate);
            visit_module(&db, &crate_def_map, module.local_id, &mut |def| {
                db.infer(def);
            });
        });
        assert!(!format!("{:?}", events).contains("infer"), "{:#?}", events)
    }
}

#[test]
fn no_such_field_diagnostics() {
    let diagnostics = TestDB::with_files(
        r"
        //- /lib.rs
        struct S { foo: i32, bar: () }
        impl S {
            fn new() -> S {
                S {
                    foo: 92,
                    baz: 62,
                }
            }
        }
        ",
    )
    .diagnostics();

    assert_snapshot!(diagnostics, @r###"
    "baz: 62": no such field
    "{\n            foo: 92,\n            baz: 62,\n        }": Missing structure fields:
    - bar
    "###
    );
}
<|endoftext|>


<commit_before>//! Just embed git-hash to `--version`

use std::{env, path::PathBuf, process::Command};

fn main() {
    set_rerun();

    let rev =
        env::var("RUST_ANALYZER_REV").ok().or_else(rev).unwrap_or_else(|| "???????".to_string());
    println!("cargo:rustc-env=REV={}", rev)
}

fn set_rerun() {
    println!("cargo:rerun-if-env-changed=RUST_ANALYZER_REV");

    let mut manifest_dir = PathBuf::from(
        env::var("CARGO_MANIFEST_DIR").expect("`CARGO_MANIFEST_DIR` is always set by cargo."),
    );

    while manifest_dir.parent().is_some() {
        if manifest_dir.join(".git/HEAD").exists() {
            let git_dir = manifest_dir.join(".git");

            println!("cargo:rerun-if-changed={}", git_dir.join("HEAD").display());
            // current branch ref
            if let Ok(output) =
                Command::new("git").args(&["rev-parse", "--symbolic-full-name", "HEAD"]).output()
            {
                if let Ok(ref_link) = String::from_utf8(output.stdout) {
                    println!("cargo:rerun-if-changed={}", git_dir.join(ref_link).display());
                }
            }
            return;
        }

        manifest_dir.pop();
    }
    println!("cargo:warning=Could not find `.git/HEAD` from manifest dir!");
}

fn rev() -> Option<String> {
    let output = Command::new("git").args(&["describe", "--tags"]).output().ok()?;
    let stdout = String::from_utf8(output.stdout).ok()?;
    Some(stdout)
}
<commit_msg>Exclude nightly tag from git describe to fix version string<commit_after>//! Just embed git-hash to `--version`

use std::{env, path::PathBuf, process::Command};

fn main() {
    set_rerun();

    let rev =
        env::var("RUST_ANALYZER_REV").ok().or_else(rev).unwrap_or_else(|| "???????".to_string());
    println!("cargo:rustc-env=REV={}", rev)
}

fn set_rerun() {
    println!("cargo:rerun-if-env-changed=RUST_ANALYZER_REV");

    let mut manifest_dir = PathBuf::from(
        env::var("CARGO_MANIFEST_DIR").expect("`CARGO_MANIFEST_DIR` is always set by cargo."),
    );

    while manifest_dir.parent().is_some() {
        if manifest_dir.join(".git/HEAD").exists() {
            let git_dir = manifest_dir.join(".git");

            println!("cargo:rerun-if-changed={}", git_dir.join("HEAD").display());
            // current branch ref
            if let Ok(output) =
                Command::new("git").args(&["rev-parse", "--symbolic-full-name", "HEAD"]).output()
            {
                if let Ok(ref_link) = String::from_utf8(output.stdout) {
                    println!("cargo:rerun-if-changed={}", git_dir.join(ref_link).display());
                }
            }
            return;
        }

        manifest_dir.pop();
    }
    println!("cargo:warning=Could not find `.git/HEAD` from manifest dir!");
}

fn rev() -> Option<String> {
    let output =
        Command::new("git").args(&["describe", "--tags", "--exclude", "nightly"]).output().ok()?;
    let stdout = String::from_utf8(output.stdout).ok()?;
    Some(stdout)
}
<|endoftext|>


<commit_before>use buffer::Buffer;

#[cfg(test)]
pub fn data_from_str(s: &'static str) -> String {
    String::from_str(s)
}

pub fn char_width(c: char, is_cjk: bool, tab_width: uint, position: uint) -> Option<uint> {
    if c == '\t' {
        Some(tab_width - position%tab_width)
    } else if c == '\n' {
        Some(1)
    } else {
        c.width(is_cjk)
    }
}

pub fn str_width(s: &str, is_cjk: bool, tab_width: uint) -> uint {
    s.chars().fold(0, |acc, c|
        acc + char_width(c, is_cjk, tab_width, acc).unwrap_or(0)
    )
}

/// Determine if a given char is alphanumeric or an underscore
pub fn is_alpha_or_(c: char) -> bool {
    c.is_alphanumeric() || c == '_'
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn test_is_alpha_or_() {
        assert!(is_alpha_or_('a'));
        assert!(is_alpha_or_('5'));
        assert!(is_alpha_or_('_'));
    }
}
<commit_msg>Remove unused import<commit_after>#[cfg(test)]
pub fn data_from_str(s: &'static str) -> String {
    String::from_str(s)
}

pub fn char_width(c: char, is_cjk: bool, tab_width: uint, position: uint) -> Option<uint> {
    if c == '\t' {
        Some(tab_width - position%tab_width)
    } else if c == '\n' {
        Some(1)
    } else {
        c.width(is_cjk)
    }
}

pub fn str_width(s: &str, is_cjk: bool, tab_width: uint) -> uint {
    s.chars().fold(0, |acc, c|
        acc + char_width(c, is_cjk, tab_width, acc).unwrap_or(0)
    )
}

/// Determine if a given char is alphanumeric or an underscore
pub fn is_alpha_or_(c: char) -> bool {
    c.is_alphanumeric() || c == '_'
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn test_is_alpha_or_() {
        assert!(is_alpha_or_('a'));
        assert!(is_alpha_or_('5'));
        assert!(is_alpha_or_('_'));
    }
}
<|endoftext|>


<commit_before>use log::{debug};
use failure::{Error};
use flirt;
use flirt::pat;

use super::super::super::util;
use super::super::super::arch::{RVA};
use super::super::super::workspace::Workspace;
use super::super::{Analyzer};

pub struct FlirtAnalyzer {
    sigs: flirt::FlirtSignatureSet,
}

impl FlirtAnalyzer {
    pub fn new() -> FlirtAnalyzer {
        // TODO: read all pat files in this directory
        // TODO: filter signatures for:
        //   - duplicates
        //   - not specific enough (no checksum, too many wildcards, etc.)
        // TODO: add startup signatures to detect runtime/signature set
        let path = "C:/Users/user/Documents/code/Lancelot/flirt/sigs/pat/__EH_prolog3.pat";
        let buf = util::read_file(path).unwrap();  // danger
        let s = String::from_utf8(buf).unwrap(); // danger
        let sigs = flirt::FlirtSignatureSet::with_signatures(pat::parse(&s).unwrap());  // danger

        FlirtAnalyzer{
            sigs
        }
    }
}

impl Analyzer for FlirtAnalyzer {
    fn get_name(&self) -> String {
        "FLIRT function signature analyzer".to_string()
    }

    fn analyze(&self, ws: &mut Workspace) -> Result<(), Error> {
        let mut buf = [0u8; 0xFF];
        let functions: Vec<RVA> = ws.get_functions().cloned().collect();

        for &fva in functions.iter() {
            if let Ok(buf) = ws.read_bytes_into(fva, &mut buf[..]) {
                let matches = self.sigs.r#match(buf);

                match matches.len() {
                    0 => debug!("no FLIRT signature match: {}", fva),
                    1 => {
                        if let Some(name) = matches[0].get_name() {
                            debug!("FLIRT signature match: {} {}", fva, name);
                            ws.make_symbol(fva, name).unwrap();  // danger
                        }
                    }
                    _ => debug!("many FLIRT signature matches: {}", fva),
                }
            }
        }

        ws.analyze()?;

        Ok(())
    }
}
<commit_msg>analysis: flirt: read many pat files, filter down to usable sigs<commit_after>use std::path::{Path};
use std::collections::HashSet;

use log::{debug, info, trace};
use failure::{Error};

use flirt;
use flirt::pat;
use super::super::super::util;
use super::super::super::arch::{RVA};
use super::super::super::workspace::Workspace;
use super::super::{Analyzer};

pub struct FlirtAnalyzer {
    sigs: flirt::FlirtSignatureSet,
}

impl FlirtAnalyzer {
    /// parse the given file system path into a collection of FLIRT signatures.
    fn load_flirt_file(path: &Path) -> Result<Vec<flirt::FlirtSignature>, Error> {
        let buf = util::read_file(path.to_str().unwrap())?;  // danger
        let s = String::from_utf8(buf)?;
        pat::parse(&s)
    }

    /// remove sigantures from the given collection that don't match basic criteria:
    ///   - the signature must specify a name (or its no use to us)
    ///   - functions must be at least 8 bytes long
    ///   - shorter functions must not have too many wildcards
    fn filter_flirt_signatures(sigs: Vec<flirt::FlirtSignature>) -> Vec<flirt::FlirtSignature> {
        sigs.into_iter()
            .filter(|sig| {
                if let None = sig.get_name() {
                    // must have a name that we can apply.
                    return false;
                }

                let wc_count = sig.byte_sig.0.iter().take(sig.size_of_function as usize).filter(|b| match b {
                    flirt::SigElement::Wildcard => true,
                    flirt::SigElement::Byte(_) => false,
                }).count();

                if sig.size_of_function < 0x8 {
                    // lancelot specific: don't use signatures for functions less than 0x8 bytes.
                    // this just seems too short to be unique and specific.
                    trace!("sig too short: {} {:?}", sig.size_of_function, sig.get_name());
                    return false;
                } else if sig.size_of_function < 0x10 && wc_count > 0 {
                    // lancelot specific: don't allow many wildcards for short functions.
                    trace!("sig too many wildcards: {}/{} {:?}", wc_count, sig.size_of_function, sig.get_name());
                    return false;
                } else if sig.size_of_function < 0x18 && wc_count > 4 {
                    trace!("sig too many wildcards: {}/{} {:?}", wc_count, sig.size_of_function, sig.get_name());
                    return false;
                } else if sig.size_of_function < 0x20 && wc_count > 0x10 {
                    trace!("sig too many wildcards: {}/{} {:?}", wc_count, sig.size_of_function, sig.get_name());
                    return false;
                } else {
                    return true;
                }
            })
            .collect()
    }

    /// parse the given directory into a set of FLIRT signatures.
    ///
    /// applies some filtering on the signatures for better accuracy:
    ///   - the signature must specify a name (or its no use to us)
    ///   - functions must be at least 8 bytes long
    ///   - shorter functions must not have too many wildcards
    ///
    /// the directory structure should look like:
    ///   - flat directory of .pat files
    fn load_flirt_directory(path: &str) -> Result<flirt::FlirtSignatureSet, Error> {

        // TODO: need lots of error handling here.

        let mut sigs = vec![];

        for entry in std::fs::read_dir(Path::new(path))? {
            let entry = entry?;
            let path = entry.path();

            if path.is_dir() {
                continue
            }

            let ext = path.extension().and_then(|ext| ext.to_str()).unwrap_or("");

            if ext == "pat" {
                debug!("FLIRT analyzer: loading .pat file: {:?}", path);
                sigs.extend(FlirtAnalyzer::load_flirt_file(&path).unwrap()); //danger
            } else if ext == ".sig" {
                // ....
                debug!("FLIRT analyzer: skipping .sig file: {:?}", path);
            } else {
                debug!("FLIRT analyzer: skipping file: {:?}", path);
            }
        }

        debug!("loaded {} total FLIRT signatures", sigs.len());

        let sigs = FlirtAnalyzer::filter_flirt_signatures(sigs);
        info!("loaded {} FLIRT signatures", sigs.len());

        Ok(flirt::FlirtSignatureSet::with_signatures(sigs))
    }

    pub fn new() -> FlirtAnalyzer {
        // TODO: add startup signatures to detect runtime/signature set
        // TODO: use .sig rather than .pat files
        let sigs = FlirtAnalyzer::load_flirt_directory("C:/Users/user/Documents/code/Lancelot/flirt/sigs/pat/").unwrap();

        FlirtAnalyzer{
            sigs
        }
    }
}

impl Analyzer for FlirtAnalyzer {
    fn get_name(&self) -> String {
        "FLIRT function signature analyzer".to_string()
    }

    fn analyze(&self, ws: &mut Workspace) -> Result<(), Error> {
        let mut buf = [0u8; 0xFF];
        let functions: Vec<RVA> = ws.get_functions().cloned().collect();

        for &fva in functions.iter() {
            if let Ok(buf) = ws.read_bytes_into(fva, &mut buf[..]) {
                let matches = self.sigs.r#match(buf);

                // no matches
                if matches.len() == 0 {
                    continue;
                }

                // one match, so we can take its name directly.
                if matches.len() == 1 {
                    // can unwrap name cause its guaranteed to have a name due to filter above.
                    let name = matches[0].get_name().unwrap();
                    debug!("FLIRT signature match: {} {}", fva, name);
                    ws.make_symbol(fva, name).unwrap();  // danger
                    continue;
                }

                // more than one match.
                //
                // the only time this is acceptable is if we've loaded multiple signature sets
                // and these match the same functions.
                // in this case, all the names should be the same.
                // so, lets ensure their names don't conflict.
                let names: HashSet<&str> = matches
                    .iter()
                    // can unwrap name cause its guaranteed to have a name due to filter above.
                    .map(|m| m.get_name().unwrap())
                    .collect();

                if names.len() > 1 {
                    debug!("ambiguous FLIRT signature match: {}", fva);
                } else {
                    // can unwrap name cause its guaranteed to have a name due to filter above.
                    let name = matches[0].get_name().unwrap();
                    debug!("FLIRT signature match: {} {}", fva, name);
                    ws.make_symbol(fva, name).unwrap();  // danger
                }

                // TODO: apply reference names
            }
        }

        ws.analyze()?;

        Ok(())
    }
}
<|endoftext|>


<commit_before>// Copyright (C) 2016 Élisabeth HENRY.
//
// This file is part of Crowbook.
//
// Crowbook is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published
// by the Free Software Foundation, either version 2.1 of the License, or
// (at your option) any later version.
//
// Caribon is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU Lesser General Public License for more details.
//
// You should have received ba copy of the GNU Lesser General Public License
// along with Crowbook.  If not, see <http://www.gnu.org/licenses/>.

use token::Token;
use error::{Result,Error, Source};

use std::mem;
use std::fs::File;
use std::path::Path;
use std::convert::AsRef;
use std::io::Read;
use std::collections::HashMap;

use cmark::{Parser as CMParser, Event, Tag, Options, OPTION_ENABLE_FOOTNOTES, OPTION_ENABLE_TABLES};

/// A parser that reads markdown and convert it to AST (a vector of `Token`s)
///
/// This AST can then be used by various renderes.
///
/// As this Parser uses Pulldown-cmark's one, it should be able to parse most
/// *valid* CommonMark variant of Markdown.
///
/// Compared to other Markdown parser, it might fail more often on invalid code, e.g.
/// footnotes references that are not defined anywhere.
///
/// # Examples
///
/// ```
/// use crowbook::Parser;
/// let mut parser = Parser::new();
/// let result = parser.parse("Some *valid* Markdown[^1]\n\n[^1]: with a valid footnote");
/// assert!(result.is_ok());
/// ```
///
/// ```
/// use crowbook::Parser;
/// let mut parser = Parser::new();
/// let result = parser.parse("Some footnote pointing to nothing[^1] ");
/// assert!(result.is_err());
/// ```
pub struct Parser {
    footnotes: HashMap<String, Vec<Token>>,
    source: Source,
}

impl Parser {
    /// Creates a parser
    pub fn new() -> Parser {
        Parser {
            footnotes: HashMap::new(),
            source: Source::empty(),
        }
    }

    /// Sets a parser's source file
    pub fn set_source_file(&mut self, s: &str) {
        self.source = Source::new(s);
    }

    /// Parse a file and returns an AST or an error
    pub fn parse_file<P: AsRef<Path>>(&mut self, filename: P) -> Result<Vec<Token>> {
        let path: &Path = filename.as_ref();
        let mut f = try!(File::open(path).map_err(|_| Error::file_not_found(&self.source,
                                                                                "markdown file",
                                                                                format!("{}", path.display()))));
        let mut s = String::new();

        try!(f.read_to_string(&mut s).map_err(|_| Error::parser(&self.source,
                                                                    format!("file {} contains invalid UTF-8, could not parse it", path.display()))));
        self.parse(&s)
    }

    /// Parse a string and returns an AST, or an Error.
    pub fn parse(&mut self, s: &str) -> Result<Vec<Token>> {
        let mut opts = Options::empty();
        opts.insert(OPTION_ENABLE_TABLES);
        opts.insert(OPTION_ENABLE_FOOTNOTES);
        let mut p = CMParser::new_ext(s, opts);

        let mut res = vec!();
        try!(self.parse_events(&mut p, &mut res, None));

        try!(self.parse_footnotes(&mut res));

        collapse(&mut res);

        find_standalone(&mut res);
        Ok(res)
    }

    /// Parse an inline string and returns a list of `Token`.
    ///
    /// This function removes the outermost `Paragraph` in most of the
    /// cases, as it is meant to be used for an inline string (e.g. metadata)
    pub fn parse_inline(&mut self, s: &str) -> Result<Vec<Token>> {
        let mut tokens = try!(self.parse(s));
        // Unfortunately, parser will put all this in a paragraph, so we might need to remove it.
        if tokens.len() == 1 {
            let res = match tokens[0] {
                Token::Paragraph(ref mut v) => {
                    Some(mem::replace(v, vec!()))
                },
                _ => None,
            };
            match res {
                Some(tokens) => Ok(tokens),
                _ => Ok(tokens),
            }
        } else {
            Ok(tokens)
        }
    }

    /// Replace footnote reference with their definition
    fn parse_footnotes(&mut self, v: &mut Vec<Token>) ->Result<()> {
        for token in v {
            match *token {
                Token::Footnote(ref mut content) => {
                    let reference = if let Token::Str(ref text) = content[0] {
                        text.clone()
                    } else {
                        panic!("Reference is not a vector of a single Token::Str");
                    };
                    if let Some(in_vec) = self.footnotes.get(&reference) {
                        *content = in_vec.clone();
                    } else {
                        return Err(Error::parser(&self.source,
                                                 format!("footnote reference {} does not have a matching definition", &reference)));
                    }
                },
                Token::Paragraph(ref mut vec) | Token::Header(_, ref mut vec) | Token::Emphasis(ref mut vec)
                    | Token::Strong(ref mut vec) | Token::Code(ref mut vec) | Token::BlockQuote(ref mut vec)
                    | Token::CodeBlock(_, ref mut vec) | Token::List(ref mut vec) | Token::OrderedList(_, ref mut vec)
                    | Token::Item(ref mut vec) | Token::Table(_, ref mut vec) | Token::TableHead(ref mut vec)
                    | Token::TableRow(ref mut vec) | Token::TableCell(ref mut vec) | Token::Link(_, _, ref mut vec)
                    | Token::Image(_, _, ref mut vec) => try!(self.parse_footnotes(vec)),
                _ => (),
            }
        }
        Ok(())
    }

    fn parse_events<'a>(&mut self, p: &mut CMParser<'a>, v: &mut Vec<Token>, current_tag: Option<&Tag>) -> Result<()> {
        while let Some(event) = p.next() {
            match event {
                Event::Html(text) | Event::InlineHtml(text) | Event::Text(text) => {
                    if let Some(&Token::Str(_)) = v.last() {
                        if let Token::Str(ref mut s) = *v.last_mut().unwrap() {
                            s.push_str(text.as_ref());
                        } else {
                            unreachable!();
                        }
                    } else {
                        v.push(Token::Str(text.into_owned()));
                    }
                },
                Event::Start(tag) => try!(self.parse_tag(p, v, tag)),
                Event::End(tag) => {
                    debug_assert!(format!("{:?}", Some(&tag)) == format!("{:?}", current_tag),
                                  format!("Error: opening and closing tags mismatch!\n{:?} ≠ {:?}",
                                          tag,
                                          current_tag));
                    break;
                },
                Event::SoftBreak => v.push(Token::SoftBreak),
                Event::HardBreak => v.push(Token::HardBreak),
                Event::FootnoteReference(text) => v.push(Token::Footnote(vec!(Token::Str(text.into_owned())))),
            }
        }
        Ok(())
    }

    fn parse_tag<'a>(&mut self, p: &mut CMParser<'a>, v: &mut Vec<Token>, tag: Tag<'a>) -> Result<()> {
        let mut res = vec!();

        try!(self.parse_events(p, &mut res, Some(&tag)));

        let token = match tag {
            Tag::Paragraph => Token::Paragraph(res),
            Tag::Emphasis => Token::Emphasis(res),
            Tag::Strong => Token::Strong(res),
            Tag::Code => Token::Code(res),
            Tag::Header(x) => Token::Header(x, res),
            Tag::Link(url, title) => Token::Link(url.into_owned(), title.into_owned(), res),
            Tag::Image(url, title) => Token::Image(url.into_owned(), title.into_owned(), res),
            Tag::Rule => Token::Rule,
            Tag::List(opt) => {
                if let Some(n) = opt {
                    Token::OrderedList(n, res)
                } else {
                    Token::List(res)
                }},
            Tag::Item => Token::Item(res),
            Tag::BlockQuote => Token::BlockQuote(res),
            Tag::CodeBlock(language) => Token::CodeBlock(language.into_owned(), res),
            Tag::Table(n) => Token::Table(n, res),
            Tag::TableHead => Token::TableHead(res),
            Tag::TableRow => Token::TableRow(res),
            Tag::TableCell => Token::TableCell(res),
            Tag::FootnoteDefinition(reference) => {
                if self.footnotes.contains_key(reference.as_ref()) {
                    println!("Warning: footnote definition for {} but previous definition already exist, overriding it", reference);
                }
                self.footnotes.insert(reference.into_owned(), res);
                Token::SoftBreak
            },
        };
        v.push(token);
        Ok(())
    }
}

/// Replace consecutives Strs by a Str of both, collapse soft breaks to previous std and so on
fn collapse(ast: &mut Vec<Token>) {
    if ast.len() < 2 {
        return;
    }

    let mut i = 0;
    while i < ast.len() {
        if ast[i].is_str() {
            if i < ast.len() - 1 {
                if ast[i+1].is_str() {
                    // Two consecutives Str, concatenate them
                    let token = ast.remove(i+1);
                    if let (&mut Token::Str(ref mut dest), Token::Str(ref source)) = (&mut ast[i], token) {
                        dest.push(' ');
                        dest.push_str(source);
                        continue;
                    } else {
                        unreachable!();
                    }
                } else if ast[i+1] == Token::SoftBreak {
                    ast.remove(i+1);
                    if let &mut Token::Str(ref mut dest) = &mut ast[i] {
                        dest.push(' ');
                        continue;
                    } else {
                        unreachable!();
                    }
                }
            }
        }

        // If token is containing others, recurse into them
        if let Some(ref mut inner) = ast[i].inner_mut() {
            collapse(inner);
        }
        i += 1;
    }
}

/// Replace images which are alone in a paragraph by standalone images
fn find_standalone(ast: &mut Vec<Token>) {
    for token in ast {
        let res = if let &mut Token::Paragraph(ref mut inner) = token {
            if inner.len() == 1 {
                if inner[0].is_image() {
                    if let Token::Image(source, title, inner) = mem::replace(&mut inner[0], Token::Rule) {
                        Token::StandaloneImage(source, title, inner)
                    } else {
                        unreachable!();
                    }
                } else {
                    continue;
                }
            } else {
                continue;
            }
        } else {
            continue;
        };

        *token = res;
    }
}
<commit_msg>Remove pseudo-collapsing done in Str construction as it is done more efficiently later<commit_after>// Copyright (C) 2016 Élisabeth HENRY.
//
// This file is part of Crowbook.
//
// Crowbook is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published
// by the Free Software Foundation, either version 2.1 of the License, or
// (at your option) any later version.
//
// Caribon is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU Lesser General Public License for more details.
//
// You should have received ba copy of the GNU Lesser General Public License
// along with Crowbook.  If not, see <http://www.gnu.org/licenses/>.

use token::Token;
use error::{Result,Error, Source};

use std::mem;
use std::fs::File;
use std::path::Path;
use std::convert::AsRef;
use std::io::Read;
use std::collections::HashMap;

use cmark::{Parser as CMParser, Event, Tag, Options, OPTION_ENABLE_FOOTNOTES, OPTION_ENABLE_TABLES};

/// A parser that reads markdown and convert it to AST (a vector of `Token`s)
///
/// This AST can then be used by various renderes.
///
/// As this Parser uses Pulldown-cmark's one, it should be able to parse most
/// *valid* CommonMark variant of Markdown.
///
/// Compared to other Markdown parser, it might fail more often on invalid code, e.g.
/// footnotes references that are not defined anywhere.
///
/// # Examples
///
/// ```
/// use crowbook::Parser;
/// let mut parser = Parser::new();
/// let result = parser.parse("Some *valid* Markdown[^1]\n\n[^1]: with a valid footnote");
/// assert!(result.is_ok());
/// ```
///
/// ```
/// use crowbook::Parser;
/// let mut parser = Parser::new();
/// let result = parser.parse("Some footnote pointing to nothing[^1] ");
/// assert!(result.is_err());
/// ```
pub struct Parser {
    footnotes: HashMap<String, Vec<Token>>,
    source: Source,
}

impl Parser {
    /// Creates a parser
    pub fn new() -> Parser {
        Parser {
            footnotes: HashMap::new(),
            source: Source::empty(),
        }
    }

    /// Sets a parser's source file
    pub fn set_source_file(&mut self, s: &str) {
        self.source = Source::new(s);
    }

    /// Parse a file and returns an AST or an error
    pub fn parse_file<P: AsRef<Path>>(&mut self, filename: P) -> Result<Vec<Token>> {
        let path: &Path = filename.as_ref();
        let mut f = try!(File::open(path).map_err(|_| Error::file_not_found(&self.source,
                                                                                "markdown file",
                                                                                format!("{}", path.display()))));
        let mut s = String::new();

        try!(f.read_to_string(&mut s).map_err(|_| Error::parser(&self.source,
                                                                    format!("file {} contains invalid UTF-8, could not parse it", path.display()))));
        self.parse(&s)
    }

    /// Parse a string and returns an AST, or an Error.
    pub fn parse(&mut self, s: &str) -> Result<Vec<Token>> {
        let mut opts = Options::empty();
        opts.insert(OPTION_ENABLE_TABLES);
        opts.insert(OPTION_ENABLE_FOOTNOTES);
        let mut p = CMParser::new_ext(s, opts);

        let mut res = vec!();
        try!(self.parse_events(&mut p, &mut res, None));

        try!(self.parse_footnotes(&mut res));

        collapse(&mut res);

        find_standalone(&mut res);
        Ok(res)
    }

    /// Parse an inline string and returns a list of `Token`.
    ///
    /// This function removes the outermost `Paragraph` in most of the
    /// cases, as it is meant to be used for an inline string (e.g. metadata)
    pub fn parse_inline(&mut self, s: &str) -> Result<Vec<Token>> {
        let mut tokens = try!(self.parse(s));
        // Unfortunately, parser will put all this in a paragraph, so we might need to remove it.
        if tokens.len() == 1 {
            let res = match tokens[0] {
                Token::Paragraph(ref mut v) => {
                    Some(mem::replace(v, vec!()))
                },
                _ => None,
            };
            match res {
                Some(tokens) => Ok(tokens),
                _ => Ok(tokens),
            }
        } else {
            Ok(tokens)
        }
    }

    /// Replace footnote reference with their definition
    fn parse_footnotes(&mut self, v: &mut Vec<Token>) ->Result<()> {
        for token in v {
            match *token {
                Token::Footnote(ref mut content) => {
                    let reference = if let Token::Str(ref text) = content[0] {
                        text.clone()
                    } else {
                        panic!("Reference is not a vector of a single Token::Str");
                    };
                    if let Some(in_vec) = self.footnotes.get(&reference) {
                        *content = in_vec.clone();
                    } else {
                        return Err(Error::parser(&self.source,
                                                 format!("footnote reference {} does not have a matching definition", &reference)));
                    }
                },
                Token::Paragraph(ref mut vec) | Token::Header(_, ref mut vec) | Token::Emphasis(ref mut vec)
                    | Token::Strong(ref mut vec) | Token::Code(ref mut vec) | Token::BlockQuote(ref mut vec)
                    | Token::CodeBlock(_, ref mut vec) | Token::List(ref mut vec) | Token::OrderedList(_, ref mut vec)
                    | Token::Item(ref mut vec) | Token::Table(_, ref mut vec) | Token::TableHead(ref mut vec)
                    | Token::TableRow(ref mut vec) | Token::TableCell(ref mut vec) | Token::Link(_, _, ref mut vec)
                    | Token::Image(_, _, ref mut vec) => try!(self.parse_footnotes(vec)),
                _ => (),
            }
        }
        Ok(())
    }

    fn parse_events<'a>(&mut self, p: &mut CMParser<'a>, v: &mut Vec<Token>, current_tag: Option<&Tag>) -> Result<()> {
        while let Some(event) = p.next() {
            match event {
                Event::Html(text) | Event::InlineHtml(text) | Event::Text(text) => {
                    v.push(Token::Str(text.into_owned()));
                },
                Event::Start(tag) => try!(self.parse_tag(p, v, tag)),
                Event::End(tag) => {
                    debug_assert!(format!("{:?}", Some(&tag)) == format!("{:?}", current_tag),
                                  format!("Error: opening and closing tags mismatch!\n{:?} ≠ {:?}",
                                          tag,
                                          current_tag));
                    break;
                },
                Event::SoftBreak => v.push(Token::SoftBreak),
                Event::HardBreak => v.push(Token::HardBreak),
                Event::FootnoteReference(text) => v.push(Token::Footnote(vec!(Token::Str(text.into_owned())))),
            }
        }
        Ok(())
    }

    fn parse_tag<'a>(&mut self, p: &mut CMParser<'a>, v: &mut Vec<Token>, tag: Tag<'a>) -> Result<()> {
        let mut res = vec!();

        try!(self.parse_events(p, &mut res, Some(&tag)));

        let token = match tag {
            Tag::Paragraph => Token::Paragraph(res),
            Tag::Emphasis => Token::Emphasis(res),
            Tag::Strong => Token::Strong(res),
            Tag::Code => Token::Code(res),
            Tag::Header(x) => Token::Header(x, res),
            Tag::Link(url, title) => Token::Link(url.into_owned(), title.into_owned(), res),
            Tag::Image(url, title) => Token::Image(url.into_owned(), title.into_owned(), res),
            Tag::Rule => Token::Rule,
            Tag::List(opt) => {
                if let Some(n) = opt {
                    Token::OrderedList(n, res)
                } else {
                    Token::List(res)
                }},
            Tag::Item => Token::Item(res),
            Tag::BlockQuote => Token::BlockQuote(res),
            Tag::CodeBlock(language) => Token::CodeBlock(language.into_owned(), res),
            Tag::Table(n) => Token::Table(n, res),
            Tag::TableHead => Token::TableHead(res),
            Tag::TableRow => Token::TableRow(res),
            Tag::TableCell => Token::TableCell(res),
            Tag::FootnoteDefinition(reference) => {
                if self.footnotes.contains_key(reference.as_ref()) {
                    println!("Warning: footnote definition for {} but previous definition already exist, overriding it", reference);
                }
                self.footnotes.insert(reference.into_owned(), res);
                Token::SoftBreak
            },
        };
        v.push(token);
        Ok(())
    }
}

/// Replace consecutives Strs by a Str of both, collapse soft breaks to previous std and so on
fn collapse(ast: &mut Vec<Token>) {
    if ast.len() < 2 {
        return;
    }

    let mut i = 0;
    while i < ast.len() {
        if ast[i].is_str() {
            if i < ast.len() - 1 {
                if ast[i+1].is_str() {
                    // Two consecutives Str, concatenate them
                    let token = ast.remove(i+1);
                    if let (&mut Token::Str(ref mut dest), Token::Str(ref source)) = (&mut ast[i], token) {
                        dest.push(' ');
                        dest.push_str(source);
                        continue;
                    } else {
                        unreachable!();
                    }
                } else if ast[i+1] == Token::SoftBreak {
                    ast.remove(i+1);
                    if let &mut Token::Str(ref mut dest) = &mut ast[i] {
                        dest.push(' ');
                        continue;
                    } else {
                        unreachable!();
                    }
                }
            }
        }

        // If token is containing others, recurse into them
        if let Some(ref mut inner) = ast[i].inner_mut() {
            collapse(inner);
        }
        i += 1;
    }
}

/// Replace images which are alone in a paragraph by standalone images
fn find_standalone(ast: &mut Vec<Token>) {
    for token in ast {
        let res = if let &mut Token::Paragraph(ref mut inner) = token {
            if inner.len() == 1 {
                if inner[0].is_image() {
                    if let Token::Image(source, title, inner) = mem::replace(&mut inner[0], Token::Rule) {
                        Token::StandaloneImage(source, title, inner)
                    } else {
                        unreachable!();
                    }
                } else {
                    continue;
                }
            } else {
                continue;
            }
        } else {
            continue;
        };

        *token = res;
    }
}
<|endoftext|>


<commit_before>use std::sync::Arc;

use hyper::server::{Server, Request, Response};
use hyper::status::StatusCode;
use hyper::method::Method;
use hyper::header::ContentType;
use hyper::uri::RequestUri;
use serde_json;
use serde::Serialize;

use super::config::Config;
use super::memdbstash::MemDbStash;
use super::{Result, ResultExt};

struct ServerContext {
    pub config: Config,
    pub stash: MemDbStash,
}

pub struct ApiServer {
    ctx: Arc<ServerContext>,
}

pub struct ApiResponse {
    body: Vec<u8>,
    status: StatusCode,
}

#[derive(Serialize)]
struct ApiError {
    #[serde(rename="type")]
    pub ty: String,
    pub message: String,
}

#[derive(Serialize)]
struct HealthCheckResult {
    is_healthy: bool,
    sync_lag: u32,
}

impl ApiResponse {
    pub fn new<S: Serialize>(data: S, status: StatusCode) -> Result<ApiResponse> {
        let mut body : Vec<u8> = vec![];
        serde_json::to_writer(&mut body, &data)
            .chain_err(|| "Failed to serialize response for client")?;
        Ok(ApiResponse {
            body: body,
            status: status,
        })
    }

    pub fn write_to_response(&self, mut resp: Response) -> Result<()> {
        *resp.status_mut() = self.status;
        resp.headers_mut().set(ContentType::json());
        resp.send(&self.body[..])?;
        Ok(())
    }
}

impl ApiServer {
    pub fn new(config: &Config) -> Result<ApiServer> {
        Ok(ApiServer {
            ctx: Arc::new(ServerContext {
                config: config.clone(),
                stash: MemDbStash::new(config)?,
            }),
        })
    }

    pub fn run(&self) -> Result<()> {
        let ctx = self.ctx.clone();
        Server::http(self.ctx.config.get_http_socket_addr()?)?
            .handle(move |req: Request, resp: Response|
        {
            let handler = match req.method {
                Method::Get => {
                    if let RequestUri::AbsolutePath(ref path) = req.uri {
                        match path.as_str() {
                            "/health" => healthcheck_handler,
                            _ => not_found_handler,
                        }
                    } else {
                        bad_request_handler
                    }
                }
                _ => {
                    method_not_allowed_handler
                }
            };
            match handler(&*ctx.clone(), req) {
                Ok(result) => result,
                Err(err) => {
                    // XXX: better logging here
                    println!("INTERNAL SERVER ERROR: {}", err);
                    ApiResponse::new(ApiError {
                        ty: "internal_server_error".into(),
                        message: "The server failed with an internal error".into()
                    }, StatusCode::InternalServerError).unwrap()
                }
            }.write_to_response(resp).unwrap();
        })?;
        Ok(())
    }
}

fn healthcheck_handler(ctx: &ServerContext, _: Request) -> Result<ApiResponse>
{
    // TODO: cache this
    let state = ctx.stash.get_sync_status()?;
    ApiResponse::new(HealthCheckResult {
        is_healthy: state.is_healthy(),
        sync_lag: state.lag(),
    }, if state.is_healthy() {
        StatusCode::Ok
    } else {
        StatusCode::ServiceUnavailable
    })
}

fn not_found_handler(_: &ServerContext, _: Request) -> Result<ApiResponse>
{
    ApiResponse::new(ApiError {
        ty: "not_found".into(),
        message: "The requested resource was not found".into()
    }, StatusCode::NotFound)
}

fn bad_request_handler(_: &ServerContext, _: Request) -> Result<ApiResponse>
{
    ApiResponse::new(ApiError {
        ty: "bad_request".into(),
        message: "The request could not be handled".into()
    }, StatusCode::BadRequest)
}

fn method_not_allowed_handler(_: &ServerContext, _: Request) -> Result<ApiResponse>
{
    ApiResponse::new(ApiError {
        ty: "method_not_allowed".into(),
        message: "The server cannot handle this method".into()
    }, StatusCode::MethodNotAllowed)
}
<commit_msg>Also send error message to the client<commit_after>use std::sync::Arc;

use hyper::server::{Server, Request, Response};
use hyper::status::StatusCode;
use hyper::method::Method;
use hyper::header::ContentType;
use hyper::uri::RequestUri;
use serde_json;
use serde::Serialize;

use super::config::Config;
use super::memdbstash::MemDbStash;
use super::{Result, ResultExt};

struct ServerContext {
    pub config: Config,
    pub stash: MemDbStash,
}

pub struct ApiServer {
    ctx: Arc<ServerContext>,
}

pub struct ApiResponse {
    body: Vec<u8>,
    status: StatusCode,
}

#[derive(Serialize)]
struct ApiError {
    #[serde(rename="type")]
    pub ty: String,
    pub message: String,
}

#[derive(Serialize)]
struct HealthCheckResult {
    is_healthy: bool,
    sync_lag: u32,
}

impl ApiResponse {
    pub fn new<S: Serialize>(data: S, status: StatusCode) -> Result<ApiResponse> {
        let mut body : Vec<u8> = vec![];
        serde_json::to_writer(&mut body, &data)
            .chain_err(|| "Failed to serialize response for client")?;
        Ok(ApiResponse {
            body: body,
            status: status,
        })
    }

    pub fn write_to_response(&self, mut resp: Response) -> Result<()> {
        *resp.status_mut() = self.status;
        resp.headers_mut().set(ContentType::json());
        resp.send(&self.body[..])?;
        Ok(())
    }
}

impl ApiServer {
    pub fn new(config: &Config) -> Result<ApiServer> {
        Ok(ApiServer {
            ctx: Arc::new(ServerContext {
                config: config.clone(),
                stash: MemDbStash::new(config)?,
            }),
        })
    }

    pub fn run(&self) -> Result<()> {
        let ctx = self.ctx.clone();
        Server::http(self.ctx.config.get_http_socket_addr()?)?
            .handle(move |req: Request, resp: Response|
        {
            let handler = match req.method {
                Method::Get => {
                    if let RequestUri::AbsolutePath(ref path) = req.uri {
                        match path.as_str() {
                            "/health" => healthcheck_handler,
                            _ => not_found_handler,
                        }
                    } else {
                        bad_request_handler
                    }
                }
                _ => {
                    method_not_allowed_handler
                }
            };
            match handler(&*ctx.clone(), req) {
                Ok(result) => result,
                Err(err) => {
                    // XXX: better logging here
                    println!("INTERNAL SERVER ERROR: {}", &err);
                    ApiResponse::new(ApiError {
                        ty: "internal_server_error".into(),
                        message: format!("The server failed with an internal error: {}",
                            &err),
                    }, StatusCode::InternalServerError).unwrap()
                }
            }.write_to_response(resp).unwrap();
        })?;
        Ok(())
    }
}

fn healthcheck_handler(ctx: &ServerContext, _: Request) -> Result<ApiResponse>
{
    // TODO: cache this
    let state = ctx.stash.get_sync_status()?;
    ApiResponse::new(HealthCheckResult {
        is_healthy: state.is_healthy(),
        sync_lag: state.lag(),
    }, if state.is_healthy() {
        StatusCode::Ok
    } else {
        StatusCode::ServiceUnavailable
    })
}

fn not_found_handler(_: &ServerContext, _: Request) -> Result<ApiResponse>
{
    ApiResponse::new(ApiError {
        ty: "not_found".into(),
        message: "The requested resource was not found".into()
    }, StatusCode::NotFound)
}

fn bad_request_handler(_: &ServerContext, _: Request) -> Result<ApiResponse>
{
    ApiResponse::new(ApiError {
        ty: "bad_request".into(),
        message: "The request could not be handled".into()
    }, StatusCode::BadRequest)
}

fn method_not_allowed_handler(_: &ServerContext, _: Request) -> Result<ApiResponse>
{
    ApiResponse::new(ApiError {
        ty: "method_not_allowed".into(),
        message: "The server cannot handle this method".into()
    }, StatusCode::MethodNotAllowed)
}
<|endoftext|>


<commit_before>///! Implements truncated singular value decomposition
///

use std::ops::DivAssign;
use ndarray::prelude::*;
use ndarray_rand::rand_distr::Uniform;
use ndarray_rand::RandomExt;
use num_traits::{Float, NumCast};
use crate::{Scalar, Lapack};
use super::lobpcg::{lobpcg, EigResult, Order};
use crate::error::Result;

#[derive(Debug)]
pub struct TruncatedSvdResult<A> {
    eigvals: Array1<A>,
    eigvecs: Array2<A>,
    problem: Array2<A>,
    ngm: bool
}

impl<A: Float + PartialOrd + DivAssign<A> + 'static> TruncatedSvdResult<A> {
    fn singular_values_with_indices(&self) -> (Array1<A>, Vec<usize>) {
        let mut a = self.eigvals.iter()
            .map(|x| x.sqrt())
            .enumerate()
            .collect::<Vec<_>>();

        a.sort_by(|(_,x), (_, y)| x.partial_cmp(&y).unwrap().reverse());

        let (values, indices): (Vec<A>, Vec<usize>) = a.into_iter()
            .filter(|(_,x)| *x > NumCast::from(1e-5).unwrap())
            .map(|(a,b)| (b,a))
            .unzip();

        (Array1::from(values), indices)
    }

    pub fn values(&self) -> Array1<A> {
        let (values, _) = self.singular_values_with_indices();

        values
    }

    pub fn values_vecs(&self) -> (Array2<A>, Array1<A>, Array2<A>) {
        let (values, indices) = self.singular_values_with_indices();

        let (u, v) = if self.ngm {
            let vlarge = self.eigvecs.select(Axis(1), &indices);
            let mut ularge = self.problem.dot(&vlarge);

            ularge.gencolumns_mut().into_iter()
                .zip(values.iter())
                .for_each(|(mut a,b)| a.mapv_inplace(|x| x / *b));

            (ularge, vlarge)
        } else {
            let ularge = self.eigvecs.select(Axis(1), &indices);

            let mut vlarge = self.problem.t().dot(&ularge);
            vlarge.gencolumns_mut().into_iter()
                .zip(values.iter())
                .for_each(|(mut a,b)| a.mapv_inplace(|x| x / *b));

            (ularge, vlarge)
        };

        (u, values, v.reversed_axes())
    }
}

/// Truncated singular value decomposition
///
/// This struct wraps the LOBPCG algorithm and provides convenient builder-pattern access to
/// parameter like maximal iteration, precision and constraint matrix. Furthermore it allows
/// conversion into a iterative solver where each iteration step yields a new eigenvalue/vector
/// pair.
pub struct TruncatedSvd<A: Scalar> {
    order: Order,
    problem: Array2<A>,
    precision: A::Real,
    maxiter: usize
}

impl<A: Scalar + Lapack + PartialOrd + Default> TruncatedSvd<A> {
    pub fn new(problem: Array2<A>, order: Order) -> TruncatedSvd<A> {
        TruncatedSvd {
            precision: NumCast::from(1e-5).unwrap(),
            maxiter: problem.len_of(Axis(0)) * 2,
            order,
            problem
        }
    }

    pub fn precision(mut self, precision: A::Real) -> Self {
        self.precision = precision;

        self
    }

    pub fn maxiter(mut self, maxiter: usize) -> Self {
        self.maxiter = maxiter;

        self

    }

    // calculate the eigenvalues once
    pub fn once(&self, num: usize) -> Result<TruncatedSvdResult<A>> {
        let (n,m) = (self.problem.nrows(), self.problem.ncols());

        let x = Array2::random((usize::min(n,m), num), Uniform::new(0.0, 1.0))
            .mapv(|x| NumCast::from(x).unwrap());

        let res = if n > m {
            lobpcg(|y| self.problem.t().dot(&self.problem.dot(&y)), x, None, None, self.precision, self.maxiter, self.order.clone())
        } else {
            lobpcg(|y| self.problem.dot(&self.problem.t().dot(&y)), x, None, None, self.precision, self.maxiter, self.order.clone())
        };

        match res {
            EigResult::Ok(vals, vecs, _) | EigResult::Err(vals, vecs, _, _) => {
                Ok(TruncatedSvdResult {
                    problem: self.problem.clone(),
                    eigvals: vals,
                    eigvecs: vecs,
                    ngm: n > m
                })
            },
            EigResult::NoResult(err) => Err(err)
        }
    }
}

#[cfg(test)]
mod tests {
    use crate::close_l2;
    use super::TruncatedSvd;
    use super::Order;
    use ndarray::{arr1, arr2};

    #[test]
    fn test_truncated_svd() {
        let a = arr2(&[[3., 2., 2.],
                       [2., 3., -2.]]);

        let res = TruncatedSvd::new(a, Order::Largest)
            .precision(1e-5)
            .maxiter(500)
            .once(2)
            .unwrap();

        let (u, sigma, v_t) = res.values_vecs();

        close_l2(&sigma, &arr1(&[5.0, 3.0]), 1e-5);
    }
}
<commit_msg>Add test for random matrix reconstruction with SVD<commit_after>///! Implements truncated singular value decomposition
///

use std::ops::DivAssign;
use ndarray::prelude::*;
use ndarray_rand::rand_distr::Uniform;
use ndarray_rand::RandomExt;
use num_traits::{Float, NumCast};
use crate::{Scalar, Lapack};
use super::lobpcg::{lobpcg, EigResult, Order};
use crate::error::Result;

/// The result of an eigenvalue decomposition for SVD
///
/// Provides methods for either calculating just the singular values with reduced cost or the
/// vectors as well.
#[derive(Debug)]
pub struct TruncatedSvdResult<A> {
    eigvals: Array1<A>,
    eigvecs: Array2<A>,
    problem: Array2<A>,
    ngm: bool
}

impl<A: Float + PartialOrd + DivAssign<A> + 'static> TruncatedSvdResult<A> {
    /// Returns singular values ordered by magnitude with indices.
    fn singular_values_with_indices(&self) -> (Array1<A>, Vec<usize>) {
        // numerate and square root eigenvalues
        let mut a = self.eigvals.iter()
            .map(|x| x.sqrt())
            .enumerate()
            .collect::<Vec<_>>();

        // sort by magnitude
        a.sort_by(|(_,x), (_, y)| x.partial_cmp(&y).unwrap().reverse());

        // filter low singular values away
        let (values, indices): (Vec<A>, Vec<usize>) = a.into_iter()
            .filter(|(_,x)| *x > NumCast::from(1e-5).unwrap())
            .map(|(a,b)| (b,a))
            .unzip();

        (Array1::from(values), indices)
    }

    /// Returns singular values orderd by magnitude
    pub fn values(&self) -> Array1<A> {
        let (values, _) = self.singular_values_with_indices();

        values
    }

    /// Returns singular values, left-singular vectors and right-singular vectors
    pub fn values_vecs(&self) -> (Array2<A>, Array1<A>, Array2<A>) {
        let (values, indices) = self.singular_values_with_indices();

        // branch n > m (for A is [n x m])
        let (u, v) = if self.ngm {
            let vlarge = self.eigvecs.select(Axis(1), &indices);
            let mut ularge = self.problem.dot(&vlarge);

            ularge.gencolumns_mut().into_iter()
                .zip(values.iter())
                .for_each(|(mut a,b)| a.mapv_inplace(|x| x / *b));

            (ularge, vlarge)
        } else {
            let ularge = self.eigvecs.select(Axis(1), &indices);

            let mut vlarge = self.problem.t().dot(&ularge);
            vlarge.gencolumns_mut().into_iter()
                .zip(values.iter())
                .for_each(|(mut a,b)| a.mapv_inplace(|x| x / *b));

            (ularge, vlarge)
        };

        (u, values, v.reversed_axes())
    }
}

/// Truncated singular value decomposition
///
/// This struct wraps the LOBPCG algorithm and provides convenient builder-pattern access to
/// parameter like maximal iteration, precision and constraint matrix. Furthermore it allows
/// conversion into a iterative solver where each iteration step yields a new eigenvalue/vector
/// pair.
pub struct TruncatedSvd<A: Scalar> {
    order: Order,
    problem: Array2<A>,
    precision: A::Real,
    maxiter: usize
}

impl<A: Scalar + Lapack + PartialOrd + Default> TruncatedSvd<A> {
    pub fn new(problem: Array2<A>, order: Order) -> TruncatedSvd<A> {
        TruncatedSvd {
            precision: NumCast::from(1e-5).unwrap(),
            maxiter: problem.len_of(Axis(0)) * 2,
            order,
            problem
        }
    }

    pub fn precision(mut self, precision: A::Real) -> Self {
        self.precision = precision;

        self
    }

    pub fn maxiter(mut self, maxiter: usize) -> Self {
        self.maxiter = maxiter;

        self

    }

    // calculate the eigenvalues once
    pub fn once(self, num: usize) -> Result<TruncatedSvdResult<A>> {
        let (n,m) = (self.problem.nrows(), self.problem.ncols());

        let x = Array2::random((usize::min(n,m), num), Uniform::new(0.0, 1.0))
            .mapv(|x| NumCast::from(x).unwrap());

        // square precision because the SVD squares the eigenvalue as well
        let precision = self.precision * self.precision;

        // use problem definition with less operations required
        let res = if n > m {
            lobpcg(|y| self.problem.t().dot(&self.problem.dot(&y)), x, None, None, precision, self.maxiter, self.order.clone())
        } else {
            lobpcg(|y| self.problem.dot(&self.problem.t().dot(&y)), x, None, None, precision, self.maxiter, self.order.clone())
        };

        // convert into TruncatedSvdResult
        match res {
            EigResult::Ok(vals, vecs, _) | EigResult::Err(vals, vecs, _, _) => {
                Ok(TruncatedSvdResult {
                    problem: self.problem.clone(),
                    eigvals: vals,
                    eigvecs: vecs,
                    ngm: n > m
                })
            },
            EigResult::NoResult(err) => Err(err)
        }
    }
}

#[cfg(test)]
mod tests {
    use crate::close_l2;
    use super::TruncatedSvd;
    use super::Order;
    use ndarray::{arr1, arr2, Array2};
    use ndarray_rand::rand_distr::Uniform;
    use ndarray_rand::RandomExt;

    #[test]
    fn test_truncated_svd() {
        let a = arr2(&[[3., 2., 2.],
                       [2., 3., -2.]]);

        let res = TruncatedSvd::new(a, Order::Largest)
            .precision(1e-5)
            .maxiter(10)
            .once(2)
            .unwrap();

        let (_, sigma, _) = res.values_vecs();

        close_l2(&sigma, &arr1(&[5.0, 3.0]), 1e-5);
    }

    #[test]
    fn test_truncated_svd_random() {
        let a: Array2<f64> = Array2::random((50, 10), Uniform::new(0.0, 1.0));

        let res = TruncatedSvd::new(a.clone(), Order::Largest)
            .precision(1e-5)
            .maxiter(10)
            .once(10)
            .unwrap();

        let (u, sigma, v_t) = res.values_vecs();
        let reconstructed = u.dot(&Array2::from_diag(&sigma).dot(&v_t));

        close_l2(&a, &reconstructed, 1e-5);
    }
}
<|endoftext|>