<commit_before>//! A type that can be treated as a difference.
//!
//! Differential dataflow most commonly tracks the counts associated with records in a multiset, but it
//! generalizes to tracking any map from the records to an Abelian group. The most common generalization
//! is when we maintain both a count and another accumulation, for example height. The differential
//! dataflow collections would then track for each record the total of counts and heights, which allows
//! us to track something like the average.

use std::ops::{Add, Sub, Neg, Mul};

use ::Data;

/// A type that can be treated as a difference.
///
/// The mathematical requirements are, I believe, an Abelian group, in that we require addition, inverses,
/// and almost certainly use commutativity somewhere (it isn't clear if it is a requirement, as it isn't
/// clear that there are semantics other than "we accumulate your differences"; I suspect we don't always
/// accumulate them in the right order, so commutativity is important until we conclude otherwise).
pub trait Diff : Add<Self, Output=Self> + Sub<Self, Output=Self> + Neg<Output=Self> + ::std::marker::Sized + Data + Copy {
	/// Returns true if the element is the additive identity.
	///
	/// This is primarily used by differential dataflow to know when it is safe to delete and update.
	/// When a difference accumulates to zero, the difference has no effect on any accumulation and can
	/// be removed.
	fn is_zero(&self) -> bool;
	/// The additive identity.
	///
	/// This method is primarily used by differential dataflow internals as part of consolidation, when
	/// one value is accumulated elsewhere and must be replaced by valid but harmless value.
	fn zero() -> Self;
}

impl Diff for isize {
	#[inline(always)] fn is_zero(&self) -> bool { *self == 0 }
	#[inline(always)] fn zero() -> Self { 0 }
}

impl Diff for i64 {
	#[inline(always)] fn is_zero(&self) -> bool { *self == 0 }
	#[inline(always)] fn zero() -> Self { 0 }
}

impl Diff for i32 {
	#[inline(always)] fn is_zero(&self) -> bool { *self == 0 }
	#[inline(always)] fn zero() -> Self { 0 }
}

/// The difference defined by a pair of difference elements.
///
/// This type is essentially a "pair", though in Rust the tuple types do not derive the numeric
/// traits we require, and so we need to emulate the types ourselves. In the interest of ergonomics,
/// we may eventually replace the numeric traits with our own, so that we can implement them for
/// tuples and allow users to ignore details like these.
#[derive(Abomonation, Copy, Ord, PartialOrd, Eq, PartialEq, Debug, Clone, Serialize, Deserialize)]
pub struct DiffPair<R1, R2> {
	/// The first element in the pair.
	pub element1: R1,
	/// The second element in the pair.
	pub element2: R2,
}

impl<R1: Diff, R2: Diff> DiffPair<R1, R2> {
	/// Creates a new Diff pair from two elements.
	#[inline(always)] pub fn new(elt1: R1, elt2: R2) -> Self {
		DiffPair {
			element1: elt1,
			element2: elt2,
		}
	}
}

impl<R1: Diff, R2: Diff> Diff for DiffPair<R1, R2> {
	#[inline(always)] fn is_zero(&self) -> bool { self.element1.is_zero() && self.element2.is_zero() }
	#[inline(always)] fn zero() -> Self { DiffPair { element1: R1::zero(), element2: R2::zero() } }
}

impl<R1: Diff, R2: Diff> Add<DiffPair<R1, R2>> for DiffPair<R1, R2> {
	type Output = Self;
	#[inline(always)] fn add(self, rhs: Self) -> Self {
		DiffPair {
			element1: self.element1 + rhs.element1,
			element2: self.element2 + rhs.element2,
		}
	}
}

impl<R1: Diff, R2: Diff> Sub<DiffPair<R1, R2>> for DiffPair<R1, R2> {
	type Output = DiffPair<R1, R2>;
	#[inline(always)] fn sub(self, rhs: Self) -> Self {
		DiffPair {
			element1: self.element1 - rhs.element1,
			element2: self.element2 - rhs.element2,
		}
	}
}

impl<R1: Diff, R2: Diff> Neg for DiffPair<R1, R2> {
	type Output = Self;
	#[inline(always)] fn neg(self) -> Self {
		DiffPair {
			element1: -self.element1,
			element2: -self.element2,
		}
	}
}

impl<T: Copy, R1: Diff+Mul<T>, R2: Diff+Mul<T>> Mul<T> for DiffPair<R1,R2>
where <R1 as Mul<T>>::Output: Diff, <R2 as Mul<T>>::Output: Diff {
	type Output = DiffPair<<R1 as Mul<T>>::Output, <R2 as Mul<T>>::Output>;
	fn mul(self, other: T) -> Self::Output {
		DiffPair::new(
			self.element1 * other,
			self.element2 * other,
		)
	}
}

// // TODO: This currently causes rustc to trip a recursion limit, because who knows why.
// impl<R1: Diff, R2: Diff> Mul<DiffPair<R1,R2>> for isize
// where isize: Mul<R1>, isize: Mul<R2>, <isize as Mul<R1>>::Output: Diff, <isize as Mul<R2>>::Output: Diff {
// 	type Output = DiffPair<<isize as Mul<R1>>::Output, <isize as Mul<R2>>::Output>;
// 	fn mul(self, other: DiffPair<R1,R2>) -> Self::Output {
// 		DiffPair::new(
// 			self * other.element1,
// 			self * other.element2,
// 		)
// 	}
// }
<commit_msg>relax Diff requirements<commit_after>//! A type that can be treated as a difference.
//!
//! Differential dataflow most commonly tracks the counts associated with records in a multiset, but it
//! generalizes to tracking any map from the records to an Abelian group. The most common generalization
//! is when we maintain both a count and another accumulation, for example height. The differential
//! dataflow collections would then track for each record the total of counts and heights, which allows
//! us to track something like the average.

use std::ops::{Add, Sub, Neg, Mul};

use ::Data;

/// A type that can be treated as a difference.
///
/// The mathematical requirements are, I believe, an Abelian group, in that we require addition, inverses,
/// and almost certainly use commutativity somewhere (it isn't clear if it is a requirement, as it isn't
/// clear that there are semantics other than "we accumulate your differences"; I suspect we don't always
/// accumulate them in the right order, so commutativity is important until we conclude otherwise).
pub trait Diff : Add<Self, Output=Self> + Sub<Self, Output=Self> + Neg<Output=Self> + ::std::marker::Sized + Data + Copy {
	/// Returns true if the element is the additive identity.
	///
	/// This is primarily used by differential dataflow to know when it is safe to delete and update.
	/// When a difference accumulates to zero, the difference has no effect on any accumulation and can
	/// be removed.
	fn is_zero(&self) -> bool;
	/// The additive identity.
	///
	/// This method is primarily used by differential dataflow internals as part of consolidation, when
	/// one value is accumulated elsewhere and must be replaced by valid but harmless value.
	fn zero() -> Self;
}

impl Diff for isize {
	#[inline(always)] fn is_zero(&self) -> bool { *self == 0 }
	#[inline(always)] fn zero() -> Self { 0 }
}

impl Diff for i64 {
	#[inline(always)] fn is_zero(&self) -> bool { *self == 0 }
	#[inline(always)] fn zero() -> Self { 0 }
}

impl Diff for i32 {
	#[inline(always)] fn is_zero(&self) -> bool { *self == 0 }
	#[inline(always)] fn zero() -> Self { 0 }
}

/// The difference defined by a pair of difference elements.
///
/// This type is essentially a "pair", though in Rust the tuple types do not derive the numeric
/// traits we require, and so we need to emulate the types ourselves. In the interest of ergonomics,
/// we may eventually replace the numeric traits with our own, so that we can implement them for
/// tuples and allow users to ignore details like these.
#[derive(Abomonation, Copy, Ord, PartialOrd, Eq, PartialEq, Debug, Clone, Serialize, Deserialize)]
pub struct DiffPair<R1, R2> {
	/// The first element in the pair.
	pub element1: R1,
	/// The second element in the pair.
	pub element2: R2,
}

impl<R1, R2> DiffPair<R1, R2> {
	/// Creates a new Diff pair from two elements.
	#[inline(always)] pub fn new(elt1: R1, elt2: R2) -> Self {
		DiffPair {
			element1: elt1,
			element2: elt2,
		}
	}
}

impl<R1: Diff, R2: Diff> Diff for DiffPair<R1, R2> {
	#[inline(always)] fn is_zero(&self) -> bool { self.element1.is_zero() && self.element2.is_zero() }
	#[inline(always)] fn zero() -> Self { DiffPair { element1: R1::zero(), element2: R2::zero() } }
}

impl<R1: Add, R2: Add> Add<DiffPair<R1, R2>> for DiffPair<R1, R2> {
	type Output = DiffPair<<R1 as Add>::Output, <R2 as Add>::Output>;
	#[inline(always)] fn add(self, rhs: Self) -> Self::Output {
		DiffPair {
			element1: self.element1 + rhs.element1,
			element2: self.element2 + rhs.element2,
		}
	}
}

impl<R1: Sub, R2: Sub> Sub<DiffPair<R1, R2>> for DiffPair<R1, R2> {
	type Output = DiffPair<<R1 as Sub>::Output, <R2 as Sub>::Output>;
	#[inline(always)] fn sub(self, rhs: Self) -> Self::Output {
		DiffPair {
			element1: self.element1 - rhs.element1,
			element2: self.element2 - rhs.element2,
		}
	}
}

impl<R1: Neg, R2: Neg> Neg for DiffPair<R1, R2> {
	type Output = DiffPair<<R1 as Neg>::Output, <R2 as Neg>::Output>;
	#[inline(always)] fn neg(self) -> Self::Output {
		DiffPair {
			element1: -self.element1,
			element2: -self.element2,
		}
	}
}

impl<T: Copy, R1: Mul<T>, R2: Mul<T>> Mul<T> for DiffPair<R1,R2> {
	type Output = DiffPair<<R1 as Mul<T>>::Output, <R2 as Mul<T>>::Output>;
	fn mul(self, other: T) -> Self::Output {
		DiffPair::new(
			self.element1 * other,
			self.element2 * other,
		)
	}
}

// // TODO: This currently causes rustc to trip a recursion limit, because who knows why.
// impl<R1: Diff, R2: Diff> Mul<DiffPair<R1,R2>> for isize
// where isize: Mul<R1>, isize: Mul<R2>, <isize as Mul<R1>>::Output: Diff, <isize as Mul<R2>>::Output: Diff {
// 	type Output = DiffPair<<isize as Mul<R1>>::Output, <isize as Mul<R2>>::Output>;
// 	fn mul(self, other: DiffPair<R1,R2>) -> Self::Output {
// 		DiffPair::new(
// 			self * other.element1,
// 			self * other.element2,
// 		)
// 	}
// }
<|endoftext|>


<commit_before>// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at https://mozilla.org/MPL/2.0/.

use log::trace;
use ra_ap_cfg::CfgExpr;
use ra_ap_hir::{self as hir, HasAttrs};
use ra_ap_ide_db::RootDatabase;

use crate::graph::{walker::GraphWalker, Graph, NodeIndex};

pub fn idx_of_node_with_path(
    graph: &Graph,
    path: &[String],
    _db: &RootDatabase,
) -> anyhow::Result<NodeIndex> {
    let mut node_indices = graph.node_indices();

    let node_idx = node_indices.find(|node_idx| {
        let node = &graph[*node_idx];
        node.path == path
    });

    node_idx.ok_or_else(|| anyhow::anyhow!("No node found with path {:?}", path))
}

pub fn shrink_graph(graph: &mut Graph, focus_node_idx: NodeIndex, max_depth: usize) {
    let mut walker = GraphWalker::new();

    trace!(
        "Walking graph from focus node up to depth {} ...",
        max_depth
    );

    walker.walk_graph(graph, focus_node_idx, max_depth);

    graph.retain_nodes(|_graph, node_idx| walker.nodes_visited.contains(&node_idx));
}

pub(crate) fn krate_name(krate: hir::Crate, db: &RootDatabase) -> String {
    // Obtain the crate's declaration name:
    let display_name = &krate.display_name(db).unwrap();

    // Since a crate's name may contain `-` we canonicalize it by replacing with `_`:
    display_name.replace("-", "_")
}

pub(crate) fn krate(module_def: hir::ModuleDef, db: &RootDatabase) -> Option<hir::Crate> {
    module(module_def, db).map(|module| module.krate())
}

pub(crate) fn module(module_def: hir::ModuleDef, db: &RootDatabase) -> Option<hir::Module> {
    match module_def {
        hir::ModuleDef::Module(module) => Some(module),
        module_def => module_def.module(db),
    }
}

pub(crate) fn path(module_def: hir::ModuleDef, db: &RootDatabase) -> String {
    let mut path = String::new();

    let krate = krate(module_def, db);

    // Obtain the module's krate's name (unless it's a builtin type, which have no crate):
    if let Some(krate_name) = krate.map(|krate| krate_name(krate, db)) {
        path.push_str(krate_name.as_str());
    }

    // Obtain the module's canonicalized name:
    if let Some(relative_canonical_path) = module_def.canonical_path(db) {
        path.push_str("::");
        path.push_str(relative_canonical_path.as_str());
    }

    assert!(!path.is_empty());

    path
}

// #[test] fn
// it_works() { … }
pub(crate) fn is_test_function(function: hir::Function, db: &RootDatabase) -> bool {
    let attrs = function.attrs(db);
    attrs.by_key("test").exists()
}

pub fn cfgs(hir: hir::ModuleDef, db: &RootDatabase) -> Vec<CfgExpr> {
    let cfg = match cfg(hir, db) {
        Some(cfg) => cfg,
        None => return vec![],
    };

    match cfg {
        CfgExpr::Invalid => vec![],
        cfg @ CfgExpr::Atom(_) => vec![cfg],
        CfgExpr::All(cfgs) => cfgs,
        cfg @ CfgExpr::Any(_) => vec![cfg],
        cfg @ CfgExpr::Not(_) => vec![cfg],
    }
}

pub fn cfg(hir: hir::ModuleDef, db: &RootDatabase) -> Option<CfgExpr> {
    match hir {
        hir::ModuleDef::Module(r#mod) => r#mod.attrs(db).cfg(),
        hir::ModuleDef::Function(r#fn) => r#fn.attrs(db).cfg(),
        hir::ModuleDef::Adt(r#adt) => r#adt.attrs(db).cfg(),
        hir::ModuleDef::Variant(r#variant) => r#variant.attrs(db).cfg(),
        hir::ModuleDef::Const(r#const) => r#const.attrs(db).cfg(),
        hir::ModuleDef::Static(r#static) => r#static.attrs(db).cfg(),
        hir::ModuleDef::Trait(r#trait) => r#trait.attrs(db).cfg(),
        hir::ModuleDef::TypeAlias(r#type) => r#type.attrs(db).cfg(),
        hir::ModuleDef::BuiltinType(_) => None,
    }
}
<commit_msg>Fix clippy warnings<commit_after>// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at https://mozilla.org/MPL/2.0/.

use log::trace;
use ra_ap_cfg::CfgExpr;
use ra_ap_hir::{self as hir, HasAttrs};
use ra_ap_ide_db::RootDatabase;

use crate::graph::{walker::GraphWalker, Graph, NodeIndex};

pub fn idx_of_node_with_path(
    graph: &Graph,
    path: &[String],
    _db: &RootDatabase,
) -> anyhow::Result<NodeIndex> {
    let mut node_indices = graph.node_indices();

    let node_idx = node_indices.find(|node_idx| {
        let node = &graph[*node_idx];
        node.path == path
    });

    node_idx.ok_or_else(|| anyhow::anyhow!("No node found with path {:?}", path))
}

pub fn shrink_graph(graph: &mut Graph, focus_node_idx: NodeIndex, max_depth: usize) {
    let mut walker = GraphWalker::new();

    trace!(
        "Walking graph from focus node up to depth {} ...",
        max_depth
    );

    walker.walk_graph(graph, focus_node_idx, max_depth);

    graph.retain_nodes(|_graph, node_idx| walker.nodes_visited.contains(&node_idx));
}

pub(crate) fn krate_name(krate: hir::Crate, db: &RootDatabase) -> String {
    // Obtain the crate's declaration name:
    let display_name = &krate.display_name(db).unwrap();

    // Since a crate's name may contain `-` we canonicalize it by replacing with `_`:
    display_name.replace('-', "_")
}

pub(crate) fn krate(module_def: hir::ModuleDef, db: &RootDatabase) -> Option<hir::Crate> {
    module(module_def, db).map(|module| module.krate())
}

pub(crate) fn module(module_def: hir::ModuleDef, db: &RootDatabase) -> Option<hir::Module> {
    match module_def {
        hir::ModuleDef::Module(module) => Some(module),
        module_def => module_def.module(db),
    }
}

pub(crate) fn path(module_def: hir::ModuleDef, db: &RootDatabase) -> String {
    let mut path = String::new();

    let krate = krate(module_def, db);

    // Obtain the module's krate's name (unless it's a builtin type, which have no crate):
    if let Some(krate_name) = krate.map(|krate| krate_name(krate, db)) {
        path.push_str(krate_name.as_str());
    }

    // Obtain the module's canonicalized name:
    if let Some(relative_canonical_path) = module_def.canonical_path(db) {
        path.push_str("::");
        path.push_str(relative_canonical_path.as_str());
    }

    assert!(!path.is_empty());

    path
}

// #[test] fn
// it_works() { … }
pub(crate) fn is_test_function(function: hir::Function, db: &RootDatabase) -> bool {
    let attrs = function.attrs(db);
    attrs.by_key("test").exists()
}

pub fn cfgs(hir: hir::ModuleDef, db: &RootDatabase) -> Vec<CfgExpr> {
    let cfg = match cfg(hir, db) {
        Some(cfg) => cfg,
        None => return vec![],
    };

    match cfg {
        CfgExpr::Invalid => vec![],
        cfg @ CfgExpr::Atom(_) => vec![cfg],
        CfgExpr::All(cfgs) => cfgs,
        cfg @ CfgExpr::Any(_) => vec![cfg],
        cfg @ CfgExpr::Not(_) => vec![cfg],
    }
}

pub fn cfg(hir: hir::ModuleDef, db: &RootDatabase) -> Option<CfgExpr> {
    match hir {
        hir::ModuleDef::Module(r#mod) => r#mod.attrs(db).cfg(),
        hir::ModuleDef::Function(r#fn) => r#fn.attrs(db).cfg(),
        hir::ModuleDef::Adt(r#adt) => r#adt.attrs(db).cfg(),
        hir::ModuleDef::Variant(r#variant) => r#variant.attrs(db).cfg(),
        hir::ModuleDef::Const(r#const) => r#const.attrs(db).cfg(),
        hir::ModuleDef::Static(r#static) => r#static.attrs(db).cfg(),
        hir::ModuleDef::Trait(r#trait) => r#trait.attrs(db).cfg(),
        hir::ModuleDef::TypeAlias(r#type) => r#type.attrs(db).cfg(),
        hir::ModuleDef::BuiltinType(_) => None,
    }
}
<|endoftext|>


<commit_before>use std::ops;
use std::mem;
use std::slice;
use gl;

pub trait Image2d {
    fn width(&self) -> usize;
    fn height(&self) -> usize;
    fn format(&self) -> ImageFormat;
    fn textel_bytes(&self) -> &[u8];
}

#[derive(Debug, Clone, Copy)]
#[repr(C)]
pub struct Pixel {
    r: u8,
    g: u8,
    b: u8,
    a: u8
}

impl Pixel {
    pub fn r_g_b_a(r: u8, g: u8, b: u8, a: u8) -> Self {
        Pixel { r: r, g: g, b: b, a: a }
    }

    pub fn r_g_b(r: u8, g: u8, b: u8) -> Self {
        Pixel::r_g_b_a(r, g, b, 0xFF)
    }

    pub fn rgb(rgb: u32) -> Self {
        Pixel::rgb_a(rgb, 0xFF)
    }

    pub fn argb(argb: u32) -> Self {
        let a = (argb & 0xFF000000) >> 24;
        Pixel::rgb_a(argb, a as u8)
    }

    pub fn rgba(rgba: u32) -> Self {
        let r = (rgba & 0xFF000000) >> 24;
        let g = (rgba & 0x00FF0000) >> 16;
        let b = (rgba & 0x0000FF00) >> 8;
        let a =  rgba & 0x000000FF;
        Pixel::r_g_b_a(r as u8, g as u8, b as u8, a as u8)
    }

    pub fn rgb_a(rgb: u32, a: u8) -> Self {
        let r = (rgb & 0xFF0000) >> 16;
        let g = (rgb & 0x00FF00) >> 8;
        let b =  rgb & 0x0000FF;
        Pixel::r_g_b_a(r as u8, g as u8, b as u8, a)
    }
}

pub struct Pixels {
    width: usize,
    height: usize,
    pixels: Vec<Pixel>
}

impl Pixels {
    pub fn new(width: usize, height: usize) -> Self {
        Pixels {
            width: width,
            height: height,
            pixels: vec![Pixel::rgb(0x0000FF); width * height]
        }
    }
}

impl ops::Index<usize> for Pixels {
    type Output = [Pixel];

    fn index(&self, row: usize) -> &[Pixel] {
        debug_assert!(row < self.height);

        &self.pixels[(row*self.width)..((row+1)*self.width)]
    }
}

impl ops::IndexMut<usize> for Pixels {
    fn index_mut(&mut self, row: usize) -> &mut [Pixel] {
        debug_assert!(row < self.height);

        &mut self.pixels[(row*self.width)..((row+1)*self.width)]
    }
}

impl Image2d for Pixels {
    fn width(&self) -> usize {
        self.width
    }

    fn height(&self) -> usize {
        self.height
    }

    fn format(&self) -> ImageFormat {
        ImageFormat::rgba8()
    }

    fn textel_bytes(&self) -> &[u8] {
        let len = self.pixels.len() * mem::size_of::<Pixel>();
        unsafe {
            slice::from_raw_parts(mem::transmute(&self.pixels[0]), len)
        }
    }
}

gl_enum! {
    pub gl_enum TextelType {
        UnsignedByte as UNSIGNED_BYTE_TEXTEL = gl::UNSIGNED_BYTE,
        UnsignedShort565 as UNSIGNED_SHORT_5_6_5 = gl::UNSIGNED_SHORT_5_6_5,
        UnsignedShort4444 as UNSIGNED_SHORT_4_4_4_4 =
            gl::UNSIGNED_SHORT_4_4_4_4,
        UnsignedShort5551 as UNSIGNED_SHORT_5_5_5_1 =
            gl::UNSIGNED_SHORT_5_5_5_1
    }
}

gl_enum! {
    pub gl_enum TextelFormat {
        Alpha as ALPHA = gl::ALPHA,
        RGB as RGB = gl::RGB,
        RGBA as RGBA = gl::RGBA
    }
}

#[derive(Debug, Clone, Copy)]
pub struct ImageFormat {
    pub textel_type: TextelType,
    pub textel_format: TextelFormat
}

impl ImageFormat {
    fn rgba8() -> Self {
        ImageFormat {
            textel_type: TextelType::UnsignedByte,
            textel_format: TextelFormat::RGBA
        }
    }
}
<commit_msg>Mark `ImageFormat::rgba8()` as public<commit_after>use std::ops;
use std::mem;
use std::slice;
use gl;

pub trait Image2d {
    fn width(&self) -> usize;
    fn height(&self) -> usize;
    fn format(&self) -> ImageFormat;
    fn textel_bytes(&self) -> &[u8];
}

#[derive(Debug, Clone, Copy)]
#[repr(C)]
pub struct Pixel {
    r: u8,
    g: u8,
    b: u8,
    a: u8
}

impl Pixel {
    pub fn r_g_b_a(r: u8, g: u8, b: u8, a: u8) -> Self {
        Pixel { r: r, g: g, b: b, a: a }
    }

    pub fn r_g_b(r: u8, g: u8, b: u8) -> Self {
        Pixel::r_g_b_a(r, g, b, 0xFF)
    }

    pub fn rgb(rgb: u32) -> Self {
        Pixel::rgb_a(rgb, 0xFF)
    }

    pub fn argb(argb: u32) -> Self {
        let a = (argb & 0xFF000000) >> 24;
        Pixel::rgb_a(argb, a as u8)
    }

    pub fn rgba(rgba: u32) -> Self {
        let r = (rgba & 0xFF000000) >> 24;
        let g = (rgba & 0x00FF0000) >> 16;
        let b = (rgba & 0x0000FF00) >> 8;
        let a =  rgba & 0x000000FF;
        Pixel::r_g_b_a(r as u8, g as u8, b as u8, a as u8)
    }

    pub fn rgb_a(rgb: u32, a: u8) -> Self {
        let r = (rgb & 0xFF0000) >> 16;
        let g = (rgb & 0x00FF00) >> 8;
        let b =  rgb & 0x0000FF;
        Pixel::r_g_b_a(r as u8, g as u8, b as u8, a)
    }
}

pub struct Pixels {
    width: usize,
    height: usize,
    pixels: Vec<Pixel>
}

impl Pixels {
    pub fn new(width: usize, height: usize) -> Self {
        Pixels {
            width: width,
            height: height,
            pixels: vec![Pixel::rgb(0x0000FF); width * height]
        }
    }
}

impl ops::Index<usize> for Pixels {
    type Output = [Pixel];

    fn index(&self, row: usize) -> &[Pixel] {
        debug_assert!(row < self.height);

        &self.pixels[(row*self.width)..((row+1)*self.width)]
    }
}

impl ops::IndexMut<usize> for Pixels {
    fn index_mut(&mut self, row: usize) -> &mut [Pixel] {
        debug_assert!(row < self.height);

        &mut self.pixels[(row*self.width)..((row+1)*self.width)]
    }
}

impl Image2d for Pixels {
    fn width(&self) -> usize {
        self.width
    }

    fn height(&self) -> usize {
        self.height
    }

    fn format(&self) -> ImageFormat {
        ImageFormat::rgba8()
    }

    fn textel_bytes(&self) -> &[u8] {
        let len = self.pixels.len() * mem::size_of::<Pixel>();
        unsafe {
            slice::from_raw_parts(mem::transmute(&self.pixels[0]), len)
        }
    }
}

gl_enum! {
    pub gl_enum TextelType {
        UnsignedByte as UNSIGNED_BYTE_TEXTEL = gl::UNSIGNED_BYTE,
        UnsignedShort565 as UNSIGNED_SHORT_5_6_5 = gl::UNSIGNED_SHORT_5_6_5,
        UnsignedShort4444 as UNSIGNED_SHORT_4_4_4_4 =
            gl::UNSIGNED_SHORT_4_4_4_4,
        UnsignedShort5551 as UNSIGNED_SHORT_5_5_5_1 =
            gl::UNSIGNED_SHORT_5_5_5_1
    }
}

gl_enum! {
    pub gl_enum TextelFormat {
        Alpha as ALPHA = gl::ALPHA,
        RGB as RGB = gl::RGB,
        RGBA as RGBA = gl::RGBA
    }
}

#[derive(Debug, Clone, Copy)]
pub struct ImageFormat {
    pub textel_type: TextelType,
    pub textel_format: TextelFormat
}

impl ImageFormat {
    pub fn rgba8() -> Self {
        ImageFormat {
            textel_type: TextelType::UnsignedByte,
            textel_format: TextelFormat::RGBA
        }
    }
}
<|endoftext|>


<commit_before>use l0::ast::{Atom, Term, FactInstruction, QueryInstruction, Var};
use l0::iterators::{BreadthFirstIterator, PostOrderIterator};

use std::collections::{HashSet};
use std::fmt;
use std::vec::{Vec};

impl fmt::Display for QueryInstruction {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        match self {
            &QueryInstruction::PutStructure(ref a, ref s, ref r) =>
                write!(f, "put_structure {}/{}, X{}", a, s, r),
            &QueryInstruction::SetVariable(ref r) =>
                write!(f, "set_variable X{}", r),
            &QueryInstruction::SetValue(ref r) =>
                write!(f, "set_value X{}", r),
        }
    }
}

impl fmt::Display for FactInstruction {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        match self {
            &FactInstruction::GetStructure(ref a, ref s, ref r) =>
                write!(f, "get_structure {}/{}, X{}", a, s, r),
            &FactInstruction::UnifyVariable(ref r) =>
                write!(f, "unify_variable X{}", r),
            &FactInstruction::UnifyValue(ref r) =>
                write!(f, "unify_value X{}", r)
        }
    }
}

pub trait CompilationTarget<'a> where Self : Sized {
    type Iterator : Iterator<Item=&'a Term>;

    fn iter(term: &'a Term) -> Self::Iterator;

    fn to_structure(name: Atom, arity: usize, cell_num: usize) -> Self;
    fn to_value(cell_num: usize) -> Self;
    fn to_variable(cell_num: usize) -> Self;
}

impl<'a> CompilationTarget<'a> for FactInstruction {
    type Iterator = BreadthFirstIterator<'a>;

    fn iter(term: &'a Term) -> Self::Iterator {
        term.breadth_first_iter()
    }

    fn to_structure(name: Atom, arity: usize, cell_num: usize) -> Self {
        FactInstruction::GetStructure(name, arity, cell_num)
    }

    fn to_value(cell_num: usize) -> Self {
        FactInstruction::UnifyValue(cell_num)
    }

    fn to_variable(cell_num: usize) -> Self {
        FactInstruction::UnifyVariable(cell_num)
    }
}

impl<'a> CompilationTarget<'a> for QueryInstruction {
    type Iterator = PostOrderIterator<'a>;

    fn iter(term: &'a Term) -> Self::Iterator {
        term.post_order_iter()
    }

    fn to_structure(name: Atom, arity: usize, cell_num: usize) -> Self {
        QueryInstruction::PutStructure(name, arity, cell_num)
    }

    fn to_value(cell_num: usize) -> Self {
        QueryInstruction::SetValue(cell_num)
    }

    fn to_variable(cell_num: usize) -> Self {
        QueryInstruction::SetVariable(cell_num)
    }
}

fn subterm_to_instr<'a, Target>(subterm:  &'a Term,
                                bindings: &mut HashSet<&'a Var>)
                                -> Target
    where Target: CompilationTarget<'a>
{
    match subterm {
        &Term::Atom(ref cell_num, _) =>
            Target::to_value(cell_num.get()),
        &Term::Var(ref cell_num, ref atom) if bindings.contains(atom) =>
            Target::to_value(cell_num.get()),
        &Term::Var(ref cell_num, ref atom) => {
            bindings.insert(atom);
            Target::to_variable(cell_num.get())
        },
        &Term::Clause(ref cell_num, _, _) =>
            Target::to_value(cell_num.get())
    }
}

pub fn compile_target<'a, Target>(term: &'a Term) -> Vec<Target>
    where Target: CompilationTarget<'a>
{
    let mut iter     = Target::iter(term);
    let mut target   = Vec::<Target>::new();
    let mut bindings = HashSet::new();

    while let Some(term) = iter.next() {
        match term {
            &Term::Atom(ref cell_num, ref atom) =>
                target.push(Target::to_structure(atom.clone(), 0, cell_num.get())),
            &Term::Clause(ref cell_num, ref atom, ref terms) => {
                target.push(Target::to_structure(atom.clone(), 0, cell_num.get()));

                for subterm in terms {
                    target.push(subterm_to_instr(subterm.as_ref(), &mut bindings));
                }
            },
            _ => {},
        };
    }

    target
}
<commit_msg>remove unnecessary Sized trait on CompilationTarget<commit_after>use l0::ast::{Atom, Term, FactInstruction, QueryInstruction, Var};
use l0::iterators::{BreadthFirstIterator, PostOrderIterator};

use std::collections::{HashSet};
use std::fmt;
use std::vec::{Vec};

impl fmt::Display for QueryInstruction {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        match self {
            &QueryInstruction::PutStructure(ref a, ref s, ref r) =>
                write!(f, "put_structure {}/{}, X{}", a, s, r),
            &QueryInstruction::SetVariable(ref r) =>
                write!(f, "set_variable X{}", r),
            &QueryInstruction::SetValue(ref r) =>
                write!(f, "set_value X{}", r),
        }
    }
}

impl fmt::Display for FactInstruction {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        match self {
            &FactInstruction::GetStructure(ref a, ref s, ref r) =>
                write!(f, "get_structure {}/{}, X{}", a, s, r),
            &FactInstruction::UnifyVariable(ref r) =>
                write!(f, "unify_variable X{}", r),
            &FactInstruction::UnifyValue(ref r) =>
                write!(f, "unify_value X{}", r)
        }
    }
}

pub trait CompilationTarget<'a> {
    type Iterator : Iterator<Item=&'a Term>;

    fn iter(term: &'a Term) -> Self::Iterator;

    fn to_structure(name: Atom, arity: usize, cell_num: usize) -> Self;
    fn to_value(cell_num: usize) -> Self;
    fn to_variable(cell_num: usize) -> Self;
}

impl<'a> CompilationTarget<'a> for FactInstruction {
    type Iterator = BreadthFirstIterator<'a>;

    fn iter(term: &'a Term) -> Self::Iterator {
        term.breadth_first_iter()
    }

    fn to_structure(name: Atom, arity: usize, cell_num: usize) -> Self {
        FactInstruction::GetStructure(name, arity, cell_num)
    }

    fn to_value(cell_num: usize) -> Self {
        FactInstruction::UnifyValue(cell_num)
    }

    fn to_variable(cell_num: usize) -> Self {
        FactInstruction::UnifyVariable(cell_num)
    }
}

impl<'a> CompilationTarget<'a> for QueryInstruction {
    type Iterator = PostOrderIterator<'a>;

    fn iter(term: &'a Term) -> Self::Iterator {
        term.post_order_iter()
    }

    fn to_structure(name: Atom, arity: usize, cell_num: usize) -> Self {
        QueryInstruction::PutStructure(name, arity, cell_num)
    }

    fn to_value(cell_num: usize) -> Self {
        QueryInstruction::SetValue(cell_num)
    }

    fn to_variable(cell_num: usize) -> Self {
        QueryInstruction::SetVariable(cell_num)
    }
}

fn subterm_to_instr<'a, Target>(subterm:  &'a Term,
                                bindings: &mut HashSet<&'a Var>)
                                -> Target
    where Target: CompilationTarget<'a>
{
    match subterm {
        &Term::Atom(ref cell_num, _) =>
            Target::to_value(cell_num.get()),
        &Term::Var(ref cell_num, ref atom) if bindings.contains(atom) =>
            Target::to_value(cell_num.get()),
        &Term::Var(ref cell_num, ref atom) => {
            bindings.insert(atom);
            Target::to_variable(cell_num.get())
        },
        &Term::Clause(ref cell_num, _, _) =>
            Target::to_value(cell_num.get())
    }
}

pub fn compile_target<'a, Target>(term: &'a Term) -> Vec<Target>
    where Target: CompilationTarget<'a>
{
    let mut iter     = Target::iter(term);
    let mut target   = Vec::<Target>::new();
    let mut bindings = HashSet::new();

    while let Some(term) = iter.next() {
        match term {
            &Term::Atom(ref cell_num, ref atom) =>
                target.push(Target::to_structure(atom.clone(), 0, cell_num.get())),
            &Term::Clause(ref cell_num, ref atom, ref terms) => {
                target.push(Target::to_structure(atom.clone(), 0, cell_num.get()));

                for subterm in terms {
                    target.push(subterm_to_instr(subterm.as_ref(), &mut bindings));
                }
            },
            _ => {},
        };
    }

    target
}
<|endoftext|>


<commit_before>use af;
use af::{Dim4, Array};
use af::MatProp;

use activations;
use initializations;
use layer::{Layer, Input};

pub enum LSTMIndex {
  Input,
  Forget,
  Output,
  CellTilda,
}

pub enum ActivationIndex {
  Input,
  Inner,
  Outer,
}

pub enum DataIndex {
  Input,
  Recurrence,
}

pub struct LSTM {
  weights: Vec<Array>,
  recurrent_weights: Vec<Array>,
  bias: Vec<Array>,
  inner_activation: &'static str,
  outer_activation: &'static str,
  return_sequences: bool,
  delta: Array,
  inputs: Input,
}

impl LSTM {
  pub fn new(input_size: u64, output_size: u64
             , outer_activation: &'static str
             , inner_activation: &'static str
             , w_init: &'static str
             , w_inner_init: &'static str
             , bias_init: &'static str
             , forget_bias_init: &'static str
             , return_sequences: bool) -> LSTM
  {
    LSTM{
      weights: vec![initializations::get_initialization(w_init, Dim4::new(&[output_size, input_size, 1, 1])).unwrap()],
      recurrent_weights: vec![initializations::get_initialization(w_inner_init, Dim4::new(&[output_size, output_size, 1, 1])).unwrap(); 4],
      bias: vec![initializations::get_initialization(b_init, Dim4::new(&[output_size, 1, 1, 1])).unwrap()
                 , initializations::get_initialization(bias_init, Dim4::new(&[output_size, 1, 1, 1])).unwrap()
                 , initializations::get_initialization(bias_init, Dim4::new(&[output_size, 1, 1, 1])).unwrap()
                 , initializations::get_initialization(forget_bias_init, Dim4::new(&[output_size, 1, 1, 1])).unwrap()],
      inner_activation: inner_activation,
      outer_activation: outer_activation,
      return_sequences: return_sequences,
    }
  }
}

impl RTRL for LSTM {
  pub fn rtrl(&self, dW_tm1: &mut Array  // previous W derivatives for [I, F, Ct]
              , dU_tm1: &mut Array       // previous U derivatives for [I, F, Ct]
              , db_tm1: &mut Array       // previous b derivatives for [I, F, Ct]
              , z_t: &Array              // current time activation
              , inputs: &Input)          // x_t & h_{t-1}
  {
    let block_size = z_t.dims().unwrap()[0];
    assert!(block_size as f32 % 5.0f32 == 0); // there are 5 data pieces we need
    let chunk_size = block_size / 5;
    // chunk out i, f, ct, c_tm1
    let if_t = activations::get_activation(self.inner_activation,
                                            , &af::rows(z_t, 0, 2*chunk_size).unwrap).unwrap();
    let ct_t = activations::get_activation(self.outer_activation,
                                           , &af::rows(z_t, 3*chunk_size, 4*chunk_size).unwrap).unwrap();
    let c_tm1 = af::rows(z_t, 4*chunk_size, 5*chunk_size).unwrap();

    // diff(z_i), diff(z_f), diff(z_ct)
    let dz = vec![&activations::get_activation_derivative(self.inner_activation, &af::rows(z_t, 0, chunk_size).unwrap()).unwrap() //d(z)
                  , &activations::get_activation_derivative(self.inner_activation, &af::rows(z_t, chunk_size, 2*chunk_size).unwrap()).unwrap()
                  , &activations::get_activation_derivative(self.outer_activation, &af::rows(z_t, 3*chunk_size, 4*chunk_size).unwrap()).unwrap()];
    let ct_ctm1_i = vec![&ct_t, &c_tm1, &af::rows(if_t, 0, chunk_size).unwrap()];

    // [Ct_t; C_{t-1}; i_t] * dz
    let dzprod = af::mul(&af::join_many(0, ct_ctm1_i).unwrap()
                         , af::join_many(0, dzvec).unwrap(), false).unwrap();

    // dC_t/dWi  = (dC_{t-1}/dWi  * f_t) + ct_t  * inner_activation(z_i) * x_t
    // dC_t/dWf  = (dC_{t-1}/dWf  * f_t) + c_tm1 * inner_activation(z_f) * x_t
    // dC_t/dWct = (dC_{t-1}/dWct * f_t) + i_t   * outer_activation(Ct)  * x_t
    let w_lhs = af::mul(dW_tm1, &af::rows(if_t, chunk_size, 2*chunk_size).unwrap(), true).unwrap(); // dC_{t-1}/dW * f_t
    let w_rhs = af::mul(&dzprod, &inputs.data[DataIndex::Input]);
    dW_tm1 = af::add(&w_lhs, &w_rhs).unwrap();

    let u_lhs = af::mul(dU_tm1, &af::rows(if_t, chunk_size, 2*chunk_size).unwrap(), true).unwrap(); // dC_{t-1}/dU * f_t
    let u_rhs = af::mul(&dzprod, &inputs.data[DataIndex::Recurrence]);
    dU_tm1 = af::add(&u_lhs, &u_rhs).unwrap();

    let b_lhs = af::mul(db_tm1, &af::rows(if_t, chunk_size, 2*chunk_size).unwrap(), true).unwrap(); // dC_{t-1}/db * f_t
    dW_tm1 = af::add(&b_lhs, &dzprod).unwrap();

    // //TODO: Optimize this to be all [W together, all U together, all b together]

    // // input gate calculations
    // // dC_t/dWi = (dC_{t-1}/dWi * f_t) + ct_t * inner_activation(z_i) * x_t
    // // dC_t/dUi = (dC_{t-1}/dUi * f_t) + ct_t * inner_activation(z_i) * h_{t-1}
    // // dC_t/dbi = (dC_{t-1}/dbi * f_t) + ct_t * inner_activation(z_i)
    // let di_rhs = af::mul(&activations::get_activation(self.inner_activation, &ct_t).unwrap()
    //                      , &af::rows(&ifo_t, 0, chunk_size).unwrap()).unwrap();
    // d_tm1[0] = af::add(&dc_lhs[0], &af::mul(&di_rhs, inputs.data[DataIndex::Input]).unwrap).unwrap();
    // d_tm1[1] = af::add(&dc_lhs[1], &af::mul(&di_rhs, inputs.data[DataIndex::Recurrence]).unwrap).unwrap();
    // d_tm1[2] = af::add(&dc_lhs[2], &di_rhs);

    // // forget gate calculations
    // // dC_t/dWf = (dC_{t-1}/dWf * f_t) + c_tm1 * inner_activation(z_f) * x_t
    // // dC_t/dUf = (dC_{t-1}/dUf * f_t) + c_tm1 * inner_activation(z_f) * h_{t-1}
    // // dC_t/dbf = (dC_{t-1}/dbf * f_t) + c_tm1 * inner_activation(z_f)
    // let df_rhs = af::mul(&activations::get_activation(self.inner_activation, &ct_t).unwrap()
    //                      , &af::rows(&ifo_t, chunk_size, 2*chunk_size).unwrap()).unwrap();
    // let df_dWi = af::add(&dc_lhs[0], &af::mul(&di_rhs, inputs.data[DataIndex::Input]).unwrap).unwrap();
    // let df_dUi = af::add(&dc_lhs[1], &af::mul(&di_rhs, inputs.data[DataIndex::Recurrence]).unwrap).unwrap();
    // let df_dbi = af::add(&dc_lhs[2], &di_rhs);

    // // cell calculations
    // // dC_t/dWct = (dC_{t-1}/dWct * f_t) + outer_activation(Ct) * x_t * i_t
    // // dC_t/dUct = (dC_{t-1}/dUct * f_t) + outer_activation(Ct) * x_t * h_{t-1}
    // // dC_t/dbct = (dC_{t-1}/dbct * f_t) + outer_activation(Ct)
    // let dct_lhs = af::mul(&d_tm1[0], &af::rows(ifo_t, chunk_size, 2*chunk_size).unwrap()).unwrap();
    // let dct_rhs = af::mul(&activations::get_activation(self.outer_activation, &ct_t).unwrap()
    //                       , &af::rows(&ifo_t, 0, chunk_size).unwrap()).unwrap();
    // let dc_dwct = af::add(&dct_lhs, &af::mul(&dct_rhs, inputs.data[DataIndex::Input]).unwrap).unwrap();
    // let dc_duct = af::add(&dct_lhs, &af::mul(&dct_rhs, inputs.data[DataIndex::Recurrence]).unwrap).unwrap();
    // let dc_dbct = af::add(&dct_lhs, &dct_rhs);
  }
}

impl Layer for LSTM {

  pub fn forward(&mut self, inputs:& Input) {
    // keep previous layer's outputs
    self.inputs = inputs.clone();

    // apply the activation to the previous layer [Optimization: Memory saving]
    let activated_input = activations::get_activation(inputs.activation[ActivationIndex::Inner]
                                                      , &inputs.data[DataIndex::Input]).unwrap();

    // extract the sub-block of each gate [i_tm1, f_tm1, o_tm1, ct_tm1, c_tm2]
    let block_size = inputs.data[DataIndex::Recurrence].dims().unwrap()[0];
    assert!(block_size as f32 % 5.0f32 == 0); // there are 5 data pieces we need
    let chunk_size = block_size / 5;
    let ifo_tm1 = activations::get_activation(inputs.activation[ActivationIndex::Inner]
                                              , &af::rows(&inputs.data[DataIndex::Recurrence], 0, 3 * chunk_size).unwrap).unwrap();
    let ct_tm1 = activations::get_activation(inputs.activation[ActivationIndex::Outer]
                                             , &af::rows(&inputs.data[DataIndex::Recurrence], 3 * chunk_size, 4 * chunk_size).unwrap).unwrap();
    let c_tm2 = af::rows(&inputs.data[DataIndex::Recurrence], 4 * chunk_size, 5 * chunk_size).unwrap();

    // calculate c_tm1 & h_tm1
    let c_tm1 = af::add(&af::mul(&af::rows(&ifo_tm1, 0, chunk_size).unwrap(), &ct_tm1, false).unwrap()
                        , &af::mul(&af::rows(&ifo_tm1, chunk_size, 2 * chunk_size).unwrap(), &c_tm2, false).unwrap()
                        , false).unwrap();
    let h_tm1 = af::mul(&o_tm1, activations::get_activation(inputs.activation[ActivationIndex::Outer]
                                                            , &c_tm1), false).unwrap();

    // forward pass in a batch for performance
    let weights_ref    = vec![&self.weights[LSTMIndex::Input]
                              , &self.weights[LSTMIndex::Forget]
                              , &self.weights[LSTMIndex::Output]
                              , &self.weights[LSTMIndex::CellTilda]];
    let recurrents_ref = vec![&self.recurrent_weights[LSTMIndex::Input]
                              , &self.recurrent_weights[LSTMIndex::Forget]
                              , &self.recurrent_weights[LSTMIndex::Output]
                              , &self.recurrent_weights[LSTMIndex::CellTilda]];
    let bias_ref       = vec![&self.bias[LSTMIndex::Input]
                              , &self.bias[LSTMIndex::Forget]
                              , &self.bias[LSTMIndex::Output]
                              , &self.bias[LSTMIndex::CellTilda]];
    // [ifo_ct] = W*x + U*h_tm1 + b
    let z_t = af::add(&af::add(&af::matmul(&af::join_many(0, weights_ref).unwrap(), &activated_input).unwrap()
                               , &af::matmul(&af::join_many(0, recurrents_ref).unwrap(), &h_tm1).unwrap(), false).unwrap()
                      , &af::join_many(0, bias_ref).unwrap(), true).unwrap();
    rtrl(&d_tm1, &z_t, )
      if self.return_sequences {
        Input { data: af::join_many(0, vec![&z_t, &c_tm1]).unwrap()
                , activation: vec![self.inner_activation, self.outer_activation] }
      }else { //TODO: Fix this
        Input { data: af::join_many(0, vec![&ifo_tm1, &ct_tm1, &c_tm1]).unwrap()
                , activation: vec![self.inner_activation, self.outer_activation] }
      }
  }

  fn backward(&mut self, delta: &Array) -> Array {
  self.delta = delta.clone();

  let activation_prev = activations::get_activation(self.inputs.activation[0], &self.inputs.data[DataIndex::Input]).unwrap();
  let d_activation_prev = activations::get_activation_derivative(self.inputs.activation[0], &activation_prev).unwrap();
  let delta_prev = af::mul(&af::matmul(&self.weights[0], delta, af::MatProp::TRANS, af::MatProp::NONE).unwrap()
                           , &d_activation_prev, false).unwrap();
  delta_prev
}

fn get_delta(&self) -> Array {
self.delta.clone()
  }

  fn get_weights(&self) -> Vec<Array> {
    self.recurrent_weights.extend(self.weights.iter().cloned()).clone()
  }

  fn set_weights(&mut self, weights: &Array, index: usize) {
    self.weights[index] = weights.clone();
  }

  fn get_bias(&self) -> Vec<Array> {
    self.bias.clone()
  }

  fn set_bias(&mut self, bias: &Array, index: usize) {
    self.bias[index] = bias.clone();
  }

  fn get_bias_dims(&self) -> Vec<Dim4> {
    let mut dims = Vec::new();
    for b in &self.bias {
      dims.push(b.dims().unwrap().clone())
    }
    dims
  }

  fn get_weight_dims(&self) -> Vec<Dim4> {
    let mut dims = Vec::new();
    for w in &self.weights {
      dims.push(w.dims().unwrap().clone())
    }
    for w in &self.recurrent_weights {
      dims.push(w.dims().unwrap().clone())
    }
    dims
  }

  fn get_input(&self) -> Input {
    self.inputs.clone()
  }

  fn output_size(&self) -> u64 {
    let weight_dims = self.get_weight_dims();
    weight_dims[weight_dims.len() - 1][1]
  }

  fn input_size(&self) -> u64 {
    let weight_dims = self.get_weight_dims();
    weight_dims[0][0]
  }

  fn get_activation_type(&self) -> &'static str {
    &self.activation
  }
}
<commit_msg>just wanted the non-batched version in need it later. remove it now<commit_after>use af;
use af::{Dim4, Array};
use af::MatProp;

use activations;
use initializations;
use layer::{Layer, Input};

pub enum LSTMIndex {
  Input,
  Forget,
  Output,
  CellTilda,
}

pub enum ActivationIndex {
  Input,
  Inner,
  Outer,
}

pub enum DataIndex {
  Input,
  Recurrence,
}

pub struct LSTM {
  weights: Vec<Array>,
  recurrent_weights: Vec<Array>,
  bias: Vec<Array>,
  inner_activation: &'static str,
  outer_activation: &'static str,
  return_sequences: bool,
  delta: Array,
  inputs: Input,
}

impl LSTM {
  pub fn new(input_size: u64, output_size: u64
             , outer_activation: &'static str
             , inner_activation: &'static str
             , w_init: &'static str
             , w_inner_init: &'static str
             , bias_init: &'static str
             , forget_bias_init: &'static str
             , return_sequences: bool) -> LSTM
  {
    LSTM{
      weights: vec![initializations::get_initialization(w_init, Dim4::new(&[output_size, input_size, 1, 1])).unwrap()],
      recurrent_weights: vec![initializations::get_initialization(w_inner_init, Dim4::new(&[output_size, output_size, 1, 1])).unwrap(); 4],
      bias: vec![initializations::get_initialization(b_init, Dim4::new(&[output_size, 1, 1, 1])).unwrap()
                 , initializations::get_initialization(bias_init, Dim4::new(&[output_size, 1, 1, 1])).unwrap()
                 , initializations::get_initialization(bias_init, Dim4::new(&[output_size, 1, 1, 1])).unwrap()
                 , initializations::get_initialization(forget_bias_init, Dim4::new(&[output_size, 1, 1, 1])).unwrap()],
      inner_activation: inner_activation,
      outer_activation: outer_activation,
      return_sequences: return_sequences,
    }
  }
}

impl RTRL for LSTM {
  pub fn rtrl(&self, dW_tm1: &mut Array  // previous W derivatives for [I, F, Ct]
              , dU_tm1: &mut Array       // previous U derivatives for [I, F, Ct]
              , db_tm1: &mut Array       // previous b derivatives for [I, F, Ct]
              , z_t: &Array              // current time activation
              , inputs: &Input)          // x_t & h_{t-1}
  {
    let block_size = z_t.dims().unwrap()[0];
    assert!(block_size as f32 % 5.0f32 == 0); // there are 5 data pieces we need
    let chunk_size = block_size / 5;
    // chunk out i, f, ct, c_tm1
    let if_t = activations::get_activation(self.inner_activation,
                                            , &af::rows(z_t, 0, 2*chunk_size).unwrap).unwrap();
    let ct_t = activations::get_activation(self.outer_activation,
                                           , &af::rows(z_t, 3*chunk_size, 4*chunk_size).unwrap).unwrap();
    let c_tm1 = af::rows(z_t, 4*chunk_size, 5*chunk_size).unwrap();

    // diff(z_i), diff(z_f), diff(z_ct)
    let dz = vec![&activations::get_activation_derivative(self.inner_activation, &af::rows(z_t, 0, chunk_size).unwrap()).unwrap() //d(z)
                  , &activations::get_activation_derivative(self.inner_activation, &af::rows(z_t, chunk_size, 2*chunk_size).unwrap()).unwrap()
                  , &activations::get_activation_derivative(self.outer_activation, &af::rows(z_t, 3*chunk_size, 4*chunk_size).unwrap()).unwrap()];
    let ct_ctm1_i = vec![&ct_t, &c_tm1, &af::rows(if_t, 0, chunk_size).unwrap()];

    // [Ct_t; C_{t-1}; i_t] * dz
    let dzprod = af::mul(&af::join_many(0, ct_ctm1_i).unwrap()
                         , af::join_many(0, dzvec).unwrap(), false).unwrap();

    // dC_t/dWi  = (dC_{t-1}/dWi  * f_t) + ct_t  * inner_activation(z_i) * x_t
    // dC_t/dWf  = (dC_{t-1}/dWf  * f_t) + c_tm1 * inner_activation(z_f) * x_t
    // dC_t/dWct = (dC_{t-1}/dWct * f_t) + i_t   * outer_activation(Ct)  * x_t
    let w_lhs = af::mul(dW_tm1, &af::rows(if_t, chunk_size, 2*chunk_size).unwrap(), true).unwrap(); // dC_{t-1}/dW * f_t
    let w_rhs = af::mul(&dzprod, &inputs.data[DataIndex::Input]);
    dW_tm1 = af::add(&w_lhs, &w_rhs).unwrap();

    // dC_t/dUi = (dC_{t-1}/dUi * f_t) + ct_t * inner_activation(z_i) * h_{t-1}
    // dC_t/dUf = (dC_{t-1}/dUf * f_t) + c_tm1 * inner_activation(z_f) * h_{t-1}
    // dC_t/dUct = (dC_{t-1}/dUct * f_t) + outer_activation(Ct) * x_t * h_{t-1}
    let u_lhs = af::mul(dU_tm1, &af::rows(if_t, chunk_size, 2*chunk_size).unwrap(), true).unwrap(); // dC_{t-1}/dU * f_t
    let u_rhs = af::mul(&dzprod, &inputs.data[DataIndex::Recurrence]);
    dU_tm1 = af::add(&u_lhs, &u_rhs).unwrap();

    // dC_t/dbi = (dC_{t-1}/dbi * f_t) + ct_t * inner_activation(z_i)
    // dC_t/dbf = (dC_{t-1}/dbf * f_t) + c_tm1 * inner_activation(z_f)
    // dC_t/dbct = (dC_{t-1}/dbct * f_t) + outer_activation(Ct)
    let b_lhs = af::mul(db_tm1, &af::rows(if_t, chunk_size, 2*chunk_size).unwrap(), true).unwrap(); // dC_{t-1}/db * f_t
    dW_tm1 = af::add(&b_lhs, &dzprod).unwrap();
  }
}

impl Layer for LSTM {

  pub fn forward(&mut self, inputs:& Input) {
    // keep previous layer's outputs
    self.inputs = inputs.clone();

    // apply the activation to the previous layer [Optimization: Memory saving]
    let activated_input = activations::get_activation(inputs.activation[ActivationIndex::Inner]
                                                      , &inputs.data[DataIndex::Input]).unwrap();

    // extract the sub-block of each gate [i_tm1, f_tm1, o_tm1, ct_tm1, c_tm2]
    let block_size = inputs.data[DataIndex::Recurrence].dims().unwrap()[0];
    assert!(block_size as f32 % 5.0f32 == 0); // there are 5 data pieces we need
    let chunk_size = block_size / 5;
    let ifo_tm1 = activations::get_activation(inputs.activation[ActivationIndex::Inner]
                                              , &af::rows(&inputs.data[DataIndex::Recurrence], 0, 3 * chunk_size).unwrap).unwrap();
    let ct_tm1 = activations::get_activation(inputs.activation[ActivationIndex::Outer]
                                             , &af::rows(&inputs.data[DataIndex::Recurrence], 3 * chunk_size, 4 * chunk_size).unwrap).unwrap();
    let c_tm2 = af::rows(&inputs.data[DataIndex::Recurrence], 4 * chunk_size, 5 * chunk_size).unwrap();

    // calculate c_tm1 & h_tm1
    let c_tm1 = af::add(&af::mul(&af::rows(&ifo_tm1, 0, chunk_size).unwrap(), &ct_tm1, false).unwrap()
                        , &af::mul(&af::rows(&ifo_tm1, chunk_size, 2 * chunk_size).unwrap(), &c_tm2, false).unwrap()
                        , false).unwrap();
    let h_tm1 = af::mul(&o_tm1, activations::get_activation(inputs.activation[ActivationIndex::Outer]
                                                            , &c_tm1), false).unwrap();

    // forward pass in a batch for performance
    let weights_ref    = vec![&self.weights[LSTMIndex::Input]
                              , &self.weights[LSTMIndex::Forget]
                              , &self.weights[LSTMIndex::Output]
                              , &self.weights[LSTMIndex::CellTilda]];
    let recurrents_ref = vec![&self.recurrent_weights[LSTMIndex::Input]
                              , &self.recurrent_weights[LSTMIndex::Forget]
                              , &self.recurrent_weights[LSTMIndex::Output]
                              , &self.recurrent_weights[LSTMIndex::CellTilda]];
    let bias_ref       = vec![&self.bias[LSTMIndex::Input]
                              , &self.bias[LSTMIndex::Forget]
                              , &self.bias[LSTMIndex::Output]
                              , &self.bias[LSTMIndex::CellTilda]];
    // [ifo_ct] = W*x + U*h_tm1 + b
    let z_t = af::add(&af::add(&af::matmul(&af::join_many(0, weights_ref).unwrap(), &activated_input).unwrap()
                               , &af::matmul(&af::join_many(0, recurrents_ref).unwrap(), &h_tm1).unwrap(), false).unwrap()
                      , &af::join_many(0, bias_ref).unwrap(), true).unwrap();
    rtrl(&d_tm1, &z_t, )
      if self.return_sequences {
        Input { data: af::join_many(0, vec![&z_t, &c_tm1]).unwrap()
                , activation: vec![self.inner_activation, self.outer_activation] }
      }else { //TODO: Fix this
        Input { data: af::join_many(0, vec![&ifo_tm1, &ct_tm1, &c_tm1]).unwrap()
                , activation: vec![self.inner_activation, self.outer_activation] }
      }
  }

  fn backward(&mut self, delta: &Array) -> Array {
  self.delta = delta.clone();

  let activation_prev = activations::get_activation(self.inputs.activation[0], &self.inputs.data[DataIndex::Input]).unwrap();
  let d_activation_prev = activations::get_activation_derivative(self.inputs.activation[0], &activation_prev).unwrap();
  let delta_prev = af::mul(&af::matmul(&self.weights[0], delta, af::MatProp::TRANS, af::MatProp::NONE).unwrap()
                           , &d_activation_prev, false).unwrap();
  delta_prev
}

fn get_delta(&self) -> Array {
self.delta.clone()
  }

  fn get_weights(&self) -> Vec<Array> {
    self.recurrent_weights.extend(self.weights.iter().cloned()).clone()
  }

  fn set_weights(&mut self, weights: &Array, index: usize) {
    self.weights[index] = weights.clone();
  }

  fn get_bias(&self) -> Vec<Array> {
    self.bias.clone()
  }

  fn set_bias(&mut self, bias: &Array, index: usize) {
    self.bias[index] = bias.clone();
  }

  fn get_bias_dims(&self) -> Vec<Dim4> {
    let mut dims = Vec::new();
    for b in &self.bias {
      dims.push(b.dims().unwrap().clone())
    }
    dims
  }

  fn get_weight_dims(&self) -> Vec<Dim4> {
    let mut dims = Vec::new();
    for w in &self.weights {
      dims.push(w.dims().unwrap().clone())
    }
    for w in &self.recurrent_weights {
      dims.push(w.dims().unwrap().clone())
    }
    dims
  }

  fn get_input(&self) -> Input {
    self.inputs.clone()
  }

  fn output_size(&self) -> u64 {
    let weight_dims = self.get_weight_dims();
    weight_dims[weight_dims.len() - 1][1]
  }

  fn input_size(&self) -> u64 {
    let weight_dims = self.get_weight_dims();
    weight_dims[0][0]
  }

  fn get_activation_type(&self) -> &'static str {
    &self.activation
  }
}
<|endoftext|>


<commit_before>// (c) 2016-2017 Productize SPRL <joost@productize.be>

// extension: .kicad_pcb
// format: new-style

use std::fmt;
use std::result;

// from parent
use Sexp;
use symbolic_expressions::IntoSexp;
use layout::data::*;

impl IntoSexp for Layout {
    fn into_sexp(&self) -> Sexp {
        let mut v = vec![];
        v.push("kicad_pcb".into());

        v.push(("version", &self.version).into());

        let mut v2 = vec![];
        v2.push("host".into());
        v2.push(self.host.tool.clone().into());
        v2.push(self.host.build.clone().into());
        v.push(v2.into());

        v.push(self.general.into_sexp());

        v.push(("page", &self.page).into());

        let mut v2 = vec![];
        v2.push("layers".into());
        for layer in &self.layers {
            v2.push(layer.into_sexp());
        }
        v.push(v2.into());

        v.push(self.setup.into_sexp());

        for element in &self.elements {
            v.push(element.into_sexp());
        }
        v.into()
    }
}

impl IntoSexp for General {
    fn into_sexp(&self) -> Sexp {
        let mut v = vec![];
        v.push("general".into());
        v.push(("links", &self.links).into());
        v.push(("no_connects", &self.no_connects).into());
        v.push(self.area.into_sexp());
        v.push(("thickness", &self.thickness).into());
        v.push(("drawings", &self.drawings).into());
        v.push(("tracks", &self.tracks).into());
        v.push(("zones", &self.zones).into());
        v.push(("modules", &self.modules).into());
        v.push(("nets", &self.nets).into());
        v.into()
    }
}

impl IntoSexp for Area {
    fn into_sexp(&self) -> Sexp {
        let mut v = vec![];
        v.push("area".into());
        v.push(self.x1.into());
        v.push(self.y1.into());
        v.push(self.x2.into());
        v.push(self.y2.into());
        v.into()
    }
}

impl IntoSexp for Element {
    fn into_sexp(&self) -> Sexp {
        match *self {
            Element::Other(ref s) => s.clone(),
            Element::Module(ref s) => s.into_sexp(),
            Element::Net(ref s) => s.into_sexp(),
            Element::NetClass(ref s) => s.into_sexp(),
            Element::GrText(ref s) => s.into_sexp(),
            Element::GrLine(ref s) => s.into_sexp(),
            Element::GrArc(ref s) => s.into_sexp(),
            Element::Dimension(ref s) => s.into_sexp(),
            Element::Zone(ref s) => s.into_sexp(),
        }
    }
}

impl IntoSexp for Zone {
    fn into_sexp(&self) -> Sexp {
        let mut v = vec![];
        v.push("zone".into());
        v.push(("net", &self.net).into());
        v.push(("net_name", &self.net_name).into());
        for o in &self.other {
            v.push(o.clone());
        }
        v.into()
    }
}

impl IntoSexp for Net {
    fn into_sexp(&self) -> Sexp {
        let mut v = vec![];
        v.push("net".into());
        v.push(self.num.into());
        v.push(self.name.clone().into());
        v.into()
    }
}

impl IntoSexp for NetClass {
    fn into_sexp(&self) -> Sexp {
        let mut v = vec![];
        v.push("net_class".into());
        v.push(self.name.clone().into());
        v.push(self.desc.clone().into());
        v.push(("clearance", &self.clearance).into());
        v.push(("trace_width", &self.trace_width).into());
        v.push(("via_dia", &self.via_dia).into());
        v.push(("via_drill", &self.via_drill).into());
        v.push(("uvia_dia", &self.uvia_dia).into());
        v.push(("uvia_drill", &self.uvia_drill).into());
        if let Some(diff_pair_gap) = self.diff_pair_gap {
            v.push(("diff_pair_gap", &diff_pair_gap).into());
        }
        if let Some(diff_pair_width) = self.diff_pair_width {
            v.push(("diff_pair_width", &diff_pair_width).into());
        }
        for net in &self.nets {
            v.push(("add_net", net).into());
        }
        v.into()
    }
}

impl IntoSexp for Layer {
    fn into_sexp(&self) -> Sexp {
        let mut v = vec![];
        v.push(self.num.into());
        v.push(format!("{}", self.layer).into());
        v.push(format!("{}", self.layer_type).into());
        if self.hide {
            v.push("hide".into());
        }
        v.into()
    }
}

impl fmt::Display for LayerType {
    fn fmt(&self, f: &mut fmt::Formatter) -> result::Result<(), fmt::Error> {
        let s = match *self {
            LayerType::Signal => "signal",
            LayerType::User => "user",
        };
        write!(f, "{}", s)
    }
}

impl IntoSexp for SetupElement {
    fn into_sexp(&self) -> Sexp {
        let mut v = vec![];
        v.push(self.name.clone().into());
        v.push(self.value1.clone().into());
        if let Some(ref x) = self.value2 {
            v.push(x.clone().into())
        }
        v.into()
    }
}

impl IntoSexp for Setup {
    fn into_sexp(&self) -> Sexp {
        let mut v = vec![];
        v.push("setup".into());
        for ref k in &self.elements {
            v.push(k.into_sexp())
        }
        let mut v2 = vec![];
        v2.push("pcbplotparams".into());
        for ref k in &self.pcbplotparams {
            v2.push(k.into_sexp())
        }
        v.push(v2.into());
        v.into()
    }
}

impl IntoSexp for GrText {
    fn into_sexp(&self) -> Sexp {
        let mut v = vec![];
        v.push("gr_text".into());
        v.push(self.value.clone().into());
        v.push(self.at.into_sexp());
        v.push(("layer", &self.layer).into());
        v.push(self.effects.into_sexp());
        v.into()
    }
}

impl IntoSexp for GrLine {
    fn into_sexp(&self) -> Sexp {
        let mut v = vec![];
        v.push("gr_line".into());
        v.push(self.start.into_sexp());
        v.push(self.end.into_sexp());
        v.push(("angle", &self.angle).into());
        v.push(("layer", &self.layer).into());
        v.push(("width", &self.width).into());
        v.into()
    }
}

impl IntoSexp for GrArc {
    fn into_sexp(&self) -> Sexp {
        let mut v = vec![];
        v.push("gr_arc".into());
        v.push(self.start.into_sexp());
        v.push(self.end.into_sexp());
        v.push(("angle", &self.angle).into());
        v.push(("layer", &self.layer).into());
        v.push(("width", &self.width).into());
        v.into()
    }
}

impl IntoSexp for Dimension {
    fn into_sexp(&self) -> Sexp {
        let mut v = vec![];
        v.push("dimension".into());
        v.push(self.name.clone().into());
        v.push(("width", &self.width).into());
        v.push(("layer", &self.layer).into());
        match self.tstamp {
            None => (),
            Some(ref tstamp) => {
                v.push(("tstamp", tstamp).into());
            }
        }
        v.push(self.text.into_sexp());
        v.push(Sexp::new_named_sexp("feature1", &self.feature1));
        v.push(Sexp::new_named_sexp("feature2", &self.feature2));
        v.push(Sexp::new_named_sexp("crossbar", &self.crossbar));
        v.push(Sexp::new_named_sexp("arrow1a", &self.arrow1a));
        v.push(Sexp::new_named_sexp("arrow1b", &self.arrow1b));
        v.push(Sexp::new_named_sexp("arrow2a", &self.arrow2a));
        v.push(Sexp::new_named_sexp("arrow2b", &self.arrow2b));
        v.into()
    }
}
<commit_msg>update layout ser code to use more compacted list handling<commit_after>// (c) 2016-2017 Productize SPRL <joost@productize.be>

// extension: .kicad_pcb
// format: new-style

use std::fmt;
use std::result;

// from parent
use Sexp;
use symbolic_expressions::IntoSexp;
use layout::data::*;

impl IntoSexp for Layout {
    fn into_sexp(&self) -> Sexp {
        let mut v = Sexp::start("kicad_pcb");

        v.push(("version", &self.version));

        let mut v2 = Sexp::start("host");
        v2.push(&self.host.tool);
        v2.push(&self.host.build);
        v.push(v2);

        v.push(self.general.into_sexp());

        v.push(("page", &self.page));

        let mut v2 = Sexp::start("layers");
        for layer in &self.layers {
            v2.push(layer.into_sexp());
        }
        v.push(v2);

        v.push(self.setup.into_sexp());

        for element in &self.elements {
            v.push(element.into_sexp());
        }
        v
    }
}

impl IntoSexp for General {
    fn into_sexp(&self) -> Sexp {
        let mut v = Sexp::start("general");
        v.push(("links", &self.links));
        v.push(("no_connects", &self.no_connects));
        v.push(self.area.into_sexp());
        v.push(("thickness", &self.thickness));
        v.push(("drawings", &self.drawings));
        v.push(("tracks", &self.tracks));
        v.push(("zones", &self.zones));
        v.push(("modules", &self.modules));
        v.push(("nets", &self.nets));
        v
    }
}

impl IntoSexp for Area {
    fn into_sexp(&self) -> Sexp {
        let mut v = Sexp::start("area");
        v.push(self.x1);
        v.push(self.y1);
        v.push(self.x2);
        v.push(self.y2);
        v
    }
}

impl IntoSexp for Element {
    fn into_sexp(&self) -> Sexp {
        match *self {
            Element::Other(ref s) => s.clone(),
            Element::Module(ref s) => s.into_sexp(),
            Element::Net(ref s) => s.into_sexp(),
            Element::NetClass(ref s) => s.into_sexp(),
            Element::GrText(ref s) => s.into_sexp(),
            Element::GrLine(ref s) => s.into_sexp(),
            Element::GrArc(ref s) => s.into_sexp(),
            Element::Dimension(ref s) => s.into_sexp(),
            Element::Zone(ref s) => s.into_sexp(),
        }
    }
}

impl IntoSexp for Zone {
    fn into_sexp(&self) -> Sexp {
        let mut v = Sexp::start("zone");
        v.push(("net", &self.net));
        v.push(("net_name", &self.net_name));
        for o in &self.other {
            v.push(o.clone());
        }
        v
    }
}

impl IntoSexp for Net {
    fn into_sexp(&self) -> Sexp {
        let mut v = Sexp::start("net");
        v.push(self.num);
        v.push(&self.name);
        v
    }
}

impl IntoSexp for NetClass {
    fn into_sexp(&self) -> Sexp {
        let mut v = Sexp::start("net_class");
        v.push(&self.name);
        v.push(&self.desc);
        v.push(("clearance", &self.clearance));
        v.push(("trace_width", &self.trace_width));
        v.push(("via_dia", &self.via_dia));
        v.push(("via_drill", &self.via_drill));
        v.push(("uvia_dia", &self.uvia_dia));
        v.push(("uvia_drill", &self.uvia_drill));
        if let Some(diff_pair_gap) = self.diff_pair_gap {
            v.push(("diff_pair_gap", &diff_pair_gap));
        }
        if let Some(diff_pair_width) = self.diff_pair_width {
            v.push(("diff_pair_width", &diff_pair_width));
        }
        for net in &self.nets {
            v.push(("add_net", net));
        }
        v
    }
}

impl IntoSexp for Layer {
    fn into_sexp(&self) -> Sexp {
        let mut v = vec![];
        v.push(self.num.into());
        v.push(format!("{}", self.layer).into());
        v.push(format!("{}", self.layer_type).into());
        if self.hide {
            v.push("hide".into());
        }
        v.into()
    }
}

impl fmt::Display for LayerType {
    fn fmt(&self, f: &mut fmt::Formatter) -> result::Result<(), fmt::Error> {
        let s = match *self {
            LayerType::Signal => "signal",
            LayerType::User => "user",
        };
        write!(f, "{}", s)
    }
}

impl IntoSexp for SetupElement {
    fn into_sexp(&self) -> Sexp {
        let mut v = vec![];
        v.push(self.name.clone().into());
        v.push(self.value1.clone().into());
        if let Some(ref x) = self.value2 {
            v.push(x.clone().into())
        }
        v.into()
    }
}

impl IntoSexp for Setup {
    fn into_sexp(&self) -> Sexp {
        let mut v = Sexp::start("setup");
        for ref k in &self.elements {
            v.push(k.into_sexp())
        }
        let mut v2 = Sexp::start("pcbplotparams");
        for ref k in &self.pcbplotparams {
            v2.push(k.into_sexp())
        }
        v.push(v2);
        v
    }
}

impl IntoSexp for GrText {
    fn into_sexp(&self) -> Sexp {
        let mut v = Sexp::start("gr_text");
        v.push(&self.value);
        v.push(self.at.into_sexp());
        v.push(("layer", &self.layer));
        v.push(self.effects.into_sexp());
        v
    }
}

impl IntoSexp for GrLine {
    fn into_sexp(&self) -> Sexp {
        let mut v = Sexp::start("gr_line");
        v.push(self.start.into_sexp());
        v.push(self.end.into_sexp());
        v.push(("angle", &self.angle));
        v.push(("layer", &self.layer));
        v.push(("width", &self.width));
        v
    }
}

impl IntoSexp for GrArc {
    fn into_sexp(&self) -> Sexp {
        let mut v = Sexp::start("gr_arc");
        v.push(self.start.into_sexp());
        v.push(self.end.into_sexp());
        v.push(("angle", &self.angle));
        v.push(("layer", &self.layer));
        v.push(("width", &self.width));
        v
    }
}

impl IntoSexp for Dimension {
    fn into_sexp(&self) -> Sexp {
        let mut v = Sexp::start("dimension");
        v.push(&self.name);
        v.push(("width", &self.width));
        v.push(("layer", &self.layer));
        match self.tstamp {
            None => (),
            Some(ref tstamp) => {
                v.push(("tstamp", tstamp));
            }
        }
        v.push(self.text.into_sexp());
        v.push(("feature1", self.feature1.into_sexp()));
        v.push(("feature2", self.feature2.into_sexp()));
        v.push(("crossbar", self.crossbar.into_sexp()));
        v.push(("arrow1a", self.arrow1a.into_sexp()));
        v.push(("arrow1b", self.arrow1b.into_sexp()));
        v.push(("arrow2a", self.arrow2a.into_sexp()));
        v.push(("arrow2b", self.arrow2b.into_sexp()));
        v
    }
}
<|endoftext|>


<commit_before>use cmark::{Parser as CMParser, Event, Tag};
use token::Token;
use cleaner::Cleaner;
use error::{Result,Error};

/// A parser that reads markdown and convert it to AST (a vector of `Token`s)
pub struct Parser {
    numbering: Option<String>, // None for no numbering, or a String with the name you want
    cleaner: Option<Box<Cleaner>>, // An optional parameter to clean source code

    verbatim: bool, // set to true when in e.g. a code block
}

impl Parser {
    /// Creates a parser with the default options
    pub fn new() -> Parser {
        Parser {
            verbatim: false,
            numbering: None,
            cleaner: Some(Box::new(())),
        }
    }

    pub fn with_cleaner(mut self, cleaner: Box<Cleaner>) -> Parser {
        self.cleaner = Some(cleaner);
        self
    }

    /// Parse a string and returns an AST, that is a vector of `Token`s
    ///
    /// Returns a result, at this method might fail.
    pub fn parse<'a>(&mut self, s: &'a str) -> Result<Vec<Token<'a>>> {
        let mut p = CMParser::new(s);

        let mut res = vec!();
        try!(self.parse_events(&mut p, &mut res, None));
        Ok(res)
    }

    fn parse_events<'a>(&mut self, p: &mut CMParser<'a>, v: &mut Vec<Token<'a>>, current_tag: Option<&Tag>) -> Result<()> {
        while let Some(event) = p.next() {
            match event {
                Event::Text(mut text) => {
                    if let Some(ref cleaner) = self.cleaner {
                        if !self.verbatim {
                            cleaner.clean(&mut text);
                        }
                    }
                    v.push(Token::Str(text));
                },
                Event::Start(tag) => try!(self.parse_tag(p, v, tag)),
                Event::End(tag) => {
                    debug_assert!(format!("{:?}", Some(&tag)) == format!("{:?}", current_tag),
                                  format!("Error: opening and closing tags mismatch!\n{:?} ≠ {:?}",
                                          tag,
                                          current_tag));
                    break;
                },
                Event::SoftBreak => v.push(Token::SoftBreak),
                Event::HardBreak => v.push(Token::HardBreak),
                Event::Html(_) | Event::InlineHtml(_) => return Err(Error::Parser("No support for HTML code inside of Markdown, sorry.")),
                Event::FootnoteReference(_) => return Err(Error::Parser("No support for footnotes yet."))
            }
        }
        Ok(())
    }

    fn parse_tag<'a>(&mut self, p: &mut CMParser<'a>, v: &mut Vec<Token<'a>>, tag: Tag<'a>) -> Result<()> {
        let mut res = vec!();

        match tag {
            Tag::Code | Tag::CodeBlock(_) => self.verbatim = true,
            _ => (),
        }

        try!(self.parse_events(p, &mut res, Some(&tag)));

        self.verbatim = false;

        let token = match tag {
            Tag::Paragraph => Token::Paragraph(res),
            Tag::Emphasis => Token::Emphasis(res),
            Tag::Strong => Token::Strong(res),
            Tag::Code => Token::Code(res),
            Tag::Header(x) => Token::Header(x, res),
            Tag::Link(url, title) => Token::Link(url, title, res),
            Tag::Image(url, title) => Token::Image(url, title, res),
            Tag::Rule => Token::Rule,
            Tag::List(opt) => {
                if let Some(n) = opt {
                    Token::OrderedList(n, res)
                } else {
                    Token::List(res)
                }},
            Tag::Item => Token::Item(res),
            Tag::BlockQuote => Token::BlockQuote(res),
            Tag::CodeBlock(language) => Token::CodeBlock(language, res),
            Tag::Table(_) | Tag::TableHead | Tag::TableRow | Tag::TableCell => return Err(Error::Parser("No support for tables yet")),
            Tag::FootnoteDefinition(_) => return Err(Error::Parser("No support for footnotes")),
        };
        v.push(token);
        Ok(())
    }
}

<commit_msg>Parser: multiple Str(_) in a row are now merged.<commit_after>use cmark::{Parser as CMParser, Event, Tag};
use token::Token;
use cleaner::Cleaner;
use error::{Result,Error};

/// A parser that reads markdown and convert it to AST (a vector of `Token`s)
pub struct Parser {
    cleaner: Option<Box<Cleaner>>, // An optional parameter to clean source code

    verbatim: bool, // set to true when in e.g. a code block
}

impl Parser {
    /// Creates a parser with the default options
    pub fn new() -> Parser {
        Parser {
            verbatim: false,
            cleaner: Some(Box::new(())),
        }
    }

    pub fn with_cleaner(mut self, cleaner: Box<Cleaner>) -> Parser {
        self.cleaner = Some(cleaner);
        self
    }

    /// Parse a string and returns an AST, that is a vector of `Token`s
    ///
    /// Returns a result, at this method might fail.
    pub fn parse<'a>(&mut self, s: &'a str) -> Result<Vec<Token<'a>>> {
        let mut p = CMParser::new(s);

        let mut res = vec!();
        try!(self.parse_events(&mut p, &mut res, None));
        Ok(res)
    }

    fn parse_events<'a>(&mut self, p: &mut CMParser<'a>, v: &mut Vec<Token<'a>>, current_tag: Option<&Tag>) -> Result<()> {
        while let Some(event) = p.next() {
            match event {
                Event::Text(mut text) => {
                    if let Some(ref cleaner) = self.cleaner {
                        if !self.verbatim {
                            cleaner.clean(&mut text);
                        }
                    }
                    if let Some(&Token::Str(_)) = v.last() {
                        if let &mut Token::Str(ref mut s) = v.last_mut().unwrap() {
                            s.to_mut().push_str(&text);
                        } else {
                            unreachable!();
                        }
                    } else {
                        v.push(Token::Str(text));
                    }
                },
                Event::Start(tag) => try!(self.parse_tag(p, v, tag)),
                Event::End(tag) => {
                    debug_assert!(format!("{:?}", Some(&tag)) == format!("{:?}", current_tag),
                                  format!("Error: opening and closing tags mismatch!\n{:?} ≠ {:?}",
                                          tag,
                                          current_tag));
                    break;
                },
                Event::SoftBreak => v.push(Token::SoftBreak),
                Event::HardBreak => v.push(Token::HardBreak),
                Event::Html(_) | Event::InlineHtml(_) => return Err(Error::Parser("No support for HTML code inside of Markdown, sorry.")),
                Event::FootnoteReference(_) => return Err(Error::Parser("No support for footnotes yet."))
            }
        }
        Ok(())
    }

    fn parse_tag<'a>(&mut self, p: &mut CMParser<'a>, v: &mut Vec<Token<'a>>, tag: Tag<'a>) -> Result<()> {
        let mut res = vec!();

        match tag {
            Tag::Code | Tag::CodeBlock(_) => self.verbatim = true,
            _ => (),
        }

        try!(self.parse_events(p, &mut res, Some(&tag)));

        self.verbatim = false;

        let token = match tag {
            Tag::Paragraph => Token::Paragraph(res),
            Tag::Emphasis => Token::Emphasis(res),
            Tag::Strong => Token::Strong(res),
            Tag::Code => Token::Code(res),
            Tag::Header(x) => Token::Header(x, res),
            Tag::Link(url, title) => Token::Link(url, title, res),
            Tag::Image(url, title) => Token::Image(url, title, res),
            Tag::Rule => Token::Rule,
            Tag::List(opt) => {
                if let Some(n) = opt {
                    Token::OrderedList(n, res)
                } else {
                    Token::List(res)
                }},
            Tag::Item => Token::Item(res),
            Tag::BlockQuote => Token::BlockQuote(res),
            Tag::CodeBlock(language) => Token::CodeBlock(language, res),
            Tag::Table(_) | Tag::TableHead | Tag::TableRow | Tag::TableCell => return Err(Error::Parser("No support for tables yet")),
            Tag::FootnoteDefinition(_) => return Err(Error::Parser("No support for footnotes")),
        };
        v.push(token);
        Ok(())
    }
}

<|endoftext|>


<commit_before>// Copyright (C) 2016 Élisabeth HENRY.
//
// This file is part of Crowbook.
//
// Crowbook is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published
// by the Free Software Foundation, either version 2.1 of the License, or
// (at your option) any later version.
//
// Caribon is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU Lesser General Public License for more details.
//
// You should have received ba copy of the GNU Lesser General Public License
// along with Crowbook.  If not, see <http://www.gnu.org/licenses/>.

use crate::error::{Error, Result};

use std::path::{Path, PathBuf};
use std::io;
use std::io::Write;
use std::process::Command;
use std::fs::{self, File, DirBuilder};
use uuid;
use std::ops::Drop;

/// Struct used to create zip (using filesystem and zip command)
pub struct Zipper {
    args: Vec<String>,
    path: PathBuf,
}

impl Zipper {
    /// Creates new zipper
    ///
    /// # Arguments
    /// * `path`: the path to a temporary directory
    /// (zipper will create a random dir in it and clean it later)
    pub fn new(path: &str) -> Result<Zipper> {
        let uuid = uuid::Uuid::new_v4();
        let zipper_path = Path::new(path).join(uuid.to_simple().to_string());

        DirBuilder::new()
            .recursive(true)
            .create(&zipper_path)
            .map_err(|_| {
                Error::zipper(lformat!("could not create temporary directory in {path}",
                                       path = path))
            })?;

        Ok(Zipper {
            args: vec![],
            path: zipper_path,
        })
    }

    /// writes a content to a temporary file
    pub fn write<P:AsRef<Path>>(&mut self, path: P, content: &[u8], add_args: bool) -> Result<()> {
        let path = path.as_ref();
        let file = format!("{}", path.display());
        if path.starts_with("..") || path.is_absolute() {
            return Err(Error::zipper(lformat!("file {file} refers to an absolute or a parent \
                                               path.
This is forbidden because we are supposed \
                                               to create a temporary file in a temporary dir.",
                                              file = file)));
        }
        let dest_file = self.path.join(path);
        let dest_dir = dest_file.parent().unwrap();
        if !fs::metadata(dest_dir).is_ok() {
            // dir does not exist, create it
            DirBuilder::new()
                .recursive(true)
                .create(&dest_dir)
                .map_err(|_| {
                    Error::zipper(lformat!("could not create temporary directory in {path}",
                                           path = dest_dir.display()))
                })?;
        }

        if let Ok(mut f) = File::create(&dest_file) {
            if f.write_all(content).is_ok() {
                if add_args {
                    self.args.push(String::from(file));
                }
                Ok(())
            } else {
                Err(Error::zipper(lformat!("could not write to temporary file {file}",
                                           file = file)))
            }
        } else {
            Err(Error::zipper(lformat!("could not create temporary file {file}", file = file)))
        }
    }

    /// Unzip a file and deletes it afterwards
    pub fn unzip(&mut self, file: &str) -> Result<()> {
        let output = Command::new("unzip")
            .current_dir(&self.path)
            .arg(file)
            .output()
            .map_err(|e| {
                Error::zipper(lformat!("failed to execute unzip on {file}: {error}",
                                       file = file,
                                       error = e))
            });

        output?;

        fs::remove_file(self.path.join(file))
            .map_err(|_| Error::zipper(lformat!("failed to remove file {file}", file = file)))
    }

    /// run command and copy content of file output (supposed to result from the command) to current dir
    pub fn run_command(&mut self,
                       mut command: Command,
                       command_name: &str,
                       in_file: &str,
                       out: &mut dyn Write)
                       -> Result<String> {
        let res_output = command.args(&self.args)
            .current_dir(&self.path)
            .output()
            .map_err(|e| {
                debug!("{}", lformat!("output for command {name}:\n{error}",
                                        name = command_name,
                                        error = e));
                Error::zipper(lformat!("failed to run command '{name}'",
                                       name = command_name))
            });
        let output = res_output?;
        if output.status.success() {
            let mut file = File::open(self.path.join(in_file))
                .map_err(|_| {
                    debug!("{}", lformat!("could not open result of command '{command}'\n\
                                           Command output:\n\
                                           {output}'",
                                          command = command_name,
                                          output = String::from_utf8_lossy(&output.stdout)));
                    Error::zipper(lformat!("could not open result of command '{command}'",
                                           command = command_name))
                })?;
            io::copy(&mut file, out)
                .map_err(|_| Error::zipper(lformat!("error copying file '{file}'",
                                                    file = in_file)))?;

            Ok(String::from_utf8_lossy(&output.stdout).into_owned())
        } else {
            debug!("{}",
                   lformat!("{command} didn't return succesfully: {output}",
                            command = command_name,
                            output = String::from_utf8_lossy(&output.stdout)));
            Err(Error::zipper(lformat!("{command} didn't return succesfully",
                                       command = command_name)))
        }
    }

    /// zip all files in zipper's tmp dir to a given file name and write to odt file
    pub fn generate_odt(&mut self, command_name: &str, odt_file: &mut dyn Write) -> Result<String> {
        let mut command = Command::new(command_name);
        command.arg("-r");
        command.arg("result.odt");
        command.arg(".");
        self.run_command(command, command_name, "result.odt", odt_file)
    }

    /// generate a pdf file into given file name
    pub fn generate_pdf(&mut self,
                        command_name: &str,
                        tex_file: &str,
                        pdf_file: &mut dyn Write)
                        -> Result<String> {
        // first pass
        let _ = Command::new(command_name)
            .current_dir(&self.path)
            .arg(tex_file)
            .output();

        // second pass
        let mut command = Command::new(command_name);
        command.arg(tex_file);
        self.run_command(command, command_name, "result.pdf", pdf_file)
    }
}

impl Drop for Zipper {
    fn drop(&mut self) {
        if let Err(err) = fs::remove_dir_all(&self.path) {
            println!("Error in zipper: could not delete temporary directory {}, error: {}",
                     self.path.to_string_lossy(),
                     err);
        }
    }
}
<commit_msg>Add third pass to latex generation to fix table of contents<commit_after>// Copyright (C) 2016, 2020 Élisabeth HENRY.
//
// This file is part of Crowbook.
//
// Crowbook is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published
// by the Free Software Foundation, either version 2.1 of the License, or
// (at your option) any later version.
//
// Caribon is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU Lesser General Public License for more details.
//
// You should have received ba copy of the GNU Lesser General Public License
// along with Crowbook.  If not, see <http://www.gnu.org/licenses/>.

use crate::error::{Error, Result};

use std::path::{Path, PathBuf};
use std::io;
use std::io::Write;
use std::process::Command;
use std::fs::{self, File, DirBuilder};
use uuid;
use std::ops::Drop;

/// Struct used to create zip (using filesystem and zip command)
pub struct Zipper {
    args: Vec<String>,
    path: PathBuf,
}

impl Zipper {
    /// Creates new zipper
    ///
    /// # Arguments
    /// * `path`: the path to a temporary directory
    /// (zipper will create a random dir in it and clean it later)
    pub fn new(path: &str) -> Result<Zipper> {
        let uuid = uuid::Uuid::new_v4();
        let zipper_path = Path::new(path).join(uuid.to_simple().to_string());

        DirBuilder::new()
            .recursive(true)
            .create(&zipper_path)
            .map_err(|_| {
                Error::zipper(lformat!("could not create temporary directory in {path}",
                                       path = path))
            })?;

        Ok(Zipper {
            args: vec![],
            path: zipper_path,
        })
    }

    /// writes a content to a temporary file
    pub fn write<P:AsRef<Path>>(&mut self, path: P, content: &[u8], add_args: bool) -> Result<()> {
        let path = path.as_ref();
        let file = format!("{}", path.display());
        if path.starts_with("..") || path.is_absolute() {
            return Err(Error::zipper(lformat!("file {file} refers to an absolute or a parent \
                                               path.
This is forbidden because we are supposed \
                                               to create a temporary file in a temporary dir.",
                                              file = file)));
        }
        let dest_file = self.path.join(path);
        let dest_dir = dest_file.parent().unwrap();
        if !fs::metadata(dest_dir).is_ok() {
            // dir does not exist, create it
            DirBuilder::new()
                .recursive(true)
                .create(&dest_dir)
                .map_err(|_| {
                    Error::zipper(lformat!("could not create temporary directory in {path}",
                                           path = dest_dir.display()))
                })?;
        }

        if let Ok(mut f) = File::create(&dest_file) {
            if f.write_all(content).is_ok() {
                if add_args {
                    self.args.push(String::from(file));
                }
                Ok(())
            } else {
                Err(Error::zipper(lformat!("could not write to temporary file {file}",
                                           file = file)))
            }
        } else {
            Err(Error::zipper(lformat!("could not create temporary file {file}", file = file)))
        }
    }

    /// Unzip a file and deletes it afterwards
    pub fn unzip(&mut self, file: &str) -> Result<()> {
        let output = Command::new("unzip")
            .current_dir(&self.path)
            .arg(file)
            .output()
            .map_err(|e| {
                Error::zipper(lformat!("failed to execute unzip on {file}: {error}",
                                       file = file,
                                       error = e))
            });

        output?;

        fs::remove_file(self.path.join(file))
            .map_err(|_| Error::zipper(lformat!("failed to remove file {file}", file = file)))
    }

    /// run command and copy content of file output (supposed to result from the command) to current dir
    pub fn run_command(&mut self,
                       mut command: Command,
                       command_name: &str,
                       in_file: &str,
                       out: &mut dyn Write)
                       -> Result<String> {
        let res_output = command.args(&self.args)
            .current_dir(&self.path)
            .output()
            .map_err(|e| {
                debug!("{}", lformat!("output for command {name}:\n{error}",
                                        name = command_name,
                                        error = e));
                Error::zipper(lformat!("failed to run command '{name}'",
                                       name = command_name))
            });
        let output = res_output?;
        if output.status.success() {
            let mut file = File::open(self.path.join(in_file))
                .map_err(|_| {
                    debug!("{}", lformat!("could not open result of command '{command}'\n\
                                           Command output:\n\
                                           {output}'",
                                          command = command_name,
                                          output = String::from_utf8_lossy(&output.stdout)));
                    Error::zipper(lformat!("could not open result of command '{command}'",
                                           command = command_name))
                })?;
            io::copy(&mut file, out)
                .map_err(|_| Error::zipper(lformat!("error copying file '{file}'",
                                                    file = in_file)))?;

            Ok(String::from_utf8_lossy(&output.stdout).into_owned())
        } else {
            debug!("{}",
                   lformat!("{command} didn't return succesfully: {output}",
                            command = command_name,
                            output = String::from_utf8_lossy(&output.stdout)));
            Err(Error::zipper(lformat!("{command} didn't return succesfully",
                                       command = command_name)))
        }
    }

    /// zip all files in zipper's tmp dir to a given file name and write to odt file
    pub fn generate_odt(&mut self, command_name: &str, odt_file: &mut dyn Write) -> Result<String> {
        let mut command = Command::new(command_name);
        command.arg("-r");
        command.arg("result.odt");
        command.arg(".");
        self.run_command(command, command_name, "result.odt", odt_file)
    }

    /// generate a pdf file into given file name
    pub fn generate_pdf(&mut self,
                        command_name: &str,
                        tex_file: &str,
                        pdf_file: &mut dyn Write)
                        -> Result<String> {
        // first pass
        let mut command = Command::new(command_name);
        command.current_dir(&self.path)
            .arg(tex_file);
        let _ = command.output();

        // second pass
//        let _ = Command::new(command_name)
  //          .current_dir(&self.path)
    //        .arg(tex_file)
        command.output();

        // third pass
        // let mut command = Command::new(command_name);
        // command.current_dir(&self.path);
        // command.arg(tex_file);
        self.run_command(command, command_name, "result.pdf", pdf_file)
    }
}

impl Drop for Zipper {
    fn drop(&mut self) {
        if let Err(err) = fs::remove_dir_all(&self.path) {
            println!("Error in zipper: could not delete temporary directory {}, error: {}",
                     self.path.to_string_lossy(),
                     err);
        }
    }
}
<|endoftext|>