<commit_before>// pest. The Elegant Parser
// Copyright (c) 2018 Dragoș Tiselice
//
// Licensed under the Apache License, Version 2.0
// <LICENSE-APACHE or http://www.apache.org/licenses/LICENSE-2.0> or the MIT
// license <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. All files in the project carrying such notice may not be copied,
// modified, or distributed except according to those terms.

use ast::*;

pub fn list(rule: Rule) -> Rule {
    match rule {
        Rule { name, ty, expr } => Rule {
            name,
            ty,
            expr: expr.map_bottom_up(|expr| {
                // TODO: Use box syntax when it gets stabilized.
                match expr {
                    Expr::Seq(l, r) => match *l {
                        Expr::Rep(l) => match *l {
                            Expr::Seq(l1, l2) => match *r {
                                // Converts `(rule ~ rest)* ~ rule?` to `rule ~ (rest ~ rule)* ~ rest?`
                                Expr::Opt(r) if l1 == r => {
                                    Expr::Seq(
                                        l1,
                                        Box::new(Expr::Seq(
                                            Box::new(Expr::Rep(Box::new(Expr::Seq(
                                                l2.clone(),
                                                r
                                            )))),
                                            Box::new(Expr::Opt(l2))
                                        ))
                                    )
                                },
                                // Converts `(rule ~ rest)* ~ rule` to `rule ~ (rest ~ rule)*`
                                r if *l1 == r => {
                                    Expr::Seq(
                                        l1,
                                        Box::new(Expr::Rep(Box::new(Expr::Seq(
                                            l2,
                                            Box::new(r)
                                        ))))
                                    )
                                }
                                r => {
                                    Expr::Seq(
                                        Box::new(Expr::Rep(Box::new(Expr::Seq(l1, l2)))),
                                        Box::new(r)
                                    )
                                }
                            }
                            expr => Expr::Seq(Box::new(Expr::Rep(Box::new(expr))), r)
                        }
                        expr => Expr::Seq(Box::new(expr), r)
                    }
                    expr => expr
                }
            })
        }
    }
}
<commit_msg>Make the lister.rs comments explicit on what it optimizes exactly<commit_after>// pest. The Elegant Parser
// Copyright (c) 2018 Dragoș Tiselice
//
// Licensed under the Apache License, Version 2.0
// <LICENSE-APACHE or http://www.apache.org/licenses/LICENSE-2.0> or the MIT
// license <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. All files in the project carrying such notice may not be copied,
// modified, or distributed except according to those terms.

use ast::*;

pub fn list(rule: Rule) -> Rule {
    match rule {
        Rule { name, ty, expr } => Rule {
            name,
            ty,
            expr: expr.map_bottom_up(|expr| {
                // TODO: Use box syntax when it gets stabilized.
                match expr {
                    Expr::Seq(l, r) => match *l {
                        Expr::Rep(l) => match *l {
                            Expr::Seq(l1, l2) => match *r {
                                // Converts `(rule ~ rest)* ~ rule?` to `rule ~ (rest ~ rule)* ~ rest?`,
                                // avoiding matching the last `rule` twice if the input doesn't end with `rest`.
                                Expr::Opt(r) if l1 == r => {
                                    Expr::Seq(
                                        l1,
                                        Box::new(Expr::Seq(
                                            Box::new(Expr::Rep(Box::new(Expr::Seq(
                                                l2.clone(),
                                                r
                                            )))),
                                            Box::new(Expr::Opt(l2))
                                        ))
                                    )
                                },
                                // Converts `(rule ~ rest)* ~ rule` to `rule ~ (rest ~ rule)*`,
                                // avoiding matching the last `rule` twice.
                                r if *l1 == r => {
                                    Expr::Seq(
                                        l1,
                                        Box::new(Expr::Rep(Box::new(Expr::Seq(
                                            l2,
                                            Box::new(r)
                                        ))))
                                    )
                                }
                                r => {
                                    Expr::Seq(
                                        Box::new(Expr::Rep(Box::new(Expr::Seq(l1, l2)))),
                                        Box::new(r)
                                    )
                                }
                            }
                            expr => Expr::Seq(Box::new(Expr::Rep(Box::new(expr))), r)
                        }
                        expr => Expr::Seq(Box::new(expr), r)
                    }
                    expr => expr
                }
            })
        }
    }
}
<|endoftext|>


<commit_before>use ::internal_prelude::works_everywhere::*;
use std::sync::atomic::{AtomicU64, Ordering, ATOMIC_U64_INIT};

static NEXT_FLUENT_NODE_ID: AtomicU64 = ATOMIC_U64_INIT;

pub fn fluently() -> FluentNode {
    FluentNode::empty()
}

#[derive(Clone,Debug)]
pub struct FluentNode {
    input: Option<Box<FluentNode>>,
    canvas: Option<Box<FluentNode>>,
    data: Option<s::Node>,
    uid: u64,
}

impl FluentNode {
    fn next_uid() -> u64 {
        NEXT_FLUENT_NODE_ID.fetch_add(1, Ordering::SeqCst)
    }
    fn new(node: s::Node,
           input_node: Option<FluentNode>,
           canvas_node: Option<FluentNode>)
           -> FluentNode {
        FluentNode {
            input: input_node.and_then(|v| match v.is_empty() {
                true => None,
                false => Some(Box::new(v)),
            }),
            canvas: canvas_node.and_then(|v| match v.is_empty() {
                true => None,
                false => Some(Box::new(v)),
            }),
            data: Some(node),
            uid: FluentNode::next_uid(),
        }
    }
    pub fn empty() -> FluentNode {
        FluentNode {
            input: None,
            canvas: None,
            data: None,
            uid: FluentNode::next_uid(),
        }
    }

    pub fn is_empty(&self) -> bool {
        self.data.is_none()
    }

    pub fn to(self, v: s::Node) -> FluentNode {
        FluentNode::new(v, Some(self), None)
    }
    pub fn to_canvas(self, canvas: FluentNode, v: s::Node) -> FluentNode {
        FluentNode::new(v, Some(self), Some(canvas))
    }
    pub fn branch(&self) -> FluentNode {
        self.clone()
    }
    pub fn builder(self) -> FluentGraphBuilder {
        FluentGraphBuilder::new_with(self)
    }

    /// Injects placeholders
    pub fn to_build_0_1(self) -> s::Build001{
        self.builder().to_framewise().wrap_in_build_0_1()
    }

    pub fn constrain_within(self, w: Option<u32>, h: Option<u32>, resampling_hints: Option<s::ConstraintResamplingHints>) -> FluentNode{
        self.to(s::Node::Constrain(s::Constraint::Within{ w: w, h: h, hints: resampling_hints}))
    }

    pub fn canvas_bgra32(self,
                         w: usize,
                         // camelCased: #[serde(rename="fromY")]
                         h: usize,
                         color: s::Color)
                         -> FluentNode {
        self.to(s::Node::CreateCanvas {
            w: w,
            h: h,
            format: s::PixelFormat::Bgra32,
            color: color,
        })
    }

    pub fn create_canvas(self,
                         w: usize,
                         // camelCased: #[serde(rename="fromY")]
                         h: usize,
                         format: s::PixelFormat,
                         color: s::Color)
                         -> FluentNode {
        self.to(s::Node::CreateCanvas {
            w: w,
            h: h,
            format: format,
            color: color,
        })
    }

    pub fn decode(self, io_id: i32) -> FluentNode {
        self.to(s::Node::Decode {
            io_id: io_id,
            commands: None,
        })
    }
    pub fn encode(self, io_id: i32, preset: s::EncoderPreset) -> FluentNode {
        self.to(s::Node::Encode {
            io_id: io_id,
            preset: preset
        })
    }

    pub fn flip_vertical(self) -> FluentNode {
        self.to(s::Node::FlipV)
    }

    pub fn flip_horizontal(self) -> FluentNode {
        self.to(s::Node::FlipH)
    }

    pub fn rotate_90(self) -> FluentNode {
        self.to(s::Node::Rotate90)
    }
    pub fn rotate_180(self) -> FluentNode {
        self.to(s::Node::Rotate180)
    }
    pub fn rotate_270(self) -> FluentNode {
        self.to(s::Node::Rotate270)
    }

    pub fn transpose(self) -> FluentNode {
        self.to(s::Node::Transpose)
    }
    pub fn copy_rect_from(self,
                          from: FluentNode,
                          from_x: u32,
                          // camelCased: #[serde(rename="fromY")]
                          from_y: u32,
                          width: u32,
                          height: u32,
                          x: u32,
                          y: u32)
                          -> FluentNode {
        from.to_canvas(self,
                       s::Node::CopyRectToCanvas {
                           from_x: from_x,
                           from_y: from_y,
                           width: width,
                           height: height,
                           x: x,
                           y: y,
                       })
    }
}
impl PartialEq for FluentNode {
    fn eq(&self, other: &FluentNode) -> bool {
        self.uid == other.uid
    }
}

pub struct FluentGraphBuilder {
    output_nodes: Vec<Box<FluentNode>>,
}

impl FluentGraphBuilder {
    pub fn new() -> FluentGraphBuilder {
        FluentGraphBuilder { output_nodes: vec![] }
    }
    pub fn new_with(n: FluentNode) -> FluentGraphBuilder {
        FluentGraphBuilder { output_nodes: vec![Box::new(n)] }
    }

    pub fn with(self, n: FluentNode) -> FluentGraphBuilder {
        let mut new_vec = self.output_nodes.clone();
        new_vec.push(Box::new(n));
        FluentGraphBuilder { output_nodes: new_vec }
    }

    fn collect_unique(&self) -> Vec<&Box<FluentNode>> {
        let mut set = HashSet::new();
        let mut todo = Vec::new();
        let mut unique = Vec::new();
        for end in self.output_nodes.as_slice().iter() {
            todo.push(end);
        }

        loop {
            if todo.len() == 0 {
                break;
            }
            let next = todo.pop().unwrap();
            if !set.contains(&next.uid) {
                set.insert(next.uid);
                unique.push(next);
                if let Some(ref c) = next.canvas {
                    todo.push(&c);
                }
                if let Some(ref c) = next.input {
                    todo.push(&c);
                }
            }
        }

        unique
    }

    fn collect_edges(&self, for_nodes: &[&Box<FluentNode>]) -> Vec<(u64, u64, s::EdgeKind)> {
        let mut edges = vec![];
        for n in for_nodes {
            if let Some(ref parent) = n.canvas {
                edges.push((parent.uid, n.uid, s::EdgeKind::Canvas));
            }
            if let Some(ref parent) = n.input {
                edges.push((parent.uid, n.uid, s::EdgeKind::Input));
            }
        }
        edges
    }
    fn lowest_uid(for_nodes: &[&Box<FluentNode>]) -> Option<u64> {
        for_nodes.iter().map(|n| n.uid).min()
    }
    pub fn to_framewise(&self) -> s::Framewise {
        let mut nodes = self.collect_unique();
        if self.output_nodes.len() == 1 && nodes.as_slice().iter().all(|n| n.canvas == None) {
            nodes.sort_by(|a,b|a.uid.cmp(&b.uid));
            s::Framewise::Steps(nodes.into_iter().map(|b| b.data.clone().unwrap()).collect::<Vec<s::Node>>())
        }else{
            self.to_framewise_graph()
        }
    }
    pub fn to_framewise_graph(&self) -> s::Framewise {
        let nodes = self.collect_unique();
        let lowest_uid = FluentGraphBuilder::lowest_uid(&nodes).unwrap_or(0);
        let edges = self.collect_edges(&nodes);
        let framewise_edges = edges.into_iter()
            .map(|(from, to, kind)| {
                s::Edge {
                    from: (from - lowest_uid) as i32,
                    to: (to - lowest_uid) as i32,
                    kind: kind,
                }
            })
            .collect::<Vec<s::Edge>>();
        let mut framewise_nodes = HashMap::new();
        for n in nodes {
            let _ =
                framewise_nodes.insert((n.uid - lowest_uid).to_string(), n.data.clone().unwrap());
        }
        s::Framewise::Graph(s::Graph {
            edges: framewise_edges,
            nodes: framewise_nodes,
        })
    }

    //    pub fn to_graph(&self) -> ::Graph {
    //        let mut uid_map = HashMap::new();
    //        let from_list = self.collect_unique();
    //
    //        let mut g = ::Graph::with_capacity(from_list.len(), from_list.len() + 8);
    //        for n in from_list.as_slice(){
    //            if let Some(ref data) = n.data {
    //                let ix = g.add_node(::flow::definitions::Node::from(data.clone()));
    //                uid_map.insert(n.uid, ix);
    //            }
    //        }
    //
    //        for n in from_list.as_slice(){
    //            if let Some(ref parent) = n.canvas{
    //                g.add_edge(uid_map[&parent.uid], uid_map[&n.uid],::ffi::EdgeKind::Canvas).unwrap();
    //            }
    //            if let Some(ref parent) = n.input{
    //                g.add_edge(uid_map[&parent.uid], uid_map[&n.uid],::ffi::EdgeKind::Input).unwrap();
    //            }
    //        }
    //        g
    //    }
}

#[test]
fn test_graph_builder() {

    // let d = fluently().decode(0).flip_horizontal().rotate_90().
    let a = fluently()
        .to(s::Node::CreateCanvas {
            w: 200,
            h: 200,
            format: s::PixelFormat::Bgra32,
            color: s::Color::Black,
        })
        .to(s::Node::FlipV);
    let b = a.branch().to(s::Node::Encode {
        preset: s::EncoderPreset::libjpegturbo(),
        io_id: 0,
    });
    let c = a.branch()
        .to(s::Node::Resample2D {
            w: 100,
            h: 100,
            down_filter: None,
            up_filter: None,
            hints: None,
        })
        .to(s::Node::Encode {
            preset: s::EncoderPreset::libpng32(),
            io_id: 1,
        });
    b.builder().with(c).to_framewise();
}
<commit_msg>test(fluent): improve unit tests<commit_after>use ::internal_prelude::works_everywhere::*;
use std::sync::atomic::{AtomicU64, Ordering, ATOMIC_U64_INIT};

static NEXT_FLUENT_NODE_ID: AtomicU64 = ATOMIC_U64_INIT;

pub fn fluently() -> FluentNode {
    FluentNode::empty()
}

#[derive(Clone,Debug)]
pub struct FluentNode {
    input: Option<Box<FluentNode>>,
    canvas: Option<Box<FluentNode>>,
    data: Option<s::Node>,
    uid: u64,
}

impl FluentNode {
    fn next_uid() -> u64 {
        NEXT_FLUENT_NODE_ID.fetch_add(1, Ordering::SeqCst)
    }
    fn new(node: s::Node,
           input_node: Option<FluentNode>,
           canvas_node: Option<FluentNode>)
           -> FluentNode {
        FluentNode {
            input: input_node.and_then(|v| match v.is_empty() {
                true => None,
                false => Some(Box::new(v)),
            }),
            canvas: canvas_node.and_then(|v| match v.is_empty() {
                true => None,
                false => Some(Box::new(v)),
            }),
            data: Some(node),
            uid: FluentNode::next_uid(),
        }
    }
    pub fn empty() -> FluentNode {
        FluentNode {
            input: None,
            canvas: None,
            data: None,
            uid: FluentNode::next_uid(),
        }
    }

    pub fn is_empty(&self) -> bool {
        self.data.is_none()
    }

    pub fn to(self, v: s::Node) -> FluentNode {
        FluentNode::new(v, Some(self), None)
    }
    pub fn to_canvas(self, canvas: FluentNode, v: s::Node) -> FluentNode {
        FluentNode::new(v, Some(self), Some(canvas))
    }
    pub fn branch(&self) -> FluentNode {
        self.clone()
    }
    pub fn builder(self) -> FluentGraphBuilder {
        FluentGraphBuilder::new_with(self)
    }

    /// Injects placeholders
    pub fn to_build_0_1(self) -> s::Build001{
        self.builder().to_framewise().wrap_in_build_0_1()
    }

    pub fn constrain_within(self, w: Option<u32>, h: Option<u32>, resampling_hints: Option<s::ConstraintResamplingHints>) -> FluentNode{
        self.to(s::Node::Constrain(s::Constraint::Within{ w: w, h: h, hints: resampling_hints}))
    }

    pub fn canvas_bgra32(self,
                         w: usize,
                         // camelCased: #[serde(rename="fromY")]
                         h: usize,
                         color: s::Color)
                         -> FluentNode {
        self.to(s::Node::CreateCanvas {
            w: w,
            h: h,
            format: s::PixelFormat::Bgra32,
            color: color,
        })
    }

    pub fn create_canvas(self,
                         w: usize,
                         // camelCased: #[serde(rename="fromY")]
                         h: usize,
                         format: s::PixelFormat,
                         color: s::Color)
                         -> FluentNode {
        self.to(s::Node::CreateCanvas {
            w: w,
            h: h,
            format: format,
            color: color,
        })
    }

    pub fn decode(self, io_id: i32) -> FluentNode {
        self.to(s::Node::Decode {
            io_id: io_id,
            commands: None,
        })
    }
    pub fn encode(self, io_id: i32, preset: s::EncoderPreset) -> FluentNode {
        self.to(s::Node::Encode {
            io_id: io_id,
            preset: preset
        })
    }

    pub fn flip_vertical(self) -> FluentNode {
        self.to(s::Node::FlipV)
    }

    pub fn flip_horizontal(self) -> FluentNode {
        self.to(s::Node::FlipH)
    }

    pub fn rotate_90(self) -> FluentNode {
        self.to(s::Node::Rotate90)
    }
    pub fn rotate_180(self) -> FluentNode {
        self.to(s::Node::Rotate180)
    }
    pub fn rotate_270(self) -> FluentNode {
        self.to(s::Node::Rotate270)
    }

    pub fn transpose(self) -> FluentNode {
        self.to(s::Node::Transpose)
    }
    pub fn copy_rect_from(self,
                          from: FluentNode,
                          from_x: u32,
                          // camelCased: #[serde(rename="fromY")]
                          from_y: u32,
                          width: u32,
                          height: u32,
                          x: u32,
                          y: u32)
                          -> FluentNode {
        from.to_canvas(self,
                       s::Node::CopyRectToCanvas {
                           from_x: from_x,
                           from_y: from_y,
                           width: width,
                           height: height,
                           x: x,
                           y: y,
                       })
    }
}
impl PartialEq for FluentNode {
    fn eq(&self, other: &FluentNode) -> bool {
        self.uid == other.uid
    }
}

pub struct FluentGraphBuilder {
    output_nodes: Vec<Box<FluentNode>>,
}

impl FluentGraphBuilder {
    pub fn new() -> FluentGraphBuilder {
        FluentGraphBuilder { output_nodes: vec![] }
    }
    pub fn new_with(n: FluentNode) -> FluentGraphBuilder {
        FluentGraphBuilder { output_nodes: vec![Box::new(n)] }
    }

    pub fn with(self, n: FluentNode) -> FluentGraphBuilder {
        let mut new_vec = self.output_nodes.clone();
        new_vec.push(Box::new(n));
        FluentGraphBuilder { output_nodes: new_vec }
    }

    fn collect_unique(&self) -> Vec<&Box<FluentNode>> {
        let mut set = HashSet::new();
        let mut todo = Vec::new();
        let mut unique = Vec::new();
        for end in self.output_nodes.as_slice().iter() {
            todo.push(end);
        }

        loop {
            if todo.len() == 0 {
                break;
            }
            let next = todo.pop().unwrap();
            if !set.contains(&next.uid) {
                set.insert(next.uid);
                unique.push(next);
                if let Some(ref c) = next.canvas {
                    todo.push(&c);
                }
                if let Some(ref c) = next.input {
                    todo.push(&c);
                }
            }
        }

        unique
    }

    fn collect_edges(&self, for_nodes: &[&Box<FluentNode>]) -> Vec<(u64, u64, s::EdgeKind)> {
        let mut edges = vec![];
        for n in for_nodes {
            if let Some(ref parent) = n.canvas {
                edges.push((parent.uid, n.uid, s::EdgeKind::Canvas));
            }
            if let Some(ref parent) = n.input {
                edges.push((parent.uid, n.uid, s::EdgeKind::Input));
            }
        }
        edges
    }
    fn lowest_uid(for_nodes: &[&Box<FluentNode>]) -> Option<u64> {
        for_nodes.iter().map(|n| n.uid).min()
    }
    pub fn to_framewise(&self) -> s::Framewise {
        let mut nodes = self.collect_unique();
        if self.output_nodes.len() == 1 && nodes.as_slice().iter().all(|n| n.canvas == None) {
            nodes.sort_by(|a,b|a.uid.cmp(&b.uid));
            s::Framewise::Steps(nodes.into_iter().map(|b| b.data.clone().unwrap()).collect::<Vec<s::Node>>())
        }else{
            self.to_framewise_graph()
        }
    }
    pub fn to_framewise_graph(&self) -> s::Framewise {
        let nodes = self.collect_unique();
        let lowest_uid = FluentGraphBuilder::lowest_uid(&nodes).unwrap_or(0);
        let edges = self.collect_edges(&nodes);
        let framewise_edges = edges.into_iter()
            .map(|(from, to, kind)| {
                s::Edge {
                    from: (from - lowest_uid) as i32,
                    to: (to - lowest_uid) as i32,
                    kind: kind,
                }
            })
            .collect::<Vec<s::Edge>>();
        let mut framewise_nodes = HashMap::new();
        for n in nodes {
            let _ =
                framewise_nodes.insert((n.uid - lowest_uid).to_string(), n.data.clone().unwrap());
        }
        s::Framewise::Graph(s::Graph {
            edges: framewise_edges,
            nodes: framewise_nodes,
        })
    }

    //    pub fn to_graph(&self) -> ::Graph {
    //        let mut uid_map = HashMap::new();
    //        let from_list = self.collect_unique();
    //
    //        let mut g = ::Graph::with_capacity(from_list.len(), from_list.len() + 8);
    //        for n in from_list.as_slice(){
    //            if let Some(ref data) = n.data {
    //                let ix = g.add_node(::flow::definitions::Node::from(data.clone()));
    //                uid_map.insert(n.uid, ix);
    //            }
    //        }
    //
    //        for n in from_list.as_slice(){
    //            if let Some(ref parent) = n.canvas{
    //                g.add_edge(uid_map[&parent.uid], uid_map[&n.uid],::ffi::EdgeKind::Canvas).unwrap();
    //            }
    //            if let Some(ref parent) = n.input{
    //                g.add_edge(uid_map[&parent.uid], uid_map[&n.uid],::ffi::EdgeKind::Input).unwrap();
    //            }
    //        }
    //        g
    //    }
}

#[test]
fn smoke_test_chaining(){

    let chain = fluently()
        .decode(0)
        .constrain_within(Some(1400), Some(1400), Some(s::ConstraintResamplingHints::with(Some(s::Filter::CatmullRom), Some(40f32))))
        .flip_horizontal()
        .flip_vertical()
        .transpose()
        .rotate_90()
        .rotate_180()
        .rotate_270().encode(1, s::EncoderPreset::libpng32()).builder().to_framewise();
}

#[test]
fn smoke_test_many_operations(){

    let chain = fluently()
        .to(s::Node::Decode{io_id:0, commands: Some(vec![s::DecoderCommand::JpegDownscaleHints(s::JpegIDCTDownscaleHints{
            gamma_correct_for_srgb_during_spatial_luma_scaling: Some(false),
            scale_luma_spatially: Some(false),
            width: 1600,
            height:1600
        })])})
        .constrain_within(Some(1400), None,None)
        .constrain_within(Some(1400), Some(1400), Some(s::ConstraintResamplingHints::with(Some(s::Filter::CatmullRom), Some(40f32))))
        .to(s::Node::Resample2D {
            w: 800,
            h: 800,
            down_filter: Some(s::Filter::Robidoux),
            up_filter: Some(s::Filter::Ginseng),
            hints: Some(s::ResampleHints {
                sharpen_percent: Some(10f32),
                prefer_1d_twice: None,
            }),
        })
        .to(s::Node::ApplyOrientation{flag: 7}).flip_horizontal().flip_vertical().transpose().rotate_90().rotate_180().rotate_270()
        .to(s::Node::FillRect {
            x1: 0,
            y1: 0,
            x2: 8,
            y2: 8,
            color: s::Color::Transparent,
        }).to(                              s::Node::ExpandCanvas {
        left: 10,
        top: 10,
        right: 10,
        bottom: 10,
        color: s::Color::Srgb(s::ColorSrgb::Hex("FFEECCFF".to_owned())),
    }).to(s::Node::Crop {
        x1: 10,
        y1: 10,
        x2: 650,
        y2: 490,
    }).encode(1, s::EncoderPreset::Libpng{
        depth: Some(s::PngBitDepth::Png24),
        matte: Some(s::Color::Srgb(s::ColorSrgb::Hex("9922FF".to_owned()))),
        zlib_compression: Some(7)
    });

    let framewise = chain.builder().to_framewise();
}
#[test]
fn smoke_test_graph_builder() {

    // let d = fluently().decode(0).flip_horizontal().rotate_90().
    let a = fluently()
        .to(s::Node::CreateCanvas {
            w: 200,
            h: 200,
            format: s::PixelFormat::Bgra32,
            color: s::Color::Black,
        })
        .to(s::Node::FlipV);
    let b = a.branch().to(s::Node::Encode {
        preset: s::EncoderPreset::libjpegturbo(),
        io_id: 0,
    });
    let c = a.branch()
        .to(s::Node::Resample2D {
            w: 100,
            h: 100,
            down_filter: None,
            up_filter: None,
            hints: None,
        })
        .to(s::Node::Encode {
            preset: s::EncoderPreset::libpng32(),
            io_id: 1,
        });
    b.builder().with(c).to_framewise();
}
<|endoftext|>


<commit_before>use libc::{int32_t,size_t};
use super::graph::Graph;
use ffi::{Context,Job,PixelFormat, EdgeKind, NodeType, BitmapBgra};
use daggy::{Dag,EdgeIndex,NodeIndex};
use petgraph::{EdgeDirection};
mod rotate_flip_transpose;
mod clone_crop_fill_expand;
mod scale_render;
mod create_canvas;
mod codecs_and_pointer;

extern crate imageflow_serde as s;
use super::definitions::*;

//TODO: Implement Scale1D
//TODO: Implement Decoder + APPLY_ORIENTATION, Encoder
//TODO: implement TRANSPOSE, APPLY_ORIENTATION
//TODO: Implement EXPAND_CANVAS, CROP

pub use self::codecs_and_pointer::PRIMITIVE_DECODER;
pub use self::codecs_and_pointer::DECODER;
pub use self::codecs_and_pointer::BITMAP_BGRA_POINTER;
pub use self::create_canvas::CREATE_CANVAS;
pub use self::clone_crop_fill_expand::CLONE;
pub use self::clone_crop_fill_expand::COPY_RECT;
pub use self::clone_crop_fill_expand::FILL_RECT;
pub use self::clone_crop_fill_expand::EXPAND_CANVAS;
pub use self::scale_render::SCALE;
pub use self::rotate_flip_transpose::FLIP_V;
pub use self::rotate_flip_transpose::FLIP_V_PRIMITIVE;
pub use self::rotate_flip_transpose::FLIP_H;
pub use self::rotate_flip_transpose::FLIP_H_PRIMITIVE;
pub use self::rotate_flip_transpose::ROTATE_90;
pub use self::rotate_flip_transpose::ROTATE_180;
pub use self::rotate_flip_transpose::ROTATE_270;
pub use self::rotate_flip_transpose::TRANSPOSE;
pub use self::rotate_flip_transpose::APPLY_ORIENTATION;
pub use self::rotate_flip_transpose::NO_OP;

struct NodeDefHelpers {}
impl NodeDefHelpers {
    fn copy_frame_est_from_first_input(ctx: &mut OpCtxMut, ix: NodeIndex<u32>) {
        ctx.copy_frame_est_from_first_input(ix);
    }
    fn copy_frame_est_from_first_canvas(ctx: &mut OpCtxMut, ix: NodeIndex<u32>) {
        ctx.copy_frame_est_from_first_canvas(ix);
    }
    fn rotate_frame_info(ctx: &mut OpCtxMut, ix: NodeIndex<u32>) {
        ctx.rotate_frame_est_from_first_input(ix);
    }
    fn flatten_flip_v(ctx: &mut OpCtxMut, ix: NodeIndex<u32>) {
        // ctx.graph.node_weight_mut(ix).unwrap()
    }

    fn delete_node_and_snap_together(ctx: &mut OpCtxMut, ix: NodeIndex<u32>) {
        ctx.delete_node_and_snap_together(ix);
    }
}

impl<'c> OpCtxMut<'c> {
    pub fn first_parent_of_kind<'a>(&'a self, of_node: NodeIndex<u32>, filter_by_kind: EdgeKind) -> Option<NodeIndex<u32>> {
        self.graph
            .graph()
            .edges_directed(of_node, EdgeDirection::Incoming)
            .filter(|&(node, &kind)| kind == filter_by_kind)
            .map(|(node, kind)| node)
            .nth(0)
    }

    pub fn first_parent_input<'a>(&'a self, of_node: NodeIndex<u32>) -> Option<NodeIndex<u32>> {
        self.first_parent_of_kind(of_node, EdgeKind::Input)
    }
    pub fn first_parent_canvas<'a>(&'a self, of_node: NodeIndex<u32>) -> Option<NodeIndex<u32>> {
        self.first_parent_of_kind(of_node, EdgeKind::Input)
    }

    pub fn first_parent_input_weight<'a>(&'a self, of_node: NodeIndex<u32>) -> Option<Node> {
        self.first_parent_input(of_node).map(|ix| self.graph.node_weight(ix).unwrap().clone())
    }

    pub fn first_parent_frame_info_some<'a>(&'a self, of_node: NodeIndex<u32>) -> Option<FrameInfo> {
        self.first_parent_input(of_node).and_then(|ix| {
            self.graph.node_weight(ix).and_then(|w| {
                match w.frame_est{
                    FrameEstimate::Some(ref frame_info) => Some(*frame_info),
                    _ => None
                }
            })
        })
    }

    pub fn get_json_params<'a>(&'a self, ix: NodeIndex<u32>) -> Option<s::Node> {
        self.graph.node_weight(ix).and_then(|w| {
            match w.params {
                NodeParams::Json(ref node) => Some(node.clone()),
                _ => None
            }
        })
    }

    pub fn first_parent_canvas_weight<'a>(&'a self, of_node: NodeIndex<u32>) -> Option<&Node> {
        self.first_parent_canvas(of_node).map(|ix| self.graph.node_weight(ix).unwrap())
    }

    pub fn first_parent_result_frame<'a, 'b>(&'a self, of_node: NodeIndex<u32>, kind: EdgeKind) -> Option<*mut BitmapBgra> {
        self.first_parent_of_kind(of_node, kind)
            .and_then(|ix| self.graph.node_weight(ix))
            .and_then(|w|
            match w.result {
                NodeResult::Frame(ptr) => Some(ptr),
                _ => None
            }
        )
    }

    pub fn first_parent_input_weight_mut<'a>(&'a mut self,
                                             of_node: NodeIndex<u32>)
                                             -> Option<&mut Node> {
        self.first_parent_input(of_node).map(move |ix| self.graph.node_weight_mut(ix).unwrap())
    }

    pub fn has_other_children<'a>(&'a self,
                                  of_node: NodeIndex<u32>,
                                  except_child: NodeIndex<u32>)
                                  -> bool {
        self.graph
            .graph()
            .neighbors_directed(of_node, EdgeDirection::Outgoing)
            .any(|n| n != except_child)
    }

    pub fn weight<'a>(&'a mut self, node_to_update: NodeIndex<u32>) -> &'a Node {
        self.graph.node_weight(node_to_update).unwrap()
    }

    pub fn weight_mut<'a>(&'a mut self, node_to_update: NodeIndex<u32>) -> &'a mut Node {
        self.graph.node_weight_mut(node_to_update).unwrap()
    }
    pub fn copy_frame_est_from_first_input<'a>(&'a mut self, node_to_update: NodeIndex<u32>) {
        match self.first_parent_input(node_to_update) {
            Some(input_ix) => {
                if self.graph.node_weight(input_ix).unwrap().frame_est == FrameEstimate::None{
                    panic!("Parent frame {} is not estimated", input_ix.index());
                }
                self.graph.node_weight_mut(node_to_update).unwrap().frame_est =
                    self.graph.node_weight(input_ix).unwrap().frame_est.clone();
            }
            None => {}
        }
    }
    pub fn copy_frame_est_from_first_canvas<'a>(&'a mut self, node_to_update: NodeIndex<u32>) {
        match self.first_parent_canvas(node_to_update) {
            Some(input_ix) => {
                self.graph.node_weight_mut(node_to_update).unwrap().frame_est =
                    self.graph.node_weight(input_ix).unwrap().frame_est.clone();
            }
            None => {}
        }
    }

    pub fn assert_ok(&self){
        unsafe {
            ::ContextPtr::from_ptr(self.c).assert_ok(None);
        }
    }

    pub fn rotate_frame_est_from_first_input<'a, 'b>(&'a mut self,
                                                     node_to_update: NodeIndex<u32>) {
        // TODO: select by EdgeKind=Input
        let input = self.graph
            .graph()
            .neighbors_directed(node_to_update, EdgeDirection::Incoming)
            .nth(0);
        match input {
            Some(input_ix) => {
                let input_est = self.graph.node_weight(input_ix).unwrap().frame_est.clone();
                let mut w = self.graph.node_weight_mut(node_to_update).unwrap();
                w.frame_est = match input_est {
                    FrameEstimate::Some(info) => {
                        FrameEstimate::Some(FrameInfo {
                            w: info.h,
                            h: info.w,
                            ..info
                        })
                    }
                    FrameEstimate::UpperBound(info) => {
                        FrameEstimate::UpperBound(FrameInfo {
                            w: info.h,
                            h: info.w,
                            ..info
                        })
                    }
                    other => other,
                };
            }
            None => {}
        }
    }

    pub fn copy_edges_to<'a>(&'a mut self,
                             from_node: NodeIndex<u32>,
                             to_node: NodeIndex<u32>,
                             direction: EdgeDirection) {
        let edges = self.graph
            .graph()
            .edges_directed(from_node, direction)
            .map(|(a, b)| (a, b.clone()))
            .collect::<Vec<_>>();

        for (other_node, weight) in edges {
            match direction {
                EdgeDirection::Incoming => {
                    self.graph.add_edge(other_node, to_node, weight.clone()).unwrap()
                }
                EdgeDirection::Outgoing => {
                    self.graph.add_edge(to_node, other_node, weight.clone()).unwrap()
                }
            };
        }
    }
    pub fn delete_node_and_snap_together<'a>(&'a mut self, node_to_delete: NodeIndex<u32>) {
        // Prefer EdgeKind=Input
        let input = self.graph
            .graph()
            .neighbors_directed(node_to_delete, EdgeDirection::Incoming)
            .nth(0);
        match input {
            None => {}
            Some(from_node) => {
                self.copy_edges_to(node_to_delete, from_node, EdgeDirection::Outgoing);
                self.graph.remove_node(node_to_delete).unwrap();
            }
        };
    }

    pub fn replace_node<'a>(&'a mut self, index: NodeIndex<u32>, with_list: Vec<Node>) {
        let mut with = with_list.clone();
        match with.len() {
            0 => self.delete_node_and_snap_together(index),
            n => {
                with.reverse();
                let mut last_ix = self.graph.add_node(with.pop().unwrap());
                self.copy_edges_to(index, last_ix, EdgeDirection::Incoming);
                while with.len() > 0 {
                    last_ix = self.graph.add_node(with.pop().unwrap());
                }
                self.copy_edges_to(index, last_ix, EdgeDirection::Outgoing);
                self.graph.remove_node(index).unwrap();
            }
        }
    }

    pub fn replace_node_with_existing<'a>(&'a mut self,
                                          index: NodeIndex<u32>,
                                          with_index: NodeIndex<u32>) {
        self.copy_edges_to(index, with_index, EdgeDirection::Incoming);
        self.copy_edges_to(index, with_index, EdgeDirection::Outgoing);
        self.graph.remove_node(index).unwrap();
    }
}<commit_msg>Fix replace_nodes<commit_after>use libc::{int32_t,size_t};
use super::graph::Graph;
use ffi::{Context,Job,PixelFormat, EdgeKind, NodeType, BitmapBgra};
use daggy::{Dag,EdgeIndex,NodeIndex};
use petgraph::{EdgeDirection};
mod rotate_flip_transpose;
mod clone_crop_fill_expand;
mod scale_render;
mod create_canvas;
mod codecs_and_pointer;

extern crate imageflow_serde as s;
use super::definitions::*;

//TODO: Implement Scale1D
//TODO: Implement Decoder + APPLY_ORIENTATION, Encoder
//TODO: implement TRANSPOSE, APPLY_ORIENTATION
//TODO: Implement EXPAND_CANVAS, CROP

pub use self::codecs_and_pointer::PRIMITIVE_DECODER;
pub use self::codecs_and_pointer::DECODER;
pub use self::codecs_and_pointer::BITMAP_BGRA_POINTER;
pub use self::create_canvas::CREATE_CANVAS;
pub use self::clone_crop_fill_expand::CLONE;
pub use self::clone_crop_fill_expand::COPY_RECT;
pub use self::clone_crop_fill_expand::FILL_RECT;
pub use self::clone_crop_fill_expand::EXPAND_CANVAS;
pub use self::scale_render::SCALE;
pub use self::rotate_flip_transpose::FLIP_V;
pub use self::rotate_flip_transpose::FLIP_V_PRIMITIVE;
pub use self::rotate_flip_transpose::FLIP_H;
pub use self::rotate_flip_transpose::FLIP_H_PRIMITIVE;
pub use self::rotate_flip_transpose::ROTATE_90;
pub use self::rotate_flip_transpose::ROTATE_180;
pub use self::rotate_flip_transpose::ROTATE_270;
pub use self::rotate_flip_transpose::TRANSPOSE;
pub use self::rotate_flip_transpose::APPLY_ORIENTATION;
pub use self::rotate_flip_transpose::NO_OP;

struct NodeDefHelpers {}
impl NodeDefHelpers {
    fn copy_frame_est_from_first_input(ctx: &mut OpCtxMut, ix: NodeIndex<u32>) {
        ctx.copy_frame_est_from_first_input(ix);
    }
    fn copy_frame_est_from_first_canvas(ctx: &mut OpCtxMut, ix: NodeIndex<u32>) {
        ctx.copy_frame_est_from_first_canvas(ix);
    }
    fn rotate_frame_info(ctx: &mut OpCtxMut, ix: NodeIndex<u32>) {
        ctx.rotate_frame_est_from_first_input(ix);
    }
    fn flatten_flip_v(ctx: &mut OpCtxMut, ix: NodeIndex<u32>) {
        // ctx.graph.node_weight_mut(ix).unwrap()
    }

    fn delete_node_and_snap_together(ctx: &mut OpCtxMut, ix: NodeIndex<u32>) {
        ctx.delete_node_and_snap_together(ix);
    }
}

impl<'c> OpCtxMut<'c> {
    pub fn first_parent_of_kind<'a>(&'a self, of_node: NodeIndex<u32>, filter_by_kind: EdgeKind) -> Option<NodeIndex<u32>> {
        self.graph
            .graph()
            .edges_directed(of_node, EdgeDirection::Incoming)
            .filter(|&(node, &kind)| kind == filter_by_kind)
            .map(|(node, kind)| node)
            .nth(0)
    }

    pub fn first_parent_input<'a>(&'a self, of_node: NodeIndex<u32>) -> Option<NodeIndex<u32>> {
        self.first_parent_of_kind(of_node, EdgeKind::Input)
    }
    pub fn first_parent_canvas<'a>(&'a self, of_node: NodeIndex<u32>) -> Option<NodeIndex<u32>> {
        self.first_parent_of_kind(of_node, EdgeKind::Input)
    }

    pub fn first_parent_input_weight<'a>(&'a self, of_node: NodeIndex<u32>) -> Option<Node> {
        self.first_parent_input(of_node).map(|ix| self.graph.node_weight(ix).unwrap().clone())
    }

    pub fn first_parent_frame_info_some<'a>(&'a self, of_node: NodeIndex<u32>) -> Option<FrameInfo> {
        self.first_parent_input(of_node).and_then(|ix| {
            self.graph.node_weight(ix).and_then(|w| {
                match w.frame_est{
                    FrameEstimate::Some(ref frame_info) => Some(*frame_info),
                    _ => None
                }
            })
        })
    }

    pub fn get_json_params<'a>(&'a self, ix: NodeIndex<u32>) -> Option<s::Node> {
        self.graph.node_weight(ix).and_then(|w| {
            match w.params {
                NodeParams::Json(ref node) => Some(node.clone()),
                _ => None
            }
        })
    }

    pub fn first_parent_canvas_weight<'a>(&'a self, of_node: NodeIndex<u32>) -> Option<&Node> {
        self.first_parent_canvas(of_node).map(|ix| self.graph.node_weight(ix).unwrap())
    }

    pub fn first_parent_result_frame<'a, 'b>(&'a self, of_node: NodeIndex<u32>, kind: EdgeKind) -> Option<*mut BitmapBgra> {
        self.first_parent_of_kind(of_node, kind)
            .and_then(|ix| self.graph.node_weight(ix))
            .and_then(|w|
            match w.result {
                NodeResult::Frame(ptr) => Some(ptr),
                _ => None
            }
        )
    }

    pub fn first_parent_input_weight_mut<'a>(&'a mut self,
                                             of_node: NodeIndex<u32>)
                                             -> Option<&mut Node> {
        self.first_parent_input(of_node).map(move |ix| self.graph.node_weight_mut(ix).unwrap())
    }

    pub fn has_other_children<'a>(&'a self,
                                  of_node: NodeIndex<u32>,
                                  except_child: NodeIndex<u32>)
                                  -> bool {
        self.graph
            .graph()
            .neighbors_directed(of_node, EdgeDirection::Outgoing)
            .any(|n| n != except_child)
    }

    pub fn weight<'a>(&'a mut self, node_to_update: NodeIndex<u32>) -> &'a Node {
        self.graph.node_weight(node_to_update).unwrap()
    }

    pub fn weight_mut<'a>(&'a mut self, node_to_update: NodeIndex<u32>) -> &'a mut Node {
        self.graph.node_weight_mut(node_to_update).unwrap()
    }
    pub fn copy_frame_est_from_first_input<'a>(&'a mut self, node_to_update: NodeIndex<u32>) {
        match self.first_parent_input(node_to_update) {
            Some(input_ix) => {
                if self.graph.node_weight(input_ix).unwrap().frame_est == FrameEstimate::None{
                    panic!("Parent frame {} is not estimated", input_ix.index());
                }
                self.graph.node_weight_mut(node_to_update).unwrap().frame_est =
                    self.graph.node_weight(input_ix).unwrap().frame_est.clone();
            }
            None => {}
        }
    }
    pub fn copy_frame_est_from_first_canvas<'a>(&'a mut self, node_to_update: NodeIndex<u32>) {
        match self.first_parent_canvas(node_to_update) {
            Some(input_ix) => {
                self.graph.node_weight_mut(node_to_update).unwrap().frame_est =
                    self.graph.node_weight(input_ix).unwrap().frame_est.clone();
            }
            None => {}
        }
    }

    pub fn assert_ok(&self){
        unsafe {
            ::ContextPtr::from_ptr(self.c).assert_ok(None);
        }
    }

    pub fn rotate_frame_est_from_first_input<'a, 'b>(&'a mut self,
                                                     node_to_update: NodeIndex<u32>) {
        // TODO: select by EdgeKind=Input
        let input = self.graph
            .graph()
            .neighbors_directed(node_to_update, EdgeDirection::Incoming)
            .nth(0);
        match input {
            Some(input_ix) => {
                let input_est = self.graph.node_weight(input_ix).unwrap().frame_est.clone();
                let mut w = self.graph.node_weight_mut(node_to_update).unwrap();
                w.frame_est = match input_est {
                    FrameEstimate::Some(info) => {
                        FrameEstimate::Some(FrameInfo {
                            w: info.h,
                            h: info.w,
                            ..info
                        })
                    }
                    FrameEstimate::UpperBound(info) => {
                        FrameEstimate::UpperBound(FrameInfo {
                            w: info.h,
                            h: info.w,
                            ..info
                        })
                    }
                    other => other,
                };
            }
            None => {}
        }
    }

    pub fn copy_edges_to<'a>(&'a mut self,
                             from_node: NodeIndex<u32>,
                             to_node: NodeIndex<u32>,
                             direction: EdgeDirection) {
        let edges = self.graph
            .graph()
            .edges_directed(from_node, direction)
            .map(|(a, b)| (a, b.clone()))
            .collect::<Vec<_>>();

        for (other_node, weight) in edges {
            match direction {
                EdgeDirection::Incoming => {
                    self.graph.add_edge(other_node, to_node, weight.clone()).unwrap()
                }
                EdgeDirection::Outgoing => {
                    self.graph.add_edge(to_node, other_node, weight.clone()).unwrap()
                }
            };
        }
    }
    pub fn delete_node_and_snap_together<'a>(&'a mut self, node_to_delete: NodeIndex<u32>) {
        // Prefer EdgeKind=Input
        let input = self.graph
            .graph()
            .neighbors_directed(node_to_delete, EdgeDirection::Incoming)
            .nth(0);
        match input {
            None => {}
            Some(from_node) => {
                self.copy_edges_to(node_to_delete, from_node, EdgeDirection::Outgoing);
                self.graph.remove_node(node_to_delete).unwrap();
            }
        };
    }

    //Links nodes with Input edges
    pub fn replace_node<'a>(&'a mut self, index: NodeIndex<u32>, with_list: Vec<Node>) {
        let mut with = with_list.clone();
        match with.len() {
            0 => self.delete_node_and_snap_together(index),
            n => {
                with.reverse();
                let mut last_ix = self.graph.add_node(with.pop().unwrap());
                self.copy_edges_to(index, last_ix, EdgeDirection::Incoming);
                while with.len() > 0 {
                    let new_ix = self.graph.add_node(with.pop().unwrap());
                    self.graph.add_edge(last_ix, new_ix, EdgeKind::Input).unwrap();
                    last_ix = new_ix;
                }
                self.copy_edges_to(index, last_ix, EdgeDirection::Outgoing);
                self.graph.remove_node(index).unwrap();
            }
        }
    }

    pub fn replace_node_with_existing<'a>(&'a mut self,
                                          index: NodeIndex<u32>,
                                          with_index: NodeIndex<u32>) {
        self.copy_edges_to(index, with_index, EdgeDirection::Incoming);
        self.copy_edges_to(index, with_index, EdgeDirection::Outgoing);
        self.graph.remove_node(index).unwrap();
    }
}<|endoftext|>


<commit_before>//  Copyright (C) 2020 Éloïs SANCHEZ.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as
// published by the Free Software Foundation, either version 3 of the
// License, or (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program.  If not, see <https://www.gnu.org/licenses/>.

use anyhow::Result;
use std::process::{Command, Output};
use structopt::StructOpt;

const MIN_RUST_VERTION: &str = "1.47.0";
const NODE_VERSION: &str = "10.22.1";
const NVM_VERSION: &str = "0.35.3";

#[derive(StructOpt)]
struct DuniterXTask {
    #[structopt(long)]
    skip_npm: bool,
    #[structopt(subcommand)]
    command: DuniterXTaskCommand,
}

#[derive(StructOpt)]
enum DuniterXTaskCommand {
    Build {
        #[structopt(long)]
        production: bool,
    },
    Test,
}

fn main() -> Result<()> {
    let args = DuniterXTask::from_args();

    if !version_check::is_min_version(MIN_RUST_VERTION).unwrap_or(false)
        && exec_should_success(Command::new("rustup").args(&["update", "stable"])).is_err()
    {
        eprintln!(
                "Duniter requires Rust {} or higher. If you installed the Rust toolchain via rustup, please execute the command `rustup update stable`.",
                MIN_RUST_VERTION
            );
        std::process::exit(1);
    }
    Command::new("rustc").arg("--version").status()?;
    Command::new("cargo").arg("--version").status()?;

    if !args.skip_npm {
        println!("Check node version …");
        if exec_and_get_stdout(Command::new("node").arg("-v"))
            .unwrap_or_default()
            .trim_end()
            != format!("v{}", NODE_VERSION)
        {
            println!("Install node v{} …", NODE_VERSION);
            install_and_use_node_version()?;
        } else {
            println!("Node v{} already installed ✔", NODE_VERSION);
        }
    }
    match args.command {
        DuniterXTaskCommand::Build { production } => build(args.skip_npm, production),
        DuniterXTaskCommand::Test => test(args.skip_npm),
    }
}

fn install_and_use_node_version() -> Result<()> {
    if exec_should_success(Command::new("nvm").arg("--version")).is_err() {
        println!("Duniter need nvm to build, please install it: https://raw.githubusercontent.com/nvm-sh/nvm/v{}/install.sh", NVM_VERSION);
        std::process::abort();
    }
    exec_should_success(Command::new("nvm").args(&["install", NODE_VERSION]))?;
    exec_should_success(Command::new("nvm").args(&["use", NODE_VERSION]))
}

fn exec_and_get_stdout(command: &mut Command) -> Result<String> {
    let Output {
        status,
        stdout,
        stderr,
    } = command.output()?;

    if status.success() {
        Ok(String::from_utf8(stdout)
            .unwrap_or_else(|_| "Output is not a valid utf8 string".to_owned()))
    } else {
        eprintln!(
            "{}",
            String::from_utf8(stderr)
                .unwrap_or_else(|_| "Error message is not a valid utf8 string".to_owned())
        );
        std::process::exit(1);
    }
}

fn exec_should_success(command: &mut Command) -> Result<()> {
    if !command.status()?.success() {
        std::process::exit(1);
    } else {
        Ok(())
    }
}

fn build(skip_npm: bool, production: bool) -> Result<()> {
    if !skip_npm {
        exec_should_success(Command::new("npm").args(&["add", "duniter-ui"]))?;
        exec_should_success(
            Command::new("npm")
                .env("NEON_BUILD_RELEASE", "true")
                .arg("install"),
        )?;
        if production {
            exec_should_success(Command::new("npm").args(&["prune", "--production"]))?;
        }
    }
    println!("Build duniter-launcher …");
    exec_should_success(Command::new("cargo").args(&["build", "--release"]))?;
    std::fs::copy("target/release/duniter", "bin/duniter")?;
    Ok(())
}

fn test(skip_npm: bool) -> Result<()> {
    exec_should_success(Command::new("cargo").args(&["test", "--all"]))?;
    if !skip_npm {
        exec_should_success(Command::new("npm").arg("test"))?;
    }
    Ok(())
}
<commit_msg>[build] xtask: upgrade min rust version<commit_after>//  Copyright (C) 2020 Éloïs SANCHEZ.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as
// published by the Free Software Foundation, either version 3 of the
// License, or (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program.  If not, see <https://www.gnu.org/licenses/>.

use anyhow::Result;
use std::process::{Command, Output};
use structopt::StructOpt;

const MIN_RUST_VERTION: &str = "1.50.0";
const NODE_VERSION: &str = "10.22.1";
const NVM_VERSION: &str = "0.35.3";

#[derive(StructOpt)]
struct DuniterXTask {
    #[structopt(long)]
    skip_npm: bool,
    #[structopt(subcommand)]
    command: DuniterXTaskCommand,
}

#[derive(StructOpt)]
enum DuniterXTaskCommand {
    Build {
        #[structopt(long)]
        production: bool,
    },
    Test,
}

fn main() -> Result<()> {
    let args = DuniterXTask::from_args();

    if !version_check::is_min_version(MIN_RUST_VERTION).unwrap_or(false)
        && exec_should_success(Command::new("rustup").args(&["update", "stable"])).is_err()
    {
        eprintln!(
                "Duniter requires Rust {} or higher. If you installed the Rust toolchain via rustup, please execute the command `rustup update stable`.",
                MIN_RUST_VERTION
            );
        std::process::exit(1);
    }
    Command::new("rustc").arg("--version").status()?;
    Command::new("cargo").arg("--version").status()?;

    if !args.skip_npm {
        println!("Check node version …");
        if exec_and_get_stdout(Command::new("node").arg("-v"))
            .unwrap_or_default()
            .trim_end()
            != format!("v{}", NODE_VERSION)
        {
            println!("Install node v{} …", NODE_VERSION);
            install_and_use_node_version()?;
        } else {
            println!("Node v{} already installed ✔", NODE_VERSION);
        }
    }
    match args.command {
        DuniterXTaskCommand::Build { production } => build(args.skip_npm, production),
        DuniterXTaskCommand::Test => test(args.skip_npm),
    }
}

fn install_and_use_node_version() -> Result<()> {
    if exec_should_success(Command::new("nvm").arg("--version")).is_err() {
        println!("Duniter need nvm to build, please install it: https://raw.githubusercontent.com/nvm-sh/nvm/v{}/install.sh", NVM_VERSION);
        std::process::abort();
    }
    exec_should_success(Command::new("nvm").args(&["install", NODE_VERSION]))?;
    exec_should_success(Command::new("nvm").args(&["use", NODE_VERSION]))
}

fn exec_and_get_stdout(command: &mut Command) -> Result<String> {
    let Output {
        status,
        stdout,
        stderr,
    } = command.output()?;

    if status.success() {
        Ok(String::from_utf8(stdout)
            .unwrap_or_else(|_| "Output is not a valid utf8 string".to_owned()))
    } else {
        eprintln!(
            "{}",
            String::from_utf8(stderr)
                .unwrap_or_else(|_| "Error message is not a valid utf8 string".to_owned())
        );
        std::process::exit(1);
    }
}

fn exec_should_success(command: &mut Command) -> Result<()> {
    if !command.status()?.success() {
        std::process::exit(1);
    } else {
        Ok(())
    }
}

fn build(skip_npm: bool, production: bool) -> Result<()> {
    if !skip_npm {
        exec_should_success(Command::new("npm").args(&["add", "duniter-ui"]))?;
        exec_should_success(
            Command::new("npm")
                .env("NEON_BUILD_RELEASE", "true")
                .arg("install"),
        )?;
        if production {
            exec_should_success(Command::new("npm").args(&["prune", "--production"]))?;
        }
    }
    println!("Build duniter-launcher …");
    exec_should_success(Command::new("cargo").args(&["build", "--release"]))?;
    std::fs::copy("target/release/duniter", "bin/duniter")?;
    Ok(())
}

fn test(skip_npm: bool) -> Result<()> {
    exec_should_success(Command::new("cargo").args(&["test", "--all"]))?;
    if !skip_npm {
        exec_should_success(Command::new("npm").arg("test"))?;
    }
    Ok(())
}
<|endoftext|>


<commit_before>use bifrost::rpc::*;
use bifrost::raft::RaftService;
use bifrost::raft::client::RaftClient;
use bifrost::raft::state_machine::master::ExecError;
use model::edge::{EdgeType};
use model::edge;
use chashmap::CHashMap;
use std::sync::Arc;
use neb::ram::schema::{Field, Schema};
use neb::client::{Client as NebClient};
use neb::server::{ServerMeta as NebServerMeta};
use server::schema::sm::schema_types::client::SMClient;
use model::vertex::VERTEX_TEMPLATE;

mod sm;

#[derive(Serialize, Deserialize, Debug, Clone, Copy)]
pub enum SchemaType {
    Vertex,
    Edge(EdgeType)
}

#[derive(Serialize, Deserialize, Debug, Clone)]
pub enum SchemaError {
    NewNebSchemaExecError(ExecError),
    NewMorpheusSchemaExecError(ExecError),
    SimpleEdgeShouldNotHaveSchema,
}

pub struct SchemaContainer {
    map: CHashMap<u32, SchemaType>,
    sm_client: Arc<SMClient>,
    neb_client: Arc<NebClient>,
    neb_mata: Arc<NebServerMeta>,
}

pub struct MorpheusSchema {
    pub id: u32,
    pub name: String,
    pub schema_type: SchemaType,
    pub key_field: Option<Vec<String>>,
    pub fields: Vec<Field>
}

pub fn head_fields(schema_type: SchemaType) -> Result<Vec<Field>, SchemaError> {
    Ok(match schema_type {
        SchemaType::Vertex => VERTEX_TEMPLATE.clone(),
        SchemaType::Edge(edge_type) => match edge_type {
            EdgeType::Direct => edge::direct::EDGE_TEMPLATE.clone(),
            EdgeType::Indirect => edge::indirect::EDGE_TEMPLATE.clone(),
            EdgeType::Hyper => edge::hyper::EDGE_TEMPLATE.clone(),
            EdgeType::Simple => return Err(SchemaError::SimpleEdgeShouldNotHaveSchema),
        }
    })
}

impl SchemaContainer {

    pub fn new_meta_service(raft_service: &Arc<RaftService>) {
        let mut container_sm = sm::schema_types::Map::new(sm::DEFAULT_RAFT_ID);
        container_sm.init_callback(raft_service);
        raft_service.register_state_machine(Box::new(container_sm));
    }

    pub fn new_client(
        raft_client: &Arc<RaftClient>,
        neb_client: &Arc<NebClient>,
        neb_meta: &Arc<NebServerMeta>
    ) -> Result<Arc<SchemaContainer>, ExecError> {
        let sm_client = Arc::new(SMClient::new(sm::DEFAULT_RAFT_ID, &raft_client));
        let sm_entries = sm_client.entries()?.unwrap();
        let container = SchemaContainer {
            map: CHashMap::new(),
            sm_client: sm_client.clone(),
            neb_client: neb_client.clone(),
            neb_mata: neb_meta.clone()
        };
        let container_ref = Arc::new(container);
        let container_ref1 = container_ref.clone();
        let container_ref2 = container_ref.clone();
        for (schema_id, schema_type) in sm_entries {
            container_ref.map.insert(schema_id, schema_type);
        }
        sm_client.on_inserted(move |res| {
            if let Ok((id, schema_type)) = res {
                container_ref1.map.insert(id, schema_type);
            }
        })?;
        sm_client.on_removed(move |res| {
            if let Ok((id, schema_type)) = res {
                container_ref2.map.remove(&id);
            }
        })?;
        return Ok(container_ref);
    }

    pub fn new_schema(&self, schema: &mut MorpheusSchema) -> Result<(), SchemaError> {
        let schema_type = schema.schema_type;
        let mut schema_fields = head_fields(schema_type)?;
        schema_fields.append(&mut schema.fields);
        let mut neb_schema = Schema::new(
            schema.name.clone(),
            schema.key_field.clone(),
            Field::new(&String::from("*"), 0, false, false, Some(schema_fields)));
        match self.neb_client.new_schema(&mut neb_schema) {
            Ok(()) => {},
            Err(e) => return Err(SchemaError::NewNebSchemaExecError(e))
        };
        let schema_id = neb_schema.id;
        match self.sm_client.insert(&schema_id, &schema_type) {
            Ok(_) => {},
            Err(e) => return Err(SchemaError::NewMorpheusSchemaExecError(e))
        }
        schema.id = schema_id;
        Ok(())
    }

    pub fn schema_type(&self, schema_id: u32) -> Option<SchemaType> {
        match self.map.get(&schema_id) {
            Some(t) => Some(*t),
            None => None
        }
    }

    pub fn get_neb_schema(&self, schema_id: u32) -> Option<Arc<Schema>> {
        self.neb_mata.schemas.get(&schema_id)
    }

    pub fn neb_to_morpheus_schema(&self, schema: &Arc<Schema>) -> Option<MorpheusSchema> {
        if let Some(schema_type) = self.schema_type(schema.id) {
            if let Some(ref fields) = schema.fields.sub_fields {
                Some(MorpheusSchema {
                    id: schema.id,
                    name: schema.name.clone(),
                    schema_type: schema_type,
                    key_field: schema.str_key_field.clone(),
                    fields: fields.clone(),
                })
            } else { None }
        } else { None }
    }
}<commit_msg>list all morpheus schemas<commit_after>use bifrost::rpc::*;
use bifrost::raft::RaftService;
use bifrost::raft::client::RaftClient;
use bifrost::raft::state_machine::master::ExecError;
use model::edge::{EdgeType};
use model::edge;
use chashmap::CHashMap;
use std::sync::Arc;
use neb::ram::schema::{Field, Schema};
use neb::client::{Client as NebClient};
use neb::server::{ServerMeta as NebServerMeta};
use server::schema::sm::schema_types::client::SMClient;
use model::vertex::VERTEX_TEMPLATE;

mod sm;

#[derive(Serialize, Deserialize, Debug, Clone, Copy)]
pub enum SchemaType {
    Vertex,
    Edge(EdgeType)
}

#[derive(Serialize, Deserialize, Debug, Clone)]
pub enum SchemaError {
    NewNebSchemaExecError(ExecError),
    NewMorpheusSchemaExecError(ExecError),
    SimpleEdgeShouldNotHaveSchema,
}

pub struct SchemaContainer {
    map: CHashMap<u32, SchemaType>,
    sm_client: Arc<SMClient>,
    neb_client: Arc<NebClient>,
    neb_mata: Arc<NebServerMeta>,
}

pub struct MorpheusSchema {
    pub id: u32,
    pub name: String,
    pub schema_type: SchemaType,
    pub key_field: Option<Vec<String>>,
    pub fields: Vec<Field>
}

pub fn head_fields(schema_type: SchemaType) -> Result<Vec<Field>, SchemaError> {
    Ok(match schema_type {
        SchemaType::Vertex => VERTEX_TEMPLATE.clone(),
        SchemaType::Edge(edge_type) => match edge_type {
            EdgeType::Direct => edge::direct::EDGE_TEMPLATE.clone(),
            EdgeType::Indirect => edge::indirect::EDGE_TEMPLATE.clone(),
            EdgeType::Hyper => edge::hyper::EDGE_TEMPLATE.clone(),
            EdgeType::Simple => return Err(SchemaError::SimpleEdgeShouldNotHaveSchema),
        }
    })
}

impl SchemaContainer {

    pub fn new_meta_service(raft_service: &Arc<RaftService>) {
        let mut container_sm = sm::schema_types::Map::new(sm::DEFAULT_RAFT_ID);
        container_sm.init_callback(raft_service);
        raft_service.register_state_machine(Box::new(container_sm));
    }

    pub fn new_client(
        raft_client: &Arc<RaftClient>,
        neb_client: &Arc<NebClient>,
        neb_meta: &Arc<NebServerMeta>
    ) -> Result<Arc<SchemaContainer>, ExecError> {
        let sm_client = Arc::new(SMClient::new(sm::DEFAULT_RAFT_ID, &raft_client));
        let sm_entries = sm_client.entries()?.unwrap();
        let container = SchemaContainer {
            map: CHashMap::new(),
            sm_client: sm_client.clone(),
            neb_client: neb_client.clone(),
            neb_mata: neb_meta.clone()
        };
        let container_ref = Arc::new(container);
        let container_ref1 = container_ref.clone();
        let container_ref2 = container_ref.clone();
        for (schema_id, schema_type) in sm_entries {
            container_ref.map.insert(schema_id, schema_type);
        }
        sm_client.on_inserted(move |res| {
            if let Ok((id, schema_type)) = res {
                container_ref1.map.insert(id, schema_type);
            }
        })?;
        sm_client.on_removed(move |res| {
            if let Ok((id, schema_type)) = res {
                container_ref2.map.remove(&id);
            }
        })?;
        return Ok(container_ref);
    }

    pub fn new_schema(&self, schema: &mut MorpheusSchema) -> Result<(), SchemaError> {
        let schema_type = schema.schema_type;
        let mut schema_fields = head_fields(schema_type)?;
        schema_fields.append(&mut schema.fields);
        let mut neb_schema = Schema::new(
            schema.name.clone(),
            schema.key_field.clone(),
            Field::new(&String::from("*"), 0, false, false, Some(schema_fields)));
        match self.neb_client.new_schema(&mut neb_schema) {
            Ok(()) => {},
            Err(e) => return Err(SchemaError::NewNebSchemaExecError(e))
        };
        let schema_id = neb_schema.id;
        match self.sm_client.insert(&schema_id, &schema_type) {
            Ok(_) => {},
            Err(e) => return Err(SchemaError::NewMorpheusSchemaExecError(e))
        }
        schema.id = schema_id;
        Ok(())
    }

    pub fn schema_type(&self, schema_id: u32) -> Option<SchemaType> {
        match self.map.get(&schema_id) {
            Some(t) => Some(*t),
            None => None
        }
    }

    pub fn get_neb_schema(&self, schema_id: u32) -> Option<Arc<Schema>> {
        self.neb_mata.schemas.get(&schema_id)
    }

    pub fn neb_to_morpheus_schema(&self, schema: &Arc<Schema>) -> Option<MorpheusSchema> {
        if let Some(schema_type) = self.schema_type(schema.id) {
            if let Some(ref fields) = schema.fields.sub_fields {
                Some(MorpheusSchema {
                    id: schema.id,
                    name: schema.name.clone(),
                    schema_type: schema_type,
                    key_field: schema.str_key_field.clone(),
                    fields: fields.clone(),
                })
            } else { None }
        } else { None }
    }
    pub fn all_morpheus_schemas(&self) -> Result<Vec<MorpheusSchema>, ExecError> {
        let neb_schemas = self.neb_client.get_all_schema()?;
        Ok(neb_schemas
            .into_iter()
            .map(|schema| self.neb_to_morpheus_schema(&Arc::new(schema)))
            .filter_map(|ms| ms)
            .collect())
    }
}<|endoftext|>


<commit_before>// A semaphore implementation based on the rust Mutex<T> type

use core::sync::atomic::{AtomicIsize, Ordering};
use core::cell::UnsafeCell;
use core::option::Option::Some;

use alloc::boxed::Box;

use super::super::ProcessQueue;

use super::super::super::process::{ready_queue, proc_yield};

use super::super::super::interrupts::{on,off};

pub struct Semaphore<T> {
    // box the inner semaphore so it has a constant address
    // and can be safely moved
    inner: Box<StaticSemaphore>,
    data: UnsafeCell<T>,
}

// static semaphore that is not RAII
pub struct StaticSemaphore {
    count: AtomicIsize,
    queue: ProcessQueue,
}

// RAII native SemaphoreGuard
pub struct SemaphoreGuard<'a, T: 'a> {
    semaphore: &'a Semaphore<T>,
}

unsafe impl<T: Send> Send for Semaphore<T> {}
unsafe impl<T: Send> Sync for Semaphore<T> {}

impl<T> Semaphore<T> {
    pub fn new(val: T, i: isize) -> Semaphore<T> {
        Semaphore {
            inner: box StaticSemaphore::new(i),
            data: UnsafeCell::new(val),
        }
    }

    // TODO
}

impl<T> Drop for Semaphore<T> {
    fn drop(&mut self) {
        self.inner.destroy();
    }
}

impl StaticSemaphore {
    pub const fn new(i: isize) -> StaticSemaphore {
        StaticSemaphore {
            count: AtomicIsize::new(i),
            queue: ProcessQueue::new(),
        }
    }

    pub fn down(&mut self) {
        off();
        if self.count.fetch_sub(1, Ordering::AcqRel) <= 0 {
            self.count.fetch_add(1, Ordering::AcqRel);
            // block
            proc_yield(Some(&mut self.queue));
        }
        on();
    }

    pub fn up(&mut self) {
        off();
        let next = self.queue.pop_head();
        if next.is_null() {
            self.count.fetch_add(1,Ordering::AcqRel);
        } else {
            ready_queue::make_ready(next);
        }
        on();
    }

    // clean up
    // cannot implement Drop here because we want to be able
    // to create a static semaphore
    pub fn destroy(&mut self) {
        if !self.queue.pop_head().is_null() {
            // TODO: intead, just kill the processes or do zombie detection
            panic!("Semaphore destroyed with processes waiting!");
        }
    }
}

impl<'a, T> SemaphoreGuard<'a, T> {

}
<commit_msg>Added prototype of RAII native semaphore  - I cannot test this until I implement fork or something like so    that processes can share a semaphore.<commit_after>// A semaphore implementation based on the rust Mutex<T> type

use core::sync::atomic::{AtomicIsize, Ordering};
use core::cell::UnsafeCell;
use core::option::Option::Some;

use core::ops::{Deref, DerefMut};

use alloc::boxed::Box;

use super::super::ProcessQueue;

use super::super::super::process::{ready_queue, proc_yield};

use super::super::super::interrupts::{on,off};

pub struct Semaphore<T> {
    // box the inner semaphore so it has a constant address
    // and can be safely moved
    inner: Box<StaticSemaphore>,
    data: UnsafeCell<T>,
}

// static semaphore that is not RAII
pub struct StaticSemaphore {
    count: AtomicIsize,
    queue: ProcessQueue,
}

// RAII native SemaphoreGuard
pub struct SemaphoreGuard<'semaphore, T: 'semaphore> {
    semaphore: &'semaphore mut StaticSemaphore,
    data: &'semaphore UnsafeCell<T>,
}

unsafe impl<T: Send> Send for Semaphore<T> {}
unsafe impl<T: Send> Sync for Semaphore<T> {}

impl<T> Semaphore<T> {
    pub fn new(val: T, i: isize) -> Semaphore<T> {
        Semaphore {
            inner: box StaticSemaphore::new(i),
            data: UnsafeCell::new(val),
        }
    }

    // acquire
    // returns an RAII guard, so no need for up()
    pub fn down<'semaphore>(&'semaphore mut self) -> SemaphoreGuard<'semaphore, T> {
        // acquire semaphore
        self.inner.down();
        SemaphoreGuard::new(&mut *self.inner, &self.data)
    }
}

impl<T> Drop for Semaphore<T> {
    fn drop(&mut self) {
        self.inner.destroy();
    }
}

impl StaticSemaphore {
    pub const fn new(i: isize) -> StaticSemaphore {
        StaticSemaphore {
            count: AtomicIsize::new(i),
            queue: ProcessQueue::new(),
        }
    }

    // acquire
    pub fn down(&mut self) {
        off();
        if self.count.fetch_sub(1, Ordering::AcqRel) <= 0 {
            self.count.fetch_add(1, Ordering::AcqRel);
            // block
            proc_yield(Some(&mut self.queue));
        }
        on();
    }

    // release
    pub fn up(&mut self) {
        off();
        let next = self.queue.pop_head();
        if next.is_null() {
            self.count.fetch_add(1,Ordering::AcqRel);
        } else {
            ready_queue::make_ready(next);
        }
        on();
    }

    // clean up
    // cannot implement Drop here because we want to be able
    // to create a static semaphore
    pub fn destroy(&mut self) {
        if !self.queue.pop_head().is_null() {
            // TODO: intead, just kill the processes or do zombie detection
            panic!("Semaphore destroyed with processes waiting!");
        }
    }
}

impl<'semaphore, T> SemaphoreGuard<'semaphore, T> {
    pub fn new(semaphore: &'semaphore mut StaticSemaphore,
               data: &'semaphore UnsafeCell<T>) -> SemaphoreGuard<'semaphore, T>{
        SemaphoreGuard {
            semaphore: semaphore,
            data: data,
        }
    }
}

impl<'semaphore, T> Drop for SemaphoreGuard<'semaphore, T> {
    fn drop(&mut self) {
        self.semaphore.up();
    }
}

impl<'semaphore, T> Deref for SemaphoreGuard<'semaphore, T> {
    type Target = T;

    fn deref(&self) -> &T {
        unsafe { &*self.data.get() }
    }
}

impl<'semaphore, T> DerefMut for SemaphoreGuard<'semaphore, T> {
    fn deref_mut(&mut self) -> &mut T {
        unsafe { &mut *self.data.get() }
    }
}

<|endoftext|>


<commit_before>#![feature(plugin_registrar)]
#![feature(quote)]

extern crate rustc;
extern crate syntax;

use std::gc::{GC, Gc};
use syntax::parse::token;
use syntax::ast;
use syntax::ext::build::AstBuilder;
use syntax::ext::base;
use syntax::ext::quote::rt::ToSource;
use syntax::codemap;

#[plugin_registrar]
#[doc(hidden)]
pub fn plugin_registrar(reg: &mut ::rustc::plugin::Registry) {
    reg.register_syntax_extension(token::intern("export_lua_module"), base::ItemModifier(expand_lua_module));
}

// handler for export_lua_module
pub fn expand_lua_module(ecx: &mut base::ExtCtxt, span: codemap::Span, _: Gc<ast::MetaItem>, input_item: Gc<ast::Item>)
    -> Gc<ast::Item>
{
    let ecx: &base::ExtCtxt = &*ecx;

    // checking that the input item is a module
    let module = match input_item.node {
        ast::ItemMod(ref module) => module,
        _ => {
            ecx.span_err(input_item.span,
                "`export_lua_module` extension is only allowed on modules");
            return input_item
        }
    };

    // creating the new item that will be returned by the function
    // it is just a clone of the input with more elements added to it
    let mut new_item = input_item.deref().clone();
    new_item.vis = ast::Public;
    if input_item.vis != ast::Public {
        ecx.span_warn(input_item.span,
            "`export_lua_module` will turn the module into a public module");
    }

    // creating an array of the statements to add to the main Lua entry point
    let module_handler_body: Vec<Gc<ast::Stmt>> = {
        let mut module_handler_body = Vec::new();

        for moditem in module.items.iter() {
            let moditem_name = moditem.ident.to_source();

            match moditem.node {
                ast::ItemFn(..) | ast::ItemStatic(..) => {
                    let moditem_name_slice = moditem_name.as_slice();
                    let moditem_name_item = ecx.ident_of(moditem_name_slice);
                    module_handler_body.push(
                        quote_stmt!(&*ecx,
                            table.set($moditem_name_slice.to_string(), $moditem_name_item);
                        )
                    )
                },

                _ => {
                    ecx.span_warn(moditem.span,
                        format!("item `{}` is neiter a function nor a static
                            and will thus be ignored by `export_lua_module`",
                            moditem_name).as_slice());
                    continue
                }
            };
        }

        module_handler_body
    };

    // adding extern crate declarations
    {
        let view_items = quote_item!(ecx,
            mod x {
                extern crate lua = "rust-hl-lua";
                extern crate libc;
            }
        );

        let view_items = match view_items {
            Some(a) => a,
            None => {
                ecx.span_err(input_item.span,
                    "internal error in the library (could not parse view items)");
                return input_item;
            }
        };

        // getting all the items
        let view_items = match view_items.node {
            ast::ItemMod(ref m) => m,
            _ => { ecx.span_err(span, "internal error in the library"); return input_item; }
        };

        let ref mut mut_new_item = match &mut new_item.node {
            &ast::ItemMod(ref mut m) => m,
            _ => { ecx.span_err(span, "internal error in the library"); return input_item; }
        };

        for i in view_items.view_items.iter() {
            mut_new_item.view_items.unshift(i.clone())
        }
    }

    // generating the function that we will add inside the module
    {
        let function_body = {
            let mut function_body = Vec::new();

            function_body.push(quote_stmt!(ecx,
                let mut lua = self::lua::Lua::from_existing_state(lua, false);
            ));

            function_body.push(quote_stmt!(ecx,
                let mut table = lua.load_new_table();
            ));

            function_body.push_all_move(module_handler_body);

            function_body.push(quote_stmt!(ecx,
                ::std::mem::forget(table);
            ));

            ecx.block(span.clone(), function_body, Some(
                quote_expr!(ecx, 1)
            ))
        };

        // identifier for "luaopen_mylib"
        let luaopen_id = ecx.ident_of(format!("luaopen_{}", input_item.ident.to_source())
            .as_slice());

        // building the function itself
        let function = quote_item!(ecx,
            #[no_mangle]
            pub extern "C" fn $luaopen_id(lua: *mut self::libc::c_void) -> self::libc::c_int {{
                unsafe {{
                    $function_body
               }}
            }}
        );

        let function = match function {
            Some(f) => f,
            None => {
                ecx.span_err(input_item.span,
                    "internal error in the library (could not parse function body)");
                return input_item;
            }
        };

        // adding the function to the module
        match &mut new_item.node {
            &ast::ItemMod(ref mut m) => {
                m.items.push(function)
            },
            _ => { ecx.span_err(span, "internal error in the library"); return input_item; }
        };
    }

    // returning
    box(GC) new_item
}
<commit_msg>Minor style change in modules<commit_after>#![feature(plugin_registrar)]
#![feature(quote)]

extern crate rustc;
extern crate syntax;

use std::gc::{GC, Gc};
use syntax::parse::token;
use syntax::ast;
use syntax::ext::build::AstBuilder;
use syntax::ext::base;
use syntax::ext::quote::rt::ToSource;
use syntax::codemap;

#[plugin_registrar]
#[doc(hidden)]
pub fn plugin_registrar(reg: &mut ::rustc::plugin::Registry) {
    reg.register_syntax_extension(token::intern("export_lua_module"),
        base::ItemModifier(expand_lua_module));
}

// handler for export_lua_module
pub fn expand_lua_module(ecx: &mut base::ExtCtxt, span: codemap::Span,
                        _: Gc<ast::MetaItem>, input_item: Gc<ast::Item>)
    -> Gc<ast::Item>
{
    let ecx: &base::ExtCtxt = &*ecx;

    // checking that the input item is a module
    let module = match input_item.node {
        ast::ItemMod(ref module) => module,
        _ => {
            ecx.span_err(input_item.span,
                "`export_lua_module` extension is only allowed on modules");
            return input_item
        }
    };

    // creating the new item that will be returned by the function
    // it is just a clone of the input with more elements added to it
    let mut new_item = input_item.deref().clone();
    new_item.vis = ast::Public;
    if input_item.vis != ast::Public {
        ecx.span_warn(input_item.span,
            "`export_lua_module` will turn the module into a public module");
    }

    // creating an array of the statements to add to the main Lua entry point
    let module_handler_body: Vec<Gc<ast::Stmt>> = {
        let mut module_handler_body = Vec::new();

        for moditem in module.items.iter() {
            let moditem_name = moditem.ident.to_source();

            match moditem.node {
                ast::ItemFn(..) | ast::ItemStatic(..) => {
                    let moditem_name_slice = moditem_name.as_slice();
                    let moditem_name_item = ecx.ident_of(moditem_name_slice);
                    module_handler_body.push(
                        quote_stmt!(&*ecx,
                            table.set($moditem_name_slice.to_string(), $moditem_name_item);
                        )
                    )
                },

                _ => {
                    ecx.span_warn(moditem.span,
                        format!("item `{}` is neiter a function nor a static
                            and will thus be ignored by `export_lua_module`",
                            moditem_name).as_slice());
                    continue
                }
            };
        }

        module_handler_body
    };

    // adding extern crate declarations
    {
        let view_items = quote_item!(ecx,
            mod x {
                extern crate lua = "rust-hl-lua";
                extern crate libc;
            }
        );

        let view_items = match view_items {
            Some(a) => a,
            None => {
                ecx.span_err(input_item.span,
                    "internal error in the library (could not parse view items)");
                return input_item;
            }
        };

        // getting all the items
        let view_items = match view_items.node {
            ast::ItemMod(ref m) => m,
            _ => { ecx.span_err(span, "internal error in the library"); return input_item; }
        };

        let ref mut mut_new_item = match &mut new_item.node {
            &ast::ItemMod(ref mut m) => m,
            _ => { ecx.span_err(span, "internal error in the library"); return input_item; }
        };

        for i in view_items.view_items.iter() {
            mut_new_item.view_items.unshift(i.clone())
        }
    }

    // generating the function that we will add inside the module
    {
        let function_body = {
            let mut function_body = Vec::new();

            function_body.push(quote_stmt!(ecx,
                let mut lua = self::lua::Lua::from_existing_state(lua, false);
            ));

            function_body.push(quote_stmt!(ecx,
                let mut table = lua.load_new_table();
            ));

            function_body.push_all_move(module_handler_body);

            function_body.push(quote_stmt!(ecx,
                ::std::mem::forget(table);
            ));

            ecx.block(span.clone(), function_body, Some(
                quote_expr!(ecx, 1)
            ))
        };

        // identifier for "luaopen_mylib"
        let luaopen_id = ecx.ident_of(format!("luaopen_{}", input_item.ident.to_source())
            .as_slice());

        // building the function itself
        let function = quote_item!(ecx,
            #[no_mangle]
            pub extern "C" fn $luaopen_id(lua: *mut self::libc::c_void)
                                                    -> self::libc::c_int {
                unsafe {
                    $function_body
               }
            }
        );

        let function = match function {
            Some(f) => f,
            None => {
                ecx.span_err(input_item.span,
                    "internal error in the library (could not parse function body)");
                return input_item;
            }
        };

        // adding the function to the module
        match &mut new_item.node {
            &ast::ItemMod(ref mut m) => {
                m.items.push(function)
            },
            _ => { ecx.span_err(span, "internal error in the library"); return input_item; }
        };
    }

    // returning
    box(GC) new_item
}
<|endoftext|>


<commit_before>extern crate pgn_reader;
extern crate memmap;
extern crate madvise;
extern crate btoi;
extern crate rand;
extern crate reqwest;

use std::env;
use std::mem;
use std::str;
use std::cmp::min;
use std::fs::File;
use std::io::Read;

use memmap::Mmap;
use madvise::{AccessPattern, AdviseMemory};
use pgn_reader::{Visitor, Skip, Reader, San};
use btoi::ParseIntegerError;
use rand::{thread_rng, Rng};
use rand::distributions::OpenClosed01;

const BATCH_SIZE: usize = 50;

const MAX_PLIES: usize = 50;

#[derive(Debug)]
enum TimeControl {
    UltraBullet,
    Bullet,
    Blitz,
    Rapid,
    Classical,
    Correspondence,
}

#[derive(Debug)]
struct TimeControlError;

impl From<ParseIntegerError> for TimeControlError {
    fn from(_: ParseIntegerError) -> TimeControlError {
        TimeControlError { }
    }
}

impl TimeControl {
    fn from_seconds_and_increment(seconds: u64, increment: u64) -> TimeControl {
        let total = seconds + 40 * increment;

        if total < 30 {
            TimeControl::UltraBullet
        } else if total < 180 {
            TimeControl::Bullet
        } else if total < 480 {
            TimeControl::Blitz
        } else if total < 1500 {
            TimeControl::Rapid
        } else if total < 21_600 {
            TimeControl::Classical
        } else {
            TimeControl::Correspondence
        }
    }

    fn from_bytes(bytes: &[u8]) -> Result<TimeControl, TimeControlError> {
        if bytes == b"-" {
            return Ok(TimeControl::Correspondence);
        }

        let mut parts = bytes.splitn(2, |ch| *ch == b'+');
        let seconds = btoi::btou(parts.next().ok_or(TimeControlError)?)?;
        let increment = btoi::btou(parts.next().ok_or(TimeControlError)?)?;
        Ok(TimeControl::from_seconds_and_increment(seconds, increment))
    }
}

struct Indexer<'pgn> {
    filename: String,
    date: &'pgn [u8],

    white_elo: i16,
    black_elo: i16,
    time_control: TimeControl,
    skip: bool,

    current_game: Vec<u8>,
    plies: usize,
    standard: bool,

    batch: Vec<u8>,
    batch_size: usize,
}

impl<'pgn> Indexer<'pgn> {
    fn new(filename: &str) -> Indexer {
        Indexer {
            filename: filename.into(),
            date: b"0000.00.00",

            white_elo: 0,
            black_elo: 0,
            time_control: TimeControl::Correspondence,
            skip: true,

            current_game: Vec::new(),
            plies: 0,
            standard: true,

            batch: Vec::new(),
            batch_size: 0,
        }
    }

    fn send(&mut self) {
        if self.batch_size > 0 {
            self.batch_size = 0;

            let mut res = reqwest::blocking::Client::new()
                .put("http://localhost:9000/import/lichess")
                .body(mem::replace(&mut self.batch, Vec::new()))
                .send().expect("send batch");

            let mut answer = String::new();
            res.read_to_string(&mut answer).expect("decode response");
            println!("{} d: {} t: {}", self.filename, str::from_utf8(self.date).expect("date is valid utf8"), answer);
            assert!(res.status().is_success());
        }
    }
}

impl<'pgn> Visitor<'pgn> for Indexer<'pgn> {
    type Result = ();

    fn begin_game(&mut self) {
        self.current_game.clear();
        self.plies = 0;
    }

    fn begin_headers(&mut self) {
        self.white_elo = 0;
        self.black_elo = 0;
        self.time_control = TimeControl::Correspondence;
        self.standard = true;
    }

    fn header(&mut self, key: &'pgn [u8], value: &'pgn [u8]) {
        if key == b"WhiteElo" {
            self.white_elo = if value == b"?" { 0 } else { btoi::btoi(value).expect("WhiteElo") };
        } else if key == b"BlackElo" {
            self.black_elo = if value == b"?" { 0 } else { btoi::btoi(value).expect("BlackElo") };
        } else if key == b"TimeControl" {
            self.time_control = TimeControl::from_bytes(value).expect("TimeControl");
        } else if key == b"Variant" {
            self.standard = value == b"Standard";
            if self.standard {
                return; // we add this unconditionally later
            }
        } else if key == b"Date" || key == b"UTCDate" {
            self.date = value;
        }

        let (key, value) = if key == b"Site" {
            (&b"LichessID"[..], value.rsplitn(2, |ch| *ch == b'/').next().expect("Site"))
        } else {
            (key, value)
        };

        self.current_game.push(b'[');
        self.current_game.extend(key);
        self.current_game.extend(b" \"");
        self.current_game.extend(value);
        self.current_game.extend(b"\"]\n");
    }

    fn end_headers(&mut self) -> Skip {
        let rating = (self.white_elo + self.black_elo) / 2;

        let probability = if self.standard {
            self.current_game.extend(b"[Variant \"Standard\"]\n");

            match self.time_control {
                TimeControl::Correspondence | TimeControl::Classical => 1.0,
                TimeControl::Rapid if rating >= 2000 => 1.0,
                TimeControl::Rapid if rating >= 1800 => 2.0 / 5.0,
                TimeControl::Rapid => 1.0 / 8.0,
                TimeControl::Blitz if rating >= 2000 => 1.0,
                TimeControl::Blitz if rating >= 1800 => 1.0 / 4.0,
                TimeControl::Blitz => 1.0 / 15.0,
                TimeControl::Bullet if rating >= 2300 => 1.0,
                TimeControl::Bullet if rating >= 2200 => 4.0 / 5.0,
                TimeControl::Bullet if rating >= 2000 => 1.0 / 4.0,
                TimeControl::Bullet if rating >= 1800 => 1.0 / 7.0,
                _ => 1.0 / 20.0,
            }
        } else {
            if rating >= 1600 { 1.0 } else { 0.5 } // variant games
        };

        self.current_game.push(b'\n');

        let rnd = thread_rng().sample(OpenClosed01);
        let accept = min(self.white_elo, self.black_elo) >= 1500 && probability >= rnd;

        self.skip = !accept;
        Skip(self.skip)
    }

    fn san(&mut self, san: San) {
        if self.plies < MAX_PLIES {
            if self.plies > 0 {
                self.current_game.push(b' ');
            }

            self.current_game.extend(san.to_string().as_bytes());
            self.plies += 1;
        }
    }

    fn begin_variation(&mut self) -> Skip {
        Skip(true) // stay in the mainline
    }

    fn end_game(&mut self, _game: &'pgn [u8]) {
        if !self.skip && self.plies > 8 {
            if self.batch_size >= BATCH_SIZE {
                self.send();
            }

            if self.batch_size > 0 {
                self.batch.extend(b"\n\n\n");
            }

            self.batch.extend(&self.current_game);
            self.batch_size += 1;
        }
    }
}

fn main() {
    for arg in env::args().skip(1) {
        eprintln!("% indexing lichess games from {} ...", arg);
        let file = File::open(&arg).expect("fopen");
        if file.metadata().expect("meta").len() != 0 {
            let pgn = unsafe { Mmap::map(&file).expect("mmap") };
            pgn.advise_memory_access(AccessPattern::Sequential).expect("madvise");

            let mut indexer = Indexer::new(&arg);
            Reader::new(&mut indexer, &pgn[..]).read_all();
            indexer.send(); // send last
        }
    }
}
<commit_msg>reuse client for lichess indexing<commit_after>extern crate pgn_reader;
extern crate memmap;
extern crate madvise;
extern crate btoi;
extern crate rand;
extern crate reqwest;

use std::env;
use std::mem;
use std::str;
use std::cmp::min;
use std::fs::File;
use std::io::Read;
use std::time::Duration;

use memmap::Mmap;
use madvise::{AccessPattern, AdviseMemory};
use pgn_reader::{Visitor, Skip, Reader, San};
use btoi::ParseIntegerError;
use rand::{thread_rng, Rng};
use rand::distributions::OpenClosed01;

const BATCH_SIZE: usize = 50;

const MAX_PLIES: usize = 50;

#[derive(Debug)]
enum TimeControl {
    UltraBullet,
    Bullet,
    Blitz,
    Rapid,
    Classical,
    Correspondence,
}

#[derive(Debug)]
struct TimeControlError;

impl From<ParseIntegerError> for TimeControlError {
    fn from(_: ParseIntegerError) -> TimeControlError {
        TimeControlError { }
    }
}

impl TimeControl {
    fn from_seconds_and_increment(seconds: u64, increment: u64) -> TimeControl {
        let total = seconds + 40 * increment;

        if total < 30 {
            TimeControl::UltraBullet
        } else if total < 180 {
            TimeControl::Bullet
        } else if total < 480 {
            TimeControl::Blitz
        } else if total < 1500 {
            TimeControl::Rapid
        } else if total < 21_600 {
            TimeControl::Classical
        } else {
            TimeControl::Correspondence
        }
    }

    fn from_bytes(bytes: &[u8]) -> Result<TimeControl, TimeControlError> {
        if bytes == b"-" {
            return Ok(TimeControl::Correspondence);
        }

        let mut parts = bytes.splitn(2, |ch| *ch == b'+');
        let seconds = btoi::btou(parts.next().ok_or(TimeControlError)?)?;
        let increment = btoi::btou(parts.next().ok_or(TimeControlError)?)?;
        Ok(TimeControl::from_seconds_and_increment(seconds, increment))
    }
}

struct Indexer<'pgn> {
    client: reqwest::blocking::Client,

    filename: String,
    date: &'pgn [u8],

    white_elo: i16,
    black_elo: i16,
    time_control: TimeControl,
    skip: bool,

    current_game: Vec<u8>,
    plies: usize,
    standard: bool,

    batch: Vec<u8>,
    batch_size: usize,
}

impl<'pgn> Indexer<'pgn> {
    fn new(filename: &str) -> Indexer {
        Indexer {
            client: reqwest::blocking::Client::builder().timeout(Duration::from_secs(60)).build().expect("client"),

            filename: filename.into(),
            date: b"0000.00.00",

            white_elo: 0,
            black_elo: 0,
            time_control: TimeControl::Correspondence,
            skip: true,

            current_game: Vec::new(),
            plies: 0,
            standard: true,

            batch: Vec::new(),
            batch_size: 0,
        }
    }

    fn send(&mut self) {
        if self.batch_size > 0 {
            self.batch_size = 0;

            let mut res = self.client
                .put("http://127.0.0.1:9000/import/lichess")
                .body(mem::replace(&mut self.batch, Vec::new()))
                .send().expect("send batch");

            let mut answer = String::new();
            res.read_to_string(&mut answer).expect("decode response");
            println!("{} d: {} t: {}", self.filename, str::from_utf8(self.date).expect("date is valid utf8"), answer);
            assert!(res.status().is_success());
        }
    }
}

impl<'pgn> Visitor<'pgn> for Indexer<'pgn> {
    type Result = ();

    fn begin_game(&mut self) {
        self.current_game.clear();
        self.plies = 0;
    }

    fn begin_headers(&mut self) {
        self.white_elo = 0;
        self.black_elo = 0;
        self.time_control = TimeControl::Correspondence;
        self.standard = true;
    }

    fn header(&mut self, key: &'pgn [u8], value: &'pgn [u8]) {
        if key == b"WhiteElo" {
            self.white_elo = if value == b"?" { 0 } else { btoi::btoi(value).expect("WhiteElo") };
        } else if key == b"BlackElo" {
            self.black_elo = if value == b"?" { 0 } else { btoi::btoi(value).expect("BlackElo") };
        } else if key == b"TimeControl" {
            self.time_control = TimeControl::from_bytes(value).expect("TimeControl");
        } else if key == b"Variant" {
            self.standard = value == b"Standard";
            if self.standard {
                return; // we add this unconditionally later
            }
        } else if key == b"Date" || key == b"UTCDate" {
            self.date = value;
        }

        let (key, value) = if key == b"Site" {
            (&b"LichessID"[..], value.rsplitn(2, |ch| *ch == b'/').next().expect("Site"))
        } else {
            (key, value)
        };

        self.current_game.push(b'[');
        self.current_game.extend(key);
        self.current_game.extend(b" \"");
        self.current_game.extend(value);
        self.current_game.extend(b"\"]\n");
    }

    fn end_headers(&mut self) -> Skip {
        let rating = (self.white_elo + self.black_elo) / 2;

        let probability = if self.standard {
            self.current_game.extend(b"[Variant \"Standard\"]\n");

            match self.time_control {
                TimeControl::Correspondence | TimeControl::Classical => 1.0,
                TimeControl::Rapid if rating >= 2000 => 1.0,
                TimeControl::Rapid if rating >= 1800 => 2.0 / 5.0,
                TimeControl::Rapid => 1.0 / 8.0,
                TimeControl::Blitz if rating >= 2000 => 1.0,
                TimeControl::Blitz if rating >= 1800 => 1.0 / 4.0,
                TimeControl::Blitz => 1.0 / 15.0,
                TimeControl::Bullet if rating >= 2300 => 1.0,
                TimeControl::Bullet if rating >= 2200 => 4.0 / 5.0,
                TimeControl::Bullet if rating >= 2000 => 1.0 / 4.0,
                TimeControl::Bullet if rating >= 1800 => 1.0 / 7.0,
                _ => 1.0 / 20.0,
            }
        } else {
            if rating >= 1600 { 1.0 } else { 0.5 } // variant games
        };

        self.current_game.push(b'\n');

        let rnd = thread_rng().sample(OpenClosed01);
        let accept = min(self.white_elo, self.black_elo) >= 1500 && probability >= rnd;

        self.skip = !accept;
        Skip(self.skip)
    }

    fn san(&mut self, san: San) {
        if self.plies < MAX_PLIES {
            if self.plies > 0 {
                self.current_game.push(b' ');
            }

            self.current_game.extend(san.to_string().as_bytes());
            self.plies += 1;
        }
    }

    fn begin_variation(&mut self) -> Skip {
        Skip(true) // stay in the mainline
    }

    fn end_game(&mut self, _game: &'pgn [u8]) {
        if !self.skip && self.plies > 8 {
            if self.batch_size >= BATCH_SIZE {
                self.send();
            }

            if self.batch_size > 0 {
                self.batch.extend(b"\n\n\n");
            }

            self.batch.extend(&self.current_game);
            self.batch_size += 1;
        }
    }
}

fn main() {
    for arg in env::args().skip(1) {
        eprintln!("% indexing lichess games from {} ...", arg);
        let file = File::open(&arg).expect("fopen");
        if file.metadata().expect("meta").len() != 0 {
            let pgn = unsafe { Mmap::map(&file).expect("mmap") };
            pgn.advise_memory_access(AccessPattern::Sequential).expect("madvise");

            let mut indexer = Indexer::new(&arg);
            Reader::new(&mut indexer, &pgn[..]).read_all();
            indexer.send(); // send last
        }
    }
}
<|endoftext|>