<commit_before>use types::*;

use cookie_factory::*;

pub fn gen_value<'a>(x: (&'a mut [u8], usize), v: &AMQPValue) -> Result<(&'a mut [u8], usize), GenError> {
    match *v {
        AMQPValue::Boolean(ref b)        => do_gen!(x, gen_type(&v.get_type()) >> gen_boolean(b)),
        AMQPValue::ShortShortInt(ref i)  => do_gen!(x, gen_type(&v.get_type()) >> gen_short_short_int(i)),
        AMQPValue::ShortShortUInt(ref u) => do_gen!(x, gen_type(&v.get_type()) >> gen_short_short_uint(u)),
        AMQPValue::ShortInt(ref i)       => do_gen!(x, gen_type(&v.get_type()) >> gen_short_int(i)),
        AMQPValue::ShortUInt(ref u)      => do_gen!(x, gen_type(&v.get_type()) >> gen_short_uint(u)),
        AMQPValue::LongInt(ref i)        => do_gen!(x, gen_type(&v.get_type()) >> gen_long_int(i)),
        AMQPValue::LongUInt(ref u)       => do_gen!(x, gen_type(&v.get_type()) >> gen_long_uint(u)),
        AMQPValue::LongLongInt(ref i)    => do_gen!(x, gen_type(&v.get_type()) >> gen_long_long_int(i)),
        AMQPValue::LongLongUInt(ref u)   => do_gen!(x, gen_type(&v.get_type()) >> gen_long_long_uint(u)),
        AMQPValue::Float(ref f)          => do_gen!(x, gen_type(&v.get_type()) >> gen_float(f)),
        AMQPValue::Double(ref d)         => do_gen!(x, gen_type(&v.get_type()) >> gen_double(d)),
        AMQPValue::DecimalValue(ref d)   => do_gen!(x, gen_type(&v.get_type()) >> gen_decimal_value(d)),
        AMQPValue::ShortString(ref s)    => do_gen!(x, gen_type(&v.get_type()) >> gen_short_string(s)),
        AMQPValue::LongString(ref s)     => do_gen!(x, gen_type(&v.get_type()) >> gen_long_string(s)),
        AMQPValue::FieldArray(ref a)     => do_gen!(x, gen_type(&v.get_type()) >> gen_field_array(a)),
        AMQPValue::Timestamp(ref t)      => do_gen!(x, gen_type(&v.get_type()) >> gen_timestamp(t)),
        AMQPValue::FieldTable(ref t)     => do_gen!(x, gen_type(&v.get_type()) >> gen_field_table(t)),
        AMQPValue::Void                  => gen_type(x, &v.get_type()),
    }
}

pub fn gen_type<'a>(x: (&'a mut [u8], usize), t: &AMQPType) -> Result<(&'a mut [u8], usize), GenError> {
    gen_be_u8!(x, t.get_id() as u8)
}

pub fn gen_boolean<'a>(x: (&'a mut [u8], usize), b: &Boolean) -> Result<(&'a mut [u8], usize), GenError> {
    gen_be_u8!(x, if *b { 1 } else { 0 })
}

pub fn gen_short_short_int<'a>(x: (&'a mut [u8], usize), i: &ShortShortInt) -> Result<(&'a mut [u8], usize), GenError> {
    gen_be_i8!(x, *i)
}

pub fn gen_short_short_uint<'a>(x: (&'a mut [u8], usize), u: &ShortShortUInt) -> Result<(&'a mut [u8], usize), GenError> {
    gen_be_u8!(x, *u)
}

pub fn gen_short_int<'a>(x: (&'a mut [u8], usize), i: &ShortInt) -> Result<(&'a mut [u8], usize), GenError> {
    gen_be_i16!(x, *i)
}

pub fn gen_short_uint<'a>(x: (&'a mut [u8], usize), u: &ShortUInt) -> Result<(&'a mut [u8], usize), GenError> {
    gen_be_u16!(x, *u)
}

pub fn gen_long_int<'a>(x: (&'a mut [u8], usize), i: &LongInt) -> Result<(&'a mut [u8], usize), GenError> {
    gen_be_i32!(x, *i)
}

pub fn gen_long_uint<'a>(x: (&'a mut [u8], usize), u: &LongUInt) -> Result<(&'a mut [u8], usize), GenError> {
    gen_be_u32!(x, *u)
}

pub fn gen_long_long_int<'a>(x: (&'a mut [u8], usize), i: &LongLongInt) -> Result<(&'a mut [u8], usize), GenError> {
    gen_be_i64!(x, *i)
}

pub fn gen_long_long_uint<'a>(x: (&'a mut [u8], usize), u: &LongLongUInt) -> Result<(&'a mut [u8], usize), GenError> {
    gen_be_u64!(x, *u)
}

pub fn gen_float<'a>(x: (&'a mut [u8], usize), f: &Float) -> Result<(&'a mut [u8], usize), GenError> {
    gen_be_f32!(x, *f)
}

pub fn gen_double<'a>(x: (&'a mut [u8], usize), d: &Double) -> Result<(&'a mut [u8], usize), GenError> {
    gen_be_f64!(x, *d)
}

pub fn gen_decimal_value<'a>(x: (&'a mut [u8], usize), d: &DecimalValue) -> Result<(&'a mut [u8], usize), GenError> {
    do_gen!(x, gen_be_u8!(d.scale) >> gen_be_u32!(d.value))
}

pub fn gen_short_string<'a>(x: (&'a mut [u8], usize), s: &ShortString) -> Result<(&'a mut [u8], usize), GenError> {
    do_gen!(x, gen_be_u8!(s.len() as u8) >> gen_slice!(s.as_bytes()))
}

pub fn gen_long_string<'a>(x: (&'a mut [u8], usize), s: &LongString) -> Result<(&'a mut [u8], usize), GenError> {
    do_gen!(x, gen_be_u32!(s.len() as u32) >> gen_slice!(s.as_bytes()))
}

pub fn gen_field_array<'a>(x: (&'a mut [u8], usize), a: &FieldArray) -> Result<(&'a mut [u8], usize), GenError> {
    let (x1, index1) = x;
    gen_many_ref!((x1, index1 + 4), a, gen_value).and_then(|(x2, index2)| {
        gen_be_u32!((x2, index1), index2 - index1 - 4).and_then(|(x3, _)| Ok((x3, index2)))
    })
}

pub fn gen_timestamp<'a>(x: (&'a mut [u8], usize), t: &Timestamp) -> Result<(&'a mut [u8], usize), GenError> {
    gen_long_long_uint(x, t)
}

pub fn gen_field_table<'a>(x: (&'a mut [u8], usize), t: &FieldTable) -> Result<(&'a mut [u8], usize), GenError> {
    let (x1, index1) = x;
    gen_many_ref!((x1, index1 + 4), t, gen_field_entry).and_then(|(x2, index2)| {
        gen_be_u32!((x2, index1), index2 - index1 - 4).and_then(|(x3, _)| Ok((x3, index2)))
    })
}

pub fn gen_field_entry<'a>(x: (&'a mut [u8], usize), e: &(&ShortString, &AMQPValue)) -> Result<(&'a mut [u8], usize), GenError> {
    do_gen!(x, gen_short_string(e.0) >> gen_value(e.1))
}
<commit_msg>types: use custom generators<commit_after>use types::*;

use cookie_factory::*;

pub fn gen_value<'a>(x: (&'a mut [u8], usize), v: &AMQPValue) -> Result<(&'a mut [u8], usize), GenError> {
    match *v {
        AMQPValue::Boolean(ref b)        => do_gen!(x, gen_type(&v.get_type()) >> gen_boolean(b)),
        AMQPValue::ShortShortInt(ref i)  => do_gen!(x, gen_type(&v.get_type()) >> gen_short_short_int(i)),
        AMQPValue::ShortShortUInt(ref u) => do_gen!(x, gen_type(&v.get_type()) >> gen_short_short_uint(u)),
        AMQPValue::ShortInt(ref i)       => do_gen!(x, gen_type(&v.get_type()) >> gen_short_int(i)),
        AMQPValue::ShortUInt(ref u)      => do_gen!(x, gen_type(&v.get_type()) >> gen_short_uint(u)),
        AMQPValue::LongInt(ref i)        => do_gen!(x, gen_type(&v.get_type()) >> gen_long_int(i)),
        AMQPValue::LongUInt(ref u)       => do_gen!(x, gen_type(&v.get_type()) >> gen_long_uint(u)),
        AMQPValue::LongLongInt(ref i)    => do_gen!(x, gen_type(&v.get_type()) >> gen_long_long_int(i)),
        AMQPValue::LongLongUInt(ref u)   => do_gen!(x, gen_type(&v.get_type()) >> gen_long_long_uint(u)),
        AMQPValue::Float(ref f)          => do_gen!(x, gen_type(&v.get_type()) >> gen_float(f)),
        AMQPValue::Double(ref d)         => do_gen!(x, gen_type(&v.get_type()) >> gen_double(d)),
        AMQPValue::DecimalValue(ref d)   => do_gen!(x, gen_type(&v.get_type()) >> gen_decimal_value(d)),
        AMQPValue::ShortString(ref s)    => do_gen!(x, gen_type(&v.get_type()) >> gen_short_string(s)),
        AMQPValue::LongString(ref s)     => do_gen!(x, gen_type(&v.get_type()) >> gen_long_string(s)),
        AMQPValue::FieldArray(ref a)     => do_gen!(x, gen_type(&v.get_type()) >> gen_field_array(a)),
        AMQPValue::Timestamp(ref t)      => do_gen!(x, gen_type(&v.get_type()) >> gen_timestamp(t)),
        AMQPValue::FieldTable(ref t)     => do_gen!(x, gen_type(&v.get_type()) >> gen_field_table(t)),
        AMQPValue::Void                  => gen_type(x, &v.get_type()),
    }
}

pub fn gen_type<'a>(x: (&'a mut [u8], usize), t: &AMQPType) -> Result<(&'a mut [u8], usize), GenError> {
    gen_be_u8!(x, t.get_id() as u8)
}

pub fn gen_boolean<'a>(x: (&'a mut [u8], usize), b: &Boolean) -> Result<(&'a mut [u8], usize), GenError> {
    gen_be_u8!(x, if *b { 1 } else { 0 })
}

pub fn gen_short_short_int<'a>(x: (&'a mut [u8], usize), i: &ShortShortInt) -> Result<(&'a mut [u8], usize), GenError> {
    gen_be_i8!(x, *i)
}

pub fn gen_short_short_uint<'a>(x: (&'a mut [u8], usize), u: &ShortShortUInt) -> Result<(&'a mut [u8], usize), GenError> {
    gen_be_u8!(x, *u)
}

pub fn gen_short_int<'a>(x: (&'a mut [u8], usize), i: &ShortInt) -> Result<(&'a mut [u8], usize), GenError> {
    gen_be_i16!(x, *i)
}

pub fn gen_short_uint<'a>(x: (&'a mut [u8], usize), u: &ShortUInt) -> Result<(&'a mut [u8], usize), GenError> {
    gen_be_u16!(x, *u)
}

pub fn gen_long_int<'a>(x: (&'a mut [u8], usize), i: &LongInt) -> Result<(&'a mut [u8], usize), GenError> {
    gen_be_i32!(x, *i)
}

pub fn gen_long_uint<'a>(x: (&'a mut [u8], usize), u: &LongUInt) -> Result<(&'a mut [u8], usize), GenError> {
    gen_be_u32!(x, *u)
}

pub fn gen_long_long_int<'a>(x: (&'a mut [u8], usize), i: &LongLongInt) -> Result<(&'a mut [u8], usize), GenError> {
    gen_be_i64!(x, *i)
}

pub fn gen_long_long_uint<'a>(x: (&'a mut [u8], usize), u: &LongLongUInt) -> Result<(&'a mut [u8], usize), GenError> {
    gen_be_u64!(x, *u)
}

pub fn gen_float<'a>(x: (&'a mut [u8], usize), f: &Float) -> Result<(&'a mut [u8], usize), GenError> {
    gen_be_f32!(x, *f)
}

pub fn gen_double<'a>(x: (&'a mut [u8], usize), d: &Double) -> Result<(&'a mut [u8], usize), GenError> {
    gen_be_f64!(x, *d)
}

pub fn gen_decimal_value<'a>(x: (&'a mut [u8], usize), d: &DecimalValue) -> Result<(&'a mut [u8], usize), GenError> {
    do_gen!(x, gen_short_short_uint(&d.scale) >> gen_long_uint(&d.value))
}

pub fn gen_short_string<'a>(x: (&'a mut [u8], usize), s: &ShortString) -> Result<(&'a mut [u8], usize), GenError> {
    do_gen!(x, gen_short_short_uint(&(s.len() as ShortShortUInt)) >> gen_slice!(s.as_bytes()))
}

pub fn gen_long_string<'a>(x: (&'a mut [u8], usize), s: &LongString) -> Result<(&'a mut [u8], usize), GenError> {
    do_gen!(x, gen_long_uint(&(s.len() as LongUInt)) >> gen_slice!(s.as_bytes()))
}

pub fn gen_field_array<'a>(x: (&'a mut [u8], usize), a: &FieldArray) -> Result<(&'a mut [u8], usize), GenError> {
    let (x1, index1) = x;
    gen_many_ref!((x1, index1 + 4), a, gen_value).and_then(|(x2, index2)| {
        gen_long_uint((x2, index1), &((index2 - index1 - 4) as LongUInt)).and_then(|(x3, _)| Ok((x3, index2)))
    })
}

pub fn gen_timestamp<'a>(x: (&'a mut [u8], usize), t: &Timestamp) -> Result<(&'a mut [u8], usize), GenError> {
    gen_long_long_uint(x, t)
}

pub fn gen_field_table<'a>(x: (&'a mut [u8], usize), t: &FieldTable) -> Result<(&'a mut [u8], usize), GenError> {
    let (x1, index1) = x;
    gen_many_ref!((x1, index1 + 4), t, gen_field_entry).and_then(|(x2, index2)| {
        gen_long_uint((x2, index1), &((index2 - index1 - 4) as LongUInt)).and_then(|(x3, _)| Ok((x3, index2)))
    })
}

pub fn gen_field_entry<'a>(x: (&'a mut [u8], usize), e: &(&ShortString, &AMQPValue)) -> Result<(&'a mut [u8], usize), GenError> {
    do_gen!(x, gen_short_string(e.0) >> gen_value(e.1))
}
<|endoftext|>


<commit_before>pub mod ngram_generator;
pub mod lucene_asciifold;
pub mod registry;
pub mod tokenizers;
pub mod filters;

use term::Term;
use token::Token;

use analysis::tokenizers::TokenizerSpec;
use analysis::filters::FilterSpec;
use analysis::ngram_generator::Edge;
use analysis::tokenizers::standard::StandardTokenizer;
use analysis::filters::lowercase::LowercaseFilter;
use analysis::filters::asciifolding::ASCIIFoldingFilter;
use analysis::filters::ngram::NGramFilter;

#[derive(Debug)]
pub struct AnalyzerSpec {
    pub tokenizer: TokenizerSpec,
    pub filters: Vec<FilterSpec>,
}

impl AnalyzerSpec {
    pub fn initialise<'a>(&self, input: &'a str) -> Box<Iterator<Item=Token> + 'a> {
        let mut analyzer = self.tokenizer.initialise(input);

        for filter in self.filters.iter() {
            analyzer = filter.initialise(analyzer);
        }

        analyzer
    }
}

#[derive(Debug, PartialEq)]
pub enum Analyzer {
    None,
    Standard,
    EdgeNGram,
}

impl Analyzer {
    pub fn run(&self, input: String) -> Vec<Token> {
        match *self {
            Analyzer::None => vec![Token{term: Term::String(input), position: 1}],
            Analyzer::Standard => {
                let tokens = Box::new(StandardTokenizer::new(&input));

                // Lowercase
                let tokens = Box::new(LowercaseFilter::new(tokens));

                // ASCII Folding (not standard in Elasticsearch, but Wagtail needs it)
                let tokens = Box::new(ASCIIFoldingFilter::new(tokens));

                tokens.collect::<Vec<Token>>()
            }
            Analyzer::EdgeNGram => {
                let tokens = Box::new(StandardTokenizer::new(&input));

                // Lowercase
                let tokens = Box::new(LowercaseFilter::new(tokens));

                // ASCII Folding (not standard in Elasticsearch, but Wagtail needs it)
                let tokens = Box::new(ASCIIFoldingFilter::new(tokens));

                // Ngrams
                let tokens = Box::new(NGramFilter::new(tokens, 2, 15, Edge::Left));

                tokens.collect::<Vec<Token>>()
            }
        }
    }
}
<commit_msg>Make FieldMapping.analyzer optional<commit_after>pub mod ngram_generator;
pub mod lucene_asciifold;
pub mod registry;
pub mod tokenizers;
pub mod filters;

use term::Term;
use token::Token;

use analysis::tokenizers::TokenizerSpec;
use analysis::filters::FilterSpec;
use analysis::ngram_generator::Edge;
use analysis::tokenizers::standard::StandardTokenizer;
use analysis::filters::lowercase::LowercaseFilter;
use analysis::filters::asciifolding::ASCIIFoldingFilter;
use analysis::filters::ngram::NGramFilter;

#[derive(Debug)]
pub struct AnalyzerSpec {
    pub tokenizer: TokenizerSpec,
    pub filters: Vec<FilterSpec>,
}

impl AnalyzerSpec {
    pub fn initialise<'a>(&self, input: &'a str) -> Box<Iterator<Item=Token> + 'a> {
        let mut analyzer = self.tokenizer.initialise(input);

        for filter in self.filters.iter() {
            analyzer = filter.initialise(analyzer);
        }

        analyzer
    }
}

#[derive(Debug, PartialEq)]
pub enum Analyzer {
    Standard,
    EdgeNGram,
}

impl Analyzer {
    pub fn run(&self, input: String) -> Vec<Token> {
        match *self {
            Analyzer::Standard => {
                let tokens = Box::new(StandardTokenizer::new(&input));

                // Lowercase
                let tokens = Box::new(LowercaseFilter::new(tokens));

                // ASCII Folding (not standard in Elasticsearch, but Wagtail needs it)
                let tokens = Box::new(ASCIIFoldingFilter::new(tokens));

                tokens.collect::<Vec<Token>>()
            }
            Analyzer::EdgeNGram => {
                let tokens = Box::new(StandardTokenizer::new(&input));

                // Lowercase
                let tokens = Box::new(LowercaseFilter::new(tokens));

                // ASCII Folding (not standard in Elasticsearch, but Wagtail needs it)
                let tokens = Box::new(ASCIIFoldingFilter::new(tokens));

                // Ngrams
                let tokens = Box::new(NGramFilter::new(tokens, 2, 15, Edge::Left));

                tokens.collect::<Vec<Token>>()
            }
        }
    }
}
<|endoftext|>


<commit_before>// Copyright (c) 2016 The vulkano developers
// Licensed under the Apache License, Version 2.0
// <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT
// license <LICENSE-MIT or http://opensource.org/licenses/MIT>,
// at your option. All files in the project carrying such
// notice may not be copied, modified, or distributed except
// according to those terms.

use std::iter;
use std::sync::Arc;

use buffer::TrackedBuffer;
use command_buffer::DynamicState;
use command_buffer::pool::CommandPool;
use command_buffer::states_manager::StatesManager;
use command_buffer::submit::Submit;
use command_buffer::submit::SubmitInfo;
use command_buffer::sys::PipelineBarrierBuilder;
use command_buffer::sys::UnsafeCommandBuffer;
use command_buffer::sys::UnsafeCommandBufferBuilder;
use descriptor::PipelineLayout;
use descriptor::descriptor_set::collection::TrackedDescriptorSetsCollection;
use device::Queue;
use framebuffer::traits::TrackedFramebuffer;
use framebuffer::RenderPass;
use framebuffer::RenderPassClearValues;
use instance::QueueFamily;
use pipeline::ComputePipeline;
use pipeline::GraphicsPipeline;
use pipeline::vertex::Source;
use sync::Fence;

pub use self::empty::PrimaryCb;
pub use self::empty::PrimaryCbBuilder;

pub mod dispatch;
pub mod draw;
pub mod empty;
pub mod execute;
pub mod fill_buffer;
pub mod render_pass;
pub mod update_buffer;

/// A list of commands that can be turned into a command buffer.
pub unsafe trait CommandsListBase {
    /// Adds a command that writes the content of a buffer.
    ///
    /// After this command is executed, the content of `buffer` will become `data`.
    #[inline]
    fn update_buffer<'a, B, D: ?Sized>(self, buffer: B, data: &'a D)
                                       -> update_buffer::UpdateCommand<'a, Self, B, D>
        where Self: Sized + CommandsListPossibleOutsideRenderPass, B: TrackedBuffer, D: Copy + 'static
    {
        update_buffer::UpdateCommand::new(self, buffer, data)
    }

    /// Adds a command that writes the content of a buffer.
    #[inline]
    fn fill_buffer<B>(self, buffer: B, data: u32) -> fill_buffer::FillCommand<Self, B>
        where Self: Sized + CommandsListPossibleOutsideRenderPass, B: TrackedBuffer
    {
        fill_buffer::FillCommand::new(self, buffer, data)
    }

    /// Adds a command that executes a secondary command buffer.
    ///
    /// When you create a command buffer, you have the possibility to create either a primary
    /// command buffer or a secondary command buffer. Secondary command buffers can't be executed
    /// directly, but can be executed from a primary command buffer.
    ///
    /// A secondary command buffer can't execute another secondary command buffer. The only way
    /// you can use `execute` is to make a primary command buffer call a secondary command buffer.
    #[inline]
    fn execute<Cb>(self, command_buffer: Cb) -> execute::ExecuteCommand<Cb, Self>
        where Self: Sized, Cb: CommandsListOutput       /* FIXME: */
    {
        execute::ExecuteCommand::new(self, command_buffer)
    }

    /// Adds a command that executes a compute shader.
    ///
    /// The `dimensions` are the number of working groups to start. The GPU will execute the
    /// compute shader `dimensions[0] * dimensions[1] * dimensions[2]` times.
    ///
    /// The `pipeline` is the compute pipeline that will be executed, and the sets and push
    /// constants will be accessible to all the invocations.
    #[inline]
    fn dispatch<'a, Pl, S, Pc>(self, pipeline: Arc<ComputePipeline<Pl>>, sets: S,
                               dimensions: [u32; 3], push_constants: &'a Pc)
                               -> dispatch::DispatchCommand<'a, Self, Pl, S, Pc>
        where Self: Sized + CommandsListBase + CommandsListPossibleOutsideRenderPass, Pl: PipelineLayout,
              S: TrackedDescriptorSetsCollection, Pc: 'a
    {
        dispatch::DispatchCommand::new(self, pipeline, sets, dimensions, push_constants)
    }

    /// Adds a command that starts a render pass.
    ///
    /// If `secondary` is true, then you will only be able to add secondary command buffers while
    /// you're inside the first subpass on the render pass. If `secondary` is false, you will only
    /// be able to add inline draw commands and not secondary command buffers.
    ///
    /// You must call this before you can add draw commands.
    #[inline]
    fn begin_render_pass<F, C>(self, framebuffer: F, secondary: bool, clear_values: C)
                               -> render_pass::BeginRenderPassCommand<Self, F::RenderPass, F>
        where Self: Sized + CommandsListPossibleOutsideRenderPass,
              F: TrackedFramebuffer, F::RenderPass: RenderPass + RenderPassClearValues<C>
    {
        render_pass::BeginRenderPassCommand::new(self, framebuffer, secondary, clear_values)
    }

    /// Adds a command that jumps to the next subpass of the current render pass.
    fn next_subpass(self, secondary: bool) -> render_pass::NextSubpassCommand<Self>
        where Self: Sized + CommandsListPossibleInsideRenderPass
    {
        render_pass::NextSubpassCommand::new(self, secondary)
    }

    /// Adds a command that ends the current render pass.
    ///
    /// This must be called after you went through all the subpasses and before you can build
    /// the command buffer or add further commands.
    fn end_render_pass(self) -> render_pass::EndRenderPassCommand<Self>
        where Self: Sized + CommandsListPossibleInsideRenderPass
    {
        render_pass::EndRenderPassCommand::new(self)
    }

    /// Adds a command that draws.
    ///
    /// Can only be used from inside a render pass.
    #[inline]
    fn draw<'a, Pv, Pl, Prp, S, Pc, V>(self, pipeline: Arc<GraphicsPipeline<Pv, Pl, Prp>>,
                                       dynamic: &DynamicState, vertices: V, sets: S,
                                       push_constants: &'a Pc)
                                       -> draw::DrawCommand<'a, Self, Pv, Pl, Prp, S, Pc>
        where Self: Sized + CommandsListBase + CommandsListPossibleInsideRenderPass, Pl: PipelineLayout,
              S: TrackedDescriptorSetsCollection, Pc: 'a, Pv: Source<V>
    {
        draw::DrawCommand::regular(self, pipeline, dynamic, vertices, sets, push_constants)
    }

    /// Returns true if the command buffer can be built. This function should always return true,
    /// except when we're building a primary command buffer that is inside a render pass.
    fn buildable_state(&self) -> bool;

    /// Returns the number of commands in the commands list.
    ///
    /// Note that multiple actual commands may count for just 1.
    fn num_commands(&self) -> usize;

    /// Checks whether the command can be executed on the given queue family.
    // TODO: error type?
    fn check_queue_validity(&self, queue: QueueFamily) -> Result<(), ()>;

    /// Extracts the object that contains the states of all the resources of the commands list.
    ///
    /// Panics if the states were already extracted.
    fn extract_states(&mut self) -> StatesManager;

    /// Returns true if the given compute pipeline is currently binded in the commands list.
    fn is_compute_pipeline_bound<Pl>(&self, pipeline: &Arc<ComputePipeline<Pl>>) -> bool;

    /// Returns true if the given graphics pipeline is currently binded in the commands list.
    fn is_graphics_pipeline_bound<Pv, Pl, Prp>(&self, pipeline: &Arc<GraphicsPipeline<Pv, Pl, Prp>>)
                                                -> bool;
}

pub unsafe trait CommandsList: CommandsListBase {
    type Pool: CommandPool;
    /// The type of the command buffer that will be generated.
    type Output: CommandsListOutput;

    /// Turns the commands list into a command buffer.
    ///
    /// This function accepts additional arguments that will customize the output:
    ///
    /// - `additional_elements` is a closure that must be called on the command buffer builder
    ///   after it has finished building and before `final_barrier` are added.
    /// - `barriers` is a list of pipeline barriers accompanied by a command number. The
    ///   pipeline barrier must happen after the given command number. Usually you want all the
    ///   the command numbers to be inferior to `num_commands`.
    /// - `final_barrier` is a pipeline barrier that must be added at the end of the
    ///   command buffer builder.
    ///
    /// This function doesn't check that `buildable_state` returns true.
    unsafe fn raw_build<I, F>(self, in_s: &mut StatesManager, out: &mut StatesManager,
                              additional_elements: F, barriers: I,
                              final_barrier: PipelineBarrierBuilder) -> Self::Output
        where F: FnOnce(&mut UnsafeCommandBufferBuilder<Self::Pool>),
              I: Iterator<Item = (usize, PipelineBarrierBuilder)>;

    /// Turns the commands list into a command buffer that can be submitted.
    // This function isn't inline because `raw_build` implementations usually are inline.
    fn build(mut self) -> CommandBuffer<Self::Output> where Self: Sized {
        assert!(self.buildable_state(), "Tried to build a command buffer still inside a \
                                         render pass");

        let mut states_in = self.extract_states();
        let mut states_out = StatesManager::new();

        let output = unsafe {
            self.raw_build(&mut states_in, &mut states_out, |_| {},
                           iter::empty(), PipelineBarrierBuilder::new())
        };

        CommandBuffer {
            states: states_out,
            commands: output,
        }
    }
}

/*pub unsafe trait AbstractCommandsList: CommandsListBase {
    /// Turns the commands list into a command buffer that can be submitted.
    fn abstract_build(self) -> CommandBuffer where Self: Sized;
}

unsafe impl<C> AbstractCommandsList for C where C: CommandsList {
    #[inline]
    fn abstract_build(self) -> CommandBuffer {
        Box::new(self.build())
    }
}*/

/// Extension trait for both `CommandsListBase` and `CommandsListOutput` that indicates that we're
/// possibly outside a render pass.
///
/// In other words, if this trait is *not* implemented then we're guaranteed *not* to be outside
/// of a render pass. If it is implemented, then we maybe are but that's not sure.
pub unsafe trait CommandsListPossibleOutsideRenderPass {
    /// Returns `true` if we're outside a render pass.
    fn is_outside_render_pass(&self) -> bool;
}

/// Extension trait for both `CommandsListBase` and `CommandsListOutput` that indicates that we're
/// possibly inside a render pass.
///
/// In other words, if this trait is *not* implemented then we're guaranteed *not* to be inside
/// a render pass. If it is implemented, then we maybe are but that's not sure.
// TODO: make all return values optional, since we're possibly not in a render pass
pub unsafe trait CommandsListPossibleInsideRenderPass {
    type RenderPass: RenderPass;

    /// Returns the number of the subpass we're in. The value is 0-indexed, so immediately after
    /// calling `begin_render_pass` the value will be `0`.
    ///
    /// The value should always be strictly inferior to the number of subpasses in the render pass.
    fn current_subpass_num(&self) -> u32;

    /// If true, only secondary command buffers can be added inside the subpass. If false, only
    /// inline draw commands can be added.
    fn secondary_subpass(&self) -> bool;

    /// Returns the description of the render pass we're in.
    // TODO: return a trait object instead?
    fn render_pass(&self) -> &Self::RenderPass;

    //fn current_subpass(&self) -> Subpass<&Self::RenderPass>;
}

pub unsafe trait CommandsListOutput<S = StatesManager> {
    /// Type of the pool that was used to allocate the command buffer.
    type Pool: CommandPool;

    /// Returns the inner object.
    fn inner(&self) -> &UnsafeCommandBuffer<Self::Pool>;

    unsafe fn on_submit<F>(&self, states: &S, queue: &Arc<Queue>, fence: F) -> SubmitInfo
        where F: FnMut() -> Arc<Fence>;
}

pub struct CommandBuffer<C /* = Box<CommandsListOutput>*/> {
    states: StatesManager,
    commands: C,
}

unsafe impl<C> Submit for CommandBuffer<C> where C: CommandsListOutput {
    type Pool = C::Pool;

    #[inline]
    fn inner(&self) -> &UnsafeCommandBuffer<Self::Pool> {
        self.commands.inner()
    }

    #[inline]
    unsafe fn on_submit<F>(&self, queue: &Arc<Queue>, fence: F) -> SubmitInfo
        where F: FnMut() -> Arc<Fence>
    {
        self.commands.on_submit(&self.states, queue, fence)
    }
}
<commit_msg>Some missing inlines<commit_after>// Copyright (c) 2016 The vulkano developers
// Licensed under the Apache License, Version 2.0
// <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT
// license <LICENSE-MIT or http://opensource.org/licenses/MIT>,
// at your option. All files in the project carrying such
// notice may not be copied, modified, or distributed except
// according to those terms.

use std::iter;
use std::sync::Arc;

use buffer::TrackedBuffer;
use command_buffer::DynamicState;
use command_buffer::pool::CommandPool;
use command_buffer::states_manager::StatesManager;
use command_buffer::submit::Submit;
use command_buffer::submit::SubmitInfo;
use command_buffer::sys::PipelineBarrierBuilder;
use command_buffer::sys::UnsafeCommandBuffer;
use command_buffer::sys::UnsafeCommandBufferBuilder;
use descriptor::PipelineLayout;
use descriptor::descriptor_set::collection::TrackedDescriptorSetsCollection;
use device::Queue;
use framebuffer::traits::TrackedFramebuffer;
use framebuffer::RenderPass;
use framebuffer::RenderPassClearValues;
use instance::QueueFamily;
use pipeline::ComputePipeline;
use pipeline::GraphicsPipeline;
use pipeline::vertex::Source;
use sync::Fence;

pub use self::empty::PrimaryCb;
pub use self::empty::PrimaryCbBuilder;

pub mod dispatch;
pub mod draw;
pub mod empty;
pub mod execute;
pub mod fill_buffer;
pub mod render_pass;
pub mod update_buffer;

/// A list of commands that can be turned into a command buffer.
pub unsafe trait CommandsListBase {
    /// Adds a command that writes the content of a buffer.
    ///
    /// After this command is executed, the content of `buffer` will become `data`.
    #[inline]
    fn update_buffer<'a, B, D: ?Sized>(self, buffer: B, data: &'a D)
                                       -> update_buffer::UpdateCommand<'a, Self, B, D>
        where Self: Sized + CommandsListPossibleOutsideRenderPass, B: TrackedBuffer, D: Copy + 'static
    {
        update_buffer::UpdateCommand::new(self, buffer, data)
    }

    /// Adds a command that writes the content of a buffer.
    #[inline]
    fn fill_buffer<B>(self, buffer: B, data: u32) -> fill_buffer::FillCommand<Self, B>
        where Self: Sized + CommandsListPossibleOutsideRenderPass, B: TrackedBuffer
    {
        fill_buffer::FillCommand::new(self, buffer, data)
    }

    /// Adds a command that executes a secondary command buffer.
    ///
    /// When you create a command buffer, you have the possibility to create either a primary
    /// command buffer or a secondary command buffer. Secondary command buffers can't be executed
    /// directly, but can be executed from a primary command buffer.
    ///
    /// A secondary command buffer can't execute another secondary command buffer. The only way
    /// you can use `execute` is to make a primary command buffer call a secondary command buffer.
    #[inline]
    fn execute<Cb>(self, command_buffer: Cb) -> execute::ExecuteCommand<Cb, Self>
        where Self: Sized, Cb: CommandsListOutput       /* FIXME: */
    {
        execute::ExecuteCommand::new(self, command_buffer)
    }

    /// Adds a command that executes a compute shader.
    ///
    /// The `dimensions` are the number of working groups to start. The GPU will execute the
    /// compute shader `dimensions[0] * dimensions[1] * dimensions[2]` times.
    ///
    /// The `pipeline` is the compute pipeline that will be executed, and the sets and push
    /// constants will be accessible to all the invocations.
    #[inline]
    fn dispatch<'a, Pl, S, Pc>(self, pipeline: Arc<ComputePipeline<Pl>>, sets: S,
                               dimensions: [u32; 3], push_constants: &'a Pc)
                               -> dispatch::DispatchCommand<'a, Self, Pl, S, Pc>
        where Self: Sized + CommandsListBase + CommandsListPossibleOutsideRenderPass, Pl: PipelineLayout,
              S: TrackedDescriptorSetsCollection, Pc: 'a
    {
        dispatch::DispatchCommand::new(self, pipeline, sets, dimensions, push_constants)
    }

    /// Adds a command that starts a render pass.
    ///
    /// If `secondary` is true, then you will only be able to add secondary command buffers while
    /// you're inside the first subpass on the render pass. If `secondary` is false, you will only
    /// be able to add inline draw commands and not secondary command buffers.
    ///
    /// You must call this before you can add draw commands.
    #[inline]
    fn begin_render_pass<F, C>(self, framebuffer: F, secondary: bool, clear_values: C)
                               -> render_pass::BeginRenderPassCommand<Self, F::RenderPass, F>
        where Self: Sized + CommandsListPossibleOutsideRenderPass,
              F: TrackedFramebuffer, F::RenderPass: RenderPass + RenderPassClearValues<C>
    {
        render_pass::BeginRenderPassCommand::new(self, framebuffer, secondary, clear_values)
    }

    /// Adds a command that jumps to the next subpass of the current render pass.
    #[inline]
    fn next_subpass(self, secondary: bool) -> render_pass::NextSubpassCommand<Self>
        where Self: Sized + CommandsListPossibleInsideRenderPass
    {
        render_pass::NextSubpassCommand::new(self, secondary)
    }

    /// Adds a command that ends the current render pass.
    ///
    /// This must be called after you went through all the subpasses and before you can build
    /// the command buffer or add further commands.
    #[inline]
    fn end_render_pass(self) -> render_pass::EndRenderPassCommand<Self>
        where Self: Sized + CommandsListPossibleInsideRenderPass
    {
        render_pass::EndRenderPassCommand::new(self)
    }

    /// Adds a command that draws.
    ///
    /// Can only be used from inside a render pass.
    #[inline]
    fn draw<'a, Pv, Pl, Prp, S, Pc, V>(self, pipeline: Arc<GraphicsPipeline<Pv, Pl, Prp>>,
                                       dynamic: &DynamicState, vertices: V, sets: S,
                                       push_constants: &'a Pc)
                                       -> draw::DrawCommand<'a, Self, Pv, Pl, Prp, S, Pc>
        where Self: Sized + CommandsListBase + CommandsListPossibleInsideRenderPass, Pl: PipelineLayout,
              S: TrackedDescriptorSetsCollection, Pc: 'a, Pv: Source<V>
    {
        draw::DrawCommand::regular(self, pipeline, dynamic, vertices, sets, push_constants)
    }

    /// Returns true if the command buffer can be built. This function should always return true,
    /// except when we're building a primary command buffer that is inside a render pass.
    fn buildable_state(&self) -> bool;

    /// Returns the number of commands in the commands list.
    ///
    /// Note that multiple actual commands may count for just 1.
    fn num_commands(&self) -> usize;

    /// Checks whether the command can be executed on the given queue family.
    // TODO: error type?
    fn check_queue_validity(&self, queue: QueueFamily) -> Result<(), ()>;

    /// Extracts the object that contains the states of all the resources of the commands list.
    ///
    /// Panics if the states were already extracted.
    fn extract_states(&mut self) -> StatesManager;

    /// Returns true if the given compute pipeline is currently binded in the commands list.
    fn is_compute_pipeline_bound<Pl>(&self, pipeline: &Arc<ComputePipeline<Pl>>) -> bool;

    /// Returns true if the given graphics pipeline is currently binded in the commands list.
    fn is_graphics_pipeline_bound<Pv, Pl, Prp>(&self, pipeline: &Arc<GraphicsPipeline<Pv, Pl, Prp>>)
                                                -> bool;
}

pub unsafe trait CommandsList: CommandsListBase {
    type Pool: CommandPool;
    /// The type of the command buffer that will be generated.
    type Output: CommandsListOutput;

    /// Turns the commands list into a command buffer.
    ///
    /// This function accepts additional arguments that will customize the output:
    ///
    /// - `additional_elements` is a closure that must be called on the command buffer builder
    ///   after it has finished building and before `final_barrier` are added.
    /// - `barriers` is a list of pipeline barriers accompanied by a command number. The
    ///   pipeline barrier must happen after the given command number. Usually you want all the
    ///   the command numbers to be inferior to `num_commands`.
    /// - `final_barrier` is a pipeline barrier that must be added at the end of the
    ///   command buffer builder.
    ///
    /// This function doesn't check that `buildable_state` returns true.
    unsafe fn raw_build<I, F>(self, in_s: &mut StatesManager, out: &mut StatesManager,
                              additional_elements: F, barriers: I,
                              final_barrier: PipelineBarrierBuilder) -> Self::Output
        where F: FnOnce(&mut UnsafeCommandBufferBuilder<Self::Pool>),
              I: Iterator<Item = (usize, PipelineBarrierBuilder)>;

    /// Turns the commands list into a command buffer that can be submitted.
    // This function isn't inline because `raw_build` implementations usually are inline.
    fn build(mut self) -> CommandBuffer<Self::Output> where Self: Sized {
        assert!(self.buildable_state(), "Tried to build a command buffer still inside a \
                                         render pass");

        let mut states_in = self.extract_states();
        let mut states_out = StatesManager::new();

        let output = unsafe {
            self.raw_build(&mut states_in, &mut states_out, |_| {},
                           iter::empty(), PipelineBarrierBuilder::new())
        };

        CommandBuffer {
            states: states_out,
            commands: output,
        }
    }
}

/*pub unsafe trait AbstractCommandsList: CommandsListBase {
    /// Turns the commands list into a command buffer that can be submitted.
    fn abstract_build(self) -> CommandBuffer where Self: Sized;
}

unsafe impl<C> AbstractCommandsList for C where C: CommandsList {
    #[inline]
    fn abstract_build(self) -> CommandBuffer {
        Box::new(self.build())
    }
}*/

/// Extension trait for both `CommandsListBase` and `CommandsListOutput` that indicates that we're
/// possibly outside a render pass.
///
/// In other words, if this trait is *not* implemented then we're guaranteed *not* to be outside
/// of a render pass. If it is implemented, then we maybe are but that's not sure.
pub unsafe trait CommandsListPossibleOutsideRenderPass {
    /// Returns `true` if we're outside a render pass.
    fn is_outside_render_pass(&self) -> bool;
}

/// Extension trait for both `CommandsListBase` and `CommandsListOutput` that indicates that we're
/// possibly inside a render pass.
///
/// In other words, if this trait is *not* implemented then we're guaranteed *not* to be inside
/// a render pass. If it is implemented, then we maybe are but that's not sure.
// TODO: make all return values optional, since we're possibly not in a render pass
pub unsafe trait CommandsListPossibleInsideRenderPass {
    type RenderPass: RenderPass;

    /// Returns the number of the subpass we're in. The value is 0-indexed, so immediately after
    /// calling `begin_render_pass` the value will be `0`.
    ///
    /// The value should always be strictly inferior to the number of subpasses in the render pass.
    fn current_subpass_num(&self) -> u32;

    /// If true, only secondary command buffers can be added inside the subpass. If false, only
    /// inline draw commands can be added.
    fn secondary_subpass(&self) -> bool;

    /// Returns the description of the render pass we're in.
    // TODO: return a trait object instead?
    fn render_pass(&self) -> &Self::RenderPass;

    //fn current_subpass(&self) -> Subpass<&Self::RenderPass>;
}

pub unsafe trait CommandsListOutput<S = StatesManager> {
    /// Type of the pool that was used to allocate the command buffer.
    type Pool: CommandPool;

    /// Returns the inner object.
    fn inner(&self) -> &UnsafeCommandBuffer<Self::Pool>;

    unsafe fn on_submit<F>(&self, states: &S, queue: &Arc<Queue>, fence: F) -> SubmitInfo
        where F: FnMut() -> Arc<Fence>;
}

pub struct CommandBuffer<C /* = Box<CommandsListOutput>*/> {
    states: StatesManager,
    commands: C,
}

unsafe impl<C> Submit for CommandBuffer<C> where C: CommandsListOutput {
    type Pool = C::Pool;

    #[inline]
    fn inner(&self) -> &UnsafeCommandBuffer<Self::Pool> {
        self.commands.inner()
    }

    #[inline]
    unsafe fn on_submit<F>(&self, queue: &Arc<Queue>, fence: F) -> SubmitInfo
        where F: FnMut() -> Arc<Fence>
    {
        self.commands.on_submit(&self.states, queue, fence)
    }
}
<|endoftext|>


<commit_before>use std::fmt::{Debug, Formatter, Error as FmtError};
use std::result::Result as RResult;

use libimagstore::store::Store;
use libimagstore::storeid::StoreIdIterator;

use diaryid::DiaryId;
use diaryid::FromStoreId;
use is_in_diary::IsInDiary;
use entry::Entry as DiaryEntry;
use error::DiaryError as DE;
use error::DiaryErrorKind as DEK;
use result::Result;

/// A iterator for iterating over diary entries
pub struct DiaryEntryIterator<'a> {
    store: &'a Store,
    name: &'a str,
    iter: StoreIdIterator,

    year: Option<i32>,
    month: Option<u32>,
    day: Option<u32>,
}

impl<'a> Debug for DiaryEntryIterator<'a> {

    fn fmt(&self, fmt: &mut Formatter) -> RResult<(), FmtError> {
        write!(fmt, "DiaryEntryIterator<name = {}, year = {:?}, month = {:?}, day = {:?}>",
               self.name, self.year, self.month, self.day)
    }

}

impl<'a> DiaryEntryIterator<'a> {

    pub fn new(diaryname: &'a str, store: &'a Store, iter: StoreIdIterator) -> DiaryEntryIterator<'a> {
        DiaryEntryIterator {
            store: store,
            name: diaryname,
            iter: iter,

            year: None,
            month: None,
            day: None,
        }
    }

    // Filter by year, get all diary entries for this year
    pub fn year(mut self, year: i32) -> DiaryEntryIterator<'a> {
        self.year = Some(year);
        self
    }

    // Filter by month, get all diary entries for this month (every year)
    pub fn month(mut self, month: u32) -> DiaryEntryIterator<'a> {
        self.month = Some(month);
        self
    }

    // Filter by day, get all diary entries for this day (every year, every year)
    pub fn day(mut self, day: u32) -> DiaryEntryIterator<'a> {
        self.day = Some(day);
        self
    }

}

impl<'a> Iterator for DiaryEntryIterator<'a> {
    type Item = Result<DiaryEntry<'a>>;

    fn next(&mut self) -> Option<Result<DiaryEntry<'a>>> {
        loop {
            let next = self.iter.next();
            debug!("Next element: {:?}", next);
            if next.is_none() {
                return None;
            }
            let next = next.unwrap();

            if next.is_in_diary(self.name) {
                debug!("Seems to be in diary: {:?}", next);
                let id = DiaryId::from_storeid(&next);
                if id.is_none() {
                    continue;
                }
                let id = id.unwrap();

                let y = match self.year  { None => true, Some(y) => y == id.year() };
                let m = match self.month { None => true, Some(m) => m == id.month() };
                let d = match self.day   { None => true, Some(d) => d == id.day() };

                if y && m && d {
                    return Some(self
                                .store
                                .retrieve(next)
                                .map(|fle| DiaryEntry::new(fle))
                                .map_err(|e| DE::new(DEK::StoreReadError, Some(Box::new(e))))
                                );
                }
            } else {
                debug!("Not in the requested diary ({}): {:?}", self.name, next);
            }
        }
    }

}

<commit_msg>Add debugging in lib-iter<commit_after>use std::fmt::{Debug, Formatter, Error as FmtError};
use std::result::Result as RResult;

use libimagstore::store::Store;
use libimagstore::storeid::StoreIdIterator;

use diaryid::DiaryId;
use diaryid::FromStoreId;
use is_in_diary::IsInDiary;
use entry::Entry as DiaryEntry;
use error::DiaryError as DE;
use error::DiaryErrorKind as DEK;
use result::Result;

/// A iterator for iterating over diary entries
pub struct DiaryEntryIterator<'a> {
    store: &'a Store,
    name: &'a str,
    iter: StoreIdIterator,

    year: Option<i32>,
    month: Option<u32>,
    day: Option<u32>,
}

impl<'a> Debug for DiaryEntryIterator<'a> {

    fn fmt(&self, fmt: &mut Formatter) -> RResult<(), FmtError> {
        write!(fmt, "DiaryEntryIterator<name = {}, year = {:?}, month = {:?}, day = {:?}>",
               self.name, self.year, self.month, self.day)
    }

}

impl<'a> DiaryEntryIterator<'a> {

    pub fn new(diaryname: &'a str, store: &'a Store, iter: StoreIdIterator) -> DiaryEntryIterator<'a> {
        DiaryEntryIterator {
            store: store,
            name: diaryname,
            iter: iter,

            year: None,
            month: None,
            day: None,
        }
    }

    // Filter by year, get all diary entries for this year
    pub fn year(mut self, year: i32) -> DiaryEntryIterator<'a> {
        self.year = Some(year);
        self
    }

    // Filter by month, get all diary entries for this month (every year)
    pub fn month(mut self, month: u32) -> DiaryEntryIterator<'a> {
        self.month = Some(month);
        self
    }

    // Filter by day, get all diary entries for this day (every year, every year)
    pub fn day(mut self, day: u32) -> DiaryEntryIterator<'a> {
        self.day = Some(day);
        self
    }

}

impl<'a> Iterator for DiaryEntryIterator<'a> {
    type Item = Result<DiaryEntry<'a>>;

    fn next(&mut self) -> Option<Result<DiaryEntry<'a>>> {
        loop {
            let next = self.iter.next();
            debug!("Next element: {:?}", next);
            if next.is_none() {
                return None;
            }
            let next = next.unwrap();

            if next.is_in_diary(self.name) {
                debug!("Seems to be in diary: {:?}", next);
                let id = DiaryId::from_storeid(&next);
                if id.is_none() {
                    debug!("Couldn't parse {:?} into DiaryId", next);
                    continue;
                }
                let id = id.unwrap();
                debug!("Success parsing id = {:?}", id);

                let y = match self.year  { None => true, Some(y) => y == id.year() };
                let m = match self.month { None => true, Some(m) => m == id.month() };
                let d = match self.day   { None => true, Some(d) => d == id.day() };

                if y && m && d {
                    debug!("Return = {:?}", id);
                    return Some(self
                                .store
                                .retrieve(next)
                                .map(|fle| DiaryEntry::new(fle))
                                .map_err(|e| DE::new(DEK::StoreReadError, Some(Box::new(e))))
                                );
                }
            } else {
                debug!("Not in the requested diary ({}): {:?}", self.name, next);
            }
        }
    }

}

<|endoftext|>


<commit_before>// Copyright © 2016, Canal TP and/or its affiliates. All rights reserved.
//
// This file is part of Navitia,
//     the software to build cool stuff with public transport.
//
// Hope you'll enjoy and contribute to this project,
//     powered by Canal TP (www.canaltp.fr).
// Help us simplify mobility and open public transport:
//     a non ending quest to the responsive locomotion way of traveling!
//
// LICENCE: This program is free software; you can redistribute it
// and/or modify it under the terms of the GNU Affero General Public
// License as published by the Free Software Foundation, either
// version 3 of the License, or (at your option) any later version.
//
// This program is distributed in the hope that it will be useful, but
// WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
// Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public
// License along with this program. If not, see
// <http://www.gnu.org/licenses/>.
//
// Stay tuned using
// twitter @navitia
// IRC #navitia on freenode
// https://groups.google.com/d/forum/navitia
// www.navitia.io

use super::objects::{DocType, EsId, Admin};
use chrono;
use regex;
use hyper;
use hyper::status::StatusCode;
use rs_es::error::EsError;
use rs_es;
use rs_es::EsResponse;
use serde;
use serde_json;
use rs_es::operations::search::ScanResult;

use super::objects::{AliasOperations, AliasOperation, AliasParameter};
use rs_es::units::{Duration};
use std::collections::BTreeMap;

// Rubber is an wrapper around elasticsearch API
pub struct Rubber {
    es_client: rs_es::Client,
    // some operation are not implemented in rs_es, we need to use a raw http client
    http_client: hyper::client::Client,
}

/// return the index associated to the given type and dataset
/// this will be an alias over another real index
fn get_main_index(doc_type: &str, dataset: &str) -> String {
    format!("munin_{}_{}", doc_type, dataset)
}

impl Rubber {
    // build a rubber with a connection string (http://host:port/)
    pub fn new(cnx: &str) -> Rubber {
        let re = regex::Regex::new(r"(?:https?://)?(?P<host>.+?):(?P<port>\d+)").unwrap();
        let cap = re.captures(cnx).unwrap();
        let host = cap.name("host").unwrap();
        let port = cap.name("port").unwrap().parse::<u32>().unwrap();
        info!("elastic search host {:?} port {:?}", host, port);

        Rubber {
            es_client: rs_es::Client::new(&host, port),
            http_client: hyper::client::Client::new(),
        }
    }

    pub fn get(&self, path: &str) -> Result<hyper::client::response::Response, EsError> {
        // Note: a bit duplicate on rs_es because some ES operations are not implemented
        debug!("doing a get on {}", path);
        let url = self.es_client.full_url(path);
        let result = try!(self.http_client
                              .get(&url)
                              .send());
        rs_es::do_req(result)
    }
    fn put(&self, path: &str, body: &str) -> Result<hyper::client::response::Response, EsError> {
        // Note: a bit duplicate on rs_es because some ES operations are not implemented
        debug!("doing a put on {} with {}", path, body);
        let url = self.es_client.full_url(path);
        let result = try!(self.http_client
                              .put(&url)
                              .body(body)
                              .send());
        rs_es::do_req(result)
    }
    fn post(&self, path: &str, body: &str) -> Result<hyper::client::response::Response, EsError> {
        // Note: a bit duplicate on rs_es because some ES operations are not implemented
        debug!("doing a post on {} with {}", path, body);
        let url = self.es_client.full_url(path);
        let result = try!(self.http_client
                              .post(&url)
                              .body(body)
                              .send());
        rs_es::do_req(result)
    }

    pub fn make_index(&self, doc_type: &str, dataset: &str) -> Result<String, String> {
        let current_time = chrono::UTC::now().format("%Y%m%d_%H%M%S_%f");
        let index_name = format!("munin_{}_{}_{}", doc_type, dataset, current_time);
        info!("creating index {}", index_name);
        self.create_index(&index_name.to_string()).map(|_| index_name)
    }

    fn create_index(&self, name: &String) -> Result<(), String> {
        debug!("creating index");
        // Note: in rs_es it can be done with MappingOperation but for the moment I think
        // storing the mapping in json is more convenient
        let analysis = include_str!("../../../json/settings.json");
        self.put(name, &analysis)
            .map_err(|e| {
                info!("Error while creating new index {}", name);
                e.to_string()
            })
            .and_then(|res| {
                if res.status == StatusCode::Ok {
                    Ok(())
                } else {
                    Err(format!("cannot create index: {:?}", res))
                }
            })
    }

    fn get_last_index(&self, doc_type: &str, dataset: &str) -> Result<Vec<String>, String> {
        debug!("get last index: {base_index}/_aliases",
               base_index = get_main_index(doc_type, dataset));
        self.get(&format!("{base_index}/_aliases",
                          base_index = get_main_index(doc_type, dataset)))
            .map_err(|e| e.to_string())
            .and_then(|res| {
                match res.status {
                    StatusCode::Ok => {
                        let value: serde_json::Value = try!(res.read_response()
                                                               .map_err(|e| e.to_string()));
                        Ok(value.as_object()
                                .and_then(|aliases| Some(aliases.keys().cloned().collect()))
                                .unwrap_or_else(|| {
                                    info!("no previous index to delete for type {} and dataset {}",
                                          doc_type,
                                          dataset);
                                    vec![]
                                }))
                    }
                    StatusCode::NotFound => {
                        info!("impossible to find alias {}, no last index to remove",
                              get_main_index(doc_type, dataset));
                        Ok(vec![])
                    }
                    _ => Err(format!("invalid elasticsearch response: {:?}", res)),
                }

            })
    }

    /// publish the index as the new index for this doc_type and this dataset
    /// move the index alias of the doc_type and the dataset to point to this indexes
    /// and remove the old index
    pub fn publish_index(&mut self,
                         doc_type: &str,
                         dataset: &str,
                         index: String)
                         -> Result<(), String> {
        debug!("publishing index");
        let last_indexes = try!(self.get_last_index(doc_type, dataset));
        let main_index = get_main_index(doc_type, dataset);
        try!(self.alias(&main_index, &vec![index.clone()], &last_indexes));
        try!(self.alias("munin", &vec![main_index.to_string()], &vec![]));
        for i in last_indexes {
            try!(self.delete_index(&i));
        }
        Ok(())
    }

    pub fn is_existing_index(&self, name: &String) -> Result<bool, String> {
        self.get(&name)
            .map_err(|e| e.to_string())
            .map(|res| res.status == StatusCode::Ok)
    }

    /// add a list of new indexes to the alias
    /// remove a list of indexes from the alias
    fn alias(&self, alias: &str, add: &Vec<String>, remove: &Vec<String>) -> Result<(), String> {
        info!("for {}, adding alias {:?}, removing {:?}",
              alias,
              add,
              remove);
        let add_operations = add.iter().map(|x| {
            AliasOperation {
                remove: None,
                add: Some(AliasParameter {
                    index: x.clone(),
                    alias: alias.to_string(),
                }),
            }
        });
        let remove_operations = remove.iter().map(|x| {
            AliasOperation {
                add: None,
                remove: Some(AliasParameter {
                    index: x.clone(),
                    alias: alias.to_string(),
                }),
            }
        });
        let operations = AliasOperations {
            actions: add_operations.chain(remove_operations).collect(),
        };
        let json = serde_json::to_string(&operations).unwrap();
        self.post("_aliases", &json)
            .map_err(|e| e.to_string())
            .and_then(|res| {
                if res.status == StatusCode::Ok {
                    Ok(())
                } else {
                    error!("failed to change aliases for {}, es response: {:?}",
                           alias,
                           res);
                    Err(format!("failed to post aliases for {}: {:?}", alias, res).to_string())
                }
            })
    }

    pub fn delete_index(&mut self, index: &String) -> Result<(), String> {
        debug!("deleting index {}", &index);
        let res = self.es_client
                      .delete_index(&index)
                      .map(|res| res.acknowledged)
                      .unwrap_or(false);
        if !res {
            Err(format!("Error deleting index {}", &index).into())
        } else {
            Ok(())
        }
    }

    pub fn bulk_index<T, I>(&mut self,
                            index: &String,
                            iter: I)
                            -> Result<usize, rs_es::error::EsError>
        where T: serde::Serialize + DocType + EsId,
              I: Iterator<Item = T>
    {
        use rs_es::operations::bulk::Action;
        let mut chunk = Vec::new();
        let mut nb = 0;
        let chunk_size = 1000;
        //fold is used for creating the action and optionally set the id of the object
        let mut actions = iter.map(|v| v.es_id()
                                        .into_iter()
                                        .fold(Action::index(v), |action, id| action.with_id(id)));
        loop {
            let chunk = actions.by_ref().take(chunk_size).collect::<Vec<_>>();
            nb += chunk.len();
            try!(self.es_client
                     .bulk(&chunk)
                     .with_index(&index)
                     .with_doc_type(T::doc_type())
                     .send());

            if chunk.len() < chunk_size {
                break;
            }
        }

        Ok(nb)
    }

    /// add all the element of 'iter' into elasticsearch
    ///
    /// To have zero downtime:
    /// first all the elements are added in a temporary index and when all has been indexed
    /// the index is published and the old index is removed
    pub fn index<T, I>(&mut self, doc_type: &str, dataset: &str, iter: I) -> Result<u32, String>
        where T: serde::Serialize + DocType,
              I: Iterator<Item = T>
    {
        // TODO better error handling
        let index = try!(self.make_index(doc_type, dataset));
        let nb_elements = try!(self.bulk_index(&index, iter).map_err(|e| e.to_string()));
        try!(self.publish_index(doc_type, dataset, index));
        Ok(nb_elements)
    }

	fn make_admin(&self, value: Option<Box<serde_json::Value>>) -> Option<Admin> {
	    value.and_then(|v| {
	    		serde_json::from_value::<Admin>(*v).ok().and_then(|o| Some(o))
	    })
	}

    pub fn get_admins(&mut self, dataset: &str) -> Result<BTreeMap<String, Admin>, rs_es::error::EsError> {
        debug!("Get Admins.");
        let mut result: BTreeMap<String, Admin> = BTreeMap::new();
        let scroll = Duration::minutes(1);
        let index = get_main_index("admin", dataset);
        let mut scan: ScanResult<serde_json::Value> =
        match self.es_client
                  .search_query()
                  .with_indexes(&[&index])
                  .with_size(10000)
                  .with_types(&[&"admin"])
                  .scan(&scroll) {
        	Ok(scan) => scan,
        	Err(e) => {
        		info!("Scan error: {:?}", e);
        		return Err(e);
        	}
        };
        loop {
        	let page = match scan.scroll(&mut self.es_client, &scroll) {
        		Ok(page) => page,
        		Err(e) => {
        			info!("scroll error: {:?}", e);
        			try!(scan.close(&mut self.es_client));
        			return Err(e);
        		}
        	};
        	if page.hits.hits.len() == 0 {
        		break;
        	}
        	let tmp: Vec<Admin> = page.hits
        	                          .hits
        	                          .into_iter()
        	                          .filter_map(|hit| self.make_admin(hit.source))
        	                          .collect();
        	for ad in tmp {
        		result.insert(ad.id.to_string(), ad);
        	}
        }
        try!(scan.close(&mut self.es_client));
        Ok(result)
	}
}

#[test]
pub fn test_valid_url() {
    Rubber::new("http://localhost:9200");
    Rubber::new("localhost:9200");
}

#[test]
#[should_panic]
pub fn test_invalid_url() {
    Rubber::new("http://bob");
}

#[test]
#[should_panic]
pub fn test_invalid_url_no_port() {
    Rubber::new("localhost");
}
<commit_msg>corrections<commit_after>// Copyright © 2016, Canal TP and/or its affiliates. All rights reserved.
//
// This file is part of Navitia,
//     the software to build cool stuff with public transport.
//
// Hope you'll enjoy and contribute to this project,
//     powered by Canal TP (www.canaltp.fr).
// Help us simplify mobility and open public transport:
//     a non ending quest to the responsive locomotion way of traveling!
//
// LICENCE: This program is free software; you can redistribute it
// and/or modify it under the terms of the GNU Affero General Public
// License as published by the Free Software Foundation, either
// version 3 of the License, or (at your option) any later version.
//
// This program is distributed in the hope that it will be useful, but
// WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
// Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public
// License along with this program. If not, see
// <http://www.gnu.org/licenses/>.
//
// Stay tuned using
// twitter @navitia
// IRC #navitia on freenode
// https://groups.google.com/d/forum/navitia
// www.navitia.io

use super::objects::{DocType, EsId, Admin};
use chrono;
use regex;
use hyper;
use hyper::status::StatusCode;
use rs_es::error::EsError;
use rs_es;
use rs_es::EsResponse;
use serde;
use serde_json;
use rs_es::operations::search::ScanResult;

use super::objects::{AliasOperations, AliasOperation, AliasParameter};
use rs_es::units::{Duration};
use std::collections::BTreeMap;

// Rubber is an wrapper around elasticsearch API
pub struct Rubber {
    es_client: rs_es::Client,
    // some operation are not implemented in rs_es, we need to use a raw http client
    http_client: hyper::client::Client,
}

/// return the index associated to the given type and dataset
/// this will be an alias over another real index
fn get_main_index(doc_type: &str, dataset: &str) -> String {
    format!("munin_{}_{}", doc_type, dataset)
}

impl Rubber {
    // build a rubber with a connection string (http://host:port/)
    pub fn new(cnx: &str) -> Rubber {
        let re = regex::Regex::new(r"(?:https?://)?(?P<host>.+?):(?P<port>\d+)").unwrap();
        let cap = re.captures(cnx).unwrap();
        let host = cap.name("host").unwrap();
        let port = cap.name("port").unwrap().parse::<u32>().unwrap();
        info!("elastic search host {:?} port {:?}", host, port);

        Rubber {
            es_client: rs_es::Client::new(&host, port),
            http_client: hyper::client::Client::new(),
        }
    }

    pub fn get(&self, path: &str) -> Result<hyper::client::response::Response, EsError> {
        // Note: a bit duplicate on rs_es because some ES operations are not implemented
        debug!("doing a get on {}", path);
        let url = self.es_client.full_url(path);
        let result = try!(self.http_client
                              .get(&url)
                              .send());
        rs_es::do_req(result)
    }
    fn put(&self, path: &str, body: &str) -> Result<hyper::client::response::Response, EsError> {
        // Note: a bit duplicate on rs_es because some ES operations are not implemented
        debug!("doing a put on {} with {}", path, body);
        let url = self.es_client.full_url(path);
        let result = try!(self.http_client
                              .put(&url)
                              .body(body)
                              .send());
        rs_es::do_req(result)
    }
    fn post(&self, path: &str, body: &str) -> Result<hyper::client::response::Response, EsError> {
        // Note: a bit duplicate on rs_es because some ES operations are not implemented
        debug!("doing a post on {} with {}", path, body);
        let url = self.es_client.full_url(path);
        let result = try!(self.http_client
                              .post(&url)
                              .body(body)
                              .send());
        rs_es::do_req(result)
    }

    pub fn make_index(&self, doc_type: &str, dataset: &str) -> Result<String, String> {
        let current_time = chrono::UTC::now().format("%Y%m%d_%H%M%S_%f");
        let index_name = format!("munin_{}_{}_{}", doc_type, dataset, current_time);
        info!("creating index {}", index_name);
        self.create_index(&index_name.to_string()).map(|_| index_name)
    }

    fn create_index(&self, name: &String) -> Result<(), String> {
        debug!("creating index");
        // Note: in rs_es it can be done with MappingOperation but for the moment I think
        // storing the mapping in json is more convenient
        let analysis = include_str!("../../../json/settings.json");
        self.put(name, &analysis)
            .map_err(|e| {
                info!("Error while creating new index {}", name);
                e.to_string()
            })
            .and_then(|res| {
                if res.status == StatusCode::Ok {
                    Ok(())
                } else {
                    Err(format!("cannot create index: {:?}", res))
                }
            })
    }

    fn get_last_index(&self, doc_type: &str, dataset: &str) -> Result<Vec<String>, String> {
        debug!("get last index: {base_index}/_aliases",
               base_index = get_main_index(doc_type, dataset));
        self.get(&format!("{base_index}/_aliases",
                          base_index = get_main_index(doc_type, dataset)))
            .map_err(|e| e.to_string())
            .and_then(|res| {
                match res.status {
                    StatusCode::Ok => {
                        let value: serde_json::Value = try!(res.read_response()
                                                               .map_err(|e| e.to_string()));
                        Ok(value.as_object()
                                .and_then(|aliases| Some(aliases.keys().cloned().collect()))
                                .unwrap_or_else(|| {
                                    info!("no previous index to delete for type {} and dataset {}",
                                          doc_type,
                                          dataset);
                                    vec![]
                                }))
                    }
                    StatusCode::NotFound => {
                        info!("impossible to find alias {}, no last index to remove",
                              get_main_index(doc_type, dataset));
                        Ok(vec![])
                    }
                    _ => Err(format!("invalid elasticsearch response: {:?}", res)),
                }

            })
    }

    /// publish the index as the new index for this doc_type and this dataset
    /// move the index alias of the doc_type and the dataset to point to this indexes
    /// and remove the old index
    pub fn publish_index(&mut self,
                         doc_type: &str,
                         dataset: &str,
                         index: String)
                         -> Result<(), String> {
        debug!("publishing index");
        let last_indexes = try!(self.get_last_index(doc_type, dataset));
        let main_index = get_main_index(doc_type, dataset);
        try!(self.alias(&main_index, &vec![index.clone()], &last_indexes));
        try!(self.alias("munin", &vec![main_index.to_string()], &vec![]));
        for i in last_indexes {
            try!(self.delete_index(&i));
        }
        Ok(())
    }

    pub fn is_existing_index(&self, name: &String) -> Result<bool, String> {
        self.get(&name)
            .map_err(|e| e.to_string())
            .map(|res| res.status == StatusCode::Ok)
    }

    /// add a list of new indexes to the alias
    /// remove a list of indexes from the alias
    fn alias(&self, alias: &str, add: &Vec<String>, remove: &Vec<String>) -> Result<(), String> {
        info!("for {}, adding alias {:?}, removing {:?}",
              alias,
              add,
              remove);
        let add_operations = add.iter().map(|x| {
            AliasOperation {
                remove: None,
                add: Some(AliasParameter {
                    index: x.clone(),
                    alias: alias.to_string(),
                }),
            }
        });
        let remove_operations = remove.iter().map(|x| {
            AliasOperation {
                add: None,
                remove: Some(AliasParameter {
                    index: x.clone(),
                    alias: alias.to_string(),
                }),
            }
        });
        let operations = AliasOperations {
            actions: add_operations.chain(remove_operations).collect(),
        };
        let json = serde_json::to_string(&operations).unwrap();
        self.post("_aliases", &json)
            .map_err(|e| e.to_string())
            .and_then(|res| {
                if res.status == StatusCode::Ok {
                    Ok(())
                } else {
                    error!("failed to change aliases for {}, es response: {:?}",
                           alias,
                           res);
                    Err(format!("failed to post aliases for {}: {:?}", alias, res).to_string())
                }
            })
    }

    pub fn delete_index(&mut self, index: &String) -> Result<(), String> {
        debug!("deleting index {}", &index);
        let res = self.es_client
                      .delete_index(&index)
                      .map(|res| res.acknowledged)
                      .unwrap_or(false);
        if !res {
            Err(format!("Error deleting index {}", &index).into())
        } else {
            Ok(())
        }
    }

    pub fn bulk_index<T, I>(&mut self,
                            index: &String,
                            iter: I)
                            -> Result<usize, rs_es::error::EsError>
        where T: serde::Serialize + DocType + EsId,
              I: Iterator<Item = T>
    {
        use rs_es::operations::bulk::Action;
        let mut nb = 0;
        let chunk_size = 1000;
        //fold is used for creating the action and optionally set the id of the object
        let mut actions = iter.map(|v| v.es_id()
                                        .into_iter()
                                        .fold(Action::index(v), |action, id| action.with_id(id)));
        loop {
            let chunk = actions.by_ref().take(chunk_size).collect::<Vec<_>>();
            nb += chunk.len();
            try!(self.es_client
                     .bulk(&chunk)
                     .with_index(&index)
                     .with_doc_type(T::doc_type())
                     .send());

            if chunk.len() < chunk_size {
                break;
            }
        }

        Ok(nb)
    }

    /// add all the element of 'iter' into elasticsearch
    ///
    /// To have zero downtime:
    /// first all the elements are added in a temporary index and when all has been indexed
    /// the index is published and the old index is removed
    pub fn index<T, I>(&mut self, doc_type: &str, dataset: &str, iter: I) -> Result<usize, String>
        where T: serde::Serialize + DocType + EsId,
              I: Iterator<Item = T>
    {
        // TODO better error handling
        let index = try!(self.make_index(doc_type, dataset));
        let nb_elements = try!(self.bulk_index(&index, iter).map_err(|e| e.to_string()));
        try!(self.publish_index(doc_type, dataset, index));
        Ok(nb_elements)
    }

	fn make_admin(&self, value: Option<Box<serde_json::Value>>) -> Option<Admin> {
	    value.and_then(|v| {
	    		serde_json::from_value::<Admin>(*v).ok().and_then(|o| Some(o))
	    })
	}

    pub fn get_admins(&mut self, dataset: &str) -> Result<BTreeMap<String, Admin>, rs_es::error::EsError> {

        let mut result: BTreeMap<String, Admin> = BTreeMap::new();
        let index = get_main_index("admin", dataset);
        let mut scan: ScanResult<serde_json::Value> =
        try!(self.es_client
                  .search_query()
                  .with_indexes(&[&index])
                  .with_size(1000)
                  .with_types(&[&"admin"])
                  .scan(&Duration::minutes(1)));
        loop {
        	let page = match scan.scroll(&mut self.es_client, &Duration::minutes(1)) {
        		Ok(page) => page,
        		Err(e) => {
        			info!("scroll error: {:?}", e);
        			try!(scan.close(&mut self.es_client));
        			return Err(e);
        		}
        	};
        	if page.hits.hits.len() == 0 {
        		break;
        	}
        	let tmp: Vec<Admin> = page.hits
        	                          .hits
        	                          .into_iter()
        	                          .filter_map(|hit| self.make_admin(hit.source))
        	                          .collect();

        	for ad in tmp {
        		result.insert(ad.id.to_string(), ad);
        	}
        }
        try!(scan.close(&mut self.es_client));
        Ok(result)
	}
}

#[test]
pub fn test_valid_url() {
    Rubber::new("http://localhost:9200");
    Rubber::new("localhost:9200");
}

#[test]
#[should_panic]
pub fn test_invalid_url() {
    Rubber::new("http://bob");
}

#[test]
#[should_panic]
pub fn test_invalid_url_no_port() {
    Rubber::new("localhost");
}
<|endoftext|>


<commit_before>//! Thread-local storage for JavaScript threads.
//!
//! At runtime, an instance of a Node.js addon can contain its own local storage,
//! which can then be shared and accessed as needed from Rust in a Neon module. This can
//! be useful for setting up long-lived state that needs to be shared between calls
//! of an addon's APIs.
//!
//! For example, an addon may wish to track the [thread ID][threadId] of each of its
//! instances:
//!
//! ```
//! # use neon::prelude::*;
//! # use neon::thread::LocalKey;
//! static THREAD_ID: LocalKey<u32> = LocalKey::new();
//!
//! pub fn thread_id<'cx, C: Context<'cx>>(cx: &mut C) -> NeonResult<u32> {
//!     THREAD_ID.get_or_try_init(cx, |cx| {
//!         let global = cx.global();
//!         let require: Handle<JsFunction> = global.get(cx, "require")?;
//!         let worker: Handle<JsObject> = require.call_with(cx)
//!             .arg(cx.string("node:worker_threads"))
//!             .apply(cx)?;
//!         let thread_id: Handle<JsNumber> = worker.get(cx, "threadId")?;
//!         Ok(thread_id.value(cx) as u32)
//!     }).cloned()
//! }
//! ```
//!
//! ### The Addon Lifecycle
//!
//! For some use cases, a single shared global constant stored in a `static` variable
//! might be sufficient:
//!
//! ```
//! static MY_CONSTANT: &'static str = "hello Neon";
//! ```
//!
//! This variable will be allocated when the addon is first loaded into the Node.js
//! process. This works fine for single-threaded applications, or global thread-safe
//! data.
//!
//! However, since the addition of [worker threads][workers] in Node v10,
//! modules can be instantiated multiple times in a single Node process. This means
//! that while the dynamically-loaded binary library (i.e., the Rust implementation of
//! the addon) is only loaded once in the running process, but its [`#[main]`](crate::main)
//! function is executed multiple times with distinct module objects, once per application
//! thread:
//!
//! ![The Node.js addon lifecycle, described in detail below.][lifecycle]
//!
//! This means that any thread-local data needs to be initialized separately for each
//! instance of the addon. This module provides a simple container type, [`LocalKey`](LocalKey),
//! for allocating and initializing thread-local data. (Technically, this data is stored in the
//! addon's [module instance][environment], which is equivalent to being thread-local.)
//!
//! A common example is when an addon needs to maintain a reference to a JavaScript value. References cannot
//! be used across separate threads. The constructor for the datatype can be [rooted](crate::handle::Root) and
//! saved in thread-local storage:
//!
//! ```
//! # use neon::prelude::*;
//! # use neon::thread::LocalKey;
//! # fn initialize_my_datatype<'cx, C: Context<'cx>>(cx: &mut C) -> JsResult<'cx, JsFunction> { unimplemented!() }
//! static MY_CONSTRUCTOR: LocalKey<Root<JsFunction>> = LocalKey::new();
//!
//! pub fn my_constructor<'cx, C: Context<'cx>>(cx: &mut C) -> JsResult<'cx, JsFunction> {
//!     let constructor = MY_CONSTRUCTOR.get_or_try_init(cx, |cx| {
//!         let constructor: Handle<JsFunction> = initialize_my_datatype(cx)?;
//!         Ok(constructor.root(cx))
//!     })?;
//!     Ok(constructor.to_inner(cx))
//! }
//! ```
//!
//! Notice that if this code were implemented without a `LocalKey`, it would panic whenever
//! one thread stores an instance of the constructor and a different thread attempts to
//! access it with the call to [`to_inner()`](crate::handle::Root::to_inner).
//!
//! ### When to Use Thread-Local Storage
//!
//! Single-threaded applications don't generally need to worry about thread-local data.
//! There are two cases where Neon apps should consider storing static data in a
//! `LocalKey` storage cell:
//!
//! - **Multi-threaded applications:** If your Node application uses the `Worker`
//!   API, you'll want to store any static data that might get access from multiple
//!   threads in thread-local data.
//! - **Libraries:** If your addon is part of a library that could be used by multiple
//!   applications, you'll want to store static data in thread-local data in case the
//!   addon ends up instantiated by multiple threads in some future application.
//!
//! ### Why Not Use Standard TLS?
//!
//! Because the JavaScript engine may not tie JavaScript threads 1:1 to system threads,
//! it is recommended to use this module instead of the Rust standard thread-local storage
//! when tying data to a JavaScript thread.
//!
//! [environment]: https://nodejs.org/api/n-api.html#environment-life-cycle-apis
//! [lifecycle]: https://raw.githubusercontent.com/neon-bindings/neon/main/doc/lifecycle.png
//! [workers]: https://nodejs.org/api/worker_threads.html
//! [threadId]: https://nodejs.org/api/worker_threads.html#workerthreadid

use std::any::Any;
use std::marker::PhantomData;
use std::sync::atomic::{AtomicUsize, Ordering};

use once_cell::sync::OnceCell;

use crate::context::Context;
use crate::lifecycle::LocalCell;

static COUNTER: AtomicUsize = AtomicUsize::new(0);

fn next_id() -> usize {
    COUNTER.fetch_add(1, Ordering::SeqCst)
}

/// A JavaScript thread-local container that owns its contents, similar to
/// [`std::thread::LocalKey`](std::thread::LocalKey) but tied to a JavaScript thread rather
/// than a system thread.
///
/// ### Initialization and Destruction
///
/// Initialization is dynamically performed on the first call to one of the `init` methods
/// of `LocalKey`, and values that implement [`Drop`](std::ops::Drop) get destructed when
/// the JavaScript thread exits, i.e. when a worker thread terminates or the main thread
/// terminates on process exit.
#[derive(Default)]
pub struct LocalKey<T> {
    _type: PhantomData<T>,
    id: OnceCell<usize>,
}

impl<T> LocalKey<T> {
    /// Creates a new local value. This method is `const`, so it can be assigned to
    /// static variables.
    pub const fn new() -> Self {
        Self {
            _type: PhantomData,
            id: OnceCell::new(),
        }
    }

    fn id(&self) -> usize {
        *self.id.get_or_init(next_id)
    }
}

impl<T: Any + Send + 'static> LocalKey<T> {
    /// Gets the current value of the cell. Returns `None` if the cell has not
    /// yet been initialized.
    pub fn get<'cx, 'a, C>(&self, cx: &'a mut C) -> Option<&'cx T>
    where
        C: Context<'cx>,
    {
        // Unwrap safety: The type bound LocalKey<T> and the fact that every LocalKey has a unique
        // id guarantees that the cell is only ever assigned instances of type T.
        let r: Option<&T> =
            LocalCell::get(cx, self.id()).map(|value| value.downcast_ref().unwrap());

        // Safety: Since the Box is immutable and heap-allocated, it's guaranteed not to
        // move or change for the duration of the context.
        unsafe { std::mem::transmute::<Option<&'a T>, Option<&'cx T>>(r) }
    }

    /// Gets the current value of the cell, initializing it with the result of
    /// calling `f` if it has not yet been initialized.
    pub fn get_or_init<'cx, 'a, C, F>(&self, cx: &'a mut C, f: F) -> &'cx T
    where
        C: Context<'cx>,
        F: FnOnce() -> T,
    {
        // Unwrap safety: The type bound LocalKey<T> and the fact that every LocalKey has a unique
        // id guarantees that the cell is only ever assigned instances of type T.
        let r: &T = LocalCell::get_or_init(cx, self.id(), || Box::new(f()))
            .downcast_ref()
            .unwrap();

        // Safety: Since the Box is immutable and heap-allocated, it's guaranteed not to
        // move or change for the duration of the context.
        unsafe { std::mem::transmute::<&'a T, &'cx T>(r) }
    }

    /// Gets the current value of the cell, initializing it with the result of
    /// calling `f` if it has not yet been initialized. Returns `Err` if the
    /// callback triggers a JavaScript exception.
    ///
    /// # Panics
    ///
    /// During the execution of `f`, calling any methods on this `LocalKey` that
    /// attempt to initialize it will panic.
    pub fn get_or_try_init<'cx, 'a, C, E, F>(&self, cx: &'a mut C, f: F) -> Result<&'cx T, E>
    where
        C: Context<'cx>,
        F: FnOnce(&mut C) -> Result<T, E>,
    {
        // Unwrap safety: The type bound LocalKey<T> and the fact that every LocalKey has a unique
        // id guarantees that the cell is only ever assigned instances of type T.
        let r: &T = LocalCell::get_or_try_init(cx, self.id(), |cx| Ok(Box::new(f(cx)?)))?
            .downcast_ref()
            .unwrap();

        // Safety: Since the Box is immutable and heap-allocated, it's guaranteed not to
        // move or change for the duration of the context.
        Ok(unsafe { std::mem::transmute::<&'a T, &'cx T>(r) })
    }
}

impl<T: Any + Send + Default + 'static> LocalKey<T> {
    /// Gets the current value of the cell, initializing it with the default value
    /// if it has not yet been initialized.
    pub fn get_or_init_default<'cx, 'a, C>(&self, cx: &'a mut C) -> &'cx T
    where
        C: Context<'cx>,
    {
        self.get_or_init(cx, Default::default)
    }
}
<commit_msg>More copy editing in the docs<commit_after>//! Thread-local storage for JavaScript threads.
//!
//! At runtime, an instance of a Node.js addon can contain its own local storage,
//! which can then be shared and accessed as needed from Rust in a Neon module. This can
//! be useful for setting up long-lived state that needs to be shared between calls
//! of an addon's APIs.
//!
//! For example, an addon may wish to track the [thread ID][threadId] of each of its
//! instances:
//!
//! ```
//! # use neon::prelude::*;
//! # use neon::thread::LocalKey;
//! static THREAD_ID: LocalKey<u32> = LocalKey::new();
//!
//! pub fn thread_id<'cx, C: Context<'cx>>(cx: &mut C) -> NeonResult<u32> {
//!     THREAD_ID.get_or_try_init(cx, |cx| {
//!         let global = cx.global();
//!         let require: Handle<JsFunction> = global.get(cx, "require")?;
//!         let worker: Handle<JsObject> = require.call_with(cx)
//!             .arg(cx.string("node:worker_threads"))
//!             .apply(cx)?;
//!         let thread_id: Handle<JsNumber> = worker.get(cx, "threadId")?;
//!         Ok(thread_id.value(cx) as u32)
//!     }).cloned()
//! }
//! ```
//!
//! ### The Addon Lifecycle
//!
//! For some use cases, a single shared global constant stored in a `static` variable
//! might be sufficient:
//!
//! ```
//! static MY_CONSTANT: &'static str = "hello Neon";
//! ```
//!
//! This variable will be allocated when the addon is first loaded into the Node.js
//! process. This works fine for single-threaded applications, or global thread-safe
//! data.
//!
//! However, since the addition of [worker threads][workers] in Node v10,
//! modules can be instantiated multiple times in a single Node process. This means
//! that while the dynamically-loaded binary library (i.e., the Rust implementation of
//! the addon) is only loaded once in the running process, but its [`#[main]`](crate::main)
//! function is executed multiple times with distinct module objects, once per application
//! thread:
//!
//! ![The Node.js addon lifecycle, described in detail below.][lifecycle]
//!
//! This means that any thread-local data needs to be initialized separately for each
//! instance of the addon. This module provides a simple container type, [`LocalKey`](LocalKey),
//! for allocating and initializing thread-local data. (Technically, this data is stored in the
//! addon's [module instance][environment], which is equivalent to being thread-local.)
//!
//! A common example is when an addon needs to maintain a reference to a JavaScript value. A
//! reference can be [rooted](crate::handle::Root) and stored in a static, but refereces cannot
//! be used across separate threads. By placing the reference in thread-local storage, an
//! addon can ensure that each thread stores its own distinct reference:
//!
//! ```
//! # use neon::prelude::*;
//! # use neon::thread::LocalKey;
//! # fn initialize_my_datatype<'cx, C: Context<'cx>>(cx: &mut C) -> JsResult<'cx, JsFunction> { unimplemented!() }
//! static MY_CONSTRUCTOR: LocalKey<Root<JsFunction>> = LocalKey::new();
//!
//! pub fn my_constructor<'cx, C: Context<'cx>>(cx: &mut C) -> JsResult<'cx, JsFunction> {
//!     let constructor = MY_CONSTRUCTOR.get_or_try_init(cx, |cx| {
//!         let constructor: Handle<JsFunction> = initialize_my_datatype(cx)?;
//!         Ok(constructor.root(cx))
//!     })?;
//!     Ok(constructor.to_inner(cx))
//! }
//! ```
//!
//! Notice that if this code were implemented without a `LocalKey`, it would panic whenever
//! one thread stores an instance of the constructor and a different thread attempts to
//! access it with the call to [`to_inner()`](crate::handle::Root::to_inner).
//!
//! ### When to Use Thread-Local Storage
//!
//! Single-threaded applications don't generally need to worry about thread-local data.
//! There are two cases where Neon apps should consider storing static data in a
//! `LocalKey` storage cell:
//!
//! - **Multi-threaded applications:** If your Node application uses the `Worker`
//!   API, you'll want to store any static data that might get access from multiple
//!   threads in thread-local data.
//! - **Libraries:** If your addon is part of a library that could be used by multiple
//!   applications, you'll want to store static data in thread-local data in case the
//!   addon ends up instantiated by multiple threads in some future application.
//!
//! ### Why Not Use Standard TLS?
//!
//! Because the JavaScript engine may not tie JavaScript threads 1:1 to system threads,
//! it is recommended to use this module instead of the Rust standard thread-local storage
//! when tying data to a JavaScript thread.
//!
//! [environment]: https://nodejs.org/api/n-api.html#environment-life-cycle-apis
//! [lifecycle]: https://raw.githubusercontent.com/neon-bindings/neon/main/doc/lifecycle.png
//! [workers]: https://nodejs.org/api/worker_threads.html
//! [threadId]: https://nodejs.org/api/worker_threads.html#workerthreadid

use std::any::Any;
use std::marker::PhantomData;
use std::sync::atomic::{AtomicUsize, Ordering};

use once_cell::sync::OnceCell;

use crate::context::Context;
use crate::lifecycle::LocalCell;

static COUNTER: AtomicUsize = AtomicUsize::new(0);

fn next_id() -> usize {
    COUNTER.fetch_add(1, Ordering::SeqCst)
}

/// A JavaScript thread-local container that owns its contents, similar to
/// [`std::thread::LocalKey`](std::thread::LocalKey) but tied to a JavaScript thread rather
/// than a system thread.
///
/// ### Initialization and Destruction
///
/// Initialization is dynamically performed on the first call to one of the `init` methods
/// of `LocalKey`, and values that implement [`Drop`](std::ops::Drop) get destructed when
/// the JavaScript thread exits, i.e. when a worker thread terminates or the main thread
/// terminates on process exit.
#[derive(Default)]
pub struct LocalKey<T> {
    _type: PhantomData<T>,
    id: OnceCell<usize>,
}

impl<T> LocalKey<T> {
    /// Creates a new local value. This method is `const`, so it can be assigned to
    /// static variables.
    pub const fn new() -> Self {
        Self {
            _type: PhantomData,
            id: OnceCell::new(),
        }
    }

    fn id(&self) -> usize {
        *self.id.get_or_init(next_id)
    }
}

impl<T: Any + Send + 'static> LocalKey<T> {
    /// Gets the current value of the cell. Returns `None` if the cell has not
    /// yet been initialized.
    pub fn get<'cx, 'a, C>(&self, cx: &'a mut C) -> Option<&'cx T>
    where
        C: Context<'cx>,
    {
        // Unwrap safety: The type bound LocalKey<T> and the fact that every LocalKey has a unique
        // id guarantees that the cell is only ever assigned instances of type T.
        let r: Option<&T> =
            LocalCell::get(cx, self.id()).map(|value| value.downcast_ref().unwrap());

        // Safety: Since the Box is immutable and heap-allocated, it's guaranteed not to
        // move or change for the duration of the context.
        unsafe { std::mem::transmute::<Option<&'a T>, Option<&'cx T>>(r) }
    }

    /// Gets the current value of the cell, initializing it with the result of
    /// calling `f` if it has not yet been initialized.
    pub fn get_or_init<'cx, 'a, C, F>(&self, cx: &'a mut C, f: F) -> &'cx T
    where
        C: Context<'cx>,
        F: FnOnce() -> T,
    {
        // Unwrap safety: The type bound LocalKey<T> and the fact that every LocalKey has a unique
        // id guarantees that the cell is only ever assigned instances of type T.
        let r: &T = LocalCell::get_or_init(cx, self.id(), || Box::new(f()))
            .downcast_ref()
            .unwrap();

        // Safety: Since the Box is immutable and heap-allocated, it's guaranteed not to
        // move or change for the duration of the context.
        unsafe { std::mem::transmute::<&'a T, &'cx T>(r) }
    }

    /// Gets the current value of the cell, initializing it with the result of
    /// calling `f` if it has not yet been initialized. Returns `Err` if the
    /// callback triggers a JavaScript exception.
    ///
    /// # Panics
    ///
    /// During the execution of `f`, calling any methods on this `LocalKey` that
    /// attempt to initialize it will panic.
    pub fn get_or_try_init<'cx, 'a, C, E, F>(&self, cx: &'a mut C, f: F) -> Result<&'cx T, E>
    where
        C: Context<'cx>,
        F: FnOnce(&mut C) -> Result<T, E>,
    {
        // Unwrap safety: The type bound LocalKey<T> and the fact that every LocalKey has a unique
        // id guarantees that the cell is only ever assigned instances of type T.
        let r: &T = LocalCell::get_or_try_init(cx, self.id(), |cx| Ok(Box::new(f(cx)?)))?
            .downcast_ref()
            .unwrap();

        // Safety: Since the Box is immutable and heap-allocated, it's guaranteed not to
        // move or change for the duration of the context.
        Ok(unsafe { std::mem::transmute::<&'a T, &'cx T>(r) })
    }
}

impl<T: Any + Send + Default + 'static> LocalKey<T> {
    /// Gets the current value of the cell, initializing it with the default value
    /// if it has not yet been initialized.
    pub fn get_or_init_default<'cx, 'a, C>(&self, cx: &'a mut C) -> &'cx T
    where
        C: Context<'cx>,
    {
        self.get_or_init(cx, Default::default)
    }
}
<|endoftext|>


<commit_before>use super::server::Server;
use crate::*;
use std::collections::HashMap;

pub mod builder;
pub mod cmd;
pub mod connect;

pub use builder::Builder;

pub struct Handlers<S: 'static, M: 'static + Metrics> {
    connected: Option<Box<Fn(&Addr<Network<S, M>>, &mut S, PeerInfo) -> () + 'static>>,
    disconnected: Option<Box<Fn(&Addr<Network<S, M>>, &mut S, PeerInfo) -> () + 'static>>,
    connect_req: Option<
        Box<
            Fn(&Addr<Network<S, M>>, &mut S, PeerInfo, Payload) -> peer::msg::HandshakeRequest
                + 'static,
        >,
    >,
    message: Box<Fn(&Addr<Network<S, M>>, &mut S, PeerId, &Payload) -> bool + 'static>,
}

pub struct Network<S: 'static, M: 'static + Metrics = DummyMetrics> {
    addr: Addr<Self>,
    state: S,
    handlers: Handlers<S, M>,
    metrics: M,
    sessions: HashMap<PeerId, (PeerInfo, Addr<Peer<S, M>>)>,
}

impl<S: 'static, M: 'static + Metrics> Network<S, M> {
    pub fn start(state: S, metrics: M, handlers: Handlers<S, M>) -> Addr<Network<S, M>> {
        Network::create(|ctx| Network {
            addr: ctx.address(),
            state,
            handlers,
            metrics,
            sessions: HashMap::with_capacity(32),
        })
    }

    fn broadcast(&mut self, msg: &Payload, skip_id: Option<PeerId>) {
        for ses in self
            .sessions
            .values()
            .filter(|ses| skip_id.map_or(true, |skip_id| ses.0.id != skip_id))
        {
            self.metrics.on_outbound_message(&msg);
            ses.1.do_send(msg.clone());
        }
    }
}

impl<S: 'static, M: 'static + Metrics> Actor for Network<S, M> {
    type Context = Context<Self>;
}

impl<S: 'static, M: 'static + Metrics> Handler<peer::msg::Handshake> for Network<S, M> {
    type Result = peer::msg::HandshakeRequest;

    fn handle(&mut self, msg: peer::msg::Handshake, _: &mut Self::Context) -> Self::Result {
        if let Some(f) = &self.handlers.connect_req {
            f(&self.addr, &mut self.state, msg.0, msg.1)
        } else {
            peer::msg::HandshakeRequest::Allow
        }
    }
}

impl<S: 'static, M: 'static + Metrics> Handler<peer::msg::Connected<S, M>> for Network<S, M> {
    type Result = ();

    fn handle(&mut self, msg: peer::msg::Connected<S, M>, ctx: &mut Self::Context) {
        let info = msg.0;
        let prev = self.sessions.insert(info.id, (info.clone(), msg.1));
        assert!(prev.is_none(), "session id already exists in network");
        if let Some(f) = &self.handlers.connected {
            f(&self.addr, &mut self.state, info);
        }
    }
}

impl<S: 'static, M: 'static + Metrics> Handler<peer::msg::Disconnected> for Network<S, M> {
    type Result = ();

    fn handle(&mut self, msg: peer::msg::Disconnected, _: &mut Self::Context) {
        let id = msg.0.id;
        let ses = self
            .sessions
            .remove(&id)
            .unwrap_or_else(|| panic!("Expected disconnected peer to exist: {}", id));
        if let Some(f) = &self.handlers.disconnected {
            f(&self.addr, &mut self.state, ses.0);
        }
    }
}

impl<S: 'static, M: 'static + Metrics> Handler<peer::msg::Message> for Network<S, M> {
    type Result = ();

    fn handle(&mut self, msg: peer::msg::Message, _: &mut Self::Context) {
        let id = msg.0;
        let pl = &msg.1;
        self.metrics.on_inbound_message(pl);
        if (self.handlers.message)(&self.addr, &mut self.state, id, pl) {
            self.broadcast(pl, Some(id));
        }
    }
}
<commit_msg>Fix unused var warning<commit_after>use super::server::Server;
use crate::*;
use std::collections::HashMap;

pub mod builder;
pub mod cmd;
pub mod connect;

pub use builder::Builder;

pub struct Handlers<S: 'static, M: 'static + Metrics> {
    connected: Option<Box<Fn(&Addr<Network<S, M>>, &mut S, PeerInfo) -> () + 'static>>,
    disconnected: Option<Box<Fn(&Addr<Network<S, M>>, &mut S, PeerInfo) -> () + 'static>>,
    connect_req: Option<
        Box<
            Fn(&Addr<Network<S, M>>, &mut S, PeerInfo, Payload) -> peer::msg::HandshakeRequest
                + 'static,
        >,
    >,
    message: Box<Fn(&Addr<Network<S, M>>, &mut S, PeerId, &Payload) -> bool + 'static>,
}

pub struct Network<S: 'static, M: 'static + Metrics = DummyMetrics> {
    addr: Addr<Self>,
    state: S,
    handlers: Handlers<S, M>,
    metrics: M,
    sessions: HashMap<PeerId, (PeerInfo, Addr<Peer<S, M>>)>,
}

impl<S: 'static, M: 'static + Metrics> Network<S, M> {
    pub fn start(state: S, metrics: M, handlers: Handlers<S, M>) -> Addr<Network<S, M>> {
        Network::create(|ctx| Network {
            addr: ctx.address(),
            state,
            handlers,
            metrics,
            sessions: HashMap::with_capacity(32),
        })
    }

    fn broadcast(&mut self, msg: &Payload, skip_id: Option<PeerId>) {
        for ses in self
            .sessions
            .values()
            .filter(|ses| skip_id.map_or(true, |skip_id| ses.0.id != skip_id))
        {
            self.metrics.on_outbound_message(&msg);
            ses.1.do_send(msg.clone());
        }
    }
}

impl<S: 'static, M: 'static + Metrics> Actor for Network<S, M> {
    type Context = Context<Self>;
}

impl<S: 'static, M: 'static + Metrics> Handler<peer::msg::Handshake> for Network<S, M> {
    type Result = peer::msg::HandshakeRequest;

    fn handle(&mut self, msg: peer::msg::Handshake, _: &mut Self::Context) -> Self::Result {
        if let Some(f) = &self.handlers.connect_req {
            f(&self.addr, &mut self.state, msg.0, msg.1)
        } else {
            peer::msg::HandshakeRequest::Allow
        }
    }
}

impl<S: 'static, M: 'static + Metrics> Handler<peer::msg::Connected<S, M>> for Network<S, M> {
    type Result = ();

    fn handle(&mut self, msg: peer::msg::Connected<S, M>, _: &mut Self::Context) {
        let info = msg.0;
        let prev = self.sessions.insert(info.id, (info.clone(), msg.1));
        assert!(prev.is_none(), "session id already exists in network");
        if let Some(f) = &self.handlers.connected {
            f(&self.addr, &mut self.state, info);
        }
    }
}

impl<S: 'static, M: 'static + Metrics> Handler<peer::msg::Disconnected> for Network<S, M> {
    type Result = ();

    fn handle(&mut self, msg: peer::msg::Disconnected, _: &mut Self::Context) {
        let id = msg.0.id;
        let ses = self
            .sessions
            .remove(&id)
            .unwrap_or_else(|| panic!("Expected disconnected peer to exist: {}", id));
        if let Some(f) = &self.handlers.disconnected {
            f(&self.addr, &mut self.state, ses.0);
        }
    }
}

impl<S: 'static, M: 'static + Metrics> Handler<peer::msg::Message> for Network<S, M> {
    type Result = ();

    fn handle(&mut self, msg: peer::msg::Message, _: &mut Self::Context) {
        let id = msg.0;
        let pl = &msg.1;
        self.metrics.on_inbound_message(pl);
        if (self.handlers.message)(&self.addr, &mut self.state, id, pl) {
            self.broadcast(pl, Some(id));
        }
    }
}
<|endoftext|>