<commit_before>use crate::{
    consumer::Consumer, message::Delivery, topology::ConsumerDefinition,
    topology_internal::ConsumerDefinitionInternal, types::ShortString, BasicProperties, Channel,
    Error, Result,
};
use parking_lot::Mutex;
use std::{borrow::Borrow, collections::HashMap, fmt, hash::Hash, sync::Arc};

#[derive(Clone, Default)]
pub(crate) struct Consumers(Arc<Mutex<HashMap<ShortString, Consumer>>>);

impl Consumers {
    pub(crate) fn register(&self, tag: ShortString, consumer: Consumer) {
        self.0.lock().insert(tag, consumer);
    }

    pub(crate) fn deregister<S: Hash + Eq + ?Sized>(&self, consumer_tag: &S) -> Result<()>
    where
        ShortString: Borrow<S>,
    {
        if let Some(consumer) = self.0.lock().remove(consumer_tag) {
            consumer.cancel()?;
        }
        Ok(())
    }

    pub(crate) fn start_delivery<S: Hash + Eq + ?Sized>(&self, consumer_tag: &S, message: Delivery)
    where
        ShortString: Borrow<S>,
    {
        if let Some(consumer) = self.0.lock().get_mut(consumer_tag) {
            consumer.start_new_delivery(message);
        }
    }

    pub(crate) fn handle_content_header_frame<S: Hash + Eq + ?Sized>(
        &self,
        channel: &Channel,
        consumer_tag: &S,
        size: u64,
        properties: BasicProperties,
    ) -> Result<()>
    where
        ShortString: Borrow<S>,
    {
        if let Some(consumer) = self.0.lock().get_mut(consumer_tag) {
            consumer.set_delivery_properties(properties);
            if size == 0 {
                consumer.new_delivery_complete(channel.clone())?;
            }
        }
        Ok(())
    }

    pub(crate) fn handle_body_frame<S: Hash + Eq + ?Sized>(
        &self,
        channel: &Channel,
        consumer_tag: &S,
        remaining_size: usize,
        payload: Vec<u8>,
    ) -> Result<()>
    where
        ShortString: Borrow<S>,
    {
        if let Some(consumer) = self.0.lock().get_mut(consumer_tag) {
            consumer.receive_delivery_content(payload);
            if remaining_size == 0 {
                consumer.new_delivery_complete(channel.clone())?;
            }
        }
        Ok(())
    }

    pub(crate) fn drop_prefetched_messages(&self) -> Result<()> {
        self.0
            .lock()
            .values()
            .map(Consumer::drop_prefetched_messages)
            .fold(Ok(()), Result::and)
    }

    pub(crate) fn cancel(&self) -> Result<()> {
        self.0
            .lock()
            .drain()
            .map(|(_, consumer)| consumer.cancel())
            .fold(Ok(()), Result::and)
    }

    pub(crate) fn error(&self, error: Error) -> Result<()> {
        self.0
            .lock()
            .drain()
            .map(|(_, consumer)| consumer.set_error(error.clone()))
            .fold(Ok(()), Result::and)
    }

    pub(crate) fn topology(&self) -> Vec<ConsumerDefinitionInternal> {
        self.0
            .lock()
            .values()
            .map(|consumer| ConsumerDefinitionInternal {
                consumer: Some(consumer.clone()),
                // TODO: drop this ?
                definition: ConsumerDefinition {
                    tag: consumer.tag(),
                    options: consumer.options(),
                    arguments: consumer.arguments(),
                    queue: consumer.queue(),
                },
            })
            .collect()
    }
}

impl fmt::Debug for Consumers {
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        let mut debug = f.debug_tuple("Consumers");
        if let Some(consumers) = self.0.try_lock() {
            debug.field(&*consumers);
        }
        debug.finish()
    }
}
<commit_msg>drop TODO<commit_after>use crate::{
    consumer::Consumer, message::Delivery, topology::ConsumerDefinition,
    topology_internal::ConsumerDefinitionInternal, types::ShortString, BasicProperties, Channel,
    Error, Result,
};
use parking_lot::Mutex;
use std::{borrow::Borrow, collections::HashMap, fmt, hash::Hash, sync::Arc};

#[derive(Clone, Default)]
pub(crate) struct Consumers(Arc<Mutex<HashMap<ShortString, Consumer>>>);

impl Consumers {
    pub(crate) fn register(&self, tag: ShortString, consumer: Consumer) {
        self.0.lock().insert(tag, consumer);
    }

    pub(crate) fn deregister<S: Hash + Eq + ?Sized>(&self, consumer_tag: &S) -> Result<()>
    where
        ShortString: Borrow<S>,
    {
        if let Some(consumer) = self.0.lock().remove(consumer_tag) {
            consumer.cancel()?;
        }
        Ok(())
    }

    pub(crate) fn start_delivery<S: Hash + Eq + ?Sized>(&self, consumer_tag: &S, message: Delivery)
    where
        ShortString: Borrow<S>,
    {
        if let Some(consumer) = self.0.lock().get_mut(consumer_tag) {
            consumer.start_new_delivery(message);
        }
    }

    pub(crate) fn handle_content_header_frame<S: Hash + Eq + ?Sized>(
        &self,
        channel: &Channel,
        consumer_tag: &S,
        size: u64,
        properties: BasicProperties,
    ) -> Result<()>
    where
        ShortString: Borrow<S>,
    {
        if let Some(consumer) = self.0.lock().get_mut(consumer_tag) {
            consumer.set_delivery_properties(properties);
            if size == 0 {
                consumer.new_delivery_complete(channel.clone())?;
            }
        }
        Ok(())
    }

    pub(crate) fn handle_body_frame<S: Hash + Eq + ?Sized>(
        &self,
        channel: &Channel,
        consumer_tag: &S,
        remaining_size: usize,
        payload: Vec<u8>,
    ) -> Result<()>
    where
        ShortString: Borrow<S>,
    {
        if let Some(consumer) = self.0.lock().get_mut(consumer_tag) {
            consumer.receive_delivery_content(payload);
            if remaining_size == 0 {
                consumer.new_delivery_complete(channel.clone())?;
            }
        }
        Ok(())
    }

    pub(crate) fn drop_prefetched_messages(&self) -> Result<()> {
        self.0
            .lock()
            .values()
            .map(Consumer::drop_prefetched_messages)
            .fold(Ok(()), Result::and)
    }

    pub(crate) fn cancel(&self) -> Result<()> {
        self.0
            .lock()
            .drain()
            .map(|(_, consumer)| consumer.cancel())
            .fold(Ok(()), Result::and)
    }

    pub(crate) fn error(&self, error: Error) -> Result<()> {
        self.0
            .lock()
            .drain()
            .map(|(_, consumer)| consumer.set_error(error.clone()))
            .fold(Ok(()), Result::and)
    }

    pub(crate) fn topology(&self) -> Vec<ConsumerDefinitionInternal> {
        self.0
            .lock()
            .values()
            .map(|consumer| ConsumerDefinitionInternal {
                consumer: Some(consumer.clone()),
                definition: ConsumerDefinition {
                    tag: consumer.tag(),
                    options: consumer.options(),
                    arguments: consumer.arguments(),
                    queue: consumer.queue(),
                },
            })
            .collect()
    }
}

impl fmt::Debug for Consumers {
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        let mut debug = f.debug_tuple("Consumers");
        if let Some(consumers) = self.0.try_lock() {
            debug.field(&*consumers);
        }
        debug.finish()
    }
}
<|endoftext|>


<commit_before>use std::ops::{Mul, Add};

use rand::{Rand, Rng, Closed01};
use num::{Float, Zero};

use rblas::{Gemm, Matrix};
use rblas::attribute::Transpose;

use ndarray::{ArrayBase, DataOwned, DataMut};
use ndarray::blas::{BlasArrayViewMut, AsBlas};

use helpers::random01;

pub trait ShapeAsTuple<T> {
    fn shape_as_tuple(&self) -> T;
}

pub type FloatT = f32;

impl<T> ShapeAsTuple<(usize, usize)> for ArrayBase<Vec<T>, (usize, usize)> {
    #[inline]
    fn shape_as_tuple(&self) -> (usize, usize) {
        (self.shape()[0], self.shape()[1])
    }
}

pub type Ix = (usize, usize);
pub type MyMatrix = ArrayBase<Vec<FloatT>, Ix>;

pub struct OrthogonalNMFBlas {
    pub hidden: MyMatrix,
    pub weights: MyMatrix,

    // these hold temporary results during an iteration.
    // kept in this struct to prevent unnecessary memory allocations.
    pub weights_multiplier: MyMatrix,
    pub weights_divisor: MyMatrix,
    pub weights_divisor_reconstruction: MyMatrix,

    pub hidden_multiplier: MyMatrix,
    pub hidden_divisor: MyMatrix,
    pub hidden_divisor_partial: MyMatrix,

    pub gamma: MyMatrix,
}

impl OrthogonalNMFBlas
    // where FloatT: 'a + Float + Mul + Zero + Clone + Gemm,
          // ArrayBase<Vec<FloatT>, (usize, usize)>: 'a + AsBlas<FloatT, ArrayBase<Vec<FloatT>, (usize, usize)>, (usize, usize)> + DataOwned<Elem=FloatT> + DataMut<Elem=FloatT>,
          // BlasArrayViewMut<'a, FloatT, (usize, usize)>: Matrix<FloatT>,
          // Closed01<FloatT>: Rand
{
    pub fn new_random01<R: Rng>(nhidden: usize, nobserved: usize, nsamples: usize, rng: &mut R) -> OrthogonalNMFBlas {
        let mut hidden = MyMatrix::zeros((nhidden, nobserved));
        for x in hidden.iter_mut() {
            *x = random01(rng);
        }

        let mut weights = MyMatrix::zeros((nsamples, nhidden));
        for x in weights.iter_mut() {
            *x = random01(rng);
        }

        Self::new(hidden, weights)
    }

    pub fn new(hidden: MyMatrix, weights: MyMatrix) -> OrthogonalNMFBlas {
        let weights_shape = weights.shape_as_tuple();
        let hidden_shape = hidden.shape_as_tuple();

        let nhidden = hidden_shape.0;
        let nobserved = hidden_shape.1;
        let nsamples = weights_shape.0;

        let samples_shape = (nsamples, nobserved);

        OrthogonalNMFBlas {
            hidden: hidden,
            weights: weights,

            weights_multiplier: MyMatrix::zeros(weights_shape),
            weights_divisor: MyMatrix::zeros(weights_shape),
            weights_divisor_reconstruction: MyMatrix::zeros(samples_shape),

            hidden_multiplier: MyMatrix::zeros(hidden_shape),
            hidden_divisor: MyMatrix::zeros(hidden_shape),
            hidden_divisor_partial: MyMatrix::zeros((nhidden, nhidden)),

            gamma: OrthogonalNMFBlas::gamma(nhidden),
        }
    }

    /// returns the number of observed variables
    #[inline]
    pub fn nobserved(&self) -> usize {
        self.hidden.shape()[1]
    }

    /// returns the number of hidden variables
    #[inline]
    pub fn nhidden(&self) -> usize {
        self.hidden.shape()[0]
    }

    /// returns the number of data points
    #[inline]
    pub fn nsamples(&self) -> usize {
        self.weights.shape()[0]
    }

    pub fn samples_shape(&self) -> (usize, usize) {
        (self.nsamples(), self.nobserved())
    }

    /// gamma is a symetric matrix with diagonal elements equal to zero
    /// and other elements equal to 1
    pub fn gamma(size: usize) -> MyMatrix {
        let mut gamma = MyMatrix::from_elem((size, size), 1.);
        for x in gamma.diag_mut().iter_mut() {
            *x = 0.
        }
        gamma
    }

    /// `samples * hidden.transpose()`
    #[inline]
    pub fn update_weights_multiplier(&mut self, samples: &mut MyMatrix) {
        Gemm::gemm(
            &1.,
            Transpose::NoTrans, &samples.blas(),
            Transpose::Trans, &self.hidden.blas(),
            &0.,
            &mut self.weights_multiplier.blas());

    }

    /// `weights * hidden * hidden.transpose()`
    #[inline]
    pub fn update_weights_divisor(&mut self) {
        Gemm::gemm(
            &1.,
            Transpose::NoTrans, &self.weights.blas(),
            Transpose::NoTrans, &self.hidden.blas(),
            &0.,
            &mut self.weights_divisor_reconstruction.blas());
        Gemm::gemm(
            &1.,
            Transpose::NoTrans, &self.weights_divisor_reconstruction.blas(),
            Transpose::Trans, &self.hidden.blas(),
            &0.,
            &mut self.weights_divisor.blas());
    }

    /// `weights.transpose() * samples`
    #[inline]
    pub fn update_hidden_multiplier(&mut self, samples: &mut MyMatrix) {
        Gemm::gemm(
            &1.,
            Transpose::Trans, &self.weights.blas(),
            Transpose::NoTrans, &samples.blas(),
            &0.,
            &mut self.hidden_multiplier.blas());
    }

    /// `weights.transpose() * weights * hidden + alpha * gamma * hidden`
    #[inline]
    pub fn update_hidden_divisor(&mut self, alpha: FloatT) {
        // we must make a copy here because we need two mutable
        // references to weights at the same time for .blas()
        // weights_copy.clone_from(&weights);
        // TODO this will be solved once
        // https://github.com/bluss/rust-ndarray/pull/40
        // is merged.
        let mut weights_copy = self.weights.clone();
        Gemm::gemm(
            &1.,
            Transpose::Trans, &self.weights.blas(),
            Transpose::NoTrans, &weights_copy.blas(),
            &0.,
            &mut self.hidden_divisor_partial.blas());
        Gemm::gemm(
            &alpha,
            Transpose::NoTrans, &self.gamma.blas(),
            Transpose::NoTrans, &self.hidden.blas(),
            &0.,
            &mut self.hidden_divisor.blas());
        Gemm::gemm(
            &1.,
            Transpose::NoTrans, &self.hidden_divisor_partial.blas(),
            Transpose::NoTrans, &self.hidden.blas(),
            // add to previous contents of self.hidden_divisor
            &1.,
            &mut self.hidden_divisor.blas());
    }

    /// `result(i,j) <- result(i,j) * multiplier(i,j) / divisor(i,j)`
    #[inline]
    pub fn update_from_multiplier_and_divisor(
        multiplier: &MyMatrix,
        divisor: &MyMatrix,
        result: &mut MyMatrix,
    ) {
        let shape = result.shape_as_tuple();
        assert_eq!(shape, multiplier.shape_as_tuple());
        assert_eq!(shape, divisor.shape_as_tuple());

        for row in 0..shape.0 {
            for col in 0..shape.1 {
                let index = (row, col);
                let mut div = unsafe { divisor.uget(index).clone() };
                // if we have any 0 in any of the matrixes
                // self.weights or self.hidden then
                // divisor will be 0.
                // we can't divide by 0.
                // so we change them to the min positive value.
                if FloatT::zero() == div {
                    div = FloatT::min_positive_value();
                }
                unsafe {
                    *result.uget_mut(index) *= multiplier.uget(index) / div;
                }
            }
        }
    }

    // TODO consider calling this something like iteration_step
    // TODO how many iterations ?
    // TODO compare this to the seoung solution
    /// does one iteration step.
    /// `weights` and `hidden` get better and better with each iteration.
    /// usually around `10000` iterations are required.
    /// `samples` contains one observed per column, one sample per row.
    pub fn iterate(&mut self, alpha: FloatT, samples: &mut MyMatrix) {
        assert_eq!(samples.shape_as_tuple(), self.samples_shape());

        // weights_multiplier <- samples * hidden.transpose()
        self.update_weights_multiplier(samples);
        // weights_divisor <- weights * hidden * hidden.transpose()
        self.update_weights_divisor();
        // hidden_multiplier <- weights.transpose() * samples
        self.update_hidden_multiplier(samples);
        // hidden_divisor <-
        // weights.transpose() * weights * hidden + alpha * gamma * hidden
        self.update_hidden_divisor(alpha);

        // weights(i,j) <-
        // weights(i,j) * weights_multiplier(i,j) / weights_divisor(i,j)
        OrthogonalNMFBlas::update_from_multiplier_and_divisor(
            &self.weights_multiplier,
            &self.weights_divisor,
            &mut self.weights);
        // hidden(i,j) <-
        // hidden(i,j) * hidden_multiplier(i,j) / hidden_divisor(i,j)
        OrthogonalNMFBlas::update_from_multiplier_and_divisor(
            &self.hidden_multiplier,
            &self.hidden_divisor,
            &mut self.hidden);
    }
}
<commit_msg>add comment for potential performance improvement<commit_after>use std::ops::{Mul, Add};

use rand::{Rand, Rng, Closed01};
use num::{Float, Zero};

use rblas::{Gemm, Matrix};
use rblas::attribute::Transpose;

use ndarray::{ArrayBase, DataOwned, DataMut};
use ndarray::blas::{BlasArrayViewMut, AsBlas};

use helpers::random01;

pub trait ShapeAsTuple<T> {
    fn shape_as_tuple(&self) -> T;
}

pub type FloatT = f32;

impl<T> ShapeAsTuple<(usize, usize)> for ArrayBase<Vec<T>, (usize, usize)> {
    #[inline]
    fn shape_as_tuple(&self) -> (usize, usize) {
        (self.shape()[0], self.shape()[1])
    }
}

pub type Ix = (usize, usize);
pub type MyMatrix = ArrayBase<Vec<FloatT>, Ix>;

pub struct OrthogonalNMFBlas {
    pub hidden: MyMatrix,
    pub weights: MyMatrix,

    // these hold temporary results during an iteration.
    // kept in this struct to prevent unnecessary memory allocations.
    pub weights_multiplier: MyMatrix,
    pub weights_divisor: MyMatrix,
    pub weights_divisor_reconstruction: MyMatrix,

    pub hidden_multiplier: MyMatrix,
    pub hidden_divisor: MyMatrix,
    pub hidden_divisor_partial: MyMatrix,

    pub gamma: MyMatrix,
}

impl OrthogonalNMFBlas
    // where FloatT: 'a + Float + Mul + Zero + Clone + Gemm,
          // ArrayBase<Vec<FloatT>, (usize, usize)>: 'a + AsBlas<FloatT, ArrayBase<Vec<FloatT>, (usize, usize)>, (usize, usize)> + DataOwned<Elem=FloatT> + DataMut<Elem=FloatT>,
          // BlasArrayViewMut<'a, FloatT, (usize, usize)>: Matrix<FloatT>,
          // Closed01<FloatT>: Rand
{
    pub fn new_random01<R: Rng>(nhidden: usize, nobserved: usize, nsamples: usize, rng: &mut R) -> OrthogonalNMFBlas {
        let mut hidden = MyMatrix::zeros((nhidden, nobserved));
        for x in hidden.iter_mut() {
            *x = random01(rng);
        }

        let mut weights = MyMatrix::zeros((nsamples, nhidden));
        for x in weights.iter_mut() {
            *x = random01(rng);
        }

        Self::new(hidden, weights)
    }

    pub fn new(hidden: MyMatrix, weights: MyMatrix) -> OrthogonalNMFBlas {
        let weights_shape = weights.shape_as_tuple();
        let hidden_shape = hidden.shape_as_tuple();

        let nhidden = hidden_shape.0;
        let nobserved = hidden_shape.1;
        let nsamples = weights_shape.0;

        let samples_shape = (nsamples, nobserved);

        OrthogonalNMFBlas {
            hidden: hidden,
            weights: weights,

            weights_multiplier: MyMatrix::zeros(weights_shape),
            weights_divisor: MyMatrix::zeros(weights_shape),
            weights_divisor_reconstruction: MyMatrix::zeros(samples_shape),

            hidden_multiplier: MyMatrix::zeros(hidden_shape),
            hidden_divisor: MyMatrix::zeros(hidden_shape),
            hidden_divisor_partial: MyMatrix::zeros((nhidden, nhidden)),

            gamma: OrthogonalNMFBlas::gamma(nhidden),
        }
    }

    /// returns the number of observed variables
    #[inline]
    pub fn nobserved(&self) -> usize {
        self.hidden.shape()[1]
    }

    /// returns the number of hidden variables
    #[inline]
    pub fn nhidden(&self) -> usize {
        self.hidden.shape()[0]
    }

    /// returns the number of data points
    #[inline]
    pub fn nsamples(&self) -> usize {
        self.weights.shape()[0]
    }

    pub fn samples_shape(&self) -> (usize, usize) {
        (self.nsamples(), self.nobserved())
    }

    /// gamma is a symetric matrix with diagonal elements equal to zero
    /// and other elements equal to 1
    pub fn gamma(size: usize) -> MyMatrix {
        let mut gamma = MyMatrix::from_elem((size, size), 1.);
        for x in gamma.diag_mut().iter_mut() {
            *x = 0.
        }
        gamma
    }

    /// `samples * hidden.transpose()`
    #[inline]
    pub fn update_weights_multiplier(&mut self, samples: &mut MyMatrix) {
        Gemm::gemm(
            &1.,
            Transpose::NoTrans, &samples.blas(),
            Transpose::Trans, &self.hidden.blas(),
            &0.,
            &mut self.weights_multiplier.blas());

    }

    /// `weights * hidden * hidden.transpose()`
    #[inline]
    pub fn update_weights_divisor(&mut self) {
        // TODO weights * hidden which we compute first is really large
        // we could first compute partial = hidden * hidden.transpose()
        // partial.shape() = (nhidden, nhidden)
        // and then in the second step compute
        // weights * partial.
        // however since we can't alias hidden we need to copy
        // it for this.
        // the question is whether the copying is faster than
        // the larger matrix multiplication. i guess it is.
        // TODO multiplying a matrix by its transpose is symetric
        // can we exploit that ?
        Gemm::gemm(
            &1.,
            Transpose::NoTrans, &self.weights.blas(),
            Transpose::NoTrans, &self.hidden.blas(),
            &0.,
            &mut self.weights_divisor_reconstruction.blas());
        Gemm::gemm(
            &1.,
            Transpose::NoTrans, &self.weights_divisor_reconstruction.blas(),
            Transpose::Trans, &self.hidden.blas(),
            &0.,
            &mut self.weights_divisor.blas());
    }

    /// `weights.transpose() * samples`
    #[inline]
    pub fn update_hidden_multiplier(&mut self, samples: &mut MyMatrix) {
        Gemm::gemm(
            &1.,
            Transpose::Trans, &self.weights.blas(),
            Transpose::NoTrans, &samples.blas(),
            &0.,
            &mut self.hidden_multiplier.blas());
    }

    /// `weights.transpose() * weights * hidden + alpha * gamma * hidden`
    #[inline]
    pub fn update_hidden_divisor(&mut self, alpha: FloatT) {
        // we must make a copy here because we need two mutable
        // references to weights at the same time for .blas()
        // weights_copy.clone_from(&weights);
        // TODO this will be solved once
        // https://github.com/bluss/rust-ndarray/pull/40
        // is merged.
        let mut weights_copy = self.weights.clone();
        Gemm::gemm(
            &1.,
            Transpose::Trans, &self.weights.blas(),
            Transpose::NoTrans, &weights_copy.blas(),
            &0.,
            &mut self.hidden_divisor_partial.blas());
        Gemm::gemm(
            &alpha,
            Transpose::NoTrans, &self.gamma.blas(),
            Transpose::NoTrans, &self.hidden.blas(),
            &0.,
            &mut self.hidden_divisor.blas());
        Gemm::gemm(
            &1.,
            Transpose::NoTrans, &self.hidden_divisor_partial.blas(),
            Transpose::NoTrans, &self.hidden.blas(),
            // add to previous contents of self.hidden_divisor
            &1.,
            &mut self.hidden_divisor.blas());
    }

    /// `result(i,j) <- result(i,j) * multiplier(i,j) / divisor(i,j)`
    #[inline]
    pub fn update_from_multiplier_and_divisor(
        multiplier: &MyMatrix,
        divisor: &MyMatrix,
        result: &mut MyMatrix,
    ) {
        let shape = result.shape_as_tuple();
        assert_eq!(shape, multiplier.shape_as_tuple());
        assert_eq!(shape, divisor.shape_as_tuple());

        for row in 0..shape.0 {
            for col in 0..shape.1 {
                let index = (row, col);
                let mut div = unsafe { divisor.uget(index).clone() };
                // if we have any 0 in any of the matrixes
                // self.weights or self.hidden then
                // divisor will be 0.
                // we can't divide by 0.
                // so we change them to the min positive value.
                if FloatT::zero() == div {
                    div = FloatT::min_positive_value();
                }
                unsafe {
                    *result.uget_mut(index) *= multiplier.uget(index) / div;
                }
            }
        }
    }

    // TODO consider calling this something like iteration_step
    // TODO how many iterations ?
    // TODO compare this to the seoung solution
    /// does one iteration step.
    /// `weights` and `hidden` get better and better with each iteration.
    /// usually around `10000` iterations are required.
    /// `samples` contains one observed per column, one sample per row.
    pub fn iterate(&mut self, alpha: FloatT, samples: &mut MyMatrix) {
        assert_eq!(samples.shape_as_tuple(), self.samples_shape());

        // weights_multiplier <- samples * hidden.transpose()
        self.update_weights_multiplier(samples);
        // weights_divisor <- weights * hidden * hidden.transpose()
        self.update_weights_divisor();
        // hidden_multiplier <- weights.transpose() * samples
        self.update_hidden_multiplier(samples);
        // hidden_divisor <-
        // weights.transpose() * weights * hidden + alpha * gamma * hidden
        self.update_hidden_divisor(alpha);

        // weights(i,j) <-
        // weights(i,j) * weights_multiplier(i,j) / weights_divisor(i,j)
        OrthogonalNMFBlas::update_from_multiplier_and_divisor(
            &self.weights_multiplier,
            &self.weights_divisor,
            &mut self.weights);
        // hidden(i,j) <-
        // hidden(i,j) * hidden_multiplier(i,j) / hidden_divisor(i,j)
        OrthogonalNMFBlas::update_from_multiplier_and_divisor(
            &self.hidden_multiplier,
            &self.hidden_divisor,
            &mut self.hidden);
    }
}
<|endoftext|>


<commit_before>use state::editor::Editor;
use io::parse::{Inst, Parameter};
use state::mode::{Mode, CommandMode, PrimitiveMode};
use edit::insert::{InsertOptions, InsertMode};
use io::redraw::RedrawTask;
use edit::buffer::TextBuffer;
use core::prompt::PromptCommand;

// TODO: Move the command definitions outta here
impl Editor {
    /// Execute an instruction
    pub fn exec(&mut self, Inst(para, cmd): Inst) {
        use io::key::Key::*;
        use state::mode::Mode::*;
        use state::mode::PrimitiveMode::*;
        use state::mode::CommandMode::*;

        let n = para.d();
        let bef = self.pos();
        let mut mov = false;

        match (self.cursor().mode, cmd.key) {
            (Primitive(Prompt), Char(' ')) if self.key_state.shift => {
                self.prompt = String::new();
                self.cursor_mut().mode = Mode::Command(CommandMode::Normal);
            }
            (Primitive(Insert(_)), Escape) => {
                let left = self.left(1);
                self.goto(left);
                self.cursor_mut().mode = Mode::Command(CommandMode::Normal);
            },
            (Primitive(Insert(_)), Char(' ')) if self.key_state.shift => {
                let left = self.left(1);
                self.goto(left);
                self.cursor_mut().mode = Mode::Command(CommandMode::Normal);
            }
            (_, Char(' ')) if self.key_state.shift => {
                self.cursor_mut().mode = Mode::Command(CommandMode::Normal)
            }
            (_, Char(' ')) if self.key_state.alt => self.next_cursor(),
            _ if self.key_state.alt => {
                if let Some(m) = self.to_motion(Inst(para, cmd)) {
                    self.goto(m);
                }
            }
            (Command(Normal), Char('i')) => {
                self.cursor_mut().mode =
                    Mode::Primitive(PrimitiveMode::Insert(InsertOptions {
                                                              mode: InsertMode::Insert,
                                                          }));

            }
            (Command(Normal), Char('a')) => {
                let pos = self.right(1, false);
                self.goto(pos);
                self.cursor_mut().mode =
                    Mode::Primitive(PrimitiveMode::Insert(InsertOptions {
                                                              mode: InsertMode::Insert,
                                                          }));

            }
            (Command(Normal), Char('o')) => {
                let y = self.y();
                let ind = if self.options.autoindent {
                    self.buffers
                        .current_buffer()
                        .get_indent(y)
                        .to_owned()
                } else {
                    String::new()
                };
                let last = ind.len();
                self.buffers.current_buffer_mut().insert_line(y + 1, ind.into());
                self.goto((last, y + 1));
                self.cursor_mut().mode =
                    Mode::Primitive(PrimitiveMode::Insert(InsertOptions {
                                                              mode: InsertMode::Insert,
                                                          }));
            }
            (Command(Normal), Char('h')) => {
                let left = self.left(n);
                self.goto(left);
                mov = true;
            }
            (Command(Normal), Char('j')) => {
                let down = self.down(n);
                self.goto(down);
                mov = true;
            }
            (Command(Normal), Char('k')) => {
                let up = self.up(n);
                self.goto(up);
                mov = true;
            }
            (Command(Normal), Char('l')) => {
                let right = self.right(n, true);
                self.goto(right);
                mov = true;
            }
            (Command(Normal), Char('J')) => {
                let down = self.down(15 * n);
                self.goto(down);
                mov = true;
            }
            (Command(Normal), Char('K')) => {
                let up = self.up(15 * n);
                self.goto(up);
                mov = true;
            }
            (Command(Normal), Char('x')) => {
                self.delete();
                let bounded = self.bound(self.pos(), true);
                self.goto(bounded);
            }
            (Command(Normal), Char('X')) => {
                self.backspace();
                let bounded = self.bound(self.pos(), true);
                self.goto(bounded);
            }
            (Command(Normal), Char('L')) => {
                let ln_end = (self.buffers.current_buffer()[self.y()].len(), self.y());
                self.goto(ln_end);
                mov = true;
            }
            (Command(Normal), Char('H')) => {
                self.cursor_mut().x = 0;
                mov = true;
            }
            (Command(Normal), Char('r')) => {
                let (x, y) = self.pos();
                let c = self.get_char();
                let current_buffer = self.buffers.current_buffer_info_mut();
                // If there is nothing in the current buffer
                // ignore the command
                if current_buffer.raw_buffer[y].len() > 0 {
                    current_buffer.raw_buffer[y].remove(x);
                }
                current_buffer.raw_buffer[y].insert(x, c);
            }
            (Command(Normal), Char('R')) => {
                self.cursor_mut().mode =
                    Mode::Primitive(PrimitiveMode::Insert(InsertOptions {
                                                              mode: InsertMode::Replace,
                                                          }));
            }
            (Command(Normal), Char('d')) => {
                let ins = self.get_inst();
                if let Some(m) = self.to_motion_unbounded(ins) {
                    self.remove_rb(m);
                }
            }
            (Command(Normal), Char('G')) => {
                let last = self.buffers.current_buffer().len() - 1;
                self.goto((0, last));
                mov = true;
            }
            (Command(Normal), Char('g')) => {
                if let Parameter::Int(n) = para {
                    self.goto((0, n.wrapping_sub(1)));
                    mov = true;
                } else {
                    let inst = self.get_inst();
                    if let Some(m) = self.to_motion(inst) {
                        self.goto(m); // fix
                        mov = true;
                    }
                }

            }
            (Command(Normal), Char('b')) => {
                // Branch cursor
                if self.buffers
                       .current_buffer_info()
                       .cursors
                       .len() < 255 {
                    let cursor = self.cursor().clone();
                    let current_cursor_index = self.buffers.current_buffer_info().current_cursor as
                                               usize;
                    self.buffers
                        .current_buffer_info_mut()
                        .cursors
                        .insert(current_cursor_index, cursor);
                    self.next_cursor();
                } else {
                    self.status_bar.msg = format!("At max 255 cursors");
                }
            }
            (Command(Normal), Char('B')) => {
                // Delete cursor
                if self.buffers
                       .current_buffer_info()
                       .cursors
                       .len() > 1 {
                    let current_cursor_index = self.buffers.current_buffer_info().current_cursor;
                    self.buffers
                        .current_buffer_info_mut()
                        .cursors
                        .remove(current_cursor_index as usize);
                    self.prev_cursor();
                } else {
                    self.status_bar.msg = format!("No other cursors!");
                }
            }
            (Command(Normal), Char('t')) => {
                let ch = self.get_char();

                let pos = self.next_ocur(ch, n);
                if let Some(p) = pos {
                    let y = self.y();
                    self.goto((p, y));
                    mov = true;
                }
            }
            (Command(Normal), Char('f')) => {
                let ch = self.get_char();

                let pos = self.previous_ocur(ch, n);
                if let Some(p) = pos {
                    let y = self.y();
                    self.goto((p, y));
                    mov = true;
                }
            }
            (Command(Normal), Char('W')) => {
                let pos = self._next_word_forward(n);
                if let Some(p) = pos {
                    let y = self.y();
                    self.goto((p, y));
                    mov = true;
                }
            }
            (Command(Normal), Char(';')) => {
                self.cursor_mut().mode = Mode::Primitive(PrimitiveMode::Prompt)
            }
            (Command(Normal), Char(' ')) => self.next_cursor(),
            (Command(Normal), Char('z')) => {
                let Inst(param, cmd) = self.get_inst();
                match param {
                    Parameter::Null => {
                        if let Some(m) = self.to_motion(Inst(param, cmd)) {
                            self.buffers.current_buffer_info_mut().scroll_y = m.1;
                            self.goto(m);
                        }
                    }
                    Parameter::Int(n) => {
                        self.buffers.current_buffer_info_mut().scroll_y = n;
                    }
                }
                self.redraw_task = RedrawTask::Full;
            }
            (Command(Normal), Char('Z')) => {
                self.buffers.current_buffer_info_mut().scroll_y = self.y() - 3;
                self.redraw_task = RedrawTask::Full;
            }
            (Command(Normal), Char('~')) => {
                self.invert_chars(n);
            }
            (Command(Normal), Char('.')) => {
                if let Some(inst) = self.previous_instruction {
                    self.exec(inst);
                } else {
                    self.status_bar.msg = "No previous command".into();
                    self.redraw_task = RedrawTask::StatusBar;
                }
            }
            (Command(Normal), Char(c)) => {
                self.status_bar.msg = format!("Unknown command: {}", c);
                self.redraw_task = RedrawTask::StatusBar;
            }
            (Primitive(Insert(opt)), k) => self.insert(k, opt),
            (Primitive(Prompt), Char('\n')) => {
                self.cursor_mut().mode = Command(Normal);
                if let Some(cmd) = PromptCommand::parse(&self.prompt.clone()) {;
                    self.invoke(cmd);
                    self.prompt = String::new();
                    self.redraw_task = RedrawTask::StatusBar;
                } else {
                    self.status_bar.msg = format!("Unknown command: {}", self.prompt);
                }
            }
            (Primitive(Prompt), Backspace) => {
                self.prompt.pop();
                self.redraw_task = RedrawTask::StatusBar;
            }
            (Primitive(Prompt), Char(c)) => {
                self.prompt.push(c);
                self.redraw_task = RedrawTask::StatusBar;
            }
            _ => {
                self.status_bar.msg = format!("Unknown command");
                self.redraw_task = RedrawTask::StatusBar;
            }
        }
        if mov {
            self.redraw_task = RedrawTask::Cursor(bef, self.pos());
        }

        if !(self.cursor().mode == Command(Normal) && cmd.key == Char('.')) {
            self.previous_instruction = Some(Inst(para, cmd));
        }
    }
}
<commit_msg>Implement 'I', 'A' and 'c' commands.<commit_after>use state::editor::Editor;
use io::parse::{Inst, Parameter};
use state::mode::{Mode, CommandMode, PrimitiveMode};
use edit::insert::{InsertOptions, InsertMode};
use io::redraw::RedrawTask;
use edit::buffer::TextBuffer;
use core::prompt::PromptCommand;

// TODO: Move the command definitions outta here
impl Editor {
    /// Execute an instruction
    pub fn exec(&mut self, Inst(para, cmd): Inst) {
        use io::key::Key::*;
        use state::mode::Mode::*;
        use state::mode::PrimitiveMode::*;
        use state::mode::CommandMode::*;

        let n = para.d();
        let bef = self.pos();
        let mut mov = false;

        match (self.cursor().mode, cmd.key) {
            (Primitive(Prompt), Char(' ')) if self.key_state.shift => {
                self.prompt = String::new();
                self.cursor_mut().mode = Mode::Command(CommandMode::Normal);
            }
            (Primitive(Insert(_)), Escape) => {
                let left = self.left(1);
                self.goto(left);
                self.cursor_mut().mode = Mode::Command(CommandMode::Normal);
            },
            (Primitive(Insert(_)), Char(' ')) if self.key_state.shift => {
                let left = self.left(1);
                self.goto(left);
                self.cursor_mut().mode = Mode::Command(CommandMode::Normal);
            }
            (_, Char(' ')) if self.key_state.shift => {
                self.cursor_mut().mode = Mode::Command(CommandMode::Normal)
            }
            (_, Char(' ')) if self.key_state.alt => self.next_cursor(),
            _ if self.key_state.alt => {
                if let Some(m) = self.to_motion(Inst(para, cmd)) {
                    self.goto(m);
                }
            }
            (Command(Normal), Char('i')) => {
                self.cursor_mut().mode =
                    Mode::Primitive(PrimitiveMode::Insert(InsertOptions {
                                                              mode: InsertMode::Insert,
                                                          }));

            }
            (Command(Normal), Char('I')) => {
                self.cursor_mut().x = 0;
                self.cursor_mut().mode =
                    Mode::Primitive(PrimitiveMode::Insert(InsertOptions {
                                                              mode: InsertMode::Insert,
                                                          }));

            }
            (Command(Normal), Char('a')) => {
                let pos = self.right(1, false);
                self.goto(pos);
                self.cursor_mut().mode =
                    Mode::Primitive(PrimitiveMode::Insert(InsertOptions {
                                                              mode: InsertMode::Insert,
                                                          }));
            }
            (Command(Normal), Char('A')) => {
                let pos = (self.buffers.current_buffer()[self.y()].len(), self.y());
                //let pos = self.right(1, false);
                self.goto(pos);
                self.cursor_mut().mode =
                    Mode::Primitive(PrimitiveMode::Insert(InsertOptions {
                                                              mode: InsertMode::Insert,
                                                          }));
            }
            (Command(Normal), Char('o')) => {
                let y = self.y();
                let ind = if self.options.autoindent {
                    self.buffers
                        .current_buffer()
                        .get_indent(y)
                        .to_owned()
                } else {
                    String::new()
                };
                let last = ind.len();
                self.buffers.current_buffer_mut().insert_line(y + 1, ind.into());
                self.goto((last, y + 1));
                self.cursor_mut().mode =
                    Mode::Primitive(PrimitiveMode::Insert(InsertOptions {
                                                              mode: InsertMode::Insert,
                                                          }));
            }
            (Command(Normal), Char('h')) => {
                let left = self.left(n);
                self.goto(left);
                mov = true;
            }
            (Command(Normal), Char('j')) => {
                let down = self.down(n);
                self.goto(down);
                mov = true;
            }
            (Command(Normal), Char('k')) => {
                let up = self.up(n);
                self.goto(up);
                mov = true;
            }
            (Command(Normal), Char('l')) => {
                let right = self.right(n, true);
                self.goto(right);
                mov = true;
            }
            (Command(Normal), Char('J')) => {
                let down = self.down(15 * n);
                self.goto(down);
                mov = true;
            }
            (Command(Normal), Char('K')) => {
                let up = self.up(15 * n);
                self.goto(up);
                mov = true;
            }
            (Command(Normal), Char('x')) => {
                self.delete();
                let bounded = self.bound(self.pos(), true);
                self.goto(bounded);
            }
            (Command(Normal), Char('X')) => {
                self.backspace();
                let bounded = self.bound(self.pos(), true);
                self.goto(bounded);
            }
            (Command(Normal), Char('L')) => {
                let ln_end = (self.buffers.current_buffer()[self.y()].len() - 1, self.y());
                self.goto(ln_end);
                mov = true;
            }
            (Command(Normal), Char('H')) => {
                self.cursor_mut().x = 0;
                mov = true;
            }
            (Command(Normal), Char('r')) => {
                let (x, y) = self.pos();
                let c = self.get_char();
                let current_buffer = self.buffers.current_buffer_info_mut();
                // If there is nothing in the current buffer
                // ignore the command
                if current_buffer.raw_buffer[y].len() > 0 {
                    current_buffer.raw_buffer[y].remove(x);
                }
                current_buffer.raw_buffer[y].insert(x, c);
            }
            (Command(Normal), Char('R')) => {
                self.cursor_mut().mode =
                    Mode::Primitive(PrimitiveMode::Insert(InsertOptions {
                                                              mode: InsertMode::Replace,
                                                          }));
            }
            (Command(Normal), Char('d')) => {
                let ins = self.get_inst();
                if let Some(m) = self.to_motion_unbounded(ins) {
                    self.remove_rb(m);
                }
            }
            (Command(Normal), Char('c')) => {
                let ins = self.get_inst();
                if let Some(m) = self.to_motion_unbounded(ins) {
                    self.remove_rb(m);
                    self.cursor_mut().mode =
                        Mode::Primitive(PrimitiveMode::Insert(InsertOptions {
                                                                  mode: InsertMode::Insert,
                                                           }));
                }
            }
            (Command(Normal), Char('G')) => {
                let last = self.buffers.current_buffer().len() - 1;
                self.goto((0, last));
                mov = true;
            }
            (Command(Normal), Char('g')) => {
                if let Parameter::Int(n) = para {
                    self.goto((0, n.wrapping_sub(1)));
                    mov = true;
                } else {
                    let inst = self.get_inst();
                    if let Some(m) = self.to_motion(inst) {
                        self.goto(m); // fix
                        mov = true;
                    }
                }

            }
            (Command(Normal), Char('b')) => {
                // Branch cursor
                if self.buffers
                       .current_buffer_info()
                       .cursors
                       .len() < 255 {
                    let cursor = self.cursor().clone();
                    let current_cursor_index = self.buffers.current_buffer_info().current_cursor as
                                               usize;
                    self.buffers
                        .current_buffer_info_mut()
                        .cursors
                        .insert(current_cursor_index, cursor);
                    self.next_cursor();
                } else {
                    self.status_bar.msg = format!("At max 255 cursors");
                }
            }
            (Command(Normal), Char('B')) => {
                // Delete cursor
                if self.buffers
                       .current_buffer_info()
                       .cursors
                       .len() > 1 {
                    let current_cursor_index = self.buffers.current_buffer_info().current_cursor;
                    self.buffers
                        .current_buffer_info_mut()
                        .cursors
                        .remove(current_cursor_index as usize);
                    self.prev_cursor();
                } else {
                    self.status_bar.msg = format!("No other cursors!");
                }
            }
            (Command(Normal), Char('t')) => {
                let ch = self.get_char();

                let pos = self.next_ocur(ch, n);
                if let Some(p) = pos {
                    let y = self.y();
                    self.goto((p, y));
                    mov = true;
                }
            }
            (Command(Normal), Char('f')) => {
                let ch = self.get_char();

                let pos = self.previous_ocur(ch, n);
                if let Some(p) = pos {
                    let y = self.y();
                    self.goto((p, y));
                    mov = true;
                }
            }
            (Command(Normal), Char('W')) => {
                let pos = self._next_word_forward(n);
                if let Some(p) = pos {
                    let y = self.y();
                    self.goto((p, y));
                    mov = true;
                }
            }
            (Command(Normal), Char(';')) => {
                self.cursor_mut().mode = Mode::Primitive(PrimitiveMode::Prompt)
            }
            (Command(Normal), Char(' ')) => self.next_cursor(),
            (Command(Normal), Char('z')) => {
                let Inst(param, cmd) = self.get_inst();
                match param {
                    Parameter::Null => {
                        if let Some(m) = self.to_motion(Inst(param, cmd)) {
                            self.buffers.current_buffer_info_mut().scroll_y = m.1;
                            self.goto(m);
                        }
                    }
                    Parameter::Int(n) => {
                        self.buffers.current_buffer_info_mut().scroll_y = n;
                    }
                }
                self.redraw_task = RedrawTask::Full;
            }
            (Command(Normal), Char('Z')) => {
                self.buffers.current_buffer_info_mut().scroll_y = self.y() - 3;
                self.redraw_task = RedrawTask::Full;
            }
            (Command(Normal), Char('~')) => {
                self.invert_chars(n);
            }
            (Command(Normal), Char('.')) => {
                if let Some(inst) = self.previous_instruction {
                    self.exec(inst);
                } else {
                    self.status_bar.msg = "No previous command".into();
                    self.redraw_task = RedrawTask::StatusBar;
                }
            }
            (Command(Normal), Char(c)) => {
                self.status_bar.msg = format!("Unknown command: {}", c);
                self.redraw_task = RedrawTask::StatusBar;
            }
            (Primitive(Insert(opt)), k) => self.insert(k, opt),
            (Primitive(Prompt), Char('\n')) => {
                self.cursor_mut().mode = Command(Normal);
                if let Some(cmd) = PromptCommand::parse(&self.prompt.clone()) {;
                    self.invoke(cmd);
                    self.prompt = String::new();
                    self.redraw_task = RedrawTask::StatusBar;
                } else {
                    self.status_bar.msg = format!("Unknown command: {}", self.prompt);
                }
            }
            (Primitive(Prompt), Backspace) => {
                self.prompt.pop();
                self.redraw_task = RedrawTask::StatusBar;
            }
            (Primitive(Prompt), Char(c)) => {
                self.prompt.push(c);
                self.redraw_task = RedrawTask::StatusBar;
            }
            _ => {
                self.status_bar.msg = format!("Unknown command");
                self.redraw_task = RedrawTask::StatusBar;
            }
        }
        if mov {
            self.redraw_task = RedrawTask::Cursor(bef, self.pos());
        }

        if !(self.cursor().mode == Command(Normal) && cmd.key == Char('.')) {
            self.previous_instruction = Some(Inst(para, cmd));
        }
    }
}
<|endoftext|>


<commit_before>//! Job management on Windows for bootstrapping
//!
//! Most of the time when you're running a build system (e.g., make) you expect
//! Ctrl-C or abnormal termination to actually terminate the entire tree of
//! process in play, not just the one at the top. This currently works "by
//! default" on Unix platforms because Ctrl-C actually sends a signal to the
//! *process group* rather than the parent process, so everything will get torn
//! down. On Windows, however, this does not happen and Ctrl-C just kills the
//! parent process.
//!
//! To achieve the same semantics on Windows we use Job Objects to ensure that
//! all processes die at the same time. Job objects have a mode of operation
//! where when all handles to the object are closed it causes all child
//! processes associated with the object to be terminated immediately.
//! Conveniently whenever a process in the job object spawns a new process the
//! child will be associated with the job object as well. This means if we add
//! ourselves to the job object we create then everything will get torn down!
//!
//! Unfortunately most of the time the build system is actually called from a
//! python wrapper (which manages things like building the build system) so this
//! all doesn't quite cut it so far. To go the last mile we duplicate the job
//! object handle into our parent process (a python process probably) and then
//! close our own handle. This means that the only handle to the job object
//! resides in the parent python process, so when python dies the whole build
//! system dies (as one would probably expect!).
//!
//! Note that this module has a #[cfg(windows)] above it as none of this logic
//! is required on Unix.

#![allow(nonstandard_style, dead_code)]

use crate::Build;
use std::env;
use std::io;
use std::mem;
use std::ptr;

use winapi::shared::minwindef::{DWORD, FALSE, LPVOID};
use winapi::um::errhandlingapi::SetErrorMode;
use winapi::um::handleapi::{CloseHandle, DuplicateHandle};
use winapi::um::jobapi2::{AssignProcessToJobObject, CreateJobObjectW, SetInformationJobObject};
use winapi::um::processthreadsapi::{GetCurrentProcess, OpenProcess};
use winapi::um::winbase::{BELOW_NORMAL_PRIORITY_CLASS, SEM_NOGPFAULTERRORBOX};
use winapi::um::winnt::{
    JobObjectExtendedLimitInformation, DUPLICATE_SAME_ACCESS, JOBOBJECT_EXTENDED_LIMIT_INFORMATION,
    JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE, JOB_OBJECT_LIMIT_PRIORITY_CLASS, PROCESS_DUP_HANDLE,
};

pub unsafe fn setup(build: &mut Build) {
    // Enable the Windows Error Reporting dialog which msys disables,
    // so we can JIT debug rustc
    let mode = SetErrorMode(0);
    SetErrorMode(mode & !SEM_NOGPFAULTERRORBOX);

    // Create a new job object for us to use
    let job = CreateJobObjectW(ptr::null_mut(), ptr::null());
    assert!(!job.is_null(), "{}", io::Error::last_os_error());

    // Indicate that when all handles to the job object are gone that all
    // process in the object should be killed. Note that this includes our
    // entire process tree by default because we've added ourselves and our
    // children will reside in the job by default.
    let mut info = mem::zeroed::<JOBOBJECT_EXTENDED_LIMIT_INFORMATION>();
    info.BasicLimitInformation.LimitFlags = JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE;
    if build.config.low_priority {
        info.BasicLimitInformation.LimitFlags |= JOB_OBJECT_LIMIT_PRIORITY_CLASS;
        info.BasicLimitInformation.PriorityClass = BELOW_NORMAL_PRIORITY_CLASS;
    }
    let r = SetInformationJobObject(
        job,
        JobObjectExtendedLimitInformation,
        &mut info as *mut _ as LPVOID,
        mem::size_of_val(&info) as DWORD,
    );
    assert!(r != 0, "{}", io::Error::last_os_error());

    // Assign our process to this job object. Note that if this fails, one very
    // likely reason is that we are ourselves already in a job object! This can
    // happen on the build bots that we've got for Windows, or if just anyone
    // else is instrumenting the build. In this case we just bail out
    // immediately and assume that they take care of it.
    //
    // Also note that nested jobs (why this might fail) are supported in recent
    // versions of Windows, but the version of Windows that our bots are running
    // at least don't support nested job objects.
    let r = AssignProcessToJobObject(job, GetCurrentProcess());
    if r == 0 {
        CloseHandle(job);
        return;
    }

    // If we've got a parent process (e.g., the python script that called us)
    // then move ownership of this job object up to them. That way if the python
    // script is killed (e.g., via ctrl-c) then we'll all be torn down.
    //
    // If we don't have a parent (e.g., this was run directly) then we
    // intentionally leak the job object handle. When our process exits
    // (normally or abnormally) it will close the handle implicitly, causing all
    // processes in the job to be cleaned up.
    let pid = match env::var("BOOTSTRAP_PARENT_ID") {
        Ok(s) => s,
        Err(..) => return,
    };

    let parent = OpenProcess(PROCESS_DUP_HANDLE, FALSE, pid.parse().unwrap());
    assert!(
        !parent.is_null(),
        "PID `{}` doesn't seem to exist: {}",
        pid,
        io::Error::last_os_error()
    );
    let mut parent_handle = ptr::null_mut();
    let r = DuplicateHandle(
        GetCurrentProcess(),
        job,
        parent,
        &mut parent_handle,
        0,
        FALSE,
        DUPLICATE_SAME_ACCESS,
    );

    // If this failed, well at least we tried! An example of DuplicateHandle
    // failing in the past has been when the wrong python2 package spawned this
    // build system (e.g., the `python2` package in MSYS instead of
    // `mingw-w64-x86_64-python2`. Not sure why it failed, but the "failure
    // mode" here is that we only clean everything up when the build system
    // dies, not when the python parent does, so not too bad.
    if r != 0 {
        CloseHandle(job);
    }
}
<commit_msg>Fix panics on Windows when the build was cancelled<commit_after>//! Job management on Windows for bootstrapping
//!
//! Most of the time when you're running a build system (e.g., make) you expect
//! Ctrl-C or abnormal termination to actually terminate the entire tree of
//! process in play, not just the one at the top. This currently works "by
//! default" on Unix platforms because Ctrl-C actually sends a signal to the
//! *process group* rather than the parent process, so everything will get torn
//! down. On Windows, however, this does not happen and Ctrl-C just kills the
//! parent process.
//!
//! To achieve the same semantics on Windows we use Job Objects to ensure that
//! all processes die at the same time. Job objects have a mode of operation
//! where when all handles to the object are closed it causes all child
//! processes associated with the object to be terminated immediately.
//! Conveniently whenever a process in the job object spawns a new process the
//! child will be associated with the job object as well. This means if we add
//! ourselves to the job object we create then everything will get torn down!
//!
//! Unfortunately most of the time the build system is actually called from a
//! python wrapper (which manages things like building the build system) so this
//! all doesn't quite cut it so far. To go the last mile we duplicate the job
//! object handle into our parent process (a python process probably) and then
//! close our own handle. This means that the only handle to the job object
//! resides in the parent python process, so when python dies the whole build
//! system dies (as one would probably expect!).
//!
//! Note that this module has a #[cfg(windows)] above it as none of this logic
//! is required on Unix.

#![allow(nonstandard_style, dead_code)]

use crate::Build;
use std::env;
use std::io;
use std::mem;
use std::ptr;

use winapi::shared::minwindef::{DWORD, FALSE, LPVOID};
use winapi::um::errhandlingapi::SetErrorMode;
use winapi::um::handleapi::{CloseHandle, DuplicateHandle};
use winapi::um::jobapi2::{AssignProcessToJobObject, CreateJobObjectW, SetInformationJobObject};
use winapi::um::processthreadsapi::{GetCurrentProcess, OpenProcess};
use winapi::um::winbase::{BELOW_NORMAL_PRIORITY_CLASS, SEM_NOGPFAULTERRORBOX};
use winapi::um::winnt::{
    JobObjectExtendedLimitInformation, DUPLICATE_SAME_ACCESS, JOBOBJECT_EXTENDED_LIMIT_INFORMATION,
    JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE, JOB_OBJECT_LIMIT_PRIORITY_CLASS, PROCESS_DUP_HANDLE,
};

pub unsafe fn setup(build: &mut Build) {
    // Enable the Windows Error Reporting dialog which msys disables,
    // so we can JIT debug rustc
    let mode = SetErrorMode(0);
    SetErrorMode(mode & !SEM_NOGPFAULTERRORBOX);

    // Create a new job object for us to use
    let job = CreateJobObjectW(ptr::null_mut(), ptr::null());
    assert!(!job.is_null(), "{}", io::Error::last_os_error());

    // Indicate that when all handles to the job object are gone that all
    // process in the object should be killed. Note that this includes our
    // entire process tree by default because we've added ourselves and our
    // children will reside in the job by default.
    let mut info = mem::zeroed::<JOBOBJECT_EXTENDED_LIMIT_INFORMATION>();
    info.BasicLimitInformation.LimitFlags = JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE;
    if build.config.low_priority {
        info.BasicLimitInformation.LimitFlags |= JOB_OBJECT_LIMIT_PRIORITY_CLASS;
        info.BasicLimitInformation.PriorityClass = BELOW_NORMAL_PRIORITY_CLASS;
    }
    let r = SetInformationJobObject(
        job,
        JobObjectExtendedLimitInformation,
        &mut info as *mut _ as LPVOID,
        mem::size_of_val(&info) as DWORD,
    );
    assert!(r != 0, "{}", io::Error::last_os_error());

    // Assign our process to this job object. Note that if this fails, one very
    // likely reason is that we are ourselves already in a job object! This can
    // happen on the build bots that we've got for Windows, or if just anyone
    // else is instrumenting the build. In this case we just bail out
    // immediately and assume that they take care of it.
    //
    // Also note that nested jobs (why this might fail) are supported in recent
    // versions of Windows, but the version of Windows that our bots are running
    // at least don't support nested job objects.
    let r = AssignProcessToJobObject(job, GetCurrentProcess());
    if r == 0 {
        CloseHandle(job);
        return;
    }

    // If we've got a parent process (e.g., the python script that called us)
    // then move ownership of this job object up to them. That way if the python
    // script is killed (e.g., via ctrl-c) then we'll all be torn down.
    //
    // If we don't have a parent (e.g., this was run directly) then we
    // intentionally leak the job object handle. When our process exits
    // (normally or abnormally) it will close the handle implicitly, causing all
    // processes in the job to be cleaned up.
    let pid = match env::var("BOOTSTRAP_PARENT_ID") {
        Ok(s) => s,
        Err(..) => return,
    };

    let parent = OpenProcess(PROCESS_DUP_HANDLE, FALSE, pid.parse().unwrap());

    // If we get a null parent pointer here, it is possible that either
    // we have got an invalid pid or the parent process has been closed.
    // Since the first case rarely happens
    // (only when wrongly setting the environmental variable),
    // so it might be better to improve the experience of the second case
    // when users have interrupted the parent process and we don't finish
    // duplicating the handle yet.
    // We just need close the job object if that occurs.
    if parent.is_null() {
        CloseHandle(job);
        return;
    }

    let mut parent_handle = ptr::null_mut();
    let r = DuplicateHandle(
        GetCurrentProcess(),
        job,
        parent,
        &mut parent_handle,
        0,
        FALSE,
        DUPLICATE_SAME_ACCESS,
    );

    // If this failed, well at least we tried! An example of DuplicateHandle
    // failing in the past has been when the wrong python2 package spawned this
    // build system (e.g., the `python2` package in MSYS instead of
    // `mingw-w64-x86_64-python2`. Not sure why it failed, but the "failure
    // mode" here is that we only clean everything up when the build system
    // dies, not when the python parent does, so not too bad.
    if r != 0 {
        CloseHandle(job);
    }
}
<|endoftext|>


<commit_before>extern crate luthor;

use std::rc::Rc;
use std::cell::RefCell;
use std::str::from_utf8;
use std::fs::File;
use std::io;
use std::io::{Read, Write};
use std::path::PathBuf;
use super::GapBuffer;
use super::gap_buffer;
use super::Position;
use super::Range;
use super::Cursor;
use super::type_detection;
use self::luthor::token::{Token, Category};
use self::luthor::lexers;

/// A UTF-8 buffer with bounds-checked cursor management and persistence.
pub struct Buffer {
    data: Rc<RefCell<GapBuffer>>,
    lexer: fn(&str) -> Vec<Token>,
    pub path: Option<PathBuf>,
    pub cursor: Cursor,
}

impl Buffer {
    /// Returns the contents of the buffer as a string.
    ///
    /// # Examples
    ///
    /// ```
    /// let mut buffer = scribe::buffer::new();
    /// buffer.insert("scribe");
    /// assert_eq!(buffer.data(), "scribe");
    /// ```
    pub fn data(&self) -> String {
        self.data.borrow().to_string()
    }

    /// Writes the contents of the buffer to its path.
    ///
    /// # Examples
    ///
    /// ```
    /// # #![feature(convert)]
    /// # use std::path::PathBuf;
    /// # use std::path::Path;
    /// # use std::fs::File;
    /// # use std::io::Read;
    ///
    /// // Set up a buffer and point it to a path.
    /// let mut buffer = scribe::buffer::new();
    /// let write_path = PathBuf::from("my_doc");
    /// buffer.path = Some(write_path.clone());
    ///
    /// // Put some data into the buffer and save it.
    /// buffer.insert("scribe");
    /// buffer.save();
    ///
    /// # let mut saved_data = String::new();
    /// # File::open(Path::new("my_doc")).unwrap().
    /// #   read_to_string(&mut saved_data).unwrap();
    /// # assert_eq!(saved_data, "scribe");
    ///
    /// # std::fs::remove_file(&write_path);
    /// ```
    pub fn save(&self) -> Option<io::Error> {
        let path = match self.path.clone() {
            Some(p) => p,
            None => PathBuf::new(),
        };

        // Try to open and write to the file, returning any errors encountered.
        let mut file = match File::create(&path) {
            Ok(f) => f,
            Err(error) => return Some(error),
        };

        // We use to_string here because we don't want to write the gap contents.
        match file.write_all(self.data().to_string().as_bytes()) {
            Ok(_) => (),
            Err(error) => return Some(error),
        }

        return None
    }

    /// Inserts `data` into the buffer at the cursor position.
    ///
    /// # Examples
    ///
    /// ```
    /// let mut buffer = scribe::buffer::new();
    /// buffer.insert("scribe");
    /// assert_eq!(buffer.data(), "scribe");
    /// ```
    pub fn insert(&mut self, data: &str) {
        self.data.borrow_mut().insert(data, &self.cursor);
    }

    /// Deletes a character at the cursor position. If at the end
    /// of the current line, it'll try to delete a newline character
    /// (joining the lines), succeeding if there's a line below.
    ///
    /// # Examples
    ///
    /// ```
    /// let mut buffer = scribe::buffer::new();
    /// buffer.insert("scribe");
    /// buffer.delete();
    /// assert_eq!(buffer.data(), "cribe");
    /// ```
    pub fn delete(&mut self) {
        // We need to specify a range to delete, so start at
        // the current offset and delete the character to the right.
        let mut end = Position{ line: self.cursor.line, offset: self.cursor.offset + 1 };

        // If there isn't a character to the right,
        // delete the newline by jumping to the start
        // of the next line. If it doesn't exist, that's okay;
        // these values are bounds-checked by delete() anyway.
        if !self.data.borrow().in_bounds(&end) {
            end.line += 1;
            end.offset = 0;
        }

        self.data.borrow_mut().delete(&Range{ start: *self.cursor, end: end});
    }

    /// Produces a set of tokens based on the buffer data
    /// suitable for colorized display, using a lexer for the
    /// buffer data's language and/or format.
    ///
    /// # Examples
    ///
    /// ```
    /// let mut buffer = scribe::buffer::new();
    /// buffer.insert("scribe data");
    ///
    /// // Build the buffer data string back by combining its token lexemes.
    /// let mut data = String::new();
    /// for token in buffer.tokens().iter() {
    ///     data.push_str(&token.lexeme);
    /// }
    /// assert_eq!(data, "scribe data");
    /// ```
    pub fn tokens(&self) -> Vec<Token> {
        (self.lexer)(&self.data())
    }

    /// Returns the file name portion of the buffer's path, if
    /// the path is set and its file name is a valid UTF-8 sequence.
    ///
    /// # Examples
    ///
    /// ```
    /// # #![feature(convert)]
    /// use std::path::PathBuf;
    ///
    /// let file_path = PathBuf::from("tests/sample/file");
    /// let buffer = scribe::buffer::from_file(file_path).unwrap();
    /// assert_eq!(buffer.file_name().unwrap(), "file");
    /// ```
    pub fn file_name(&self) -> Option<String> {
        match self.path {
            Some(ref path) => {
                match path.file_name() {
                    Some(file_name) => {
                        match file_name.to_str() {
                            Some(utf8_file_name) => Some(utf8_file_name.to_string()),
                            None => None,
                        }
                    },
                    None => None,
                }
            },
            None => None,
        }
    }
}

/// Creates a new empty buffer. The buffer's cursor is set to the beginning of the buffer.
///
/// # Examples
///
/// ```
/// let buffer = scribe::buffer::new();
/// # assert_eq!(buffer.cursor.line, 0);
/// # assert_eq!(buffer.cursor.offset, 0);
/// ```
pub fn new() -> Buffer {
    let data = Rc::new(RefCell::new(gap_buffer::new(String::new())));
    let cursor = Cursor{ data: data.clone(), position: Position{ line: 0, offset: 0 }};

    Buffer{ data: data.clone(), path: None, cursor: cursor, lexer: lexers::default::lex as fn(&str) -> Vec<Token> }
}

/// Creates a new buffer by reading the UTF-8 interpreted file contents of the specified path.
/// The buffer's cursor is set to the beginning of the buffer. The buffer data's type will be
/// inferred based on its extension, and an appropriate lexer will be used, if available (see
/// tokens method for further information on why this happens).
///
/// # Examples
///
/// ```
/// # #![feature(convert)]
/// use std::path::PathBuf;
///
/// let file_path = PathBuf::from("tests/sample/file");
/// let buffer = scribe::buffer::from_file(file_path).unwrap();
/// assert_eq!(buffer.data(), "it works!\n");
/// # assert_eq!(buffer.cursor.line, 0);
/// # assert_eq!(buffer.cursor.offset, 0);
/// ```
pub fn from_file(path: PathBuf) -> io::Result<Buffer> {
    // Try to open and read the file, returning any errors encountered.
    let mut file = match File::open(path.clone()) {
        Ok(f) => f,
        Err(error) => return Err(error),
    };
    let mut data = String::new();
    match file.read_to_string(&mut data) {
        Ok(_) => (),
        Err(error) => return Err(error),
    };

    let data = Rc::new(RefCell::new(gap_buffer::new(data)));
    let cursor = Cursor{ data: data.clone(), position: Position{ line: 0, offset: 0 }};

    // Detect the file type and use its corresponding lexer, if available.
    let lexer = match type_detection::from_path(&path) {
        Some(type_detection::Type::JSON) => lexers::json::lex as fn(&str) -> Vec<Token>,
        Some(type_detection::Type::XML) => lexers::xml::lex as fn(&str) -> Vec<Token>,
        _ => lexers::default::lex as fn(&str) -> Vec<Token>,
    };

    // Create a new buffer using the loaded data, path, and other defaults.
    Ok(Buffer{ data: data.clone(), path: Some(path), cursor: cursor, lexer: lexer })
}

#[cfg(test)]
mod tests {
    use super::new;
    use super::luthor::token::{Token, Category};

    #[test]
    fn tokens_returns_result_of_lexer() {
        let mut buffer = new();
        buffer.insert("scribe data");
        let expected_tokens = vec![
            Token{ lexeme: "scribe".to_string(), category: Category::Text },
            Token{ lexeme: " ".to_string(), category: Category::Whitespace },
            Token{ lexeme: "data".to_string(), category: Category::Text },
        ];
        assert_eq!(buffer.tokens(), expected_tokens);
    }

    #[test]
    fn delete_joins_lines_when_invoked_at_end_of_line() {
        let mut buffer = new();
        buffer.insert("scribe\n library");
        buffer.cursor.move_to_end_of_line();
        buffer.delete();
        assert_eq!(buffer.data(), "scribe library");
    }

    #[test]
    fn delete_does_nothing_when_invoked_at_the_end_of_the_document() {
        let mut buffer = new();
        buffer.insert("scribe\n library");
        buffer.cursor.move_down();
        buffer.cursor.move_to_end_of_line();
        buffer.delete();
        assert_eq!(buffer.data(), "scribe\n library");
    }
}
<commit_msg>Cache buffer string and token representations. Closes #9.<commit_after>extern crate luthor;

use std::rc::Rc;
use std::cell::RefCell;
use std::str::from_utf8;
use std::fs::File;
use std::io;
use std::io::{Read, Write};
use std::path::PathBuf;
use super::GapBuffer;
use super::gap_buffer;
use super::Position;
use super::Range;
use super::Cursor;
use super::type_detection;
use self::luthor::token::{Token, Category};
use self::luthor::lexers;

/// A UTF-8 buffer with bounds-checked cursor management and persistence.
pub struct Buffer {
    data: Rc<RefCell<GapBuffer>>,
    lexer: fn(&str) -> Vec<Token>,
    pub path: Option<PathBuf>,
    pub cursor: Cursor,
    data_cache: Option<String>,
    token_cache: Option<Vec<Token>>,
}

impl Buffer {
    /// Returns the contents of the buffer as a string. Caches
    /// this string representation to make subsequent requests
    /// to an unchanged buffer as fast as possible.
    ///
    /// # Examples
    ///
    /// ```
    /// let mut buffer = scribe::buffer::new();
    /// buffer.insert("scribe");
    /// assert_eq!(buffer.data(), "scribe");
    /// ```
    pub fn data(&mut self) -> String {
        match self.data_cache {
            Some(ref cache) => cache.clone(), // Cache hit; return a copy.
            None => {
                // Cache miss; update the cache w/ fresh data and return a copy.
                self.data_cache = Some(self.data.borrow().to_string());
                self.data_cache.clone().unwrap()
            }
        }
    }

    /// Writes the contents of the buffer to its path.
    ///
    /// # Examples
    ///
    /// ```
    /// # #![feature(convert)]
    /// # use std::path::PathBuf;
    /// # use std::path::Path;
    /// # use std::fs::File;
    /// # use std::io::Read;
    ///
    /// // Set up a buffer and point it to a path.
    /// let mut buffer = scribe::buffer::new();
    /// let write_path = PathBuf::from("my_doc");
    /// buffer.path = Some(write_path.clone());
    ///
    /// // Put some data into the buffer and save it.
    /// buffer.insert("scribe");
    /// buffer.save();
    ///
    /// # let mut saved_data = String::new();
    /// # File::open(Path::new("my_doc")).unwrap().
    /// #   read_to_string(&mut saved_data).unwrap();
    /// # assert_eq!(saved_data, "scribe");
    ///
    /// # std::fs::remove_file(&write_path);
    /// ```
    pub fn save(&mut self) -> Option<io::Error> {
        let path = match self.path.clone() {
            Some(p) => p,
            None => PathBuf::new(),
        };

        // Try to open and write to the file, returning any errors encountered.
        let mut file = match File::create(&path) {
            Ok(f) => f,
            Err(error) => return Some(error),
        };

        // We use to_string here because we don't want to write the gap contents.
        match file.write_all(self.data().to_string().as_bytes()) {
            Ok(_) => (),
            Err(error) => return Some(error),
        }

        return None
    }

    /// Inserts `data` into the buffer at the cursor position.
    ///
    /// # Examples
    ///
    /// ```
    /// let mut buffer = scribe::buffer::new();
    /// buffer.insert("scribe");
    /// assert_eq!(buffer.data(), "scribe");
    /// ```
    pub fn insert(&mut self, data: &str) {
        // Insert the data at the cursor
        self.data.borrow_mut().insert(data, &self.cursor);

        // Caches are invalid as the buffer has changed.
        self.clear_caches();
    }

    /// Deletes a character at the cursor position. If at the end
    /// of the current line, it'll try to delete a newline character
    /// (joining the lines), succeeding if there's a line below.
    ///
    /// # Examples
    ///
    /// ```
    /// let mut buffer = scribe::buffer::new();
    /// buffer.insert("scribe");
    /// buffer.delete();
    /// assert_eq!(buffer.data(), "cribe");
    /// ```
    pub fn delete(&mut self) {
        // We need to specify a range to delete, so start at
        // the current offset and delete the character to the right.
        let mut end = Position{ line: self.cursor.line, offset: self.cursor.offset + 1 };

        // If there isn't a character to the right,
        // delete the newline by jumping to the start
        // of the next line. If it doesn't exist, that's okay;
        // these values are bounds-checked by delete() anyway.
        if !self.data.borrow().in_bounds(&end) {
            end.line += 1;
            end.offset = 0;
        }

        // Delete the data.
        self.data.borrow_mut().delete(&Range{ start: *self.cursor, end: end});

        // Caches are invalid as the buffer has changed.
        self.clear_caches();
    }

    /// Produces a set of tokens based on the buffer data
    /// suitable for colorized display, using a lexer for the
    /// buffer data's language and/or format. Caches this
    /// lexed representation to make subsequent requests
    /// to an unchanged buffer as fast as possible.
    ///
    /// # Examples
    ///
    /// ```
    /// let mut buffer = scribe::buffer::new();
    /// buffer.insert("scribe data");
    ///
    /// // Build the buffer data string back by combining its token lexemes.
    /// let mut data = String::new();
    /// for token in buffer.tokens().iter() {
    ///     data.push_str(&token.lexeme);
    /// }
    /// assert_eq!(data, "scribe data");
    /// ```
    pub fn tokens(&mut self) -> Vec<Token> {
        match self.token_cache {
            Some(ref cache) => cache.clone(), // Cache hit; return a copy.
            None => {
                // Cache miss; update the cache w/ fresh tokens and return a copy.
                self.token_cache = Some((self.lexer)(&self.data()));
                self.token_cache.clone().unwrap()
            }
        }
    }

    /// Returns the file name portion of the buffer's path, if
    /// the path is set and its file name is a valid UTF-8 sequence.
    ///
    /// # Examples
    ///
    /// ```
    /// # #![feature(convert)]
    /// use std::path::PathBuf;
    ///
    /// let file_path = PathBuf::from("tests/sample/file");
    /// let buffer = scribe::buffer::from_file(file_path).unwrap();
    /// assert_eq!(buffer.file_name().unwrap(), "file");
    /// ```
    pub fn file_name(&self) -> Option<String> {
        match self.path {
            Some(ref path) => {
                match path.file_name() {
                    Some(file_name) => {
                        match file_name.to_str() {
                            Some(utf8_file_name) => Some(utf8_file_name.to_string()),
                            None => None,
                        }
                    },
                    None => None,
                }
            },
            None => None,
        }
    }

    /// Called when caches are invalidated via buffer modifications.
    fn clear_caches(&mut self) {
        self.data_cache = None;
        self.token_cache = None;
    }
}

/// Creates a new empty buffer. The buffer's cursor is set to the beginning of the buffer.
///
/// # Examples
///
/// ```
/// let buffer = scribe::buffer::new();
/// # assert_eq!(buffer.cursor.line, 0);
/// # assert_eq!(buffer.cursor.offset, 0);
/// ```
pub fn new() -> Buffer {
    let data = Rc::new(RefCell::new(gap_buffer::new(String::new())));
    let cursor = Cursor{ data: data.clone(), position: Position{ line: 0, offset: 0 }};

    Buffer{
        data: data.clone(),
        path: None,
        cursor: cursor,
        lexer: lexers::default::lex as fn(&str) -> Vec<Token>,
        data_cache: None,
        token_cache: None,
    }
}

/// Creates a new buffer by reading the UTF-8 interpreted file contents of the specified path.
/// The buffer's cursor is set to the beginning of the buffer. The buffer data's type will be
/// inferred based on its extension, and an appropriate lexer will be used, if available (see
/// tokens method for further information on why this happens).
///
/// # Examples
///
/// ```
/// # #![feature(convert)]
/// use std::path::PathBuf;
///
/// let file_path = PathBuf::from("tests/sample/file");
/// let mut buffer = scribe::buffer::from_file(file_path).unwrap();
/// assert_eq!(buffer.data(), "it works!\n");
/// # assert_eq!(buffer.cursor.line, 0);
/// # assert_eq!(buffer.cursor.offset, 0);
/// ```
pub fn from_file(path: PathBuf) -> io::Result<Buffer> {
    // Try to open and read the file, returning any errors encountered.
    let mut file = match File::open(path.clone()) {
        Ok(f) => f,
        Err(error) => return Err(error),
    };
    let mut data = String::new();
    match file.read_to_string(&mut data) {
        Ok(_) => (),
        Err(error) => return Err(error),
    };

    let data = Rc::new(RefCell::new(gap_buffer::new(data)));
    let cursor = Cursor{ data: data.clone(), position: Position{ line: 0, offset: 0 }};

    // Detect the file type and use its corresponding lexer, if available.
    let lexer = match type_detection::from_path(&path) {
        Some(type_detection::Type::JSON) => lexers::json::lex as fn(&str) -> Vec<Token>,
        Some(type_detection::Type::XML) => lexers::xml::lex as fn(&str) -> Vec<Token>,
        _ => lexers::default::lex as fn(&str) -> Vec<Token>,
    };

    // Create a new buffer using the loaded data, path, and other defaults.
    Ok(
        Buffer{
            data: data.clone(),
            path: Some(path),
            cursor: cursor,
            lexer: lexer,
            data_cache: None,
            token_cache: None,
        }
    )
}

#[cfg(test)]
mod tests {
    use super::new;
    use super::luthor::token::{Token, Category};

    #[test]
    fn tokens_returns_result_of_lexer() {
        let mut buffer = new();
        buffer.insert("scribe data");
        let expected_tokens = vec![
            Token{ lexeme: "scribe".to_string(), category: Category::Text },
            Token{ lexeme: " ".to_string(), category: Category::Whitespace },
            Token{ lexeme: "data".to_string(), category: Category::Text },
        ];
        assert_eq!(buffer.tokens(), expected_tokens);
    }

    #[test]
    fn delete_joins_lines_when_invoked_at_end_of_line() {
        let mut buffer = new();
        buffer.insert("scribe\n library");
        buffer.cursor.move_to_end_of_line();
        buffer.delete();
        assert_eq!(buffer.data(), "scribe library");
    }

    #[test]
    fn delete_does_nothing_when_invoked_at_the_end_of_the_document() {
        let mut buffer = new();
        buffer.insert("scribe\n library");
        buffer.cursor.move_down();
        buffer.cursor.move_to_end_of_line();
        buffer.delete();
        assert_eq!(buffer.data(), "scribe\n library");
    }

    #[test]
    fn insert_clears_data_and_token_caches() {
        let mut buffer = new();
        buffer.insert("scribe");

        // Trigger data and token cache storage.
        assert_eq!("scribe", buffer.data());
        assert_eq!(
            vec![Token{ lexeme: "scribe".to_string(), category: Category::Text }],
            buffer.tokens()
        );

        // Change the buffer contents using delete.
        buffer.insert("test_");

        // Ensure the cache has been busted.
        assert_eq!("test_scribe", buffer.data());
        assert_eq!(
            vec![Token{ lexeme: "test_scribe".to_string(), category: Category::Text }],
            buffer.tokens()
        );
    }

    #[test]
    fn delete_clears_data_and_token_caches() {
        let mut buffer = new();
        buffer.insert("scribe");

        // Trigger data and token cache storage.
        assert_eq!("scribe", buffer.data());
        assert_eq!(
            vec![Token{ lexeme: "scribe".to_string(), category: Category::Text }],
            buffer.tokens()
        );

        // Change the buffer contents using delete.
        buffer.delete();

        // Ensure the cache has been busted.
        assert_eq!("cribe", buffer.data());
        assert_eq!(
            vec![Token{ lexeme: "cribe".to_string(), category: Category::Text }],
            buffer.tokens()
        );
    }
}
<|endoftext|>


<commit_before>use std::ops::Deref;
use std::rc::Rc;
use std::cell::RefCell;
use super::Position;
use super::GapBuffer;

/// Read-only wrapper for a `Position`, to allow field level access to a
/// buffer's cursor while simultaneously enforcing bounds-checking when
/// updating its value.
pub struct Cursor {
    pub data: Rc<RefCell<GapBuffer>>,
    pub position: Position,
}

impl Deref for Cursor {
    type Target = Position;

    fn deref(&self) -> &Position {
        &self.position
    }
}

impl Cursor {
    /// Moves the cursor to the specified location. The location is
    /// bounds-checked against the data and the cursor will not be
    /// updated if it is out-of-bounds.
    ///
    /// # Examples
    ///
    /// ```
    /// let mut buffer = scribe::buffer::new();
    /// let in_bounds = scribe::buffer::Position{ line: 0, offset: 2 };
    /// let out_of_bounds = scribe::buffer::Position{ line: 2, offset: 2 };
    /// buffer.insert("scribe");
    ///
    /// buffer.cursor.move_to(in_bounds);
    /// assert_eq!(buffer.cursor.line, 0);
    /// assert_eq!(buffer.cursor.offset, 2);
    ///
    /// buffer.cursor.move_to(out_of_bounds);
    /// assert_eq!(buffer.cursor.line, 0);
    /// assert_eq!(buffer.cursor.offset, 2);
    /// ```
    pub fn move_to(&mut self, position: Position) {
        if self.data.borrow().in_bounds(&position) {
            self.position = position;
        }
    }

    /// Decrements the cursor line. The location is bounds-checked against
    /// the data and the cursor will not be updated if it is out-of-bounds.
    pub fn move_up(&mut self) {
        let new_position = Position{ line: self.line-1, offset: self.offset };
        self.move_to(new_position);
    }

    /// Increments the cursor line. The location is bounds-checked against
    /// the data and the cursor will not be updated if it is out-of-bounds.
    pub fn move_down(&mut self) {
        let new_position = Position{ line: self.line+1, offset: self.offset };
        self.move_to(new_position);
    }

    /// Decrements the cursor offset. The location is bounds-checked against
    /// the data and the cursor will not be updated if it is out-of-bounds.
    pub fn move_left(&mut self) {
        let new_position = Position{ line: self.line, offset: self.offset-1 };
        self.move_to(new_position);
    }

    /// Increments the cursor offset. The location is bounds-checked against
    /// the data and the cursor will not be updated if it is out-of-bounds.
    pub fn move_right(&mut self) {
        let new_position = Position{ line: self.line, offset: self.offset+1 };
        self.move_to(new_position);
    }
}
<commit_msg>Initial (inefficient) to-EOL vertical cursor movement implementation.<commit_after>use std::ops::Deref;
use std::rc::Rc;
use std::cell::RefCell;
use super::Position;
use super::GapBuffer;

/// Read-only wrapper for a `Position`, to allow field level access to a
/// buffer's cursor while simultaneously enforcing bounds-checking when
/// updating its value.
pub struct Cursor {
    pub data: Rc<RefCell<GapBuffer>>,
    pub position: Position,
}

impl Deref for Cursor {
    type Target = Position;

    fn deref(&self) -> &Position {
        &self.position
    }
}

impl Cursor {
    /// Moves the cursor to the specified location. The location is
    /// bounds-checked against the data and the cursor will not be
    /// updated if it is out-of-bounds.
    ///
    /// # Examples
    ///
    /// ```
    /// let mut buffer = scribe::buffer::new();
    /// let in_bounds = scribe::buffer::Position{ line: 0, offset: 2 };
    /// let out_of_bounds = scribe::buffer::Position{ line: 2, offset: 2 };
    /// buffer.insert("scribe");
    ///
    /// buffer.cursor.move_to(in_bounds);
    /// assert_eq!(buffer.cursor.line, 0);
    /// assert_eq!(buffer.cursor.offset, 2);
    ///
    /// buffer.cursor.move_to(out_of_bounds);
    /// assert_eq!(buffer.cursor.line, 0);
    /// assert_eq!(buffer.cursor.offset, 2);
    /// ```
    pub fn move_to(&mut self, position: Position) -> bool {
        if self.data.borrow().in_bounds(&position) {
            self.position = position;
            return true
        }
        false
    }

    /// Decrements the cursor line. The location is bounds-checked against
    /// the data and the cursor will not be updated if it is out-of-bounds.
    ///
    /// TODO: Make this more efficient by checking the
    /// target line's existence and using its length.
    pub fn move_up(&mut self) {
        let mut moved = false;
        let mut offset = self.offset;
        let mut new_position: Position;

        while moved == false && offset >= 0 {
            new_position = Position{ line: self.line-1, offset: offset };
            moved = self.move_to(new_position);
            offset -= 1;
        }
    }

    /// Increments the cursor line. The location is bounds-checked against
    /// the data and the cursor will not be updated if it is out-of-bounds.
    pub fn move_down(&mut self) {
        let new_position = Position{ line: self.line+1, offset: self.offset };
        self.move_to(new_position);
    }

    /// Decrements the cursor offset. The location is bounds-checked against
    /// the data and the cursor will not be updated if it is out-of-bounds.
    pub fn move_left(&mut self) {
        let new_position = Position{ line: self.line, offset: self.offset-1 };
        self.move_to(new_position);
    }

    /// Increments the cursor offset. The location is bounds-checked against
    /// the data and the cursor will not be updated if it is out-of-bounds.
    pub fn move_right(&mut self) {
        let new_position = Position{ line: self.line, offset: self.offset+1 };
        self.move_to(new_position);
    }
}

#[cfg(test)]
mod tests {
    use super::Cursor;
    use super::super::gap_buffer;
    use super::super::Position;
    use std::rc::Rc;
    use std::cell::RefCell;

    #[test]
    fn move_up_goes_to_EOL_if_offset_would_be_out_of_range() {
        let mut buffer = Rc::new(RefCell::new(gap_buffer::new("This is a test.\nAnother line that is longer.".to_string())));
        let position = Position{ line: 1, offset: 20 };
        let mut cursor = Cursor{ data: buffer, position: position };
        cursor.move_up();
        assert_eq!(cursor.line, 0);
        assert_eq!(cursor.offset, 15);
    }
}
<|endoftext|>


<commit_before>// Copyright lowRISC contributors.
// Licensed under the Apache License, Version 2.0, see LICENSE for details.
// SPDX-License-Identifier: Apache-2.0

//! CWT parser tests.
//!
//! These are hung off to the side to avoid cluttering the main cwt.rs.

use testutil::data::keys;

use crate::cert::Cert;
use crate::cert::CertFormat;
use crate::crypto::ring;
use crate::crypto::sig::Sign as _;

const UINT: u8 = 0;
const NINT: u8 = 1;
const UTF8: u8 = 3;
const BYTES: u8 = 2;
const ARRAY: u8 = 4;
const MAP: u8 = 5;

#[test]
#[cfg_attr(miri, ignore)]
fn self_signed() {
    let protected = raw_cbor!(MAP [
        // Algorithm.
        UINT:1  NINT:256,
        // Criticalities.
        UINT:2  ARRAY[UINT:1, UINT:2],
    ]);
    let payload = raw_cbor!(MAP [
        // Issuer.
        UINT:1  UTF8 {"my cool ca"},
        // Subject; self-signed.
        UINT:2  UTF8 {"my cool ca"},

        // DICE SPKI.
        NINT:4670551    MAP [
            // Key type, RSA.
            UINT:1  UINT:3,
            // Algorithm,i RSA.
            UINT:3  NINT:256,
            // Modulus.
            NINT:0  BYTES {h"
                a6be96e18c5c826a748b887b194ed3f6
                daaa8897cdbaee69e5a980b9dab5af7e
                5f4fed0236d78400837b28229fd4ee24
                1c0d6f3178651265a2762aa1f449dcec
                2a8b679058f9c97a3ba93a7c2bb6d62d
                3baaa70a47991ee6f0daefc55892035f
                a4f4d1c493a73bf61018c6cf86bb43aa
                151f9f0bfa5f474a7fa2bba6a1c77e17
                e160cee2387586e43ff78efd95327d22
                0fc12b0e717089b18cf667ce672d6351
                d187d9330b8777cbe2fb28fccba0b2ad
                c560c9d03e2cba50d624736545c587cc
                0f1d3ef3d7ec69f4108fff270455155b
                d9a3d72c41a397915bad3c9aba753e7b
                f91614d25642cd968816108647e2f7a8
                580e4c05ff0a7f233beba82ca8569a53
            "},
            // Exponent.
            NINT:1  BYTES {h"010001"},
        ],
        // DICE KeyUsage
        NINT:4670552    BYTES {[0b0010_0000]},
    ]);

    let tbs = raw_cbor!(ARRAY [
        UTF8 {"Signature1"},
        BYTES {protected},
        BYTES {},
        BYTES {payload},
    ]);

    let (_, mut signer) = ring::rsa::from_keypair(keys::KEY1_RSA_KEYPAIR);
    let mut sig = vec![0; signer.sig_bytes()];
    signer.sign(&[&tbs], &mut sig).unwrap();

    let data = raw_cbor! {
        BYTES {protected}
        MAP []
        BYTES {payload}
        BYTES {sig}
    };

    let cert = Cert::parse(
        &data,
        CertFormat::OpenDiceCwt,
        None,
        &mut ring::sig::Ciphers::new(),
    )
    .unwrap();

    assert_eq!(cert.subject(), cert.issuer());
    assert!(cert.supports_cert_signing());
}
<commit_msg>[cert] Add a function for generating test CWT certs<commit_after>// Copyright lowRISC contributors.
// Licensed under the Apache License, Version 2.0, see LICENSE for details.
// SPDX-License-Identifier: Apache-2.0

//! CWT parser tests.
//!
//! These are hung off to the side to avoid cluttering the main cwt.rs.

use testutil::data::keys;

use crate::cert::Cert;
use crate::cert::CertFormat;
use crate::crypto::ring;
use crate::crypto::sig::PublicKeyParams;
use crate::crypto::sig::Sign as _;

const UINT: u8 = 0;
const NINT: u8 = 1;
const UTF8: u8 = 3;
const BYTES: u8 = 2;
const ARRAY: u8 = 4;
const MAP: u8 = 5;

/// Generator for a test CWT.
pub struct TestCwt {
    issuer: &'static str,
    subject: &'static str,
    spki: PublicKeyParams<'static>,
    key_usage: &'static [u8],
    issuer_key: &'static [u8],
}

impl TestCwt {
    pub fn encode(&self) -> Vec<u8> {
        let protected = raw_cbor!(MAP [
            // Algorithm, i.e., RSA.
            UINT:1  NINT:256,
            // Criticalities.
            UINT:2  ARRAY[UINT:1, UINT:2],
        ]);

        let spki = match &self.spki {
            PublicKeyParams::Rsa { modulus, exponent } => raw_cbor!(MAP [
                // Key type, RSA.
                UINT:1  UINT:3,
                // Algorithm, RSA.
                UINT:3  NINT:256,
                // Modulus.
                NINT:0  BYTES {modulus},
                // Exponent.
                NINT:1  BYTES {exponent},
            ]),
        };

        let payload = raw_cbor!(MAP [
            // Issuer.
            UINT:1          UTF8 {(self.issuer)},
            // Subject.
            UINT:2          UTF8 {(self.subject)},
            // DICE SPKI.
            NINT:4670551    spki,
            // DICE KeyUsage
            NINT:4670552    BYTES {(self.key_usage)},
        ]);

        let tbs = raw_cbor!(ARRAY [
            UTF8 {"Signature1"},
            BYTES {protected},
            BYTES {},
            BYTES {payload},
        ]);

        let (_, mut signer) = ring::rsa::from_keypair(self.issuer_key);
        let mut sig = vec![0; signer.sig_bytes()];
        signer.sign(&[&tbs], &mut sig).unwrap();

        raw_cbor! {
            BYTES {protected}
            MAP []  // No unprotected fields.
            BYTES {payload}
            BYTES {sig}
        }
    }
}

#[test]
#[cfg_attr(miri, ignore)]
fn self_signed() {
    let cwt = TestCwt {
        issuer: "my cool ca",
        subject: "my cool ca",
        spki: PublicKeyParams::Rsa {
            modulus: keys::KEY1_RSA_MOD,
            exponent: keys::KEY1_RSA_EXP,
        },
        key_usage: &[0b0010_0000],
        issuer_key: keys::KEY1_RSA_KEYPAIR,
    };

    let data = cwt.encode();
    let cert = Cert::parse(
        &data,
        CertFormat::OpenDiceCwt,
        None,
        &mut ring::sig::Ciphers::new(),
    )
    .unwrap();

    assert_eq!(cert.subject(), cert.issuer());
    assert!(cert.supports_cert_signing());
}
<|endoftext|>