<commit_before>use crate::{Scalar, Dim, Matrix, VectorN, RowVectorN, DefaultAllocator, U1, VectorSliceN};
use alga::general::{Field, SupersetOf};
use crate::storage::Storage;
use crate::allocator::Allocator;

impl<N: Scalar, R: Dim, C: Dim, S: Storage<N, R, C>> Matrix<N, R, C, S> {
    /// Returns a row vector where each element is the result of the application of `f` on the
    /// corresponding column of the original matrix.
    #[inline]
    pub fn compress_rows(&self, f: impl Fn(VectorSliceN<N, R, S::RStride, S::CStride>) -> N) -> RowVectorN<N, C>
        where DefaultAllocator: Allocator<N, U1, C> {

        let ncols = self.data.shape().1;
        let mut res = unsafe { RowVectorN::new_uninitialized_generic(U1, ncols) };

        for i in 0..ncols.value() {
            // FIXME: avoid bound checking of column.
            unsafe { *res.get_unchecked_mut((0, i)) = f(self.column(i)); }
        }

        res
    }

    /// Returns a column vector where each element is the result of the application of `f` on the
    /// corresponding column of the original matrix.
    ///
    /// This is the same as `self.compress_rows(f).transpose()`.
    #[inline]
    pub fn compress_rows_tr(&self, f: impl Fn(VectorSliceN<N, R, S::RStride, S::CStride>) -> N) -> VectorN<N, C>
        where DefaultAllocator: Allocator<N, C> {

        let ncols = self.data.shape().1;
        let mut res = unsafe { VectorN::new_uninitialized_generic(ncols, U1) };

        for i in 0..ncols.value() {
            // FIXME: avoid bound checking of column.
            unsafe { *res.vget_unchecked_mut(i) = f(self.column(i)); }
        }

        res
    }

    /// Returns a column vector resulting from the folding of `f` on each column of this matrix.
    #[inline]
    pub fn compress_columns(&self, init: VectorN<N, R>, f: impl Fn(&mut VectorN<N, R>, VectorSliceN<N, R, S::RStride, S::CStride>)) -> VectorN<N, R>
        where DefaultAllocator: Allocator<N, R> {
        let mut res = init;

        for i in 0..self.ncols() {
            f(&mut res, self.column(i))
        }

        res
    }
}

impl<N: Scalar + Field + SupersetOf<f64>, R: Dim, C: Dim, S: Storage<N, R, C>> Matrix<N, R, C, S> {
    /*
     *
     * Sum computation.
     *
     */
    /// The sum of all the elements of this matrix.
    ///
    /// # Example
    ///
    /// ```
    /// # use nalgebra::Matrix2x3;
    ///
    /// let m = Matrix2x3::new(1.0, 2.0, 3.0,
    ///                        4.0, 5.0, 6.0);
    /// assert_eq!(m.sum(), 21.0);
    /// ```
    #[inline]
    pub fn sum(&self) -> N {
        self.iter().cloned().fold(N::zero(), |a, b| a + b)
    }

    /// The sum of all the rows of this matrix.
    ///
    /// Use `.row_variance_tr` if you need the result in a column vector instead.
    ///
    /// # Example
    ///
    /// ```
    /// # use nalgebra::{Matrix2x3, RowVector3};
    ///
    /// let m = Matrix2x3::new(1.0, 2.0, 3.0,
    ///                        4.0, 5.0, 6.0);
    /// assert_eq!(m.row_sum(), RowVector3::new(5.0, 7.0, 9.0));
    /// ```
    #[inline]
    pub fn row_sum(&self) -> RowVectorN<N, C>
        where DefaultAllocator: Allocator<N, U1, C> {
        self.compress_rows(|col| col.sum())
    }

    /// The sum of all the rows of this matrix. The result is transposed and returned as a column vector.
    ///
    /// # Example
    ///
    /// ```
    /// # use nalgebra::{Matrix2x3, Vector3};
    ///
    /// let m = Matrix2x3::new(1.0, 2.0, 3.0,
    ///                        4.0, 5.0, 6.0);
    /// assert_eq!(m.row_sum_tr(), Vector3::new(5.0, 7.0, 9.0));
    /// ```
    #[inline]
    pub fn row_sum_tr(&self) -> VectorN<N, C>
        where DefaultAllocator: Allocator<N, C> {
        self.compress_rows_tr(|col| col.sum())
    }

    /// The sum of all the columns of this matrix.
    ///
    /// # Example
    ///
    /// ```
    /// # use nalgebra::{Matrix2x3, Vector2};
    ///
    /// let m = Matrix2x3::new(1.0, 2.0, 3.0,
    ///                        4.0, 5.0, 6.0);
    /// assert_eq!(m.column_sum(), Vector2::new(6.0, 15.0));
    /// ```
    #[inline]
    pub fn column_sum(&self) -> VectorN<N, R>
        where DefaultAllocator: Allocator<N, R> {
        let nrows = self.data.shape().0;
        self.compress_columns(VectorN::zeros_generic(nrows, U1), |out, col| {
            out.axpy(N::one(), &col, N::one())
        })
    }

    /*
     *
     * Variance computation.
     *
     */
    /// The variance of all the elements of this matrix.
    ///
    /// # Example
    ///
    /// ```
    /// # #[macro_use] extern crate approx;
    /// # use nalgebra::Matrix2x3;
    ///
    /// let m = Matrix2x3::new(1.0, 2.0, 3.0,
    ///                        4.0, 5.0, 6.0);
    /// assert_relative_eq!(m.variance(), 35.0 / 12.0, epsilon = 1.0e-8);
    /// ```
    #[inline]
    pub fn variance(&self) -> N {
        if self.len() == 0 {
            N::zero()
        } else {
            let val = self.iter().cloned().fold((N::zero(), N::zero()), |a, b| (a.0 + b * b, a.1 + b));
            let denom = N::one() / crate::convert::<_, N>(self.len() as f64);
            val.0 * denom - (val.1 * denom) * (val.1 * denom)
        }
    }

    /// The variance of all the rows of this matrix.
    ///
    /// Use `.row_variance_tr` if you need the result in a column vector instead.
    /// # Example
    ///
    /// ```
    /// # use nalgebra::{Matrix2x3, RowVector3};
    ///
    /// let m = Matrix2x3::new(1.0, 2.0, 3.0,
    ///                        4.0, 5.0, 6.0);
    /// assert_eq!(m.row_variance(), RowVector3::new(2.25, 2.25, 2.25));
    /// ```
    #[inline]
    pub fn row_variance(&self) -> RowVectorN<N, C>
        where DefaultAllocator: Allocator<N, U1, C> {
        self.compress_rows(|col| col.variance())
    }

    /// The variance of all the rows of this matrix. The result is transposed and returned as a column vector.
    ///
    /// # Example
    ///
    /// ```
    /// # use nalgebra::{Matrix2x3, Vector3};
    ///
    /// let m = Matrix2x3::new(1.0, 2.0, 3.0,
    ///                        4.0, 5.0, 6.0);
    /// assert_eq!(m.row_variance_tr(), Vector3::new(2.25, 2.25, 2.25));
    /// ```
    #[inline]
    pub fn row_variance_tr(&self) -> VectorN<N, C>
        where DefaultAllocator: Allocator<N, C> {
        self.compress_rows_tr(|col| col.variance())
    }

    /// The variance of all the columns of this matrix.
    ///
    /// # Example
    ///
    /// ```
    /// # #[macro_use] extern crate approx;
    /// # use nalgebra::{Matrix2x3, Vector2};
    ///
    /// let m = Matrix2x3::new(1.0, 2.0, 3.0,
    ///                        4.0, 5.0, 6.0);
    /// assert_relative_eq!(m.column_variance(), Vector2::new(2.0 / 3.0, 2.0 / 3.0), epsilon = 1.0e-8);
    /// ```
    #[inline]
    pub fn column_variance(&self) -> VectorN<N, R>
        where DefaultAllocator: Allocator<N, R> {
        let (nrows, ncols) = self.data.shape();

        let mut mean = self.column_mean();
        mean.apply(|e| -(e * e));

        let denom = N::one() / crate::convert::<_, N>(ncols.value() as f64);
        self.compress_columns(mean, |out, col| {
            for i in 0..nrows.value() {
                unsafe {
                    let val = col.vget_unchecked(i);
                    *out.vget_unchecked_mut(i) += denom * *val * *val
                }
            }
        })
    }

    /*
     *
     * Mean computation.
     *
     */
    /// The mean of all the elements of this matrix.
    ///
    /// # Example
    ///
    /// ```
    /// # use nalgebra::Matrix2x3;
    ///
    /// let m = Matrix2x3::new(1.0, 2.0, 3.0,
    ///                        4.0, 5.0, 6.0);
    /// assert_eq!(m.mean(), 3.5);
    /// ```
    #[inline]
    pub fn mean(&self) -> N {
        if self.len() == 0 {
            N::zero()
        } else {
            self.sum() / crate::convert(self.len() as f64)
        }
    }

    /// The mean of all the rows of this matrix.
    ///
    /// Use `.row_mean_tr` if you need the result in a column vector instead.
    ///
    /// # Example
    ///
    /// ```
    /// # use nalgebra::{Matrix2x3, RowVector3};
    ///
    /// let m = Matrix2x3::new(1.0, 2.0, 3.0,
    ///                        4.0, 5.0, 6.0);
    /// assert_eq!(m.row_mean(), RowVector3::new(2.5, 3.5, 4.5));
    /// ```
    #[inline]
    pub fn row_mean(&self) -> RowVectorN<N, C>
        where DefaultAllocator: Allocator<N, U1, C> {
        self.compress_rows(|col| col.mean())
    }

    /// The mean of all the rows of this matrix. The result is transposed and returned as a column vector.
    ///
    /// # Example
    ///
    /// ```
    /// # use nalgebra::{Matrix2x3, Vector3};
    ///
    /// let m = Matrix2x3::new(1.0, 2.0, 3.0,
    ///                        4.0, 5.0, 6.0);
    /// assert_eq!(m.row_mean_tr(), Vector3::new(2.5, 3.5, 4.5));
    /// ```
    #[inline]
    pub fn row_mean_tr(&self) -> VectorN<N, C>
        where DefaultAllocator: Allocator<N, C> {
        self.compress_rows_tr(|col| col.mean())
    }

    /// The mean of all the columns of this matrix.
    ///
    /// # Example
    ///
    /// ```
    /// # use nalgebra::{Matrix2x3, Vector2};
    ///
    /// let m = Matrix2x3::new(1.0, 2.0, 3.0,
    ///                        4.0, 5.0, 6.0);
    /// assert_eq!(m.column_mean(), Vector2::new(2.0, 5.0));
    /// ```
    #[inline]
    pub fn column_mean(&self) -> VectorN<N, R>
        where DefaultAllocator: Allocator<N, R> {
        let (nrows, ncols) = self.data.shape();
        let denom = N::one() / crate::convert::<_, N>(ncols.value() as f64);
        self.compress_columns(VectorN::zeros_generic(nrows, U1), |out, col| {
            out.axpy(denom, &col, N::one())
        })
    }
}
<commit_msg>Refactor row_sum() and column_sum() to cover more cases.<commit_after>use crate::{Scalar, Dim, Matrix, VectorN, RowVectorN, DefaultAllocator, U1, VectorSliceN};
use alga::general::{AdditiveMonoid, Field, SupersetOf};
use crate::storage::Storage;
use crate::allocator::Allocator;

impl<N: Scalar, R: Dim, C: Dim, S: Storage<N, R, C>> Matrix<N, R, C, S> {
    /// Returns a row vector where each element is the result of the application of `f` on the
    /// corresponding column of the original matrix.
    #[inline]
    pub fn compress_rows(&self, f: impl Fn(VectorSliceN<N, R, S::RStride, S::CStride>) -> N) -> RowVectorN<N, C>
        where DefaultAllocator: Allocator<N, U1, C> {

        let ncols = self.data.shape().1;
        let mut res = unsafe { RowVectorN::new_uninitialized_generic(U1, ncols) };

        for i in 0..ncols.value() {
            // FIXME: avoid bound checking of column.
            unsafe { *res.get_unchecked_mut((0, i)) = f(self.column(i)); }
        }

        res
    }

    /// Returns a column vector where each element is the result of the application of `f` on the
    /// corresponding column of the original matrix.
    ///
    /// This is the same as `self.compress_rows(f).transpose()`.
    #[inline]
    pub fn compress_rows_tr(&self, f: impl Fn(VectorSliceN<N, R, S::RStride, S::CStride>) -> N) -> VectorN<N, C>
        where DefaultAllocator: Allocator<N, C> {

        let ncols = self.data.shape().1;
        let mut res = unsafe { VectorN::new_uninitialized_generic(ncols, U1) };

        for i in 0..ncols.value() {
            // FIXME: avoid bound checking of column.
            unsafe { *res.vget_unchecked_mut(i) = f(self.column(i)); }
        }

        res
    }

    /// Returns a column vector resulting from the folding of `f` on each column of this matrix.
    #[inline]
    pub fn compress_columns(&self, init: VectorN<N, R>, f: impl Fn(&mut VectorN<N, R>, VectorSliceN<N, R, S::RStride, S::CStride>)) -> VectorN<N, R>
        where DefaultAllocator: Allocator<N, R> {
        let mut res = init;

        for i in 0..self.ncols() {
            f(&mut res, self.column(i))
        }

        res
    }
}

impl<N: Scalar + AdditiveMonoid, R: Dim, C: Dim, S: Storage<N, R, C>> Matrix<N, R, C, S> {
    /*
     *
     * Sum computation.
     *
     */
    /// The sum of all the elements of this matrix.
    ///
    /// # Example
    ///
    /// ```
    /// # use nalgebra::Matrix2x3;
    ///
    /// let m = Matrix2x3::new(1.0, 2.0, 3.0,
    ///                        4.0, 5.0, 6.0);
    /// assert_eq!(m.sum(), 21.0);
    /// ```
    #[inline]
    pub fn sum(&self) -> N {
        self.iter().cloned().fold(N::zero(), |a, b| a + b)
    }

    /// The sum of all the rows of this matrix.
    ///
    /// Use `.row_variance_tr` if you need the result in a column vector instead.
    ///
    /// # Example
    ///
    /// ```
    /// # use nalgebra::{Matrix2x3, Matrix3x2};
    /// # use nalgebra::{RowVector2, RowVector3};
    ///
    /// let m = Matrix2x3::new(1.0, 2.0, 3.0,
    ///                        4.0, 5.0, 6.0);
    /// assert_eq!(m.row_sum(), RowVector3::new(5.0, 7.0, 9.0));
    ///
    /// let mint = Matrix3x2::new(1,2,3,4,5,6);
    /// assert_eq!(mint.row_sum(), RowVector2::new(9,12));
    /// ```
    #[inline]
    pub fn row_sum(&self) -> RowVectorN<N, C>
        where DefaultAllocator: Allocator<N, U1, C> {
        self.compress_rows(|col| col.sum())
    }

    /// The sum of all the rows of this matrix. The result is transposed and returned as a column vector.
    ///
    /// # Example
    ///
    /// ```
    /// # use nalgebra::{Matrix2x3, Matrix3x2};
    /// # use nalgebra::{Vector2, Vector3};
    ///
    /// let m = Matrix2x3::new(1.0, 2.0, 3.0,
    ///                        4.0, 5.0, 6.0);
    /// assert_eq!(m.row_sum_tr(), Vector3::new(5.0, 7.0, 9.0));
    ///
    /// let mint = Matrix3x2::new(1,2,3,4,5,6);
    /// assert_eq!(mint.row_sum_tr(), Vector2::new(9,12));
    /// ```
    #[inline]
    pub fn row_sum_tr(&self) -> VectorN<N, C>
        where DefaultAllocator: Allocator<N, C> {
        self.compress_rows_tr(|col| col.sum())
    }

    /// The sum of all the columns of this matrix.
    ///
    /// # Example
    ///
    /// ```
    /// # use nalgebra::{Matrix2x3, Matrix3x2};
    /// # use nalgebra::{Vector2, Vector3};
    ///
    /// let m = Matrix2x3::new(1.0, 2.0, 3.0,
    ///                        4.0, 5.0, 6.0);
    /// assert_eq!(m.column_sum(), Vector2::new(6.0, 15.0));
    ///
    /// let mint = Matrix3x2::new(1,2,3,4,5,6);
    /// assert_eq!(mint.column_sum(), Vector3::new(3,7,11));
    /// ```
    #[inline]
    pub fn column_sum(&self) -> VectorN<N, R>
        where DefaultAllocator: Allocator<N, R> {
        let nrows = self.data.shape().0;
        self.compress_columns(VectorN::zeros_generic(nrows, U1), |out, col| {
            *out += col;
        })
    }
}

impl<N: Scalar + Field + SupersetOf<f64>, R: Dim, C: Dim, S: Storage<N, R, C>> Matrix<N, R, C, S> {
    /*
     *
     * Variance computation.
     *
     */
    /// The variance of all the elements of this matrix.
    ///
    /// # Example
    ///
    /// ```
    /// # #[macro_use] extern crate approx;
    /// # use nalgebra::Matrix2x3;
    ///
    /// let m = Matrix2x3::new(1.0, 2.0, 3.0,
    ///                        4.0, 5.0, 6.0);
    /// assert_relative_eq!(m.variance(), 35.0 / 12.0, epsilon = 1.0e-8);
    /// ```
    #[inline]
    pub fn variance(&self) -> N {
        if self.len() == 0 {
            N::zero()
        } else {
            let val = self.iter().cloned().fold((N::zero(), N::zero()), |a, b| (a.0 + b * b, a.1 + b));
            let denom = N::one() / crate::convert::<_, N>(self.len() as f64);
            val.0 * denom - (val.1 * denom) * (val.1 * denom)
        }
    }

    /// The variance of all the rows of this matrix.
    ///
    /// Use `.row_variance_tr` if you need the result in a column vector instead.
    /// # Example
    ///
    /// ```
    /// # use nalgebra::{Matrix2x3, RowVector3};
    ///
    /// let m = Matrix2x3::new(1.0, 2.0, 3.0,
    ///                        4.0, 5.0, 6.0);
    /// assert_eq!(m.row_variance(), RowVector3::new(2.25, 2.25, 2.25));
    /// ```
    #[inline]
    pub fn row_variance(&self) -> RowVectorN<N, C>
        where DefaultAllocator: Allocator<N, U1, C> {
        self.compress_rows(|col| col.variance())
    }

    /// The variance of all the rows of this matrix. The result is transposed and returned as a column vector.
    ///
    /// # Example
    ///
    /// ```
    /// # use nalgebra::{Matrix2x3, Vector3};
    ///
    /// let m = Matrix2x3::new(1.0, 2.0, 3.0,
    ///                        4.0, 5.0, 6.0);
    /// assert_eq!(m.row_variance_tr(), Vector3::new(2.25, 2.25, 2.25));
    /// ```
    #[inline]
    pub fn row_variance_tr(&self) -> VectorN<N, C>
        where DefaultAllocator: Allocator<N, C> {
        self.compress_rows_tr(|col| col.variance())
    }

    /// The variance of all the columns of this matrix.
    ///
    /// # Example
    ///
    /// ```
    /// # #[macro_use] extern crate approx;
    /// # use nalgebra::{Matrix2x3, Vector2};
    ///
    /// let m = Matrix2x3::new(1.0, 2.0, 3.0,
    ///                        4.0, 5.0, 6.0);
    /// assert_relative_eq!(m.column_variance(), Vector2::new(2.0 / 3.0, 2.0 / 3.0), epsilon = 1.0e-8);
    /// ```
    #[inline]
    pub fn column_variance(&self) -> VectorN<N, R>
        where DefaultAllocator: Allocator<N, R> {
        let (nrows, ncols) = self.data.shape();

        let mut mean = self.column_mean();
        mean.apply(|e| -(e * e));

        let denom = N::one() / crate::convert::<_, N>(ncols.value() as f64);
        self.compress_columns(mean, |out, col| {
            for i in 0..nrows.value() {
                unsafe {
                    let val = col.vget_unchecked(i);
                    *out.vget_unchecked_mut(i) += denom * *val * *val
                }
            }
        })
    }

    /*
     *
     * Mean computation.
     *
     */
    /// The mean of all the elements of this matrix.
    ///
    /// # Example
    ///
    /// ```
    /// # use nalgebra::Matrix2x3;
    ///
    /// let m = Matrix2x3::new(1.0, 2.0, 3.0,
    ///                        4.0, 5.0, 6.0);
    /// assert_eq!(m.mean(), 3.5);
    /// ```
    #[inline]
    pub fn mean(&self) -> N {
        if self.len() == 0 {
            N::zero()
        } else {
            self.sum() / crate::convert(self.len() as f64)
        }
    }

    /// The mean of all the rows of this matrix.
    ///
    /// Use `.row_mean_tr` if you need the result in a column vector instead.
    ///
    /// # Example
    ///
    /// ```
    /// # use nalgebra::{Matrix2x3, RowVector3};
    ///
    /// let m = Matrix2x3::new(1.0, 2.0, 3.0,
    ///                        4.0, 5.0, 6.0);
    /// assert_eq!(m.row_mean(), RowVector3::new(2.5, 3.5, 4.5));
    /// ```
    #[inline]
    pub fn row_mean(&self) -> RowVectorN<N, C>
        where DefaultAllocator: Allocator<N, U1, C> {
        self.compress_rows(|col| col.mean())
    }

    /// The mean of all the rows of this matrix. The result is transposed and returned as a column vector.
    ///
    /// # Example
    ///
    /// ```
    /// # use nalgebra::{Matrix2x3, Vector3};
    ///
    /// let m = Matrix2x3::new(1.0, 2.0, 3.0,
    ///                        4.0, 5.0, 6.0);
    /// assert_eq!(m.row_mean_tr(), Vector3::new(2.5, 3.5, 4.5));
    /// ```
    #[inline]
    pub fn row_mean_tr(&self) -> VectorN<N, C>
        where DefaultAllocator: Allocator<N, C> {
        self.compress_rows_tr(|col| col.mean())
    }

    /// The mean of all the columns of this matrix.
    ///
    /// # Example
    ///
    /// ```
    /// # use nalgebra::{Matrix2x3, Vector2};
    ///
    /// let m = Matrix2x3::new(1.0, 2.0, 3.0,
    ///                        4.0, 5.0, 6.0);
    /// assert_eq!(m.column_mean(), Vector2::new(2.0, 5.0));
    /// ```
    #[inline]
    pub fn column_mean(&self) -> VectorN<N, R>
        where DefaultAllocator: Allocator<N, R> {
        let (nrows, ncols) = self.data.shape();
        let denom = N::one() / crate::convert::<_, N>(ncols.value() as f64);
        self.compress_columns(VectorN::zeros_generic(nrows, U1), |out, col| {
            out.axpy(denom, &col, N::one())
        })
    }
}
<|endoftext|>


<commit_before>use std::collections::HashMap;
use std::error::Error;
use std::fs::File;
use std::io::{BufWriter, Write};
use std::path::{Path, PathBuf};

use clap::{App, Arg};
use indicatif::{ProgressBar, ProgressStyle};
use rayon::prelude::*;
use stopwatch::Stopwatch;
use zim::{Cluster, DirectoryEntry, MimeType, Namespace, Target, Zim};

fn main() {
    let matches = App::new("zimextractor")
        .version("0.1")
        .about("Extract zim files")
        .arg(
            Arg::with_name("out")
                .short("o")
                .long("out")
                .help("Output directory")
                .takes_value(true),
        )
        .arg(
            Arg::with_name("skip-link")
                .long("skip-link")
                .help("Skip genrating hard links")
                .takes_value(false),
        )
        .arg(
            Arg::with_name("flatten-link")
                .long("flatten-link")
                .help("Write files to disk, instead of using hard links")
                .takes_value(false),
        )
        .arg(
            Arg::with_name("INPUT")
                .help("Set the zim file to extract")
                .required(true)
                .index(1),
        )
        .get_matches();

    let skip_link = matches.is_present("skip-link");
    let flatten_link = matches.is_present("flatten-link");
    let out = matches.value_of("out").unwrap_or("out");
    let root_output = Path::new(out);

    let input = matches.value_of("INPUT").unwrap();

    println!("Extracting file: {} to {}\n", input, out);
    println!("Generating symlinks: {}", !skip_link);
    println!("Generating copies for links: {}", flatten_link);

    let sw = Stopwatch::start_new();
    let zim_file = Zim::new(input).expect("failed to parse input");

    if let Some(main_page_idx) = zim_file.header.main_page {
        let page = zim_file
            .get_by_url_index(main_page_idx)
            .expect("failed to get main page");
        println!("Main page is {}", page.url);
    }
    println!("");

    let pb = ProgressBar::new(zim_file.article_count() as u64);
    let style = ProgressStyle::default_bar()
        .template(
            "{msg}\n{spinner:.green} [{elapsed_precise}] [{bar:40.cyan/blue}] {pos}/{len} ({eta})",
        )
        .progress_chars("#>-");
    pb.set_style(style);

    ensure_dir(root_output);

    // map between cluster and directory entry
    let mut cluster_map = HashMap::new();

    for i in 0..zim_file.header.cluster_count {
        let cluster = zim_file.get_cluster(i).expect("failed to retrieve cluster");
        cluster_map.insert(i, cluster);
    }

    let entries: Vec<_> = zim_file.iterate_by_urls().collect();
    pb.set_message("Writing entries to disk");
    entries
        .par_iter()
        .filter(|entry| {
            if let Some(Target::Cluster(_, _)) = entry.target.as_ref() {
                return true;
            }
            false
        })
        .for_each(|entry| {
            process_file(&root_output, &cluster_map, entry, &pb);
        });

    if !skip_link {
        pb.set_message("Generating links");
        entries
            .par_iter()
            .filter(|entry| {
                if let Some(Target::Redirect(_)) = entry.target.as_ref() {
                    return true;
                }
                false
            })
            .for_each(|entry| {
                process_link(&zim_file, &root_output, entry, skip_link, flatten_link, &pb);
            });
    }

    pb.finish_with_message(&format!(
        "Extraction done in {}s",
        sw.elapsed_ms() as f64 / 1000.
    ));
}

fn safe_write<T: AsRef<[u8]>>(path: &Path, data: T, count: usize) {
    let display = path.display();
    let contain_path = path.parent().unwrap();

    ensure_dir(contain_path);

    match File::create(&path) {
        Err(why) => {
            if count < 3 {
                safe_write(path, data, count + 1);
            } else {
                eprintln!(
                    "skipping: failed retry: couldn't create {}: {:?}",
                    display, why
                );
            }
        }
        Ok(file) => {
            let mut writer = BufWriter::new(&file);

            if let Err(why) = writer.write_all(data.as_ref()) {
                eprintln!(
                    "skipping: couldn't write to {}: {}",
                    display,
                    why.description()
                );
            }
        }
    }
}

fn ensure_dir(path: &Path) {
    if path.exists() {
        // already done
        return;
    }

    std::fs::create_dir_all(path)
        .unwrap_or_else(|e| ignore_exists_err(e, &format!("create: {}", path.display())));
}

fn process_file<'a>(
    root_output: &Path,
    cluster_map: &'a HashMap<u32, Cluster<'a>>,
    entry: &DirectoryEntry,
    pb: &ProgressBar,
) {
    let dst = make_path(root_output, entry.namespace, &entry.url, &entry.mime_type);
    match entry.target.as_ref() {
        Some(Target::Cluster(cluster_index, blob_idx)) => {
            let cluster = cluster_map.get(cluster_index).expect("missing cluster");

            match cluster.get_blob(*blob_idx) {
                Ok(blob) => {
                    safe_write(&dst, blob, 1);
                }
                Err(err) => {
                    eprintln!("skipping invalid blob: {}: {}", dst.display(), err);
                }
            }
            pb.inc(1);
        }
        Some(_) => unreachable!("filtered out earlier"),
        None => {
            eprintln!("skipping missing target {} {:?}", dst.display(), entry);
        }
    }
}
fn process_link(
    zim_file: &Zim,
    root_output: &Path,
    entry: &DirectoryEntry,
    skip_link: bool,
    flatten_link: bool,
    pb: &ProgressBar,
) {
    let dst = make_path(root_output, entry.namespace, &entry.url, &entry.mime_type);

    if entry.target.is_none() {
        eprintln!("skipping missing target {:?} {:?}", dst, entry);
        return;
    }

    match entry.target.as_ref() {
        Some(Target::Redirect(redir)) => {
            if !skip_link && !dst.exists() {
                pb.inc_length(1);

                let entry = {
                    zim_file
                        .get_by_url_index(*redir)
                        .expect("failed to get_by_url_index")
                };
                let src = make_path(root_output, entry.namespace, &entry.url, &entry.mime_type);
                make_link(src, dst, flatten_link);
                pb.inc(1);
            }
        }
        _ => panic!("must be filtered before"),
    }
}

fn make_link(src: PathBuf, mut dst: PathBuf, flatten_link: bool) {
    if !src.exists() {
        eprintln!("Warning: link source doesn't exist: {}", src.display());
    } else if !dst.exists() {
        let contain_path = dst.parent().unwrap();
        ensure_dir(contain_path);

        if let Some(ext) = src.extension() {
            if dst.extension().is_none() || dst.extension().unwrap() != ext {
                dst.set_extension(ext);
            }
        }

        if flatten_link {
            std::fs::copy(&src, &dst).unwrap_or_else(|e| {
                ignore_exists_err(
                    e,
                    format!("copy link: {} -> {}", src.display(), dst.display()),
                );
                0
            });
        } else {
            std::fs::hard_link(&src, &dst).unwrap_or_else(|e| {
                ignore_exists_err(
                    e,
                    format!("create link: {} -> {}", src.display(), dst.display()),
                );
            });
        }
    }
}
fn ignore_exists_err<T: AsRef<str>>(e: std::io::Error, msg: T) {
    use std::io::ErrorKind::*;

    match e.kind() {
        // do not panic if it already exists, that's fine, we just want to make
        // sure we have it before moving on
        AlreadyExists => {}
        _ => {
            eprintln!("skipping: {}: {}", msg.as_ref(), e);
        }
    }
}

fn make_path(root: &Path, namespace: Namespace, url: &str, mime_type: &MimeType) -> PathBuf {
    let mut s = String::new();
    s.push(namespace as u8 as char);
    let mut path = if url.starts_with("/") {
        // make absolute urls relative to the output folder
        let url = url.replacen("/", "", 1);
        root.join(&s).join(url)
    } else {
        root.join(&s).join(url)
    };

    if let MimeType::Type(typ) = mime_type {
        let extension = match typ.as_str() {
            "text/html" => Some("html"),
            "image/jpeg" => Some("jpeg"),
            "image/png" => Some("png"),
            "image/gif" => Some("gif"),
            "image/svg+xml" => Some("svg"),
            "application/javascript" => Some("js"),
            "text/css" => Some("css"),
            "text/plain" => Some("txt"),
            _ => None,
        };
        if let Some(extension) = extension {
            if path.extension().is_none()
                || !path
                    .extension()
                    .unwrap()
                    .to_str()
                    .unwrap_or_default()
                    .starts_with(extension)
            {
                path.set_extension(extension);
            }
        }
    }

    path
}
<commit_msg>fix(extract): write jpeg files with a '.jpg' extension<commit_after>use std::collections::HashMap;
use std::error::Error;
use std::fs::File;
use std::io::{BufWriter, Write};
use std::path::{Path, PathBuf};

use clap::{App, Arg};
use indicatif::{ProgressBar, ProgressStyle};
use rayon::prelude::*;
use stopwatch::Stopwatch;
use zim::{Cluster, DirectoryEntry, MimeType, Namespace, Target, Zim};

fn main() {
    let matches = App::new("zimextractor")
        .version("0.1")
        .about("Extract zim files")
        .arg(
            Arg::with_name("out")
                .short("o")
                .long("out")
                .help("Output directory")
                .takes_value(true),
        )
        .arg(
            Arg::with_name("skip-link")
                .long("skip-link")
                .help("Skip genrating hard links")
                .takes_value(false),
        )
        .arg(
            Arg::with_name("flatten-link")
                .long("flatten-link")
                .help("Write files to disk, instead of using hard links")
                .takes_value(false),
        )
        .arg(
            Arg::with_name("INPUT")
                .help("Set the zim file to extract")
                .required(true)
                .index(1),
        )
        .get_matches();

    let skip_link = matches.is_present("skip-link");
    let flatten_link = matches.is_present("flatten-link");
    let out = matches.value_of("out").unwrap_or("out");
    let root_output = Path::new(out);

    let input = matches.value_of("INPUT").unwrap();

    println!("Extracting file: {} to {}\n", input, out);
    println!("Generating symlinks: {}", !skip_link);
    println!("Generating copies for links: {}", flatten_link);

    let sw = Stopwatch::start_new();
    let zim_file = Zim::new(input).expect("failed to parse input");

    if let Some(main_page_idx) = zim_file.header.main_page {
        let page = zim_file
            .get_by_url_index(main_page_idx)
            .expect("failed to get main page");
        println!("Main page is {}", page.url);
    }
    println!("");

    let pb = ProgressBar::new(zim_file.article_count() as u64);
    let style = ProgressStyle::default_bar()
        .template(
            "{msg}\n{spinner:.green} [{elapsed_precise}] [{bar:40.cyan/blue}] {pos}/{len} ({eta})",
        )
        .progress_chars("#>-");
    pb.set_style(style);

    ensure_dir(root_output);

    // map between cluster and directory entry
    let mut cluster_map = HashMap::new();

    for i in 0..zim_file.header.cluster_count {
        let cluster = zim_file.get_cluster(i).expect("failed to retrieve cluster");
        cluster_map.insert(i, cluster);
    }

    let entries: Vec<_> = zim_file.iterate_by_urls().collect();
    pb.set_message("Writing entries to disk");
    entries
        .par_iter()
        .filter(|entry| {
            if let Some(Target::Cluster(_, _)) = entry.target.as_ref() {
                return true;
            }
            false
        })
        .for_each(|entry| {
            process_file(&root_output, &cluster_map, entry, &pb);
        });

    if !skip_link {
        pb.set_message("Generating links");
        entries
            .par_iter()
            .filter(|entry| {
                if let Some(Target::Redirect(_)) = entry.target.as_ref() {
                    return true;
                }
                false
            })
            .for_each(|entry| {
                process_link(&zim_file, &root_output, entry, skip_link, flatten_link, &pb);
            });
    }

    pb.finish_with_message(&format!(
        "Extraction done in {}s",
        sw.elapsed_ms() as f64 / 1000.
    ));
}

fn safe_write<T: AsRef<[u8]>>(path: &Path, data: T, count: usize) {
    let display = path.display();
    let contain_path = path.parent().unwrap();

    ensure_dir(contain_path);

    match File::create(&path) {
        Err(why) => {
            if count < 3 {
                safe_write(path, data, count + 1);
            } else {
                eprintln!(
                    "skipping: failed retry: couldn't create {}: {:?}",
                    display, why
                );
            }
        }
        Ok(file) => {
            let mut writer = BufWriter::new(&file);

            if let Err(why) = writer.write_all(data.as_ref()) {
                eprintln!(
                    "skipping: couldn't write to {}: {}",
                    display,
                    why.description()
                );
            }
        }
    }
}

fn ensure_dir(path: &Path) {
    if path.exists() {
        // already done
        return;
    }

    std::fs::create_dir_all(path)
        .unwrap_or_else(|e| ignore_exists_err(e, &format!("create: {}", path.display())));
}

fn process_file<'a>(
    root_output: &Path,
    cluster_map: &'a HashMap<u32, Cluster<'a>>,
    entry: &DirectoryEntry,
    pb: &ProgressBar,
) {
    let dst = make_path(root_output, entry.namespace, &entry.url, &entry.mime_type);
    match entry.target.as_ref() {
        Some(Target::Cluster(cluster_index, blob_idx)) => {
            let cluster = cluster_map.get(cluster_index).expect("missing cluster");

            match cluster.get_blob(*blob_idx) {
                Ok(blob) => {
                    safe_write(&dst, blob, 1);
                }
                Err(err) => {
                    eprintln!("skipping invalid blob: {}: {}", dst.display(), err);
                }
            }
            pb.inc(1);
        }
        Some(_) => unreachable!("filtered out earlier"),
        None => {
            eprintln!("skipping missing target {} {:?}", dst.display(), entry);
        }
    }
}
fn process_link(
    zim_file: &Zim,
    root_output: &Path,
    entry: &DirectoryEntry,
    skip_link: bool,
    flatten_link: bool,
    pb: &ProgressBar,
) {
    let dst = make_path(root_output, entry.namespace, &entry.url, &entry.mime_type);

    if entry.target.is_none() {
        eprintln!("skipping missing target {:?} {:?}", dst, entry);
        return;
    }

    match entry.target.as_ref() {
        Some(Target::Redirect(redir)) => {
            if !skip_link && !dst.exists() {
                pb.inc_length(1);

                let entry = {
                    zim_file
                        .get_by_url_index(*redir)
                        .expect("failed to get_by_url_index")
                };
                let src = make_path(root_output, entry.namespace, &entry.url, &entry.mime_type);
                make_link(src, dst, flatten_link);
                pb.inc(1);
            }
        }
        _ => panic!("must be filtered before"),
    }
}

fn make_link(src: PathBuf, mut dst: PathBuf, flatten_link: bool) {
    if !src.exists() {
        eprintln!("Warning: link source doesn't exist: {}", src.display());
    } else if !dst.exists() {
        let contain_path = dst.parent().unwrap();
        ensure_dir(contain_path);

        if let Some(ext) = src.extension() {
            if dst.extension().is_none() || dst.extension().unwrap() != ext {
                dst.set_extension(ext);
            }
        }

        if flatten_link {
            std::fs::copy(&src, &dst).unwrap_or_else(|e| {
                ignore_exists_err(
                    e,
                    format!("copy link: {} -> {}", src.display(), dst.display()),
                );
                0
            });
        } else {
            std::fs::hard_link(&src, &dst).unwrap_or_else(|e| {
                ignore_exists_err(
                    e,
                    format!("create link: {} -> {}", src.display(), dst.display()),
                );
            });
        }
    }
}
fn ignore_exists_err<T: AsRef<str>>(e: std::io::Error, msg: T) {
    use std::io::ErrorKind::*;

    match e.kind() {
        // do not panic if it already exists, that's fine, we just want to make
        // sure we have it before moving on
        AlreadyExists => {}
        _ => {
            eprintln!("skipping: {}: {}", msg.as_ref(), e);
        }
    }
}

fn make_path(root: &Path, namespace: Namespace, url: &str, mime_type: &MimeType) -> PathBuf {
    let mut s = String::new();
    s.push(namespace as u8 as char);
    let mut path = if url.starts_with("/") {
        // make absolute urls relative to the output folder
        let url = url.replacen("/", "", 1);
        root.join(&s).join(url)
    } else {
        root.join(&s).join(url)
    };

    if let MimeType::Type(typ) = mime_type {
        let extension = match typ.as_str() {
            "text/html" => Some("html"),
            "image/jpeg" => Some("jpg"),
            "image/png" => Some("png"),
            "image/gif" => Some("gif"),
            "image/svg+xml" => Some("svg"),
            "application/javascript" => Some("js"),
            "text/css" => Some("css"),
            "text/plain" => Some("txt"),
            _ => None,
        };
        if let Some(extension) = extension {
            if path.extension().is_none()
                || !path
                    .extension()
                    .unwrap()
                    .to_str()
                    .unwrap_or_default()
                    .starts_with(extension)
            {
                path.set_extension(extension);
            }
        }
    }

    path
}
<|endoftext|>


<commit_before>/*
Module: char

Utilities for manipulating the char type
*/

/*
Function: is_whitespace

Indicates whether a character is whitespace.

Whitespace characters include space (U+0020), tab (U+0009), line feed
(U+000A), carriage return (U+000D), and a number of less common
ASCII and unicode characters.
*/
pure fn is_whitespace(c: char) -> bool {
    const ch_space: char = '\u0020';
    const ch_ogham_space_mark: char = '\u1680';
    const ch_mongolian_vowel_sep: char = '\u180e';
    const ch_en_quad: char = '\u2000';
    const ch_em_quad: char = '\u2001';
    const ch_en_space: char = '\u2002';
    const ch_em_space: char = '\u2003';
    const ch_three_per_em_space: char = '\u2004';
    const ch_four_per_em_space: char = '\u2005';
    const ch_six_per_em_space: char = '\u2006';
    const ch_figure_space: char = '\u2007';
    const ch_punctuation_space: char = '\u2008';
    const ch_thin_space: char = '\u2009';
    const ch_hair_space: char = '\u200a';
    const ch_narrow_no_break_space: char = '\u202f';
    const ch_medium_mathematical_space: char = '\u205f';
    const ch_ideographic_space: char = '\u3000';
    const ch_line_separator: char = '\u2028';
    const ch_paragraph_separator: char = '\u2029';
    const ch_character_tabulation: char = '\u0009';
    const ch_line_feed: char = '\u000a';
    const ch_line_tabulation: char = '\u000b';
    const ch_form_feed: char = '\u000c';
    const ch_carriage_return: char = '\u000d';
    const ch_next_line: char = '\u0085';
    const ch_no_break_space: char = '\u00a0';

    if c == ch_space {
        true
    } else if c == ch_ogham_space_mark {
        true
    } else if c == ch_mongolian_vowel_sep {
        true
    } else if c == ch_en_quad {
        true
    } else if c == ch_em_quad {
        true
    } else if c == ch_en_space {
        true
    } else if c == ch_em_space {
        true
    } else if c == ch_three_per_em_space {
        true
    } else if c == ch_four_per_em_space {
        true
    } else if c == ch_six_per_em_space {
        true
    } else if c == ch_figure_space {
        true
    } else if c == ch_punctuation_space {
        true
    } else if c == ch_thin_space {
        true
    } else if c == ch_hair_space {
        true
    } else if c == ch_narrow_no_break_space {
        true
    } else if c == ch_medium_mathematical_space {
        true
    } else if c == ch_ideographic_space {
        true
    } else if c == ch_line_tabulation {
        true
    } else if c == ch_paragraph_separator {
        true
    } else if c == ch_character_tabulation {
        true
    } else if c == ch_line_feed {
        true
    } else if c == ch_line_tabulation {
        true
    } else if c == ch_form_feed {
        true
    } else if c == ch_carriage_return {
        true
    } else if c == ch_next_line {
        true
    } else if c == ch_no_break_space { true } else { false }
}

pure fn to_digit(c: char) -> u8 {
    alt c {
        '0' to '9' { c as u8 - ('0' as u8) }
        'a' to 'z' { c as u8 + 10u8 - ('a' as u8) }
        'A' to 'Z' { c as u8 + 10u8 - ('A' as u8) }
        _ { fail; }
    }
}
<commit_msg>char.rs: Added a function cmp<commit_after>/*
Module: char

Utilities for manipulating the char type
*/

/*
Function: is_whitespace

Indicates whether a character is whitespace.

Whitespace characters include space (U+0020), tab (U+0009), line feed
(U+000A), carriage return (U+000D), and a number of less common
ASCII and unicode characters.
*/
pure fn is_whitespace(c: char) -> bool {
    const ch_space: char = '\u0020';
    const ch_ogham_space_mark: char = '\u1680';
    const ch_mongolian_vowel_sep: char = '\u180e';
    const ch_en_quad: char = '\u2000';
    const ch_em_quad: char = '\u2001';
    const ch_en_space: char = '\u2002';
    const ch_em_space: char = '\u2003';
    const ch_three_per_em_space: char = '\u2004';
    const ch_four_per_em_space: char = '\u2005';
    const ch_six_per_em_space: char = '\u2006';
    const ch_figure_space: char = '\u2007';
    const ch_punctuation_space: char = '\u2008';
    const ch_thin_space: char = '\u2009';
    const ch_hair_space: char = '\u200a';
    const ch_narrow_no_break_space: char = '\u202f';
    const ch_medium_mathematical_space: char = '\u205f';
    const ch_ideographic_space: char = '\u3000';
    const ch_line_separator: char = '\u2028';
    const ch_paragraph_separator: char = '\u2029';
    const ch_character_tabulation: char = '\u0009';
    const ch_line_feed: char = '\u000a';
    const ch_line_tabulation: char = '\u000b';
    const ch_form_feed: char = '\u000c';
    const ch_carriage_return: char = '\u000d';
    const ch_next_line: char = '\u0085';
    const ch_no_break_space: char = '\u00a0';

    if c == ch_space {
        true
    } else if c == ch_ogham_space_mark {
        true
    } else if c == ch_mongolian_vowel_sep {
        true
    } else if c == ch_en_quad {
        true
    } else if c == ch_em_quad {
        true
    } else if c == ch_en_space {
        true
    } else if c == ch_em_space {
        true
    } else if c == ch_three_per_em_space {
        true
    } else if c == ch_four_per_em_space {
        true
    } else if c == ch_six_per_em_space {
        true
    } else if c == ch_figure_space {
        true
    } else if c == ch_punctuation_space {
        true
    } else if c == ch_thin_space {
        true
    } else if c == ch_hair_space {
        true
    } else if c == ch_narrow_no_break_space {
        true
    } else if c == ch_medium_mathematical_space {
        true
    } else if c == ch_ideographic_space {
        true
    } else if c == ch_line_tabulation {
        true
    } else if c == ch_paragraph_separator {
        true
    } else if c == ch_character_tabulation {
        true
    } else if c == ch_line_feed {
        true
    } else if c == ch_line_tabulation {
        true
    } else if c == ch_form_feed {
        true
    } else if c == ch_carriage_return {
        true
    } else if c == ch_next_line {
        true
    } else if c == ch_no_break_space { true } else { false }
}

pure fn to_digit(c: char) -> u8 {
    alt c {
        '0' to '9' { c as u8 - ('0' as u8) }
        'a' to 'z' { c as u8 + 10u8 - ('a' as u8) }
        'A' to 'Z' { c as u8 + 10u8 - ('A' as u8) }
        _ { fail; }
    }
}

fn cmp(a: char, b: char) -> int {
    ret  if b > a { -1 }
    else if b < a { 1 }
    else { 0 }
}<|endoftext|>


<commit_before>// Copyright 2018 Dmitry Tantsur <divius.inside@gmail.com>
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

//! Common protocol bits.

#![allow(dead_code)] // various things are unused with --no-default-features
#![allow(missing_docs)]

use reqwest::{Method, Url};
use serde::{Deserialize, Deserializer};
use serde::de::{DeserializeOwned, Error as DeserError};
use serde_json;

use super::super::{Error, ErrorKind, Result};
use super::super::auth::AuthMethod;
use super::super::session::ServiceInfo;
use super::super::utils;
use super::ApiVersion;

#[derive(Clone, Debug, Deserialize)]
pub struct Link {
    #[serde(deserialize_with = "deser_url")]
    pub href: Url,
    pub rel: String
}

#[derive(Clone, Debug, Deserialize)]
pub struct Ref {
    pub id: String,
    pub links: Vec<Link>
}

#[derive(Clone, Debug, Deserialize)]
pub struct IdAndName {
    pub id: String,
    pub name: String
}

#[derive(Clone, Debug, Deserialize)]
pub struct Version {
    pub id: String,
    pub links: Vec<Link>,
    pub status: String,
    #[serde(deserialize_with = "empty_as_none", default)]
    pub version: Option<ApiVersion>,
    #[serde(deserialize_with = "empty_as_none", default)]
    pub min_version: Option<ApiVersion>
}

#[derive(Clone, Debug, Deserialize)]
pub struct VersionsRoot {
    pub versions: Vec<Version>
}

#[derive(Clone, Debug, Deserialize)]
pub struct VersionRoot {
    pub version: Version
}

impl Version {
    pub fn into_service_info(self) -> Result<ServiceInfo> {
        let endpoint = match self.links.into_iter().find(|x| &x.rel == "self") {
            Some(link) => link.href,
            None => {
                return Err(Error::new(
                    ErrorKind::InvalidResponse,
                    "Invalid version - missing self link"));
            }
        };

        Ok(ServiceInfo {
            root_url: endpoint,
            current_version: self.version,
            minimum_version: self.min_version
        })
    }
}

/// Generic code to extract a `ServiceInfo` from a URL.
pub fn fetch_service_info(endpoint: Url, auth: &AuthMethod,
                          service_type: &str, major_version: &str)
        -> Result<ServiceInfo> {
    debug!("Fetching {} service info from {}", service_type, endpoint);

    // Workaround for old version of Nova returning HTTP endpoints even if
    // accessed via HTTP
    let secure = endpoint.scheme() == "https";

    let result = auth.request(Method::Get, endpoint.clone())?.send();
    match result {
        Ok(mut resp) => {
            let body = resp.text()?;

            // First, assume it's a versioned URL.
            let mut info = match serde_json::from_str::<VersionRoot>(&body) {
                Ok(ver) => ver.version.into_service_info(),
                Err(..) => {
                    // Second, assume it's a root URL.
                    let vers = serde_json::from_str::<VersionsRoot>(&body)
                        .map_err(|e| {
                            let msg = format!("Malformed version root of the {} service: {}",
                                              service_type, e);
                            Error::new(ErrorKind::InvalidResponse, msg)
                        })?;
                    match vers.versions.into_iter().find(|x| &x.id == major_version) {
                        Some(ver) => ver.into_service_info(),
                        None => Err(Error::new_endpoint_not_found(service_type))
                    }
                }
            }?;

            // Older Nova returns insecure URLs even for secure protocol.
            if secure {
                let _ = info.root_url.set_scheme("https").unwrap();
            }

            debug!("Received {:?} for {} service from {}",
                   info, service_type, endpoint);
            Ok(info)
        },
        Err(ref e) if e.kind() == ErrorKind::ResourceNotFound => {
            if utils::url::is_root(&endpoint) {
                Err(Error::new_endpoint_not_found(service_type))
            } else {
                debug!("Got HTTP 404 from {}, trying parent endpoint",
                       endpoint);
                fetch_service_info(utils::url::pop(endpoint, true), auth,
                                   service_type, major_version)
            }
        },
        Err(other) => Err(other)
    }
}

/// Deserialize value where empty string equals None.
pub fn empty_as_none<'de, D, T>(des: D) -> ::std::result::Result<Option<T>, D::Error>
        where D: Deserializer<'de>, T: DeserializeOwned {
    let value = serde_json::Value::deserialize(des)?;
    match &value {
        &serde_json::Value::String(ref s) if s == "" => return Ok(None),
        _ => ()
    };

    match serde_json::from_value(value) {
        Ok(value) => Ok(Some(value)),
        Err(e) => Err(DeserError::custom(e))
    }
}

/// Deserialize value where empty string equals None.
pub fn empty_as_default<'de, D, T>(des: D) -> ::std::result::Result<T, D::Error>
        where D: Deserializer<'de>, T: DeserializeOwned + Default {
    let value = serde_json::Value::deserialize(des)?;
    match &value {
        &serde_json::Value::String(ref s) if s == "" =>
            return Ok(Default::default()),
        _ => ()
    };

    serde_json::from_value(value).map_err(DeserError::custom)
}

/// Deserialize a URL.
pub fn deser_url<'de, D>(des: D) -> ::std::result::Result<Url, D::Error>
        where D: Deserializer<'de> {
    Url::parse(&String::deserialize(des)?).map_err(DeserError::custom)
}

/// Deserialize a URL.
pub fn deser_optional_url<'de, D>(des: D)
        -> ::std::result::Result<Option<Url>, D::Error>
        where D: Deserializer<'de> {
    let value: Option<String> = Deserialize::deserialize(des)?;
    match value {
        Some(s) => Url::parse(&s).map_err(DeserError::custom).map(Some),
        None => Ok(None)
    }
}
<commit_msg>refactor: simplify root endpoint handling<commit_after>// Copyright 2018 Dmitry Tantsur <divius.inside@gmail.com>
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

//! Common protocol bits.

#![allow(dead_code)] // various things are unused with --no-default-features
#![allow(missing_docs)]

use reqwest::{Method, Url};
use serde::{Deserialize, Deserializer};
use serde::de::{DeserializeOwned, Error as DeserError};
use serde_json;

use super::super::{Error, ErrorKind, Result};
use super::super::auth::AuthMethod;
use super::super::session::ServiceInfo;
use super::super::utils;
use super::ApiVersion;

#[derive(Clone, Debug, Deserialize)]
pub struct Link {
    #[serde(deserialize_with = "deser_url")]
    pub href: Url,
    pub rel: String
}

#[derive(Clone, Debug, Deserialize)]
pub struct Ref {
    pub id: String,
    pub links: Vec<Link>
}

#[derive(Clone, Debug, Deserialize)]
pub struct IdAndName {
    pub id: String,
    pub name: String
}

#[derive(Clone, Debug, Deserialize)]
pub struct Version {
    pub id: String,
    pub links: Vec<Link>,
    pub status: String,
    #[serde(deserialize_with = "empty_as_none", default)]
    pub version: Option<ApiVersion>,
    #[serde(deserialize_with = "empty_as_none", default)]
    pub min_version: Option<ApiVersion>
}

#[derive(Clone, Debug, Deserialize)]
#[serde(untagged)]
pub enum Root {
    Versions { versions: Vec<Version> },
    Version { version: Version }
}

impl Version {
    pub fn into_service_info(self) -> Result<ServiceInfo> {
        let endpoint = match self.links.into_iter().find(|x| &x.rel == "self") {
            Some(link) => link.href,
            None => {
                return Err(Error::new(
                    ErrorKind::InvalidResponse,
                    "Invalid version - missing self link"));
            }
        };

        Ok(ServiceInfo {
            root_url: endpoint,
            current_version: self.version,
            minimum_version: self.min_version
        })
    }
}

/// Generic code to extract a `ServiceInfo` from a URL.
pub fn fetch_service_info(endpoint: Url, auth: &AuthMethod,
                          service_type: &str, major_version: &str)
        -> Result<ServiceInfo> {
    debug!("Fetching {} service info from {}", service_type, endpoint);

    // Workaround for old version of Nova returning HTTP endpoints even if
    // accessed via HTTP
    let secure = endpoint.scheme() == "https";

    let result = auth.request(Method::Get, endpoint.clone())?.send();
    match result {
        Ok(mut resp) => {
            let mut info = match resp.json()? {
                Root::Version { version: ver } => ver.into_service_info(),
                Root::Versions { versions: vers } => {
                    match vers.into_iter().find(|x| &x.id == major_version) {
                        Some(ver) => ver.into_service_info(),
                        None => Err(Error::new_endpoint_not_found(service_type))
                    }
                }
            }?;

            // Older Nova returns insecure URLs even for secure protocol.
            if secure {
                let _ = info.root_url.set_scheme("https").unwrap();
            }

            debug!("Received {:?} for {} service from {}",
                   info, service_type, endpoint);
            Ok(info)
        },
        Err(ref e) if e.kind() == ErrorKind::ResourceNotFound => {
            if utils::url::is_root(&endpoint) {
                Err(Error::new_endpoint_not_found(service_type))
            } else {
                debug!("Got HTTP 404 from {}, trying parent endpoint",
                       endpoint);
                fetch_service_info(utils::url::pop(endpoint, true), auth,
                                   service_type, major_version)
            }
        },
        Err(other) => Err(other)
    }
}

/// Deserialize value where empty string equals None.
pub fn empty_as_none<'de, D, T>(des: D) -> ::std::result::Result<Option<T>, D::Error>
        where D: Deserializer<'de>, T: DeserializeOwned {
    let value = serde_json::Value::deserialize(des)?;
    match &value {
        &serde_json::Value::String(ref s) if s == "" => return Ok(None),
        _ => ()
    };

    match serde_json::from_value(value) {
        Ok(value) => Ok(Some(value)),
        Err(e) => Err(DeserError::custom(e))
    }
}

/// Deserialize value where empty string equals None.
pub fn empty_as_default<'de, D, T>(des: D) -> ::std::result::Result<T, D::Error>
        where D: Deserializer<'de>, T: DeserializeOwned + Default {
    let value = serde_json::Value::deserialize(des)?;
    match &value {
        &serde_json::Value::String(ref s) if s == "" =>
            return Ok(Default::default()),
        _ => ()
    };

    serde_json::from_value(value).map_err(DeserError::custom)
}

/// Deserialize a URL.
pub fn deser_url<'de, D>(des: D) -> ::std::result::Result<Url, D::Error>
        where D: Deserializer<'de> {
    Url::parse(&String::deserialize(des)?).map_err(DeserError::custom)
}

/// Deserialize a URL.
pub fn deser_optional_url<'de, D>(des: D)
        -> ::std::result::Result<Option<Url>, D::Error>
        where D: Deserializer<'de> {
    let value: Option<String> = Deserialize::deserialize(des)?;
    match value {
        Some(s) => Url::parse(&s).map_err(DeserError::custom).map(Some),
        None => Ok(None)
    }
}
<|endoftext|>


<commit_before>use futures::{Future, BoxFuture};
use linear_map::LinearMap;
use protobuf;

use mercury::MercuryError;
use protocol;
use session::Session;
use util::{SpotifyId, FileId, StrChunksExt};

pub use protocol::metadata::AudioFile_Format as FileFormat;

fn countrylist_contains(list: &str, country: &str) -> bool {
    list.chunks(2).any(|cc| cc == country)
}

fn parse_restrictions<'s, I>(restrictions: I, country: &str, catalogue: &str) -> bool
    where I: IntoIterator<Item = &'s protocol::metadata::Restriction>
{
    let mut forbidden = "".to_string();
    let mut has_forbidden = false;

    let mut allowed = "".to_string();
    let mut has_allowed = false;

    let rs = restrictions.into_iter().filter(|r|
        r.get_catalogue_str().contains(&catalogue.to_owned())
    );

    for r in rs {
        if r.has_countries_forbidden() {
            forbidden.push_str(r.get_countries_forbidden());
            has_forbidden = true;
        }

        if r.has_countries_allowed() {
            allowed.push_str(r.get_countries_allowed());
            has_allowed = true;
        }
    }

    (!has_forbidden || !countrylist_contains(forbidden.as_str(), country)) &&
    (!has_allowed || countrylist_contains(allowed.as_str(), country))
}

pub trait MetadataTrait : Send + 'static {
    type Message: protobuf::MessageStatic;

    fn base_url() -> &'static str;
    fn parse(msg: &Self::Message, session: &Session) -> Self;
}

#[derive(Debug, Clone)]
pub struct Track {
    pub id: SpotifyId,
    pub name: String,
    pub album: SpotifyId,
    pub artists: Vec<SpotifyId>,
    pub files: LinearMap<FileFormat, FileId>,
    pub alternatives: Vec<SpotifyId>,
    pub available: bool,
}

#[derive(Debug, Clone)]
pub struct Album {
    pub id: SpotifyId,
    pub name: String,
    pub artists: Vec<SpotifyId>,
    pub tracks: Vec<SpotifyId>,
    pub covers: Vec<FileId>,
}

#[derive(Debug, Clone)]
pub struct Artist {
    pub id: SpotifyId,
    pub name: String,
    pub top_tracks: Vec<SpotifyId>,
}

impl MetadataTrait for Track {
    type Message = protocol::metadata::Track;

    fn base_url() -> &'static str {
        "hm://metadata/3/track"
    }

    fn parse(msg: &Self::Message, session: &Session) -> Self {
        let country = session.country();

        let artists = msg.get_artist()
                         .iter()
                         .filter(|artist| artist.has_gid())
                         .map(|artist| SpotifyId::from_raw(artist.get_gid()))
                         .collect::<Vec<_>>();

        let files = msg.get_file()
                       .iter()
                       .filter(|file| file.has_file_id())
                       .map(|file| {
                           let mut dst = [0u8; 20];
                           dst.clone_from_slice(file.get_file_id());
                           (file.get_format(), FileId(dst))
                       })
                       .collect();

        Track {
            id: SpotifyId::from_raw(msg.get_gid()),
            name: msg.get_name().to_owned(),
            album: SpotifyId::from_raw(msg.get_album().get_gid()),
            artists: artists,
            files: files,
            alternatives: msg.get_alternative()
                             .iter()
                             .map(|alt| SpotifyId::from_raw(alt.get_gid()))
                             .collect(),
            available: parse_restrictions(msg.get_restriction(),
                                          &country,
                                          "premium"),
        }
    }
}

impl MetadataTrait for Album {
    type Message = protocol::metadata::Album;

    fn base_url() -> &'static str {
        "hm://metadata/3/album"
    }

    fn parse(msg: &Self::Message, _: &Session) -> Self {
        let artists = msg.get_artist()
                         .iter()
                         .filter(|artist| artist.has_gid())
                         .map(|artist| SpotifyId::from_raw(artist.get_gid()))
                         .collect::<Vec<_>>();

        let tracks = msg.get_disc()
                        .iter()
                        .flat_map(|disc| disc.get_track())
                        .filter(|track| track.has_gid())
                        .map(|track| SpotifyId::from_raw(track.get_gid()))
                        .collect::<Vec<_>>();

        let covers = msg.get_cover_group()
                        .get_image()
                        .iter()
                        .filter(|image| image.has_file_id())
                        .map(|image| {
                            let mut dst = [0u8; 20];
                            dst.clone_from_slice(image.get_file_id());
                            FileId(dst)
                        })
                        .collect::<Vec<_>>();

        Album {
            id: SpotifyId::from_raw(msg.get_gid()),
            name: msg.get_name().to_owned(),
            artists: artists,
            tracks: tracks,
            covers: covers,
        }
    }
}

impl MetadataTrait for Artist {
    type Message = protocol::metadata::Artist;

    fn base_url() -> &'static str {
        "hm://metadata/3/artist"
    }

    fn parse(msg: &Self::Message, session: &Session) -> Self {
        let country = session.country();

        let top_tracks = msg.get_top_track()
                            .iter()
                            .find(|tt| !tt.has_country() || countrylist_contains(tt.get_country(), &country))
                            .unwrap()
                            .get_track()
                            .iter()
                            .filter(|track| track.has_gid())
                            .map(|track| SpotifyId::from_raw(track.get_gid()))
                            .collect::<Vec<_>>();

        Artist {
            id: SpotifyId::from_raw(msg.get_gid()),
            name: msg.get_name().to_owned(),
            top_tracks: top_tracks
        }
    }
}

component! {
    MetadataManager : MetadataManagerInner { }
}

impl MetadataManager {
    pub fn get<T: MetadataTrait>(&self, id: SpotifyId) -> BoxFuture<T, MercuryError> {
        let session = self.session();

        let uri = format!("{}/{}", T::base_url(), id.to_base16());
        let request = session.mercury().get(uri);

        request.and_then(move |response| {
            let data = response.payload.first().expect("Empty payload");
            let msg: T::Message = protobuf::parse_from_bytes(data).unwrap();

            Ok(T::parse(&msg, &session))
        }).boxed()
    }
}
<commit_msg>Update metadata.rs<commit_after>use futures::{Future, BoxFuture};
use linear_map::LinearMap;
use protobuf;

use mercury::MercuryError;
use protocol;
use session::Session;
use util::{SpotifyId, FileId, StrChunksExt};

pub use protocol::metadata::AudioFile_Format as FileFormat;

fn countrylist_contains(list: &str, country: &str) -> bool {
    list.chunks(2).any(|cc| cc == country)
}

fn parse_restrictions<'s, I>(restrictions: I, country: &str, catalogue: &str) -> bool
    where I: IntoIterator<Item = &'s protocol::metadata::Restriction>
{
    let mut forbidden = "".to_string();
    let mut has_forbidden = false;

    let mut allowed = "".to_string();
    let mut has_allowed = false;

    let rs = restrictions.into_iter().filter(|r|
        r.get_catalogue_str().contains(&catalogue.to_owned())
    );

    for r in rs {
        if r.has_countries_forbidden() {
            forbidden.push_str(r.get_countries_forbidden());
            has_forbidden = true;
        }

        if r.has_countries_allowed() {
            allowed.push_str(r.get_countries_allowed());
            has_allowed = true;
        }
    }

    (has_forbidden || has_allowed) &&
    (!has_forbidden || !countrylist_contains(forbidden.as_str(), country)) &&
    (!has_allowed || countrylist_contains(allowed.as_str(), country))
}

pub trait MetadataTrait : Send + 'static {
    type Message: protobuf::MessageStatic;

    fn base_url() -> &'static str;
    fn parse(msg: &Self::Message, session: &Session) -> Self;
}

#[derive(Debug, Clone)]
pub struct Track {
    pub id: SpotifyId,
    pub name: String,
    pub album: SpotifyId,
    pub artists: Vec<SpotifyId>,
    pub files: LinearMap<FileFormat, FileId>,
    pub alternatives: Vec<SpotifyId>,
    pub available: bool,
}

#[derive(Debug, Clone)]
pub struct Album {
    pub id: SpotifyId,
    pub name: String,
    pub artists: Vec<SpotifyId>,
    pub tracks: Vec<SpotifyId>,
    pub covers: Vec<FileId>,
}

#[derive(Debug, Clone)]
pub struct Artist {
    pub id: SpotifyId,
    pub name: String,
    pub top_tracks: Vec<SpotifyId>,
}

impl MetadataTrait for Track {
    type Message = protocol::metadata::Track;

    fn base_url() -> &'static str {
        "hm://metadata/3/track"
    }

    fn parse(msg: &Self::Message, session: &Session) -> Self {
        let country = session.country();

        let artists = msg.get_artist()
                         .iter()
                         .filter(|artist| artist.has_gid())
                         .map(|artist| SpotifyId::from_raw(artist.get_gid()))
                         .collect::<Vec<_>>();

        let files = msg.get_file()
                       .iter()
                       .filter(|file| file.has_file_id())
                       .map(|file| {
                           let mut dst = [0u8; 20];
                           dst.clone_from_slice(file.get_file_id());
                           (file.get_format(), FileId(dst))
                       })
                       .collect();

        Track {
            id: SpotifyId::from_raw(msg.get_gid()),
            name: msg.get_name().to_owned(),
            album: SpotifyId::from_raw(msg.get_album().get_gid()),
            artists: artists,
            files: files,
            alternatives: msg.get_alternative()
                             .iter()
                             .map(|alt| SpotifyId::from_raw(alt.get_gid()))
                             .collect(),
            available: parse_restrictions(msg.get_restriction(),
                                          &country,
                                          "premium"),
        }
    }
}

impl MetadataTrait for Album {
    type Message = protocol::metadata::Album;

    fn base_url() -> &'static str {
        "hm://metadata/3/album"
    }

    fn parse(msg: &Self::Message, _: &Session) -> Self {
        let artists = msg.get_artist()
                         .iter()
                         .filter(|artist| artist.has_gid())
                         .map(|artist| SpotifyId::from_raw(artist.get_gid()))
                         .collect::<Vec<_>>();

        let tracks = msg.get_disc()
                        .iter()
                        .flat_map(|disc| disc.get_track())
                        .filter(|track| track.has_gid())
                        .map(|track| SpotifyId::from_raw(track.get_gid()))
                        .collect::<Vec<_>>();

        let covers = msg.get_cover_group()
                        .get_image()
                        .iter()
                        .filter(|image| image.has_file_id())
                        .map(|image| {
                            let mut dst = [0u8; 20];
                            dst.clone_from_slice(image.get_file_id());
                            FileId(dst)
                        })
                        .collect::<Vec<_>>();

        Album {
            id: SpotifyId::from_raw(msg.get_gid()),
            name: msg.get_name().to_owned(),
            artists: artists,
            tracks: tracks,
            covers: covers,
        }
    }
}

impl MetadataTrait for Artist {
    type Message = protocol::metadata::Artist;

    fn base_url() -> &'static str {
        "hm://metadata/3/artist"
    }

    fn parse(msg: &Self::Message, session: &Session) -> Self {
        let country = session.country();

        let top_tracks = msg.get_top_track()
                            .iter()
                            .find(|tt| !tt.has_country() || countrylist_contains(tt.get_country(), &country))
                            .unwrap()
                            .get_track()
                            .iter()
                            .filter(|track| track.has_gid())
                            .map(|track| SpotifyId::from_raw(track.get_gid()))
                            .collect::<Vec<_>>();

        Artist {
            id: SpotifyId::from_raw(msg.get_gid()),
            name: msg.get_name().to_owned(),
            top_tracks: top_tracks
        }
    }
}

component! {
    MetadataManager : MetadataManagerInner { }
}

impl MetadataManager {
    pub fn get<T: MetadataTrait>(&self, id: SpotifyId) -> BoxFuture<T, MercuryError> {
        let session = self.session();

        let uri = format!("{}/{}", T::base_url(), id.to_base16());
        let request = session.mercury().get(uri);

        request.and_then(move |response| {
            let data = response.payload.first().expect("Empty payload");
            let msg: T::Message = protobuf::parse_from_bytes(data).unwrap();

            Ok(T::parse(&msg, &session))
        }).boxed()
    }
}
<|endoftext|>


<commit_before>use uuid::Uuid;
use std::collections::BTreeMap;
use std::fmt::Write;
use std::rc::Rc;

use conditions::Conditions;
use message::{Message};
use state::State;
use timer::TimerEvent;
use context::base::BaseContext;
use context::event::EventHandler;
use dispatcher::request::{Request, InternalRequest};

pub struct MapContext {
    base: BaseContext,
    map: BTreeMap<String, State>,
    format_buffer: String
}

impl MapContext {
    pub fn new(uuid: Uuid, conditions: Conditions) -> MapContext {
        MapContext {
            base: BaseContext::new(uuid, conditions),
            map: BTreeMap::new(),
            format_buffer: String::new()
        }
    }

    pub fn on_event(&mut self, event: InternalRequest) {
        if let Request::Message(event) = event {
            self.on_message(event);
        }
    }

    pub fn on_timer(&mut self, event: &TimerEvent) {
        for (_, mut state) in self.map.iter_mut() {
            self.base.on_timer(event, &mut state);

        }
        self.remove_closed_states();
    }

    fn get_closed_state_ids(&self) -> Vec<String> {
        self.map.iter().filter_map(|(id, state)| {
            if !state.is_open() {
                Some(id.clone())
            } else {
                None
            }
        }).collect::<Vec<String>>()
    }

    fn remove_closed_states(&mut self) {
        for id in self.get_closed_state_ids() {
            let _ = self.map.remove(&id);
        }
    }

    pub fn on_message(&mut self, event: Rc<Message>) {
        self.format_context_id(&event);
        self.update_state(event);
        self.format_buffer.clear();
        self.remove_closed_states();
    }

    fn update_state(&mut self, event: Rc<Message>) {
        let id = self.format_buffer.clone();
        let state = self.map.entry(id).or_insert(State::new());
        self.base.on_message(event, state);
    }

    pub fn is_open(&mut self) -> bool {
        !self.map.is_empty()
    }

    fn format_context_id(&mut self, message: &Message) {
        let empty = String::new();
        let _ = self.format_buffer.write_str(message.get("HOST").unwrap_or(&empty));
        let _ = self.format_buffer.write_str(":");
        let _ = self.format_buffer.write_str(message.get("PROGRAM").unwrap_or(&empty));
        let _ = self.format_buffer.write_str(":");
        let _ = self.format_buffer.write_str(message.get("PID").unwrap_or(&empty));
    }

    pub fn patterns(&self) -> &[String] {
        &self.base.conditions().patterns
    }
}

impl From<BaseContext> for MapContext {
    fn from(context: BaseContext) -> MapContext {
        MapContext {
            base: context,
            map: BTreeMap::new(),
            format_buffer: String::new()
        }
    }
}

impl EventHandler<InternalRequest> for MapContext {
    fn handlers(&self) -> &[String] {
        self.patterns()
    }
    fn handle_event(&mut self, event: InternalRequest) {
        self.on_event(event);
    }
}
<commit_msg>MapContext: call on_message() when a Message request is received<commit_after>use uuid::Uuid;
use std::collections::BTreeMap;
use std::fmt::Write;
use std::rc::Rc;

use conditions::Conditions;
use message::{Message};
use state::State;
use timer::TimerEvent;
use context::base::BaseContext;
use context::event::EventHandler;
use dispatcher::request::{Request, InternalRequest};

pub struct MapContext {
    base: BaseContext,
    map: BTreeMap<String, State>,
    format_buffer: String
}

impl MapContext {
    pub fn new(uuid: Uuid, conditions: Conditions) -> MapContext {
        MapContext {
            base: BaseContext::new(uuid, conditions),
            map: BTreeMap::new(),
            format_buffer: String::new()
        }
    }

    pub fn on_event(&mut self, event: InternalRequest) {
        match event {
            Request::Timer(event) => {
                self.on_timer(&event)
            },
            Request::Message(message) => {
                self.on_message(message)
            },
            _ => {}
        }
    }

    pub fn on_timer(&mut self, event: &TimerEvent) {
        for (_, mut state) in self.map.iter_mut() {
            self.base.on_timer(event, &mut state);

        }
        self.remove_closed_states();
    }

    fn get_closed_state_ids(&self) -> Vec<String> {
        self.map.iter().filter_map(|(id, state)| {
            if !state.is_open() {
                Some(id.clone())
            } else {
                None
            }
        }).collect::<Vec<String>>()
    }

    fn remove_closed_states(&mut self) {
        for id in self.get_closed_state_ids() {
            let _ = self.map.remove(&id);
        }
    }

    pub fn on_message(&mut self, event: Rc<Message>) {
        self.format_context_id(&event);
        self.update_state(event);
        self.format_buffer.clear();
        self.remove_closed_states();
    }

    fn update_state(&mut self, event: Rc<Message>) {
        let id = self.format_buffer.clone();
        let state = self.map.entry(id).or_insert(State::new());
        self.base.on_message(event, state);
    }

    pub fn is_open(&mut self) -> bool {
        !self.map.is_empty()
    }

    fn format_context_id(&mut self, message: &Message) {
        let empty = String::new();
        let _ = self.format_buffer.write_str(message.get("HOST").unwrap_or(&empty));
        let _ = self.format_buffer.write_str(":");
        let _ = self.format_buffer.write_str(message.get("PROGRAM").unwrap_or(&empty));
        let _ = self.format_buffer.write_str(":");
        let _ = self.format_buffer.write_str(message.get("PID").unwrap_or(&empty));
    }

    pub fn patterns(&self) -> &[String] {
        &self.base.conditions().patterns
    }
}

impl From<BaseContext> for MapContext {
    fn from(context: BaseContext) -> MapContext {
        MapContext {
            base: context,
            map: BTreeMap::new(),
            format_buffer: String::new()
        }
    }
}

impl EventHandler<InternalRequest> for MapContext {
    fn handlers(&self) -> &[String] {
        self.patterns()
    }
    fn handle_event(&mut self, event: InternalRequest) {
        self.on_event(event);
    }
}
<|endoftext|>


<commit_before>use std::fmt;

pub struct Instruction(pub u32);

impl Instruction {
    #[inline(always)]
    pub fn opcode(&self) -> u8 {
        ((self.0  >> 26) & 0b11_1111) as u8
    }

    #[inline(always)]
    pub fn subopcode(&self) -> u16 {
        ((self.0 >> 1) & 0b11_1111_1111) as u16
    }

    #[inline(always)]
    pub fn d(&self) -> usize {
        ((self.0 >> 21) & 0b1_1111) as usize
    }

    #[inline(always)]
    pub fn a(&self) -> usize {
        ((self.0 >> 16) & 0b1_1111) as usize
    }

    #[inline(always)]
    pub fn b(&self) -> usize {
        ((self.0 >> 11) & 0b1_1111) as usize
    }

    #[inline(always)]
    pub fn crfd(&self) -> usize {
        ((self.0 >> 16) & 0b1_1111) as usize
    }

    #[inline(always)]
    pub fn simm(&self) -> u32 { // FIXME: should probably be signed
        (self.0 & 0b1111_1111_1111_1111) as u32
    }

    #[inline(always)]
    pub fn uimm(&self) -> u32 {
        (self.0 & 0b1111_1111_1111_1111) as u32
    }

    #[inline(always)]
    pub fn li(&self) -> u32 {
        ((self.0 >> 2) & 0b1111_1111_1111_1111_1111_1111)
    }

    #[inline(always)]
    pub fn bo(&self) -> u8 {
        ((self.0 >> 21) & 0b1_1111) as u8
    }

    #[inline(always)]
    pub fn bi(&self) -> usize {
        ((self.0 >> 16) & 0b1_1111) as usize
    }

    #[inline(always)]
    pub fn bd(&self) -> u32 {
        ((self.0 >> 2) & 0b1111_1111_1111_1111_1111_1111)
    }

    #[inline(always)]
    pub fn aa(&self) -> u8 {
        ((self.0 >> 1) & 0b1) as u8
    }

    #[inline(always)]
    pub fn lk(&self) -> u8 {
        (self.0 & 0b1) as u8
    }

    #[inline(always)]
    pub fn s(&self) -> usize {
        ((self.0 >> 21) & 0b1_1111) as usize
    }

    #[inline(always)]
    pub fn sr(&self) -> usize {
        ((self.0 >> 16) & 0b1111) as usize
    }

    #[inline(always)]
    pub fn sh(&self) -> u8 {
        ((self.0 >> 11) & 0b1_1111) as u8
    }

    #[inline(always)]
    pub fn mb(&self) -> u8 {
        ((self.0 >> 6) & 0b1_1111) as u8
    }

    #[inline(always)]
    pub fn me(&self) -> u8 {
        ((self.0 >> 1) & 0b1_1111) as u8
    }

    #[inline(always)]
    //pub fn spr(&self) -> u32 {
    //    (self.0 >> 11) & 0b11_1111_1111
    //}

    #[inline(always)]
    pub fn spr_upper(&self) -> u32 {
        (self.0 >> 11) & 0b1_1111
    }

    #[inline(always)]
    pub fn spr_lower(&self) -> u32 {
        (self.0 >> 16) & 0b1_1111
    }
}

impl fmt::Debug for Instruction {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        write!(f, "{:#b}", self.opcode())
    }
}
<commit_msg>fix wrong instruction bd<commit_after>use std::fmt;

pub struct Instruction(pub u32);

impl Instruction {
    #[inline(always)]
    pub fn opcode(&self) -> u8 {
        ((self.0  >> 26) & 0b11_1111) as u8
    }

    #[inline(always)]
    pub fn subopcode(&self) -> u16 {
        ((self.0 >> 1) & 0b11_1111_1111) as u16
    }

    #[inline(always)]
    pub fn d(&self) -> usize {
        ((self.0 >> 21) & 0b1_1111) as usize
    }

    #[inline(always)]
    pub fn a(&self) -> usize {
        ((self.0 >> 16) & 0b1_1111) as usize
    }

    #[inline(always)]
    pub fn b(&self) -> usize {
        ((self.0 >> 11) & 0b1_1111) as usize
    }

    #[inline(always)]
    pub fn crfd(&self) -> usize {
        ((self.0 >> 16) & 0b1_1111) as usize
    }

    #[inline(always)]
    pub fn simm(&self) -> u32 { // FIXME: should probably be signed
        (self.0 & 0b1111_1111_1111_1111) as u32
    }

    #[inline(always)]
    pub fn uimm(&self) -> u32 {
        (self.0 & 0b1111_1111_1111_1111) as u32
    }

    #[inline(always)]
    pub fn li(&self) -> u32 {
        ((self.0 >> 2) & 0b1111_1111_1111_1111_1111_1111)
    }

    #[inline(always)]
    pub fn bo(&self) -> u8 {
        ((self.0 >> 21) & 0b1_1111) as u8
    }

    #[inline(always)]
    pub fn bi(&self) -> usize {
        ((self.0 >> 16) & 0b1_1111) as usize
    }

    #[inline(always)]
    pub fn bd(&self) -> u32 {
        ((self.0 >> 2) & 0b1111_1111_1111_11)
    }

    #[inline(always)]
    pub fn aa(&self) -> u8 {
        ((self.0 >> 1) & 0b1) as u8
    }

    #[inline(always)]
    pub fn lk(&self) -> u8 {
        (self.0 & 0b1) as u8
    }

    #[inline(always)]
    pub fn s(&self) -> usize {
        ((self.0 >> 21) & 0b1_1111) as usize
    }

    #[inline(always)]
    pub fn sr(&self) -> usize {
        ((self.0 >> 16) & 0b1111) as usize
    }

    #[inline(always)]
    pub fn sh(&self) -> u8 {
        ((self.0 >> 11) & 0b1_1111) as u8
    }

    #[inline(always)]
    pub fn mb(&self) -> u8 {
        ((self.0 >> 6) & 0b1_1111) as u8
    }

    #[inline(always)]
    pub fn me(&self) -> u8 {
        ((self.0 >> 1) & 0b1_1111) as u8
    }

    #[inline(always)]
    //pub fn spr(&self) -> u32 {
    //    (self.0 >> 11) & 0b11_1111_1111
    //}

    #[inline(always)]
    pub fn spr_upper(&self) -> u32 {
        (self.0 >> 11) & 0b1_1111
    }

    #[inline(always)]
    pub fn spr_lower(&self) -> u32 {
        (self.0 >> 16) & 0b1_1111
    }
}

impl fmt::Debug for Instruction {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        write!(f, "{:#b}", self.opcode())
    }
}
<|endoftext|>


<commit_before>pub enum TaskPriority {
    NoPrio,
    Low,
    Medium,
    High,
}

impl<'a> From<&'a str> for TaskPriority {

    fn from(s: &str) -> TaskPriority {
        match s {
            "L" => TaskPriority::Low,
            "M" => TaskPriority::Medium,
            "H" => TaskPriority::High,
            _ => TaskPriority::NoPrio,
        }
    }
}
<commit_msg>TaskPriority: Derive Debug, Clone<commit_after>#[derive(Debug, Clone)]
pub enum TaskPriority {
    NoPrio,
    Low,
    Medium,
    High,
}

impl<'a> From<&'a str> for TaskPriority {

    fn from(s: &str) -> TaskPriority {
        match s {
            "L" => TaskPriority::Low,
            "M" => TaskPriority::Medium,
            "H" => TaskPriority::High,
            _ => TaskPriority::NoPrio,
        }
    }
}
<|endoftext|>