<commit_before>pub mod test_common;
pub mod distance;

mod partition;
mod bounds;

use self::bounds::*;
use self::distance::*;

use std::cmp;

pub trait KdtreePointTrait: Copy + PartialEq {
    fn dims(&self) -> &[f64];
}

pub struct Kdtree<KdtreePoint> {
    nodes: Vec<KdtreeNode<KdtreePoint>>,

    node_adding_dimension: usize,
    node_depth_during_last_rebuild: usize,
    current_node_depth: usize,
}

impl<KdtreePoint: KdtreePointTrait> Kdtree<KdtreePoint> {
    pub fn new(mut points: &mut [KdtreePoint]) -> Kdtree<KdtreePoint> {
        if points.len() == 0 {
            panic!("empty vector point not allowed");
        }

        let mut tree = Kdtree {
            nodes: vec![],
            node_adding_dimension: 0,
            node_depth_during_last_rebuild: 0,
            current_node_depth: 0,
        };

        tree.rebuild_tree(&mut points);

        tree
    }

    pub fn rebuild_tree(&mut self, points : &mut [KdtreePoint]) {
        self.nodes.clear();

        self.node_depth_during_last_rebuild = 0;
        self.current_node_depth = 0;

        let rect = Bounds::new_from_points(points);
        self.build_tree(points, &rect, 1);
    }

    /// Can be used if you are sure that the tree is degenerated or if you will never again insert the nodes into the tree.
    pub fn gather_points_and_rebuild(&mut self) {
        let mut points : Vec<KdtreePoint> = vec![];
        self.gather_points(0,&mut points);

        self.rebuild_tree(&mut points);
    }

    pub fn nearest_search(&self, node: &KdtreePoint) -> KdtreePoint
    {
        let mut nearest_neighbor = 0usize;
        let mut best_distance = squared_euclidean(node.dims(), &self.nodes[0].point.dims());
        self.nearest_search_impl(node, 0usize, &mut best_distance, &mut nearest_neighbor);

        self.nodes[nearest_neighbor].point
    }

    pub fn has_neighbor_in_range(&self, node: &KdtreePoint, range: f64) -> bool {
        let squared_range = range * range;

        self.distance_squared_to_nearest(node) <= squared_range
    }

    pub fn distance_squared_to_nearest(&self, node: &KdtreePoint) -> f64 {
        squared_euclidean(&self.nearest_search(node).dims(), node.dims())
    }

    pub fn insert_nodes_and_rebuild(&mut self, nodes_to_add : &mut [KdtreePoint]) {
        let mut pts : Vec<KdtreePoint> = vec![];
        self.gather_points(0, &mut pts);
        pts.extend(nodes_to_add.iter());

        self.rebuild_tree(&mut pts);
    }

    pub fn insert_node(&mut self, node_to_add : KdtreePoint) {

        let mut current_index = 0;
        let dimension = self.node_adding_dimension;
        let index_of_new_node = self.add_node(node_to_add,dimension,node_to_add.dims()[dimension]);
        self.node_adding_dimension = ( dimension + 1) % node_to_add.dims().len();
        let mut should_pop_node = false;

        let mut depth = 0;
        loop {

            depth +=1 ;
            let current_node = &mut self.nodes[current_index];

            if node_to_add.dims()[current_node.dimension] <= current_node.split_on {
                if let Some(left_node_index) = current_node.left_node {
                    current_index = left_node_index
                } else {
                    if current_node.point.eq(&node_to_add) {
                        should_pop_node = true;
                    } else {
                        current_node.left_node = Some(index_of_new_node);
                    }
                    break;
                }
            } else {
                if let Some(right_node_index) = current_node.right_node {
                    current_index = right_node_index
                } else {
                    if current_node.point.eq(&node_to_add) {
                        should_pop_node = true;
                    } else {
                        current_node.right_node = Some(index_of_new_node);
                    }
                    break;
                }
            }
        }

        if should_pop_node {
            self.nodes.pop();
        }

        if self.node_depth_during_last_rebuild as f64 * 4.0 < depth as f64  {
            self.gather_points_and_rebuild();
        }
    }

    fn nearest_search_impl(&self, p: &KdtreePoint, searched_index: usize, best_distance_squared: &mut f64, best_leaf_found: &mut usize) {
        let node = &self.nodes[searched_index];

        let splitting_value = node.split_on;
        let point_splitting_dim_value = p.dims()[node.dimension];

        let (closer_node, farther_node) = if point_splitting_dim_value <= splitting_value {
            (node.left_node, node.right_node)
        } else {
            (node.right_node, node.left_node)
        };

        if let Some(closer_node) = closer_node {
            self.nearest_search_impl(p, closer_node, best_distance_squared, best_leaf_found);
        }

        let distance = squared_euclidean(p.dims(), node.point.dims());
        if distance < *best_distance_squared {
            *best_distance_squared = distance;
            *best_leaf_found = searched_index;
        }

        if let Some(farther_node) = farther_node {
            let distance_on_single_dimension = squared_euclidean(&[splitting_value], &[point_splitting_dim_value]);

            if distance_on_single_dimension <= *best_distance_squared {
                self.nearest_search_impl(p, farther_node, best_distance_squared, best_leaf_found);
            }
        }
    }

    fn add_node(&mut self, p: KdtreePoint, dimension: usize, split_on: f64) -> usize {
        let node = KdtreeNode::new(p, dimension, split_on);

        self.nodes.push(node);
        self.nodes.len() - 1
    }

    fn build_tree(&mut self, nodes: &mut [KdtreePoint], bounds: &Bounds, depth : usize) -> usize {
        let splitting_index = partition::partition_sliding_midpoint(nodes, bounds.get_midvalue_of_widest_dim(), bounds.get_widest_dim());
        let pivot_value = nodes[splitting_index].dims()[bounds.get_widest_dim()];

        let node_id = self.add_node(nodes[splitting_index], bounds.get_widest_dim(), pivot_value);
        let nodes_len = nodes.len();

        if splitting_index > 0 {
            let left_rect = bounds.clone_moving_max(pivot_value, bounds.get_widest_dim());
            let left_child_id = self.build_tree(&mut nodes[0..splitting_index], &left_rect, depth+1);
            self.nodes[node_id].left_node = Some(left_child_id);
        }

        if splitting_index < nodes.len() - 1 {
            let right_rect = bounds.clone_moving_min(pivot_value, bounds.get_widest_dim());

            let right_child_id = self.build_tree(&mut nodes[splitting_index + 1..nodes_len], &right_rect, depth+1);
            self.nodes[node_id].right_node = Some(right_child_id);
        }

        self.node_depth_during_last_rebuild =  cmp::max(self.node_depth_during_last_rebuild,depth);

        node_id
    }

    fn gather_points(&self, current_index: usize, points : &mut Vec<KdtreePoint>){
        points.push(self.nodes[current_index].point);
        if let Some(left_index) = self.nodes[current_index].left_node {
            self.gather_points(left_index, points);
        }

        if let Some(right_index) = self.nodes[current_index].right_node {
            self.gather_points(right_index, points);
        }
    }
}

pub struct KdtreeNode<T> {
    left_node: Option<usize>,
    right_node: Option<usize>,

    point: T,
    dimension: usize,
    split_on: f64
}

impl<T: KdtreePointTrait> KdtreeNode<T> {
    fn new(p: T, splitting_dimension: usize, split_on_value: f64) -> KdtreeNode<T> {
        KdtreeNode {
            left_node: None,
            right_node: None,

            point: p,
            dimension: splitting_dimension,
            split_on: split_on_value
        }
    }
}

#[cfg(test)]
mod tests {
    use ::kdtree::test_common::Point2WithId;

    use super::*;

    #[test]
    #[should_panic(expected = "empty vector point not allowed")]
    fn should_panic_given_empty_vector() {
        let mut empty_vec: Vec<Point2WithId> = vec![];

        Kdtree::new(&mut empty_vec);
    }

    quickcheck! {
        fn tree_build_creates_tree_with_as_many_leafs_as_there_is_points(xs : Vec<f64>) -> bool {
            if xs.len() == 0 {
                return true;
            }
            let mut vec : Vec<Point2WithId> = vec![];
            for i in 0 .. xs.len() {
                let p = Point2WithId::new(i as i32, xs[i], xs[i]);

                vec.push(p);
            }

            let tree = Kdtree::new(&mut qc_value_vec_to_2d_points_vec(&xs));

            let mut to_iterate : Vec<usize> = vec![];
            to_iterate.push(0);

            while to_iterate.len() > 0 {
                let last_index = to_iterate.last().unwrap().clone();
                let ref x = tree.nodes.get(last_index).unwrap();
                to_iterate.pop();
                if x.left_node.is_some() {
                    to_iterate.push(x.left_node.unwrap());
                }
                if x.right_node.is_some() {
                    to_iterate.push(x.right_node.unwrap());
                }
            }
            xs.len() == tree.nodes.len()
        }
    }

    quickcheck! {
        fn nearest_neighbor_search_using_qc(xs : Vec<f64>) -> bool {
            if xs.len() == 0 {
                return true;
            }

            let point_vec = qc_value_vec_to_2d_points_vec(&xs);
            let tree = Kdtree::new(&mut point_vec.clone());

            for p in &point_vec {
                let found_nn = tree.nearest_search(p);

                assert_eq!(p.id,found_nn.id);
            }

            true
        }
    }

    #[test]
    fn has_neighbor_in_range() {
        let mut vec: Vec<Point2WithId> = vec![Point2WithId::new(0,2.,0.)];

        let tree = Kdtree::new(&mut vec);

        assert_eq!(false,tree.has_neighbor_in_range(&Point2WithId::new(0,0.,0.), 0.));
        assert_eq!(false,tree.has_neighbor_in_range(&Point2WithId::new(0,0.,0.), 1.));
        assert_eq!(true,tree.has_neighbor_in_range(&Point2WithId::new(0,0.,0.), 2.));
        assert_eq!(true,tree.has_neighbor_in_range(&Point2WithId::new(0,0.,0.), 300.));
    }

    #[test]
    fn incremental_add_adds_as_expected() {
        //this test is tricky because it can have problems with the automatic tree rebuild.

        let mut vec = vec![Point2WithId::new(0,0.,0.)];

        let mut tree = Kdtree::new(&mut vec);

        tree.insert_node(Point2WithId::new(0,1.,0.));
        tree.insert_node(Point2WithId::new(0,-1.,0.));

        assert_eq!(tree.nodes.len(), 3);
        assert_eq!(tree.nodes[0].dimension, 0);

        assert_eq!(tree.nodes[0].left_node.is_some(), true);
        assert_eq!(tree.nodes[1].point.dims()[0], 1.);
        assert_eq!(tree.nodes[2].point.dims()[0], -1.);

        assert_eq!(tree.nodes[0].right_node.is_some(), true);
    }

    #[test]
    fn incremental_add_filters_duplicates() {
        let mut vec = vec![Point2WithId::new(0,0.,0.)];

        let mut tree = Kdtree::new(&mut vec);

        let node = Point2WithId::new(0,1.,0.);
        tree.insert_node(node);
        tree.insert_node(node);

        assert_eq!(tree.nodes.len(), 2);
    }

    fn qc_value_vec_to_2d_points_vec(xs: &Vec<f64>) -> Vec<Point2WithId> {
        let mut vec: Vec<Point2WithId> = vec![];
        for i in 0..xs.len() {
            let mut is_duplicated_value = false;
            for j in 0..i {
                if xs[i] == xs[j] {
                    is_duplicated_value = true;
                    break;
                }
            }
            if !is_duplicated_value {
                let p = Point2WithId::new(i as i32, xs[i], xs[i]);
                vec.push(p);
            }
        }

        vec
    }
}<commit_msg>tweaking<commit_after>pub mod test_common;
pub mod distance;

mod partition;
mod bounds;

use self::bounds::*;
use self::distance::*;

use std::cmp;

pub trait KdtreePointTrait: Copy + PartialEq {
    fn dims(&self) -> &[f64];
}

pub struct Kdtree<KdtreePoint> {
    nodes: Vec<KdtreeNode<KdtreePoint>>,

    node_adding_dimension: usize,
    node_depth_during_last_rebuild: usize,
    current_node_depth: usize,
}

impl<KdtreePoint: KdtreePointTrait> Kdtree<KdtreePoint> {
    pub fn new(mut points: &mut [KdtreePoint]) -> Kdtree<KdtreePoint> {
        if points.len() == 0 {
            panic!("empty vector point not allowed");
        }

        let mut tree = Kdtree {
            nodes: vec![],
            node_adding_dimension: 0,
            node_depth_during_last_rebuild: 0,
            current_node_depth: 0,
        };

        tree.rebuild_tree(&mut points);

        tree
    }

    pub fn rebuild_tree(&mut self, points : &mut [KdtreePoint]) {
        self.nodes.clear();

        self.node_depth_during_last_rebuild = 0;
        self.current_node_depth = 0;

        let rect = Bounds::new_from_points(points);
        self.build_tree(points, &rect, 1);
    }

    /// Can be used if you are sure that the tree is degenerated or if you will never again insert the nodes into the tree.
    pub fn gather_points_and_rebuild(&mut self) {
        let mut points : Vec<KdtreePoint> = vec![];
        self.gather_points(0,&mut points);

        self.rebuild_tree(&mut points);
    }

    pub fn nearest_search(&self, node: &KdtreePoint) -> KdtreePoint
    {
        let mut nearest_neighbor = 0usize;
        let mut best_distance = squared_euclidean(node.dims(), &self.nodes[0].point.dims());
        self.nearest_search_impl(node, 0usize, &mut best_distance, &mut nearest_neighbor);

        self.nodes[nearest_neighbor].point
    }

    pub fn has_neighbor_in_range(&self, node: &KdtreePoint, range: f64) -> bool {
        let squared_range = range * range;

        self.distance_squared_to_nearest(node) <= squared_range
    }

    pub fn distance_squared_to_nearest(&self, node: &KdtreePoint) -> f64 {
        squared_euclidean(&self.nearest_search(node).dims(), node.dims())
    }

    pub fn insert_nodes_and_rebuild(&mut self, nodes_to_add : &mut [KdtreePoint]) {
        let mut pts : Vec<KdtreePoint> = vec![];
        self.gather_points(0, &mut pts);
        pts.extend(nodes_to_add.iter());

        self.rebuild_tree(&mut pts);
    }

    pub fn insert_node(&mut self, node_to_add : KdtreePoint) {

        let mut current_index = 0;
        let dimension = self.node_adding_dimension;
        let index_of_new_node = self.add_node(node_to_add,dimension,node_to_add.dims()[dimension]);
        self.node_adding_dimension = ( dimension + 1) % node_to_add.dims().len();
        let mut should_pop_node = false;

        let mut depth = 0;
        loop {

            depth +=1 ;
            let current_node = &mut self.nodes[current_index];

            if node_to_add.dims()[current_node.dimension] <= current_node.split_on {
                if let Some(left_node_index) = current_node.left_node {
                    current_index = left_node_index
                } else {
                    if current_node.point.eq(&node_to_add) {
                        should_pop_node = true;
                    } else {
                        current_node.left_node = Some(index_of_new_node);
                    }
                    break;
                }
            } else {
                if let Some(right_node_index) = current_node.right_node {
                    current_index = right_node_index
                } else {
                    if current_node.point.eq(&node_to_add) {
                        should_pop_node = true;
                    } else {
                        current_node.right_node = Some(index_of_new_node);
                    }
                    break;
                }
            }
        }

        if should_pop_node {
            self.nodes.pop();
        }

        if self.node_depth_during_last_rebuild as f64 * 8.0 < depth as f64  {
            self.gather_points_and_rebuild();
        }
    }

    fn nearest_search_impl(&self, p: &KdtreePoint, searched_index: usize, best_distance_squared: &mut f64, best_leaf_found: &mut usize) {
        let node = &self.nodes[searched_index];

        let splitting_value = node.split_on;
        let point_splitting_dim_value = p.dims()[node.dimension];

        let (closer_node, farther_node) = if point_splitting_dim_value <= splitting_value {
            (node.left_node, node.right_node)
        } else {
            (node.right_node, node.left_node)
        };

        if let Some(closer_node) = closer_node {
            self.nearest_search_impl(p, closer_node, best_distance_squared, best_leaf_found);
        }

        let distance = squared_euclidean(p.dims(), node.point.dims());
        if distance < *best_distance_squared {
            *best_distance_squared = distance;
            *best_leaf_found = searched_index;
        }

        if let Some(farther_node) = farther_node {
            let distance_on_single_dimension = squared_euclidean(&[splitting_value], &[point_splitting_dim_value]);

            if distance_on_single_dimension <= *best_distance_squared {
                self.nearest_search_impl(p, farther_node, best_distance_squared, best_leaf_found);
            }
        }
    }

    fn add_node(&mut self, p: KdtreePoint, dimension: usize, split_on: f64) -> usize {
        let node = KdtreeNode::new(p, dimension, split_on);

        self.nodes.push(node);
        self.nodes.len() - 1
    }

    fn build_tree(&mut self, nodes: &mut [KdtreePoint], bounds: &Bounds, depth : usize) -> usize {
        let splitting_index = partition::partition_sliding_midpoint(nodes, bounds.get_midvalue_of_widest_dim(), bounds.get_widest_dim());
        let pivot_value = nodes[splitting_index].dims()[bounds.get_widest_dim()];

        let node_id = self.add_node(nodes[splitting_index], bounds.get_widest_dim(), pivot_value);
        let nodes_len = nodes.len();

        if splitting_index > 0 {
            let left_rect = bounds.clone_moving_max(pivot_value, bounds.get_widest_dim());
            let left_child_id = self.build_tree(&mut nodes[0..splitting_index], &left_rect, depth+1);
            self.nodes[node_id].left_node = Some(left_child_id);
        }

        if splitting_index < nodes.len() - 1 {
            let right_rect = bounds.clone_moving_min(pivot_value, bounds.get_widest_dim());

            let right_child_id = self.build_tree(&mut nodes[splitting_index + 1..nodes_len], &right_rect, depth+1);
            self.nodes[node_id].right_node = Some(right_child_id);
        }

        self.node_depth_during_last_rebuild =  cmp::max(self.node_depth_during_last_rebuild,depth);

        node_id
    }

    fn gather_points(&self, current_index: usize, points : &mut Vec<KdtreePoint>){
        points.push(self.nodes[current_index].point);
        if let Some(left_index) = self.nodes[current_index].left_node {
            self.gather_points(left_index, points);
        }

        if let Some(right_index) = self.nodes[current_index].right_node {
            self.gather_points(right_index, points);
        }
    }
}

pub struct KdtreeNode<T> {
    left_node: Option<usize>,
    right_node: Option<usize>,

    point: T,
    dimension: usize,
    split_on: f64
}

impl<T: KdtreePointTrait> KdtreeNode<T> {
    fn new(p: T, splitting_dimension: usize, split_on_value: f64) -> KdtreeNode<T> {
        KdtreeNode {
            left_node: None,
            right_node: None,

            point: p,
            dimension: splitting_dimension,
            split_on: split_on_value
        }
    }
}

#[cfg(test)]
mod tests {
    use ::kdtree::test_common::Point2WithId;

    use super::*;

    #[test]
    #[should_panic(expected = "empty vector point not allowed")]
    fn should_panic_given_empty_vector() {
        let mut empty_vec: Vec<Point2WithId> = vec![];

        Kdtree::new(&mut empty_vec);
    }

    quickcheck! {
        fn tree_build_creates_tree_with_as_many_leafs_as_there_is_points(xs : Vec<f64>) -> bool {
            if xs.len() == 0 {
                return true;
            }
            let mut vec : Vec<Point2WithId> = vec![];
            for i in 0 .. xs.len() {
                let p = Point2WithId::new(i as i32, xs[i], xs[i]);

                vec.push(p);
            }

            let tree = Kdtree::new(&mut qc_value_vec_to_2d_points_vec(&xs));

            let mut to_iterate : Vec<usize> = vec![];
            to_iterate.push(0);

            while to_iterate.len() > 0 {
                let last_index = to_iterate.last().unwrap().clone();
                let ref x = tree.nodes.get(last_index).unwrap();
                to_iterate.pop();
                if x.left_node.is_some() {
                    to_iterate.push(x.left_node.unwrap());
                }
                if x.right_node.is_some() {
                    to_iterate.push(x.right_node.unwrap());
                }
            }
            xs.len() == tree.nodes.len()
        }
    }

    quickcheck! {
        fn nearest_neighbor_search_using_qc(xs : Vec<f64>) -> bool {
            if xs.len() == 0 {
                return true;
            }

            let point_vec = qc_value_vec_to_2d_points_vec(&xs);
            let tree = Kdtree::new(&mut point_vec.clone());

            for p in &point_vec {
                let found_nn = tree.nearest_search(p);

                assert_eq!(p.id,found_nn.id);
            }

            true
        }
    }

    #[test]
    fn has_neighbor_in_range() {
        let mut vec: Vec<Point2WithId> = vec![Point2WithId::new(0,2.,0.)];

        let tree = Kdtree::new(&mut vec);

        assert_eq!(false,tree.has_neighbor_in_range(&Point2WithId::new(0,0.,0.), 0.));
        assert_eq!(false,tree.has_neighbor_in_range(&Point2WithId::new(0,0.,0.), 1.));
        assert_eq!(true,tree.has_neighbor_in_range(&Point2WithId::new(0,0.,0.), 2.));
        assert_eq!(true,tree.has_neighbor_in_range(&Point2WithId::new(0,0.,0.), 300.));
    }

    #[test]
    fn incremental_add_adds_as_expected() {
        //this test is tricky because it can have problems with the automatic tree rebuild.

        let mut vec = vec![Point2WithId::new(0,0.,0.)];

        let mut tree = Kdtree::new(&mut vec);

        tree.insert_node(Point2WithId::new(0,1.,0.));
        tree.insert_node(Point2WithId::new(0,-1.,0.));

        assert_eq!(tree.nodes.len(), 3);
        assert_eq!(tree.nodes[0].dimension, 0);

        assert_eq!(tree.nodes[0].left_node.is_some(), true);
        assert_eq!(tree.nodes[1].point.dims()[0], 1.);
        assert_eq!(tree.nodes[2].point.dims()[0], -1.);

        assert_eq!(tree.nodes[0].right_node.is_some(), true);
    }

    #[test]
    fn incremental_add_filters_duplicates() {
        let mut vec = vec![Point2WithId::new(0,0.,0.)];

        let mut tree = Kdtree::new(&mut vec);

        let node = Point2WithId::new(0,1.,0.);
        tree.insert_node(node);
        tree.insert_node(node);

        assert_eq!(tree.nodes.len(), 2);
    }

    fn qc_value_vec_to_2d_points_vec(xs: &Vec<f64>) -> Vec<Point2WithId> {
        let mut vec: Vec<Point2WithId> = vec![];
        for i in 0..xs.len() {
            let mut is_duplicated_value = false;
            for j in 0..i {
                if xs[i] == xs[j] {
                    is_duplicated_value = true;
                    break;
                }
            }
            if !is_duplicated_value {
                let p = Point2WithId::new(i as i32, xs[i], xs[i]);
                vec.push(p);
            }
        }

        vec
    }
}<|endoftext|>


<commit_before>// Copyright (C) 2017 Élisabeth HENRY.
//
// This file is part of Crowbook.
//
// Crowbook is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published
// by the Free Software Foundation, either version 2.1 of the License, or
// (at your option) any later version.
//
// Caribon is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU Lesser General Public License for more details.
//
// You should have received ba copy of the GNU Lesser General Public License
// along with Crowbook.  If not, see <http://www.gnu.org/licenses/>.

use error::Result;
use book::Book;

use crowbook_text_processing::escape;

#[cfg(feature="syntect")]
use syntect;

/// Wrapper around syntect, so it can be more easily optionally compiled.
#[cfg(feature="syntect")]
pub struct Syntax {
    syntax_set: syntect::parsing::SyntaxSet,
    theme: syntect::highlighting::Theme,
}

#[cfg(not(feature="syntect"))]
pub struct Syntax {}

#[cfg(feature="syntect")]
impl Syntax {
    /// Creates a new Syntax wrapper
    pub fn new(book: &Book) -> Syntax {
        let mut theme_set = syntect::highlighting::ThemeSet::load_defaults();
        let theme_name = book.options.get_str("rendering.highlight.theme").unwrap();
        let theme = match theme_set.themes.remove(theme_name) {
            Some(theme) => theme,
            None => {
                book.logger.error(lformat!("could not set syntect theme to {theme}, defaulting to \"InspiredGithub\"",
                                           theme = theme_name));
                book.logger.info(lformat!("valid theme names are: {themes}",
                                          themes = theme_set.themes
                                          .keys()
                                          .map(|s| s.to_owned())
                                          .collect::<Vec<_>>()
                                          .join(", ")));
                theme_set.themes.remove("InspiredGitHub").unwrap()
            }
        };
        Syntax {
            syntax_set: syntect::parsing::SyntaxSet::load_defaults_nonewlines(),
            theme: theme,
        }
    }

    /// Convert a string containing code to HTML
    pub fn to_html(&self, code: &str, language: &str) -> Result<String> {
        let syntax = self.syntax_set.find_syntax_by_token(language)
            .unwrap_or_else(|| self.syntax_set.find_syntax_plain_text());
        let mut h = syntect::easy::HighlightLines::new(syntax, &self.theme);
        let regions = h.highlight(code);
        Ok(format!("
{}
",
                   syntect::html::styles_to_coloured_html(&regions[..],
                                                          syntect::html::IncludeBackground::No)))
    }

    pub fn to_tex(&self, code: &str, language: &str) -> Result<String> {
        use latex::insert_breaks;
        use syntect::highlighting::{BLACK, FONT_STYLE_BOLD, FONT_STYLE_ITALIC, FONT_STYLE_UNDERLINE};
        let syntax = self.syntax_set.find_syntax_by_token(language)
            .unwrap_or_else(|| self.syntax_set.find_syntax_plain_text());
        let mut h = syntect::easy::HighlightLines::new(syntax, &self.theme);
        let regions = h.highlight(code);

        let mut result = String::with_capacity(code.len());
        for (style, text) in regions {
            let mut content = escape::tex(text).into_owned();
            content = insert_breaks(&content);
            content = content.replace('\n', "\\\\\n")
                .replace(' ', "\\hphantom{ }\\allowbreak{}");
            content = format!("\\texttt{{{}}}", content);
            if style.foreground != BLACK {
                let r = style.foreground.r as f32 / 255.0;
                let g = style.foreground.g as f32 / 255.0;
                let b = style.foreground.b as f32 / 255.0;
                content = format!("\\textcolor[rgb]{{{r}, {g}, {b}}}{{{text}}}",
                                  r = r,
                                  g = g,
                                  b = b,
                                  text = content);
            }
            if style.font_style.contains(FONT_STYLE_BOLD) {
                content = format!("\\textbf{{{}}}", content);
            }
            if style.font_style.contains(FONT_STYLE_ITALIC) {
                content = format!("\\emph{{{}}}", content);
            }
            if style.font_style.contains(FONT_STYLE_UNDERLINE) {
                content = format!("\\underline{{{}}}", content);
            }
            result.push_str(&content);
        }
        Ok(format!("{{\\sloppy {}}}", result))
    }
}

#[cfg(not(feature="syntect"))]
impl Syntax {
    pub fn new(book: &Book) -> Syntax {
        book.error(lformat!("crowbook was compiled without syntect support, syntax highlighting will be disabled"));
        Syntax {}
    }

    pub fn to_html(&self, code: &str, language: &str) -> Result<String> {
        Ok(format!("
{code}
",
                code = escape::html(code),
                lang = language))
    }

    pub fn to_tex(&self, code: &str, language: &str) -> Result<String> {
        Ok(format!("\\begin{{spverbatim}}{}\\end{{spverbatim}}\n",
                code))
    }
}
<commit_msg>Fix build error when syntect feature is disactivated<commit_after>// Copyright (C) 2017 Élisabeth HENRY.
//
// This file is part of Crowbook.
//
// Crowbook is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published
// by the Free Software Foundation, either version 2.1 of the License, or
// (at your option) any later version.
//
// Caribon is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU Lesser General Public License for more details.
//
// You should have received ba copy of the GNU Lesser General Public License
// along with Crowbook.  If not, see <http://www.gnu.org/licenses/>.

use error::Result;
use book::Book;

use crowbook_text_processing::escape;

#[cfg(feature="syntect")]
use syntect;

/// Wrapper around syntect, so it can be more easily optionally compiled.
#[cfg(feature="syntect")]
pub struct Syntax {
    syntax_set: syntect::parsing::SyntaxSet,
    theme: syntect::highlighting::Theme,
}

#[cfg(not(feature="syntect"))]
pub struct Syntax {}

#[cfg(feature="syntect")]
impl Syntax {
    /// Creates a new Syntax wrapper
    pub fn new(book: &Book) -> Syntax {
        let mut theme_set = syntect::highlighting::ThemeSet::load_defaults();
        let theme_name = book.options.get_str("rendering.highlight.theme").unwrap();
        let theme = match theme_set.themes.remove(theme_name) {
            Some(theme) => theme,
            None => {
                book.logger.error(lformat!("could not set syntect theme to {theme}, defaulting to \"InspiredGithub\"",
                                           theme = theme_name));
                book.logger.info(lformat!("valid theme names are: {themes}",
                                          themes = theme_set.themes
                                          .keys()
                                          .map(|s| s.to_owned())
                                          .collect::<Vec<_>>()
                                          .join(", ")));
                theme_set.themes.remove("InspiredGitHub").unwrap()
            }
        };
        Syntax {
            syntax_set: syntect::parsing::SyntaxSet::load_defaults_nonewlines(),
            theme: theme,
        }
    }

    /// Convert a string containing code to HTML
    pub fn to_html(&self, code: &str, language: &str) -> Result<String> {
        let syntax = self.syntax_set.find_syntax_by_token(language)
            .unwrap_or_else(|| self.syntax_set.find_syntax_plain_text());
        let mut h = syntect::easy::HighlightLines::new(syntax, &self.theme);
        let regions = h.highlight(code);
        Ok(format!("
{}
",
                   syntect::html::styles_to_coloured_html(&regions[..],
                                                          syntect::html::IncludeBackground::No)))
    }

    pub fn to_tex(&self, code: &str, language: &str) -> Result<String> {
        use latex::insert_breaks;
        use syntect::highlighting::{BLACK, FONT_STYLE_BOLD, FONT_STYLE_ITALIC, FONT_STYLE_UNDERLINE};
        let syntax = self.syntax_set.find_syntax_by_token(language)
            .unwrap_or_else(|| self.syntax_set.find_syntax_plain_text());
        let mut h = syntect::easy::HighlightLines::new(syntax, &self.theme);
        let regions = h.highlight(code);

        let mut result = String::with_capacity(code.len());
        for (style, text) in regions {
            let mut content = escape::tex(text).into_owned();
            content = insert_breaks(&content);
            content = content.replace('\n', "\\\\\n")
                .replace(' ', "\\hphantom{ }\\allowbreak{}");
            content = format!("\\texttt{{{}}}", content);
            if style.foreground != BLACK {
                let r = style.foreground.r as f32 / 255.0;
                let g = style.foreground.g as f32 / 255.0;
                let b = style.foreground.b as f32 / 255.0;
                content = format!("\\textcolor[rgb]{{{r}, {g}, {b}}}{{{text}}}",
                                  r = r,
                                  g = g,
                                  b = b,
                                  text = content);
            }
            if style.font_style.contains(FONT_STYLE_BOLD) {
                content = format!("\\textbf{{{}}}", content);
            }
            if style.font_style.contains(FONT_STYLE_ITALIC) {
                content = format!("\\emph{{{}}}", content);
            }
            if style.font_style.contains(FONT_STYLE_UNDERLINE) {
                content = format!("\\underline{{{}}}", content);
            }
            result.push_str(&content);
        }
        Ok(format!("{{\\sloppy {}}}", result))
    }
}

#[cfg(not(feature="syntect"))]
impl Syntax {
    pub fn new(book: &Book) -> Syntax {
        book.logger.error(lformat!("crowbook was compiled without syntect support, syntax highlighting will be disabled"));
        Syntax {}
    }

    pub fn to_html(&self, code: &str, language: &str) -> Result<String> {
        Ok(format!("
{code}
",
                code = escape::html(code),
                lang = language))
    }

    pub fn to_tex(&self, code: &str, _: &str) -> Result<String> {
        Ok(format!("\\begin{{spverbatim}}{}\\end{{spverbatim}}\n",
                code))
    }
}
<|endoftext|>


<commit_before>extern crate libc;
extern crate groonga_sys as ffi;
use std::mem;
use ffi::groonga as groonga;
use super::commandapi;

#[derive(Clone, Debug)]
pub struct LibGroonga {
    disposed: bool
}

impl LibGroonga {
    pub fn new() -> Result<LibGroonga, String> {
        let rc =  commandapi::groonga_init();
        if rc != groonga::GRN_SUCCESS {
            return Err("Couldn't initilize Groonga.".to_string())
        }
        Ok(LibGroonga{ disposed: false })
    }

    fn close(&mut self) -> Result<(), String> {
        if self.disposed {
            return Ok(())
        }
        let rc = commandapi::groonga_fin();
        if rc != groonga::GRN_SUCCESS {
            return Err("Couldn't finalized Groonga.".to_string())
        }
        self.disposed = true;
        Ok(())
    }
}

impl Drop for LibGroonga {
    fn drop(&mut self) {
        self.close().unwrap();
    }
}

#[derive(Clone, Debug)]
pub struct Context {
    ctx: *mut groonga::grn_ctx,
    disposed: bool
}

impl Context {
    pub fn new() -> Result<Context, String> {
        let ctx = commandapi::groonga_ctx_open(0);
        if ctx.is_null() {
            return Err("Couldn't create Groonga Context.".to_string())
        }
        Ok(Context{ ctx: ctx, disposed: false })
    }

    pub fn close(&mut self) -> Result<(), String> {
        if self.disposed {
            return Ok(())
        }
        let rc = commandapi::groonga_ctx_close(self.ctx);
        if rc != groonga::GRN_SUCCESS {
            return Err("Couldn't dispose Groonga Context.".to_string())
        }
        unsafe {
            self.ctx = mem::zeroed();
        }
        self.disposed = true;
        Ok(())
    }
}

#[derive(Clone, Debug)]
pub struct Database {
    ctx: *mut groonga::grn_ctx,
    disposed: bool
}

impl Database {
    pub fn new(ctx: Context) -> Database {
        Database{ctx: ctx.ctx, disposed: false}
    }

    pub fn uses(&mut self, dbpath: &str) {
        let _ = commandapi::groonga_db_use(self.ctx, dbpath);
    }

    fn close(&mut self) {
        if self.disposed {
            return
        }
        unsafe {
            self.ctx = mem::zeroed();
        }
        self.disposed = true;
    }
}

impl Drop for Database {
    fn drop(&mut self) {
        self.close()
    }
}

#[derive(Clone, Debug)]
pub struct Command {
    ctx: *mut groonga::grn_ctx,
    disposed: bool
}

impl Command {
    pub fn new(ctx: Context) -> Command {
        Command{ctx: ctx.ctx, disposed: false}
    }

    pub fn groonga_version() -> &'static str {
        commandapi::get_groonga_version()
    }

    pub fn execute(&mut self, command: &str) -> String {
        let result = commandapi::groonga_execute_command(self.ctx, command);
        match result {
            Ok(v) => v,
            Err(_) => "".to_string()
        }
    }

    fn close(&mut self) {
        if self.disposed {
            return
        }
        unsafe {
            self.ctx = mem::zeroed();
        }
        self.disposed = true;
    }
}

impl Drop for Command {
    fn drop(&mut self) {
        self.close()
    }
}

#[cfg(test)]
mod test {
    extern crate tempdir;
    use super::*;
    use self::tempdir::TempDir;

    fn setup_dbpath() -> String {
        let dbdir = TempDir::new("ruroonga");
        let dbname = "test.db";
        let mut dbpath_buf = dbdir.unwrap().into_path();
        dbpath_buf.push(dbname);
        let dbpath = match dbpath_buf.to_str() {
            Some(v) => v,
            None => panic!("Could not get temporary database path."),
        };

        dbpath.to_owned()
    }

    #[test]
    fn test_integration_test() {
        let groonga = LibGroonga::new();
        let is_success = match groonga {
            Ok(_) => true,
            Err(_) => false,
        };

        assert_eq!(true, is_success);

        let context = Context::new();
        let is_success = match context {
            Ok(_) => true,
            Err(_) => false,
        };

        assert_eq!(true, is_success);

        let ctx = match context {
            Ok(v) => v,
            Err(_) => panic!("Could not get context."),
        };

        let mut db = Database::new(ctx.clone());
        let dbpath = setup_dbpath();
        let _ = db.uses(&*dbpath);

        let grn_command = "table_create Users TABLE_HASH_KEY ShortText";
        let mut command = Command::new(ctx.clone());
        let _ = command.execute(grn_command.clone());

        let dump = "dump";
        let result = command.execute(dump.clone());
        assert_eq!("table_create Users TABLE_HASH_KEY ShortText\n", result);
    }
}
<commit_msg>test: Tidy up<commit_after>extern crate libc;
extern crate groonga_sys as ffi;
use std::mem;
use ffi::groonga as groonga;
use super::commandapi;

#[derive(Clone, Debug)]
pub struct LibGroonga {
    disposed: bool
}

impl LibGroonga {
    pub fn new() -> Result<LibGroonga, String> {
        let rc =  commandapi::groonga_init();
        if rc != groonga::GRN_SUCCESS {
            return Err("Couldn't initilize Groonga.".to_string())
        }
        Ok(LibGroonga{ disposed: false })
    }

    fn close(&mut self) -> Result<(), String> {
        if self.disposed {
            return Ok(())
        }
        let rc = commandapi::groonga_fin();
        if rc != groonga::GRN_SUCCESS {
            return Err("Couldn't finalized Groonga.".to_string())
        }
        self.disposed = true;
        Ok(())
    }
}

impl Drop for LibGroonga {
    fn drop(&mut self) {
        self.close().unwrap();
    }
}

#[derive(Clone, Debug)]
pub struct Context {
    ctx: *mut groonga::grn_ctx,
    disposed: bool
}

impl Context {
    pub fn new() -> Result<Context, String> {
        let ctx = commandapi::groonga_ctx_open(0);
        if ctx.is_null() {
            return Err("Couldn't create Groonga Context.".to_string())
        }
        Ok(Context{ ctx: ctx, disposed: false })
    }

    pub fn close(&mut self) -> Result<(), String> {
        if self.disposed {
            return Ok(())
        }
        let rc = commandapi::groonga_ctx_close(self.ctx);
        if rc != groonga::GRN_SUCCESS {
            return Err("Couldn't dispose Groonga Context.".to_string())
        }
        unsafe {
            self.ctx = mem::zeroed();
        }
        self.disposed = true;
        Ok(())
    }
}

#[derive(Clone, Debug)]
pub struct Database {
    ctx: *mut groonga::grn_ctx,
    disposed: bool
}

impl Database {
    pub fn new(ctx: Context) -> Database {
        Database{ctx: ctx.ctx, disposed: false}
    }

    pub fn uses(&mut self, dbpath: &str) {
        let _ = commandapi::groonga_db_use(self.ctx, dbpath);
    }

    fn close(&mut self) {
        if self.disposed {
            return
        }
        unsafe {
            self.ctx = mem::zeroed();
        }
        self.disposed = true;
    }
}

impl Drop for Database {
    fn drop(&mut self) {
        self.close()
    }
}

#[derive(Clone, Debug)]
pub struct Command {
    ctx: *mut groonga::grn_ctx,
    disposed: bool
}

impl Command {
    pub fn new(ctx: Context) -> Command {
        Command{ctx: ctx.ctx, disposed: false}
    }

    pub fn groonga_version() -> &'static str {
        commandapi::get_groonga_version()
    }

    pub fn execute(&mut self, command: &str) -> String {
        let result = commandapi::groonga_execute_command(self.ctx, command);
        match result {
            Ok(v) => v,
            Err(_) => "".to_string()
        }
    }

    fn close(&mut self) {
        if self.disposed {
            return
        }
        unsafe {
            self.ctx = mem::zeroed();
        }
        self.disposed = true;
    }
}

impl Drop for Command {
    fn drop(&mut self) {
        self.close()
    }
}

#[cfg(test)]
mod test {
    extern crate tempdir;
    use super::*;
    use self::tempdir::TempDir;

    fn setup_dbpath() -> String {
        let dbdir = TempDir::new("ruroonga");
        let dbname = "test.db";
        let mut dbpath_buf = dbdir.unwrap().into_path();
        dbpath_buf.push(dbname);
        let dbpath = match dbpath_buf.to_str() {
            Some(v) => v,
            None => panic!("Could not get temporary database path."),
        };

        dbpath.to_owned()
    }

    #[test]
    fn test_integration_test() {
        let groonga = LibGroonga::new();
        let is_success = match groonga {
            Ok(_) => true,
            Err(_) => false,
        };

        assert_eq!(true, is_success);

        let context = Context::new();
        let is_success = match context {
            Ok(_) => true,
            Err(_) => false,
        };

        assert_eq!(true, is_success);

        let ctx = match context {
            Ok(v) => v,
            Err(_) => panic!("Could not get context."),
        };

        let dbpath = setup_dbpath();
        let _ = Database::new(ctx.clone()).uses(&*dbpath);

        let grn_command = "table_create Users TABLE_HASH_KEY ShortText";
        let mut command = Command::new(ctx.clone());
        let table_create_result = command.execute(grn_command.clone());
        assert_eq!("true", table_create_result);

        let dump = "dump";
        let result = command.execute(dump.clone());
        let dumped_command = "table_create Users TABLE_HASH_KEY ShortText\n";
        assert!(
            dumped_command == result ||
            format!("{}\u{fffd}\u{7f}", dumped_command) == result
        );
    }
}
<|endoftext|>


<commit_before>#[doc = "An atomically reference counted wrapper that can be used to
share immutable data between tasks."]

export arc, get, clone;

#[abi = "cdecl"]
native mod rustrt {
    #[rust_stack]
    fn rust_atomic_increment(p: &mut libc::intptr_t)
        -> libc::intptr_t;

    #[rust_stack]
    fn rust_atomic_decrement(p: &mut libc::intptr_t)
        -> libc::intptr_t;
}

type arc_data<T> = {
    mut count: libc::intptr_t,
    data: T
};

resource arc_destruct<T>(data: *arc_data<T>) {
    unsafe {
        let ptr = &mut (*data).count;

        let new_count = rustrt::rust_atomic_decrement(ptr);
        assert new_count >= 0;
        if new_count == 0 {
            let _ptr : ~arc_data<T> = unsafe::reinterpret_cast(data);
            // drop glue takes over.
        }
    }
}

type arc<T> = arc_destruct<T>;

#[doc="Create an atomically reference counted wrapper."]
fn arc<T>(-data: T) -> arc<T> {
    let data = ~{mut count: 1, data: data};
    unsafe {
        let ptr = unsafe::reinterpret_cast(data);
        unsafe::forget(data);
        arc_destruct(ptr)
    }
}

#[doc="Access the underlying data in an atomically reference counted
 wrapper."]
fn get<T>(rc: &a.arc<T>) -> &a.T {
    unsafe {
        &(***rc).data
    }
}

#[doc="Duplicate an atomically reference counted wrapper.

The resulting two `arc` objects will point to the same underlying data
object. However, one of the `arc` objects can be sent to another task,
allowing them to share the underlying data."]
fn clone<T>(rc: &arc<T>) -> arc<T> {
    let data = **rc;
    unsafe {
        rustrt::rust_atomic_increment(&mut (*data).count);
    }
    arc_destruct(**rc)
}
<commit_msg>std: Modify arc to tolerate upcoming change to uniques<commit_after>#[doc = "An atomically reference counted wrapper that can be used to
share immutable data between tasks."]

export arc, get, clone;

#[abi = "cdecl"]
native mod rustrt {
    #[rust_stack]
    fn rust_atomic_increment(p: &mut libc::intptr_t)
        -> libc::intptr_t;

    #[rust_stack]
    fn rust_atomic_decrement(p: &mut libc::intptr_t)
        -> libc::intptr_t;
}

type arc_data<T> = {
    mut count: libc::intptr_t,
    data: T
};

resource arc_destruct<T>(data: *libc::c_void) {
    unsafe {
        let data: ~arc_data<T> = unsafe::reinterpret_cast(data);
        let ref_ptr = &mut data.count;

        let new_count = rustrt::rust_atomic_decrement(ref_ptr);
        assert new_count >= 0;
        if new_count == 0 {
            // drop glue takes over.
        } else {
            unsafe::forget(data);
        }
    }
}

type arc<T> = arc_destruct<T>;

#[doc="Create an atomically reference counted wrapper."]
fn arc<T>(-data: T) -> arc<T> {
    let data = ~{mut count: 1, data: data};
    unsafe {
        let ptr = unsafe::reinterpret_cast(data);
        unsafe::forget(data);
        arc_destruct(ptr)
    }
}

#[doc="Access the underlying data in an atomically reference counted
 wrapper."]
fn get<T>(rc: &a.arc<T>) -> &a.T {
    unsafe {
        let ptr: ~arc_data<T> = unsafe::reinterpret_cast(**rc);
        // Cast us back into the correct region
        let r = unsafe::reinterpret_cast(&ptr.data);
        unsafe::forget(ptr);
        ret r;
    }
}

#[doc="Duplicate an atomically reference counted wrapper.

The resulting two `arc` objects will point to the same underlying data
object. However, one of the `arc` objects can be sent to another task,
allowing them to share the underlying data."]
fn clone<T>(rc: &arc<T>) -> arc<T> {
    unsafe {
        let ptr: ~arc_data<T> = unsafe::reinterpret_cast(**rc);
        rustrt::rust_atomic_increment(&mut ptr.count);
        unsafe::forget(ptr);
    }
    arc_destruct(**rc)
}
<|endoftext|>


<commit_before>use clap::{App, AppSettings, Arg, SubCommand};
use error::Error;
use glob::glob;
use std::path::Path;
use std::process::exit;
use std::{env, fs, u64};
use tree_sitter::Language;
use tree_sitter_cli::{
    config, error, generate, highlight, loader, logger, parse, query, test, test_highlight, wasm,
    web_ui,
};

const BUILD_VERSION: &'static str = env!("CARGO_PKG_VERSION");
const BUILD_SHA: Option<&'static str> = option_env!("BUILD_SHA");

fn main() {
    if let Err(e) = run() {
        if !e.message().is_empty() {
            println!("");
            eprintln!("{}", e.message());
        }
        exit(1);
    }
}

fn run() -> error::Result<()> {
    let version = if let Some(build_sha) = BUILD_SHA {
        format!("{} ({})", BUILD_VERSION, build_sha)
    } else {
        BUILD_VERSION.to_string()
    };

    let matches = App::new("tree-sitter")
        .version(version.as_str())
        .setting(AppSettings::SubcommandRequiredElseHelp)
        .author("Max Brunsfeld <maxbrunsfeld@gmail.com>")
        .about("Generates and tests parsers")
        .subcommand(SubCommand::with_name("init-config").about("Generate a default config file"))
        .subcommand(
            SubCommand::with_name("generate")
                .about("Generate a parser")
                .arg(Arg::with_name("grammar-path").index(1))
                .arg(Arg::with_name("log").long("log"))
                .arg(Arg::with_name("prev-abi").long("prev-abi"))
                .arg(
                    Arg::with_name("report-states-for-rule")
                        .long("report-states-for-rule")
                        .value_name("rule-name")
                        .takes_value(true),
                )
                .arg(Arg::with_name("no-minimize").long("no-minimize")),
        )
        .subcommand(
            SubCommand::with_name("parse")
                .about("Parse files")
                .arg(
                    Arg::with_name("path")
                        .index(1)
                        .multiple(true)
                        .required(true),
                )
                .arg(Arg::with_name("scope").long("scope").takes_value(true))
                .arg(Arg::with_name("debug").long("debug").short("d"))
                .arg(Arg::with_name("debug-graph").long("debug-graph").short("D"))
                .arg(Arg::with_name("quiet").long("quiet").short("q"))
                .arg(Arg::with_name("time").long("time").short("t"))
                .arg(Arg::with_name("allow-cancellation").long("cancel"))
                .arg(Arg::with_name("timeout").long("timeout").takes_value(true))
                .arg(
                    Arg::with_name("edits")
                        .long("edit")
                        .short("edit")
                        .takes_value(true)
                        .multiple(true)
                        .number_of_values(1),
                ),
        )
        .subcommand(
            SubCommand::with_name("query")
                .about("Search files using a syntax tree query")
                .arg(Arg::with_name("query-path").index(1).required(true))
                .arg(
                    Arg::with_name("path")
                        .index(2)
                        .multiple(true)
                        .required(true),
                )
                .arg(Arg::with_name("scope").long("scope").takes_value(true))
                .arg(Arg::with_name("captures").long("captures").short("c")),
        )
        .subcommand(
            SubCommand::with_name("test")
                .about("Run a parser's tests")
                .arg(
                    Arg::with_name("filter")
                        .long("filter")
                        .short("f")
                        .takes_value(true),
                )
                .arg(Arg::with_name("debug").long("debug").short("d"))
                .arg(Arg::with_name("debug-graph").long("debug-graph").short("D")),
        )
        .subcommand(
            SubCommand::with_name("highlight")
                .about("Highlight a file")
                .arg(
                    Arg::with_name("path")
                        .index(1)
                        .multiple(true)
                        .required(true),
                )
                .arg(Arg::with_name("scope").long("scope").takes_value(true))
                .arg(Arg::with_name("html").long("html").short("h"))
                .arg(Arg::with_name("time").long("time").short("t"))
                .arg(Arg::with_name("q").short("q")),
        )
        .subcommand(
            SubCommand::with_name("build-wasm")
                .about("Compile a parser to WASM")
                .arg(
                    Arg::with_name("docker")
                        .long("docker")
                        .help("Run emscripten via docker even if it is installed locally"),
                )
                .arg(Arg::with_name("path").index(1).multiple(true)),
        )
        .subcommand(
            SubCommand::with_name("web-ui").about("Test a parser interactively in the browser"),
        )
        .subcommand(
            SubCommand::with_name("dump-languages")
                .about("Print info about all known language parsers"),
        )
        .get_matches();

    let home_dir = dirs::home_dir().expect("Failed to read home directory");
    let current_dir = env::current_dir().unwrap();
    let config = config::Config::load(&home_dir);
    let mut loader = loader::Loader::new(config.binary_directory.clone());

    if matches.subcommand_matches("init-config").is_some() {
        let config = config::Config::new(&home_dir);
        config.save(&home_dir)?;
    } else if let Some(matches) = matches.subcommand_matches("generate") {
        let grammar_path = matches.value_of("grammar-path");
        let report_symbol_name = matches.value_of("report-states-for-rule").or_else(|| {
            if matches.is_present("report-states") {
                Some("")
            } else {
                None
            }
        });
        if matches.is_present("log") {
            logger::init();
        }
        let prev_abi = matches.is_present("prev-abi");
        generate::generate_parser_in_directory(
            &current_dir,
            grammar_path,
            !prev_abi,
            report_symbol_name,
        )?;
    } else if let Some(matches) = matches.subcommand_matches("test") {
        let debug = matches.is_present("debug");
        let debug_graph = matches.is_present("debug-graph");
        let filter = matches.value_of("filter");
        let languages = loader.languages_at_path(&current_dir)?;
        let language = languages
            .first()
            .ok_or_else(|| "No language found".to_string())?;
        let test_dir = current_dir.join("test");

        // Run the corpus tests. Look for them at two paths: `test/corpus` and `corpus`.
        let mut test_corpus_dir = test_dir.join("corpus");
        if !test_corpus_dir.is_dir() {
            test_corpus_dir = current_dir.join("corpus");
        }
        if test_corpus_dir.is_dir() {
            test::run_tests_at_path(*language, &test_corpus_dir, debug, debug_graph, filter)?;
        }

        // Check that all of the queries are valid.
        test::check_queries_at_path(*language, &current_dir.join("queries"))?;

        // Run the syntax highlighting tests.
        let test_highlight_dir = test_dir.join("highlight");
        if test_highlight_dir.is_dir() {
            test_highlight::test_highlights(&loader, &test_highlight_dir)?;
        }
    } else if let Some(matches) = matches.subcommand_matches("parse") {
        let debug = matches.is_present("debug");
        let debug_graph = matches.is_present("debug-graph");
        let quiet = matches.is_present("quiet");
        let time = matches.is_present("time");
        let edits = matches
            .values_of("edits")
            .map_or(Vec::new(), |e| e.collect());
        let allow_cancellation = matches.is_present("allow-cancellation");
        let timeout = matches
            .value_of("timeout")
            .map_or(0, |t| u64::from_str_radix(t, 10).unwrap());
        let paths = collect_paths(matches.values_of("path").unwrap())?;
        let max_path_length = paths.iter().map(|p| p.chars().count()).max().unwrap();
        let mut has_error = false;
        loader.find_all_languages(&config.parser_directories)?;
        for path in paths {
            let path = Path::new(&path);
            let language =
                select_language(&mut loader, path, &current_dir, matches.value_of("scope"))?;
            has_error |= parse::parse_file_at_path(
                language,
                path,
                &edits,
                max_path_length,
                quiet,
                time,
                timeout,
                debug,
                debug_graph,
                allow_cancellation,
            )?;
        }
        if has_error {
            return Error::err(String::new());
        }
    } else if let Some(matches) = matches.subcommand_matches("query") {
        let ordered_captures = matches.values_of("captures").is_some();
        let paths = matches
            .values_of("path")
            .unwrap()
            .into_iter()
            .map(Path::new)
            .collect::<Vec<&Path>>();
        loader.find_all_languages(&config.parser_directories)?;
        let language = select_language(
            &mut loader,
            paths[0],
            &current_dir,
            matches.value_of("scope"),
        )?;
        let query_path = Path::new(matches.value_of("query-path").unwrap());
        query::query_files_at_paths(language, paths, query_path, ordered_captures)?;
    } else if let Some(matches) = matches.subcommand_matches("highlight") {
        loader.configure_highlights(&config.theme.highlight_names);
        loader.find_all_languages(&config.parser_directories)?;

        let time = matches.is_present("time");
        let paths = collect_paths(matches.values_of("path").unwrap())?;
        let html_mode = matches.is_present("html");
        if html_mode {
            println!("{}", highlight::HTML_HEADER);
        }

        let language_config;
        if let Some(scope) = matches.value_of("scope") {
            language_config = loader.language_configuration_for_scope(scope)?;
            if language_config.is_none() {
                return Error::err(format!("Unknown scope '{}'", scope));
            }
        } else {
            language_config = None;
        }

        for path in paths {
            let path = Path::new(&path);
            let (language, language_config) = match language_config {
                Some(v) => v,
                None => match loader.language_configuration_for_file_name(path)? {
                    Some(v) => v,
                    None => {
                        eprintln!("No language found for path {:?}", path);
                        continue;
                    }
                },
            };

            let source = fs::read(path)?;

            if let Some(highlight_config) = language_config.highlight_config(language)? {
                if html_mode {
                    highlight::html(&loader, &config.theme, &source, highlight_config, time)?;
                } else {
                    highlight::ansi(&loader, &config.theme, &source, highlight_config, time)?;
                }
            } else {
                return Error::err(format!("No syntax highlighting query found"));
            }
        }
    } else if let Some(matches) = matches.subcommand_matches("build-wasm") {
        let grammar_path = current_dir.join(matches.value_of("path").unwrap_or(""));
        wasm::compile_language_to_wasm(&grammar_path, matches.is_present("docker"))?;
    } else if matches.subcommand_matches("web-ui").is_some() {
        web_ui::serve(&current_dir);
    } else if matches.subcommand_matches("dump-languages").is_some() {
        loader.find_all_languages(&config.parser_directories)?;
        for (configuration, language_path) in loader.get_all_language_configurations() {
            println!(
                concat!(
                    "scope: {}\n",
                    "parser: {:?}\n",
                    "highlights: {:?}\n",
                    "file_types: {:?}\n",
                    "content_regex: {:?}\n",
                    "injection_regex: {:?}\n",
                ),
                configuration.scope.as_ref().unwrap_or(&String::new()),
                language_path,
                configuration.highlights_filenames,
                configuration.file_types,
                configuration.content_regex,
                configuration.injection_regex,
            );
        }
    }

    Ok(())
}

fn collect_paths<'a>(paths: impl Iterator<Item = &'a str>) -> error::Result<Vec<String>> {
    let mut result = Vec::new();

    let mut incorporate_path = |path: &str, positive| {
        if positive {
            result.push(path.to_string());
        } else {
            if let Some(index) = result.iter().position(|p| p == path) {
                result.remove(index);
            }
        }
    };

    for mut path in paths {
        let mut positive = true;
        if path.starts_with("!") {
            positive = false;
            path = path.trim_start_matches("!");
        }

        if Path::new(path).exists() {
            incorporate_path(path, positive);
        } else {
            let paths =
                glob(path).map_err(Error::wrap(|| format!("Invalid glob pattern {:?}", path)))?;
            for path in paths {
                if let Some(path) = path?.to_str() {
                    incorporate_path(path, positive);
                }
            }
        }
    }
    Ok(result)
}

fn select_language(
    loader: &mut loader::Loader,
    path: &Path,
    current_dir: &Path,
    scope: Option<&str>,
) -> Result<Language, Error> {
    if let Some(scope) = scope {
        if let Some(config) =
            loader
                .language_configuration_for_scope(scope)
                .map_err(Error::wrap(|| {
                    format!("Failed to load language for scope '{}'", scope)
                }))?
        {
            Ok(config.0)
        } else {
            return Error::err(format!("Unknown scope '{}'", scope));
        }
    } else if let Some((lang, _)) =
        loader
            .language_configuration_for_file_name(path)
            .map_err(Error::wrap(|| {
                format!(
                    "Failed to load language for file name {:?}",
                    path.file_name().unwrap()
                )
            }))?
    {
        Ok(lang)
    } else if let Some(lang) = loader
        .languages_at_path(&current_dir)
        .map_err(Error::wrap(|| {
            "Failed to load language in current directory"
        }))?
        .first()
        .cloned()
    {
        Ok(lang)
    } else {
        eprintln!("No language found");
        Error::err("No language found".to_string())
    }
}
<commit_msg>Output also HTML_FOOTER with highlight --html (#550)<commit_after>use clap::{App, AppSettings, Arg, SubCommand};
use error::Error;
use glob::glob;
use std::path::Path;
use std::process::exit;
use std::{env, fs, u64};
use tree_sitter::Language;
use tree_sitter_cli::{
    config, error, generate, highlight, loader, logger, parse, query, test, test_highlight, wasm,
    web_ui,
};

const BUILD_VERSION: &'static str = env!("CARGO_PKG_VERSION");
const BUILD_SHA: Option<&'static str> = option_env!("BUILD_SHA");

fn main() {
    if let Err(e) = run() {
        if !e.message().is_empty() {
            println!("");
            eprintln!("{}", e.message());
        }
        exit(1);
    }
}

fn run() -> error::Result<()> {
    let version = if let Some(build_sha) = BUILD_SHA {
        format!("{} ({})", BUILD_VERSION, build_sha)
    } else {
        BUILD_VERSION.to_string()
    };

    let matches = App::new("tree-sitter")
        .version(version.as_str())
        .setting(AppSettings::SubcommandRequiredElseHelp)
        .author("Max Brunsfeld <maxbrunsfeld@gmail.com>")
        .about("Generates and tests parsers")
        .subcommand(SubCommand::with_name("init-config").about("Generate a default config file"))
        .subcommand(
            SubCommand::with_name("generate")
                .about("Generate a parser")
                .arg(Arg::with_name("grammar-path").index(1))
                .arg(Arg::with_name("log").long("log"))
                .arg(Arg::with_name("prev-abi").long("prev-abi"))
                .arg(
                    Arg::with_name("report-states-for-rule")
                        .long("report-states-for-rule")
                        .value_name("rule-name")
                        .takes_value(true),
                )
                .arg(Arg::with_name("no-minimize").long("no-minimize")),
        )
        .subcommand(
            SubCommand::with_name("parse")
                .about("Parse files")
                .arg(
                    Arg::with_name("path")
                        .index(1)
                        .multiple(true)
                        .required(true),
                )
                .arg(Arg::with_name("scope").long("scope").takes_value(true))
                .arg(Arg::with_name("debug").long("debug").short("d"))
                .arg(Arg::with_name("debug-graph").long("debug-graph").short("D"))
                .arg(Arg::with_name("quiet").long("quiet").short("q"))
                .arg(Arg::with_name("time").long("time").short("t"))
                .arg(Arg::with_name("allow-cancellation").long("cancel"))
                .arg(Arg::with_name("timeout").long("timeout").takes_value(true))
                .arg(
                    Arg::with_name("edits")
                        .long("edit")
                        .short("edit")
                        .takes_value(true)
                        .multiple(true)
                        .number_of_values(1),
                ),
        )
        .subcommand(
            SubCommand::with_name("query")
                .about("Search files using a syntax tree query")
                .arg(Arg::with_name("query-path").index(1).required(true))
                .arg(
                    Arg::with_name("path")
                        .index(2)
                        .multiple(true)
                        .required(true),
                )
                .arg(Arg::with_name("scope").long("scope").takes_value(true))
                .arg(Arg::with_name("captures").long("captures").short("c")),
        )
        .subcommand(
            SubCommand::with_name("test")
                .about("Run a parser's tests")
                .arg(
                    Arg::with_name("filter")
                        .long("filter")
                        .short("f")
                        .takes_value(true),
                )
                .arg(Arg::with_name("debug").long("debug").short("d"))
                .arg(Arg::with_name("debug-graph").long("debug-graph").short("D")),
        )
        .subcommand(
            SubCommand::with_name("highlight")
                .about("Highlight a file")
                .arg(
                    Arg::with_name("path")
                        .index(1)
                        .multiple(true)
                        .required(true),
                )
                .arg(Arg::with_name("scope").long("scope").takes_value(true))
                .arg(Arg::with_name("html").long("html").short("h"))
                .arg(Arg::with_name("time").long("time").short("t"))
                .arg(Arg::with_name("q").short("q")),
        )
        .subcommand(
            SubCommand::with_name("build-wasm")
                .about("Compile a parser to WASM")
                .arg(
                    Arg::with_name("docker")
                        .long("docker")
                        .help("Run emscripten via docker even if it is installed locally"),
                )
                .arg(Arg::with_name("path").index(1).multiple(true)),
        )
        .subcommand(
            SubCommand::with_name("web-ui").about("Test a parser interactively in the browser"),
        )
        .subcommand(
            SubCommand::with_name("dump-languages")
                .about("Print info about all known language parsers"),
        )
        .get_matches();

    let home_dir = dirs::home_dir().expect("Failed to read home directory");
    let current_dir = env::current_dir().unwrap();
    let config = config::Config::load(&home_dir);
    let mut loader = loader::Loader::new(config.binary_directory.clone());

    if matches.subcommand_matches("init-config").is_some() {
        let config = config::Config::new(&home_dir);
        config.save(&home_dir)?;
    } else if let Some(matches) = matches.subcommand_matches("generate") {
        let grammar_path = matches.value_of("grammar-path");
        let report_symbol_name = matches.value_of("report-states-for-rule").or_else(|| {
            if matches.is_present("report-states") {
                Some("")
            } else {
                None
            }
        });
        if matches.is_present("log") {
            logger::init();
        }
        let prev_abi = matches.is_present("prev-abi");
        generate::generate_parser_in_directory(
            &current_dir,
            grammar_path,
            !prev_abi,
            report_symbol_name,
        )?;
    } else if let Some(matches) = matches.subcommand_matches("test") {
        let debug = matches.is_present("debug");
        let debug_graph = matches.is_present("debug-graph");
        let filter = matches.value_of("filter");
        let languages = loader.languages_at_path(&current_dir)?;
        let language = languages
            .first()
            .ok_or_else(|| "No language found".to_string())?;
        let test_dir = current_dir.join("test");

        // Run the corpus tests. Look for them at two paths: `test/corpus` and `corpus`.
        let mut test_corpus_dir = test_dir.join("corpus");
        if !test_corpus_dir.is_dir() {
            test_corpus_dir = current_dir.join("corpus");
        }
        if test_corpus_dir.is_dir() {
            test::run_tests_at_path(*language, &test_corpus_dir, debug, debug_graph, filter)?;
        }

        // Check that all of the queries are valid.
        test::check_queries_at_path(*language, &current_dir.join("queries"))?;

        // Run the syntax highlighting tests.
        let test_highlight_dir = test_dir.join("highlight");
        if test_highlight_dir.is_dir() {
            test_highlight::test_highlights(&loader, &test_highlight_dir)?;
        }
    } else if let Some(matches) = matches.subcommand_matches("parse") {
        let debug = matches.is_present("debug");
        let debug_graph = matches.is_present("debug-graph");
        let quiet = matches.is_present("quiet");
        let time = matches.is_present("time");
        let edits = matches
            .values_of("edits")
            .map_or(Vec::new(), |e| e.collect());
        let allow_cancellation = matches.is_present("allow-cancellation");
        let timeout = matches
            .value_of("timeout")
            .map_or(0, |t| u64::from_str_radix(t, 10).unwrap());
        let paths = collect_paths(matches.values_of("path").unwrap())?;
        let max_path_length = paths.iter().map(|p| p.chars().count()).max().unwrap();
        let mut has_error = false;
        loader.find_all_languages(&config.parser_directories)?;
        for path in paths {
            let path = Path::new(&path);
            let language =
                select_language(&mut loader, path, &current_dir, matches.value_of("scope"))?;
            has_error |= parse::parse_file_at_path(
                language,
                path,
                &edits,
                max_path_length,
                quiet,
                time,
                timeout,
                debug,
                debug_graph,
                allow_cancellation,
            )?;
        }
        if has_error {
            return Error::err(String::new());
        }
    } else if let Some(matches) = matches.subcommand_matches("query") {
        let ordered_captures = matches.values_of("captures").is_some();
        let paths = matches
            .values_of("path")
            .unwrap()
            .into_iter()
            .map(Path::new)
            .collect::<Vec<&Path>>();
        loader.find_all_languages(&config.parser_directories)?;
        let language = select_language(
            &mut loader,
            paths[0],
            &current_dir,
            matches.value_of("scope"),
        )?;
        let query_path = Path::new(matches.value_of("query-path").unwrap());
        query::query_files_at_paths(language, paths, query_path, ordered_captures)?;
    } else if let Some(matches) = matches.subcommand_matches("highlight") {
        loader.configure_highlights(&config.theme.highlight_names);
        loader.find_all_languages(&config.parser_directories)?;

        let time = matches.is_present("time");
        let paths = collect_paths(matches.values_of("path").unwrap())?;
        let html_mode = matches.is_present("html");
        if html_mode {
            println!("{}", highlight::HTML_HEADER);
        }

        let language_config;
        if let Some(scope) = matches.value_of("scope") {
            language_config = loader.language_configuration_for_scope(scope)?;
            if language_config.is_none() {
                return Error::err(format!("Unknown scope '{}'", scope));
            }
        } else {
            language_config = None;
        }

        for path in paths {
            let path = Path::new(&path);
            let (language, language_config) = match language_config {
                Some(v) => v,
                None => match loader.language_configuration_for_file_name(path)? {
                    Some(v) => v,
                    None => {
                        eprintln!("No language found for path {:?}", path);
                        continue;
                    }
                },
            };

            let source = fs::read(path)?;

            if let Some(highlight_config) = language_config.highlight_config(language)? {
                if html_mode {
                    highlight::html(&loader, &config.theme, &source, highlight_config, time)?;
                } else {
                    highlight::ansi(&loader, &config.theme, &source, highlight_config, time)?;
                }
            } else {
                return Error::err(format!("No syntax highlighting query found"));
            }
        }

        if html_mode {
            println!("{}", highlight::HTML_FOOTER);
        }

    } else if let Some(matches) = matches.subcommand_matches("build-wasm") {
        let grammar_path = current_dir.join(matches.value_of("path").unwrap_or(""));
        wasm::compile_language_to_wasm(&grammar_path, matches.is_present("docker"))?;
    } else if matches.subcommand_matches("web-ui").is_some() {
        web_ui::serve(&current_dir);
    } else if matches.subcommand_matches("dump-languages").is_some() {
        loader.find_all_languages(&config.parser_directories)?;
        for (configuration, language_path) in loader.get_all_language_configurations() {
            println!(
                concat!(
                    "scope: {}\n",
                    "parser: {:?}\n",
                    "highlights: {:?}\n",
                    "file_types: {:?}\n",
                    "content_regex: {:?}\n",
                    "injection_regex: {:?}\n",
                ),
                configuration.scope.as_ref().unwrap_or(&String::new()),
                language_path,
                configuration.highlights_filenames,
                configuration.file_types,
                configuration.content_regex,
                configuration.injection_regex,
            );
        }
    }

    Ok(())
}

fn collect_paths<'a>(paths: impl Iterator<Item = &'a str>) -> error::Result<Vec<String>> {
    let mut result = Vec::new();

    let mut incorporate_path = |path: &str, positive| {
        if positive {
            result.push(path.to_string());
        } else {
            if let Some(index) = result.iter().position(|p| p == path) {
                result.remove(index);
            }
        }
    };

    for mut path in paths {
        let mut positive = true;
        if path.starts_with("!") {
            positive = false;
            path = path.trim_start_matches("!");
        }

        if Path::new(path).exists() {
            incorporate_path(path, positive);
        } else {
            let paths =
                glob(path).map_err(Error::wrap(|| format!("Invalid glob pattern {:?}", path)))?;
            for path in paths {
                if let Some(path) = path?.to_str() {
                    incorporate_path(path, positive);
                }
            }
        }
    }
    Ok(result)
}

fn select_language(
    loader: &mut loader::Loader,
    path: &Path,
    current_dir: &Path,
    scope: Option<&str>,
) -> Result<Language, Error> {
    if let Some(scope) = scope {
        if let Some(config) =
            loader
                .language_configuration_for_scope(scope)
                .map_err(Error::wrap(|| {
                    format!("Failed to load language for scope '{}'", scope)
                }))?
        {
            Ok(config.0)
        } else {
            return Error::err(format!("Unknown scope '{}'", scope));
        }
    } else if let Some((lang, _)) =
        loader
            .language_configuration_for_file_name(path)
            .map_err(Error::wrap(|| {
                format!(
                    "Failed to load language for file name {:?}",
                    path.file_name().unwrap()
                )
            }))?
    {
        Ok(lang)
    } else if let Some(lang) = loader
        .languages_at_path(&current_dir)
        .map_err(Error::wrap(|| {
            "Failed to load language in current directory"
        }))?
        .first()
        .cloned()
    {
        Ok(lang)
    } else {
        eprintln!("No language found");
        Error::err("No language found".to_string())
    }
}
<|endoftext|>


<commit_before>use std::collections::HashMap;
use std::io::{Read, Result, Write};
use std::net::TcpListener;
use std::net::TcpStream;
use std::sync::Arc;

use bufstream::BufStream;
use openssl::ssl::{SslAcceptor, SslAcceptorBuilder, SslMethod, SslStream};

use error::ImapResult;
use self::config::Config;
use user::{load_users, Email, User};

mod config;
#[macro_use]
pub mod lmtp;

pub enum Stream {
    Ssl(SslStream<TcpStream>),
    Tcp(TcpStream)
}

impl Write for Stream {
    fn write(&mut self, buf: &[u8]) -> Result<usize> {
        match *self {
            Stream::Ssl(ref mut s) => s.write(buf),
            Stream::Tcp(ref mut s) => s.write(buf)
        }
    }

    fn flush(&mut self) -> Result<()> {
        match *self {
            Stream::Ssl(ref mut s) => s.flush(),
            Stream::Tcp(ref mut s) => s.flush()
        }
    }
}

impl Read for Stream {
    fn read(&mut self, buf: &mut [u8]) -> Result<usize> {
        match *self {
            Stream::Ssl(ref mut s) => s.read(buf),
            Stream::Tcp(ref mut s) => s.read(buf)
        }
    }
}

/// Holds configuration state and email->user map
pub struct Server {
    conf: Config,
    users: HashMap<Email, User>,
    ssl_acceptor: Option<SslAcceptor>,
}

impl Server {
    pub fn new() -> ImapResult<Server> {
        Server::new_with_conf(Config::new()?)
    }

    /// Create server to hold the Config and User HashMap
    fn new_with_conf(conf: Config) -> ImapResult<Server> {
        // Load the user data from the specified user data file.
        let users = load_users(&conf.users)?;
        let ssl_acceptor = if let Ok(identity) = conf.get_ssl_keys() {
            if conf.imap_ssl_port != 0 || conf.lmtp_ssl_port != 0 {
                match SslAcceptorBuilder::mozilla_intermediate(
                        SslMethod::tls(), &identity.pkey,
                        &identity.cert, &identity.chain) {
                    Ok(a) => Some(a.build()),
                    _ => None
                }
            } else {
                None
            }
        } else {
            None
        };

        Ok(Server {
            conf: conf,
            users: users,
            ssl_acceptor: ssl_acceptor,
        })
    }

    /// Create a TCP listener on the server host and input port
    fn generic_listener(&self, port: u16) -> Option<Result<TcpListener>> {
        if port != 0 {
            Some(TcpListener::bind((&self.conf.host[..], port)))
        } else {
            None
        }
    }

    /// Create a TCP listener on the server host and imap port
    pub fn imap_listener(&self) -> Option<Result<TcpListener>> {
        self.generic_listener(self.conf.imap_port)
    }

    /// Create a TCP listener on the server host and imap ssl port
    pub fn imap_ssl_listener(&self) -> Option<Result<TcpListener>> {
        self.generic_listener(self.conf.imap_ssl_port)
    }

    /// Create a TCP listener on the server host and lmtp port
    pub fn lmtp_listener(&self) -> Option<Result<TcpListener>> {
        self.generic_listener(self.conf.lmtp_port)
    }

    /// Create a TCP listener on the server host and lmtp ssl port
    pub fn lmtp_ssl_listener(&self) -> Option<Result<TcpListener>> {
        self.generic_listener(self.conf.lmtp_ssl_port)
    }

    pub fn imap_ssl(&self, stream: TcpStream) -> Stream {
        let local_addr = stream.local_addr();
        match local_addr {
            Ok(addr) =>
                if addr.port() != self.conf.imap_ssl_port {
                    return Stream::Tcp(stream);
                },
            _ => return Stream::Tcp(stream)
        }
        if let Some(ref ssl_acceptor) = self.ssl_acceptor {
            Stream::Ssl(ssl_acceptor.accept(stream).unwrap())
        } else {
            Stream::Tcp(stream)
        }
    }

    pub fn can_starttls(&self) -> bool {
        if let Some(_) = self.ssl_acceptor {
            true
        } else {
            false
        }
    }

    pub fn starttls(&self, stream: TcpStream) -> Option<SslStream<TcpStream>> {
        if let Some(ref ssl_acceptor) = self.ssl_acceptor {
            Some(ssl_acceptor.accept(stream).unwrap())
        } else {
            None
        }
    }

    fn host(&self) -> &String {
        &self.conf.host
    }

    pub fn get_user(&self, email: &Email) -> Option<&User> {
        self.users.get(email)
    }
}

pub fn lmtp_serve(serv: Arc<Server>, stream: TcpStream) {
    lmtp::serve(serv, BufStream::new(stream))
}
<commit_msg>Close misconfigured SSL connections<commit_after>use std::collections::HashMap;
use std::io::{Read, Result, Write};
use std::net::{Shutdown, TcpListener, TcpStream};
use std::sync::Arc;

use bufstream::BufStream;
use openssl::ssl::{SslAcceptor, SslAcceptorBuilder, SslMethod, SslStream};

use error::ImapResult;
use self::config::Config;
use user::{load_users, Email, User};

mod config;
#[macro_use]
pub mod lmtp;

pub enum Stream {
    Ssl(SslStream<TcpStream>),
    Tcp(TcpStream)
}

impl Write for Stream {
    fn write(&mut self, buf: &[u8]) -> Result<usize> {
        match *self {
            Stream::Ssl(ref mut s) => s.write(buf),
            Stream::Tcp(ref mut s) => s.write(buf)
        }
    }

    fn flush(&mut self) -> Result<()> {
        match *self {
            Stream::Ssl(ref mut s) => s.flush(),
            Stream::Tcp(ref mut s) => s.flush()
        }
    }
}

impl Read for Stream {
    fn read(&mut self, buf: &mut [u8]) -> Result<usize> {
        match *self {
            Stream::Ssl(ref mut s) => s.read(buf),
            Stream::Tcp(ref mut s) => s.read(buf)
        }
    }
}

/// Holds configuration state and email->user map
pub struct Server {
    conf: Config,
    users: HashMap<Email, User>,
    ssl_acceptor: Option<SslAcceptor>,
}

impl Server {
    pub fn new() -> ImapResult<Server> {
        Server::new_with_conf(Config::new()?)
    }

    /// Create server to hold the Config and User HashMap
    fn new_with_conf(conf: Config) -> ImapResult<Server> {
        // Load the user data from the specified user data file.
        let users = load_users(&conf.users)?;
        let ssl_acceptor = if let Ok(identity) = conf.get_ssl_keys() {
            if conf.imap_ssl_port != 0 || conf.lmtp_ssl_port != 0 {
                match SslAcceptorBuilder::mozilla_intermediate(
                        SslMethod::tls(), &identity.pkey,
                        &identity.cert, &identity.chain) {
                    Ok(a) => Some(a.build()),
                    _ => None
                }
            } else {
                None
            }
        } else {
            None
        };

        Ok(Server {
            conf: conf,
            users: users,
            ssl_acceptor: ssl_acceptor,
        })
    }

    /// Create a TCP listener on the server host and input port
    fn generic_listener(&self, port: u16) -> Option<Result<TcpListener>> {
        if port != 0 {
            Some(TcpListener::bind((&self.conf.host[..], port)))
        } else {
            None
        }
    }

    /// Create a TCP listener on the server host and imap port
    pub fn imap_listener(&self) -> Option<Result<TcpListener>> {
        self.generic_listener(self.conf.imap_port)
    }

    /// Create a TCP listener on the server host and imap ssl port
    pub fn imap_ssl_listener(&self) -> Option<Result<TcpListener>> {
        self.generic_listener(self.conf.imap_ssl_port)
    }

    /// Create a TCP listener on the server host and lmtp port
    pub fn lmtp_listener(&self) -> Option<Result<TcpListener>> {
        self.generic_listener(self.conf.lmtp_port)
    }

    /// Create a TCP listener on the server host and lmtp ssl port
    pub fn lmtp_ssl_listener(&self) -> Option<Result<TcpListener>> {
        self.generic_listener(self.conf.lmtp_ssl_port)
    }

    pub fn imap_ssl(&self, stream: TcpStream) -> Stream {
        if let Ok(addr) = stream.local_addr() {
            if addr.port() == self.conf.imap_ssl_port {
                if let Some(ref ssl_acceptor) = self.ssl_acceptor {
                    return Stream::Ssl(ssl_acceptor.accept(stream).unwrap());
                } else {
                    error!("Listening on SSL port without SSL certificate configured.");
                    let _ = stream.shutdown(Shutdown::Both);
                }
            }
        }
        Stream::Tcp(stream)
    }

    pub fn can_starttls(&self) -> bool {
        if let Some(_) = self.ssl_acceptor {
            true
        } else {
            false
        }
    }

    pub fn starttls(&self, stream: TcpStream) -> Option<SslStream<TcpStream>> {
        if let Some(ref ssl_acceptor) = self.ssl_acceptor {
            Some(ssl_acceptor.accept(stream).unwrap())
        } else {
            None
        }
    }

    fn host(&self) -> &String {
        &self.conf.host
    }

    pub fn get_user(&self, email: &Email) -> Option<&User> {
        self.users.get(email)
    }
}

pub fn lmtp_serve(serv: Arc<Server>, stream: TcpStream) {
    lmtp::serve(serv, BufStream::new(stream))
}
<|endoftext|>


<commit_before>pub mod control_regs;
pub mod descriptor;
pub mod dtables;
pub mod io;
pub mod irq;
pub mod msr;
pub mod paging;
pub mod flags;
pub mod segmentation;
pub mod task;
pub mod tlb;

#[derive(Copy, Clone, PartialEq, Eq)]
#[repr(u8)]
pub enum PrivilegeLevel {
    Ring0 = 0,
    Ring1 = 1,
    Ring2 = 2,
    Ring3 = 3,
}

#[inline(always)]
pub unsafe fn halt() {
    asm!("hlt" :::: "volatile", "intel");
}
<commit_msg>Added nop as a function.<commit_after>pub mod control_regs;
pub mod descriptor;
pub mod dtables;
pub mod io;
pub mod irq;
pub mod msr;
pub mod paging;
pub mod flags;
pub mod segmentation;
pub mod task;
pub mod tlb;

#[derive(Copy, Clone, PartialEq, Eq)]
#[repr(u8)]
pub enum PrivilegeLevel {
    Ring0 = 0,
    Ring1 = 1,
    Ring2 = 2,
    Ring3 = 3,
}

#[inline(always)]
pub unsafe fn halt() {
    asm!("hlt" :::: "volatile", "intel");
}
#[inline(always)]
pub unsafe fn nop() {
    asm!("nop" :::: "volatile", "intel");
}
<|endoftext|>


<commit_before>use std::convert::From;
use std::io::{self, Read};
use std::iter::Iterator;
use std::path::{Component, Path, PathBuf};
use std::slice;

use flate2::read::GzDecoder;
use tar;
use time::Timespec;

use backend::Backend;
use collections::{CollectionsStatus, SignatureFile};

pub struct BackupFiles {
    chains: Vec<Chain>
}

pub struct Snapshot {
    pub time: Timespec
}

pub struct File {
    pub name: String,
    pub last_modified: Timespec
}

/// Iterator over a list of backup snapshots.
pub type Snapshots<'a> = slice::Iter<'a, Snapshot>;

enum DiffType {
    Signature,
    Snapshot,
    Deleted
}

/// Store separately informations about the signatures and informations about the paths in the
/// signatures. This allows to reuse informations between snapshots and avoid duplicating them.
struct Chain {
    timestamps: Vec<Timespec>,
    files: Vec<PathSnapshots>
}

struct PathSnapshots {
    // the directory or file path
    path: PathBuf,
    // all the snapshots for this path
    snapshots: Vec<PathSnapshot>
}

struct PathSnapshot {
    // info are None if the snapshot has deleted this path
    info: Option<PathInfo>,
    // the index of the snapshot in the chain
    index: u8
}

/// Informations about a path inside a snapshot.
struct PathInfo {
    mtime: Timespec
}

impl BackupFiles {
    pub fn new<B: Backend>(backend: &B) -> io::Result<BackupFiles> {
        let collection = {
            let filenames = try!(backend.get_file_names());
            CollectionsStatus::from_filenames(&filenames)
        };
        let mut chains: Vec<Chain> = Vec::new();
        let coll_chains = collection.signature_chains();
        for coll_chain in coll_chains {
            // translate collections::SignatureChain into a Chain
            let mut chain = Chain{ timestamps: Vec::new(), files: Vec::new() };
            // add to the chain the full signature and all the incremental signatures
            // if an error occurs in the full signature exit
            let file = try!(backend.open_file(coll_chain.fullsig.file_name.as_ref()));
            try!(add_sigfile_to_chain(&mut chain, file, &coll_chain.fullsig));
            for inc in coll_chain.inclist.iter() {
                // TODO: if an error occurs here, do not exit with an error, instead
                // break the iteration and store the error inside the chain
                let file = try!(backend.open_file(inc.file_name.as_ref()));
                try!(add_sigfile_to_chain(&mut chain, file, &inc));
            }
            chains.push(chain);
        }
        Ok(BackupFiles{ chains: chains })
    }

    pub fn snapshots(&self) -> Snapshots {
        unimplemented!()
    }
}

// TODO
// impl<P: AsRef<Path>> From<P> for PathSnapshot {
//     fn from(path: P) -> Self {
//         use std::io::{Error, ErrorKind};
//
//         let mut pcomps = path.as_ref().components();
//         // split the path in (first directory, the remaining path)
//         // the first is the type, the remaining is the real path
//         let pfirst = try!(pcomps.next());
//         if let Component::Normal(strfirst) = pfirst {
//             let difftype = match strfirst.to_str() {
//                 Some("signature") => DiffType::Signature,
//                 Some("snapshot")  => DiffType::Snapshot,
//                 Some("deleted")   => DiffType::Deleted,
//                 _                 => { return io::Error::new(ErrorKind::Other, "unexpected signature type"); }
//             };
//             let realpath = pcomps.as_path();
//         }
//         unimplemented!()
//     }
// }

fn add_sigfile_to_chain<R: Read>(chain: &mut Chain, file: R, sigfile: &SignatureFile) -> io::Result<()> {
    let result = {
        if sigfile.compressed {
            let gz_decoder = try!(GzDecoder::new(file));
            add_sigtar_to_chain(chain, tar::Archive::new(gz_decoder))
        }
        else {
            add_sigtar_to_chain(chain, tar::Archive::new(file))
        }
    };
    if result.is_ok() {
        // add to the list of signatures only if everything is ok
        // we do not need to cleanup the chain if someting went wrong, because if the list of
        // signatures is not updated, the change is not observable
        chain.timestamps.push(sigfile.time);
    }
    result
}

fn add_sigtar_to_chain<R: Read>(chain: &mut Chain, mut tar: tar::Archive<R>) -> io::Result<()> {
    for tarfile in try!(tar.files_mut()) {
        if let Ok(tarfile) = tarfile {
            let header = tarfile.header();
            // we can ignore paths with errors
            // the only problem here is that we miss some change in the chain, but it is better
            // than abort the whole signature
            let path = unwrap_or_continue!(header.path());
            let mut pcomps = path.components();
            // split the path in (first directory, the remaining path)
            // the first is the type, the remaining is the real path
            let pfirst = unwrap_opt_or_continue!(pcomps.next());
            if let Component::Normal(strfirst) = pfirst {
                let difftype = match strfirst.to_str() {
                    Some("signature") => DiffType::Signature,
                    Some("snapshot")  => DiffType::Snapshot,
                    Some("deleted")   => DiffType::Deleted,
                    _                 => { continue; /* unexpected diff type */ }
                };
                let realpath = pcomps.as_path();
            }
        }
    }
    Ok(())
}
<commit_msg>Refactored snapshot path parsing<commit_after>use std::convert::From;
use std::io::{self, Read};
use std::iter::Iterator;
use std::path::{Component, Path, PathBuf};
use std::slice;

use flate2::read::GzDecoder;
use tar;
use time::Timespec;

use backend::Backend;
use collections::{CollectionsStatus, SignatureFile};

pub struct BackupFiles {
    chains: Vec<Chain>
}

pub struct Snapshot {
    pub time: Timespec
}

pub struct File {
    pub name: String,
    pub last_modified: Timespec
}

/// Iterator over a list of backup snapshots.
pub type Snapshots<'a> = slice::Iter<'a, Snapshot>;

enum DiffType {
    Signature,
    Snapshot,
    Deleted
}

/// Store separately informations about the signatures and informations about the paths in the
/// signatures. This allows to reuse informations between snapshots and avoid duplicating them.
struct Chain {
    timestamps: Vec<Timespec>,
    files: Vec<PathSnapshots>
}

struct PathSnapshots {
    // the directory or file path
    path: PathBuf,
    // all the snapshots for this path
    snapshots: Vec<PathSnapshot>
}

struct PathSnapshot {
    // info are None if the snapshot has deleted this path
    info: Option<PathInfo>,
    // the index of the snapshot in the chain
    index: u8
}

/// Informations about a path inside a snapshot.
struct PathInfo {
    mtime: Timespec
}

impl BackupFiles {
    pub fn new<B: Backend>(backend: &B) -> io::Result<BackupFiles> {
        let collection = {
            let filenames = try!(backend.get_file_names());
            CollectionsStatus::from_filenames(&filenames)
        };
        let mut chains: Vec<Chain> = Vec::new();
        let coll_chains = collection.signature_chains();
        for coll_chain in coll_chains {
            // translate collections::SignatureChain into a Chain
            let mut chain = Chain{ timestamps: Vec::new(), files: Vec::new() };
            // add to the chain the full signature and all the incremental signatures
            // if an error occurs in the full signature exit
            let file = try!(backend.open_file(coll_chain.fullsig.file_name.as_ref()));
            try!(add_sigfile_to_chain(&mut chain, file, &coll_chain.fullsig));
            for inc in coll_chain.inclist.iter() {
                // TODO: if an error occurs here, do not exit with an error, instead
                // break the iteration and store the error inside the chain
                let file = try!(backend.open_file(inc.file_name.as_ref()));
                try!(add_sigfile_to_chain(&mut chain, file, &inc));
            }
            chains.push(chain);
        }
        Ok(BackupFiles{ chains: chains })
    }

    pub fn snapshots(&self) -> Snapshots {
        unimplemented!()
    }
}

fn add_sigfile_to_chain<R: Read>(chain: &mut Chain, file: R, sigfile: &SignatureFile) -> io::Result<()> {
    let result = {
        if sigfile.compressed {
            let gz_decoder = try!(GzDecoder::new(file));
            add_sigtar_to_chain(chain, tar::Archive::new(gz_decoder))
        }
        else {
            add_sigtar_to_chain(chain, tar::Archive::new(file))
        }
    };
    if result.is_ok() {
        // add to the list of signatures only if everything is ok
        // we do not need to cleanup the chain if someting went wrong, because if the list of
        // signatures is not updated, the change is not observable
        chain.timestamps.push(sigfile.time);
    }
    result
}

fn add_sigtar_to_chain<R: Read>(chain: &mut Chain, mut tar: tar::Archive<R>) -> io::Result<()> {
    for tarfile in try!(tar.files_mut()) {
        // we can ignore paths with errors
        // the only problem here is that we miss some change in the chain, but it is better
        // than abort the whole signature
        let tarfile = unwrap_or_continue!(tarfile);
        let header = tarfile.header();
        let path = unwrap_or_continue!(header.path());
        let (difftype, path) = unwrap_opt_or_continue!(parse_snapshot_path(&path));
    }
    Ok(())
}

fn parse_snapshot_path(path: &Path) -> Option<(DiffType, &Path)> {
    // split the path in (first directory, the remaining path)
    // the first is the type, the remaining is the real path
    let mut pcomps = path.components();
    let pfirst = try_opt!(pcomps.next());
    if let Component::Normal(strfirst) = pfirst {
        let difftype = match strfirst.to_str() {
            Some("signature") => DiffType::Signature,
            Some("snapshot")  => DiffType::Snapshot,
            Some("deleted")   => DiffType::Deleted,
            _                 => { return None; }
        };
        let realpath = pcomps.as_path();
        Some((difftype, realpath))
    }
    else {
        None
    }
}

<|endoftext|>