<commit_before>#![allow(non_snake_case)]
use std::fmt;
use std::slice;
use std::mem;
use num;
use approx;
use channel::{FreeChannel, FreeChannelScalar, PosNormalChannelScalar, PosNormalBoundedChannel,
              ColorChannel, ChannelFormatCast, ChannelCast};
use color::{Color, Bounded, Lerp, Flatten};
use convert::FromColor;
use xyz::Xyz;

pub struct XyYTag;

#[repr(C)]
#[derive(Copy, Clone, Debug, PartialEq, PartialOrd)]
pub struct XyY<T> {
    pub x: PosNormalBoundedChannel<T>,
    pub y: PosNormalBoundedChannel<T>,
    pub Y: FreeChannel<T>,
}

impl<T> XyY<T>
    where T: FreeChannelScalar + num::Float + PosNormalChannelScalar
{
    pub fn from_channels(x: T, y: T, Y: T) -> Self {
        let zero = num::cast(0.0).unwrap();
        if x + y > num::cast(1.0).unwrap() || x + y < zero {
            panic!("xyY `x` and `y` channels are ratios and must sum to be between 0 and 1");
        }
        assert!(x >= zero);
        assert!(y >= zero);

        XyY {
            x: PosNormalBoundedChannel::new(x),
            y: PosNormalBoundedChannel::new(y),
            Y: FreeChannel::new(Y),
        }
    }

    impl_color_color_cast_square!(XyY {x, y, Y}, chan_traits={FreeChannelScalar,
        PosNormalChannelScalar});

    pub fn x(&self) -> T {
        self.x.0.clone()
    }
    pub fn y(&self) -> T {
        self.y.0.clone()
    }
    pub fn z(&self) -> T {
        num::cast::<_, T>(1.0).unwrap() - self.x() - self.y()
    }
    pub fn Y(&self) -> T {
        self.Y.0.clone()
    }
    pub fn Y_mut(&mut self) -> &mut T {
        &mut self.Y.0
    }
    pub fn set_x(&mut self, val: T) {
        let (x, y, _) = Self::rescale_channels(val, self.y(), self.z());
        self.x.0 = x;
        self.y.0 = y;
    }
    pub fn set_y(&mut self, val: T) {
        let (y, x, _) = Self::rescale_channels(val, self.x(), self.z());
        self.x.0 = x;
        self.y.0 = y;
    }
    pub fn set_Y(&mut self, val: T) {
        let (_, x, y) = Self::rescale_channels(val, self.x(), self.y());
        self.x.0 = x;
        self.y.0 = y;
    }

    fn rescale_channels(primary: T, c2: T, c3: T) -> (T, T, T) {
        if primary > PosNormalBoundedChannel::max_bound() ||
           primary < PosNormalBoundedChannel::min_bound() {
            panic!("xyY chromaticity channels must be between 0.0 and 1.0")
        }

        let zero = num::cast(0.0).unwrap();
        let rem_scale = c2 + c3;
        let rem = num::cast::<_, T>(1.0).unwrap() - primary;
        if rem_scale > zero {
            (primary, (c2 / rem_scale) * rem, (c3 / rem_scale) * rem)
        } else {
            let one_half = num::cast(0.5).unwrap();
            (primary, rem * one_half, rem * one_half)
        }
    }
}

impl<T> Color for XyY<T>
    where T: FreeChannelScalar + PosNormalChannelScalar + num::Float
{
    type Tag = XyYTag;
    type ChannelsTuple = (T, T, T);

    #[inline]
    fn num_channels() -> u32 {
        3
    }
    fn from_tuple(values: (T, T, T)) -> Self {
        let (x, y, Y) = values;
        XyY {
            x: PosNormalBoundedChannel::new(x),
            y: PosNormalBoundedChannel::new(y),
            Y: FreeChannel::new(Y),
        }
    }
    fn to_tuple(self) -> Self::ChannelsTuple {
        (self.x.0, self.y.0, self.Y.0)
    }
}

impl<T> Bounded for XyY<T>
    where T: FreeChannelScalar + PosNormalChannelScalar + num::Float
{
    fn normalize(self) -> Self {
        self
    }
    fn is_normalized(&self) -> bool {
        true
    }
}

impl<T> Lerp for XyY<T>
    where T: FreeChannelScalar + PosNormalChannelScalar + num::Float,
          FreeChannel<T>: Lerp,
          PosNormalBoundedChannel<T>: Lerp<Position=<FreeChannel<T> as Lerp>::Position>,
{
    type Position = <FreeChannel<T> as Lerp>::Position;
    impl_color_lerp_square!(XyY {x, y, Y});
}

impl<T> Flatten for XyY<T>
    where T: FreeChannelScalar + PosNormalChannelScalar + num::Float
{
    type ScalarFormat = T;

    impl_color_as_slice!(T);
    impl_color_from_slice_square!(XyY<T> {x:PosNormalBoundedChannel - 0,
        y:PosNormalBoundedChannel - 1, Y:FreeChannel - 2});
}

impl<T> approx::ApproxEq for XyY<T>
    where T: FreeChannelScalar + PosNormalChannelScalar + approx::ApproxEq,
          T::Epsilon: Clone
{
    impl_approx_eq!({x, y, Y});
}

impl<T> Default for XyY<T>
    where T: FreeChannelScalar + PosNormalChannelScalar + num::Float
{
    impl_color_default!(XyY {x:PosNormalBoundedChannel, y:PosNormalBoundedChannel,
        Y:FreeChannel});
}

impl<T> fmt::Display for XyY<T>
    where T: FreeChannelScalar + PosNormalChannelScalar + num::Float + fmt::Display
{
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        write!(f, "xyY({}, {}, {})", self.x, self.y, self.Y)
    }
}

impl<T> FromColor<Xyz<T>> for XyY<T>
    where T: FreeChannelScalar + PosNormalChannelScalar + num::Float
{
    fn from_color(from: &Xyz<T>) -> Self {
        let zero = num::cast(0.0).unwrap();
        let sum = from.x() + from.y() + from.z();

        if sum != zero {
            let x = from.x() / sum;
            let y = from.y() / sum;
            let Y = from.y();

            XyY::from_channels(x, y, Y)
        } else {
            XyY::from_channels(zero, zero, zero)
        }
    }
}

impl<T> FromColor<XyY<T>> for Xyz<T>
    where T: FreeChannelScalar + PosNormalChannelScalar + num::Float
{
    fn from_color(from: &XyY<T>) -> Self {
        let x = (from.Y() / from.y()) * from.x();
        let y = from.Y();
        let z = (from.Y() / from.y()) * from.z();

        Xyz::from_channels(x, y, z)
    }
}
<commit_msg>Added Xyz -> XyY and XyY -> Xyz tests.<commit_after>#![allow(non_snake_case)]
use std::fmt;
use std::slice;
use std::mem;
use num;
use approx;
use channel::{FreeChannel, FreeChannelScalar, PosNormalChannelScalar, PosNormalBoundedChannel,
              ColorChannel, ChannelFormatCast, ChannelCast};
use color::{Color, Bounded, Lerp, Flatten};
use convert::FromColor;
use xyz::Xyz;

pub struct XyYTag;

#[repr(C)]
#[derive(Copy, Clone, Debug, PartialEq, PartialOrd)]
pub struct XyY<T> {
    pub x: PosNormalBoundedChannel<T>,
    pub y: PosNormalBoundedChannel<T>,
    pub Y: FreeChannel<T>,
}

impl<T> XyY<T>
    where T: FreeChannelScalar + num::Float + PosNormalChannelScalar
{
    pub fn from_channels(x: T, y: T, Y: T) -> Self {
        let zero = num::cast(0.0).unwrap();
        if x + y > num::cast(1.0).unwrap() || x + y < zero {
            panic!("xyY `x` and `y` channels are ratios and must sum to be between 0 and 1");
        }
        assert!(x >= zero);
        assert!(y >= zero);

        XyY {
            x: PosNormalBoundedChannel::new(x),
            y: PosNormalBoundedChannel::new(y),
            Y: FreeChannel::new(Y),
        }
    }

    impl_color_color_cast_square!(XyY {x, y, Y}, chan_traits={FreeChannelScalar,
        PosNormalChannelScalar});

    pub fn x(&self) -> T {
        self.x.0.clone()
    }
    pub fn y(&self) -> T {
        self.y.0.clone()
    }
    pub fn z(&self) -> T {
        num::cast::<_, T>(1.0).unwrap() - self.x() - self.y()
    }
    pub fn Y(&self) -> T {
        self.Y.0.clone()
    }
    pub fn Y_mut(&mut self) -> &mut T {
        &mut self.Y.0
    }
    pub fn set_x(&mut self, val: T) {
        let (x, y, _) = Self::rescale_channels(val, self.y(), self.z());
        self.x.0 = x;
        self.y.0 = y;
    }
    pub fn set_y(&mut self, val: T) {
        let (y, x, _) = Self::rescale_channels(val, self.x(), self.z());
        self.x.0 = x;
        self.y.0 = y;
    }
    pub fn set_Y(&mut self, val: T) {
        let (_, x, y) = Self::rescale_channels(val, self.x(), self.y());
        self.x.0 = x;
        self.y.0 = y;
    }

    fn rescale_channels(primary: T, c2: T, c3: T) -> (T, T, T) {
        if primary > PosNormalBoundedChannel::max_bound() ||
           primary < PosNormalBoundedChannel::min_bound() {
            panic!("xyY chromaticity channels must be between 0.0 and 1.0")
        }

        let zero = num::cast(0.0).unwrap();
        let rem_scale = c2 + c3;
        let rem = num::cast::<_, T>(1.0).unwrap() - primary;
        if rem_scale > zero {
            (primary, (c2 / rem_scale) * rem, (c3 / rem_scale) * rem)
        } else {
            let one_half = num::cast(0.5).unwrap();
            (primary, rem * one_half, rem * one_half)
        }
    }
}

impl<T> Color for XyY<T>
    where T: FreeChannelScalar + PosNormalChannelScalar + num::Float
{
    type Tag = XyYTag;
    type ChannelsTuple = (T, T, T);

    #[inline]
    fn num_channels() -> u32 {
        3
    }
    fn from_tuple(values: (T, T, T)) -> Self {
        let (x, y, Y) = values;
        XyY {
            x: PosNormalBoundedChannel::new(x),
            y: PosNormalBoundedChannel::new(y),
            Y: FreeChannel::new(Y),
        }
    }
    fn to_tuple(self) -> Self::ChannelsTuple {
        (self.x.0, self.y.0, self.Y.0)
    }
}

impl<T> Bounded for XyY<T>
    where T: FreeChannelScalar + PosNormalChannelScalar + num::Float
{
    fn normalize(self) -> Self {
        self
    }
    fn is_normalized(&self) -> bool {
        true
    }
}

impl<T> Lerp for XyY<T>
    where T: FreeChannelScalar + PosNormalChannelScalar + num::Float,
          FreeChannel<T>: Lerp,
          PosNormalBoundedChannel<T>: Lerp<Position=<FreeChannel<T> as Lerp>::Position>,
{
    type Position = <FreeChannel<T> as Lerp>::Position;
    impl_color_lerp_square!(XyY {x, y, Y});
}

impl<T> Flatten for XyY<T>
    where T: FreeChannelScalar + PosNormalChannelScalar + num::Float
{
    type ScalarFormat = T;

    impl_color_as_slice!(T);
    impl_color_from_slice_square!(XyY<T> {x:PosNormalBoundedChannel - 0,
        y:PosNormalBoundedChannel - 1, Y:FreeChannel - 2});
}

impl<T> approx::ApproxEq for XyY<T>
    where T: FreeChannelScalar + PosNormalChannelScalar + approx::ApproxEq,
          T::Epsilon: Clone
{
    impl_approx_eq!({x, y, Y});
}

impl<T> Default for XyY<T>
    where T: FreeChannelScalar + PosNormalChannelScalar + num::Float
{
    impl_color_default!(XyY {x:PosNormalBoundedChannel, y:PosNormalBoundedChannel,
        Y:FreeChannel});
}

impl<T> fmt::Display for XyY<T>
    where T: FreeChannelScalar + PosNormalChannelScalar + num::Float + fmt::Display
{
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        write!(f, "xyY({}, {}, {})", self.x, self.y, self.Y)
    }
}

impl<T> FromColor<Xyz<T>> for XyY<T>
    where T: FreeChannelScalar + PosNormalChannelScalar + num::Float
{
    fn from_color(from: &Xyz<T>) -> Self {
        let zero = num::cast(0.0).unwrap();
        if from.x() < zero || from.y() < zero || from.z() < zero {
            panic!("Cannot convert an XYZ color with negative channels to xyY");
        }
        let sum = from.x() + from.y() + from.z();

        if sum != zero {
            let x = from.x() / sum;
            let y = from.y() / sum;
            let Y = from.y();

            XyY::from_channels(x, y, Y)
        } else {
            XyY::from_channels(zero, zero, zero)
        }
    }
}

impl<T> FromColor<XyY<T>> for Xyz<T>
    where T: FreeChannelScalar + PosNormalChannelScalar + num::Float
{
    fn from_color(from: &XyY<T>) -> Self {
        let zero = num::cast(0.0).unwrap();
        if from.y() == zero {
            Xyz::from_channels(zero, zero, zero)
        } else {
            let x = (from.Y() / from.y()) * from.x();
            let y = from.Y();
            let z = (from.Y() / from.y()) * from.z();
            Xyz::from_channels(x, y, z)
        }
    }
}

#[cfg(test)]
mod test {
    use super::*;
    use xyz::Xyz;
    use convert::*;

    #[test]
    fn test_from_xyz() {
        let c1 = Xyz::from_channels(0.3, 0.2, 0.5);
        let t1 = XyY::from_color(&c1);
        assert_relative_eq!(t1, XyY::from_channels(0.3, 0.2, 0.2), epsilon=1e-6);
        assert_relative_eq!(Xyz::from_color(&t1), c1, epsilon=1e-6);

        let c2 = Xyz::from_channels(0.8, 0.1, 0.5);
        let t2 = XyY::from_color(&c2);
        assert_relative_eq!(t2, XyY::from_channels(0.571429, 0.071429, 0.1), epsilon=1e-6);
        assert_relative_eq!(Xyz::from_color(&t2), c2, epsilon=1e-6);

        let c3 = Xyz::from_channels(0.0, 0.0, 0.0);
        let t3 = XyY::from_color(&c3);
        assert_relative_eq!(t3, XyY::from_channels(0.0, 0.0, 0.0), epsilon=1e-6);
        assert_relative_eq!(Xyz::from_color(&t3), c3, epsilon=1e-6);

        let c4 = Xyz::from_channels(0.5, 0.5, 0.5);
        let t4 = XyY::from_color(&c4);
        assert_relative_eq!(t4, XyY::from_channels(1.0/3.0, 1.0/3.0, 0.5), epsilon=1e-6);
        assert_relative_eq!(Xyz::from_color(&t4), c4, epsilon=1e-6);

        let c5 = Xyz::from_channels(1.2, 0.3, 0.8);
        let t5 = XyY::from_color(&c5);
        assert_relative_eq!(t5, XyY::from_channels(0.521739, 0.130435, 0.3000), epsilon=1e-6);
        assert_relative_eq!(Xyz::from_color(&t5), c5, epsilon=1e-6);
    }

    #[test]
    fn test_to_xyz() {
        let c1 = XyY::from_channels(0.5, 0.2, 0.5);
        let t1 = Xyz::from_color(&c1);
        assert_relative_eq!(t1, Xyz::from_channels(1.25, 0.5, 0.75), epsilon=1e-6);
        assert_relative_eq!(XyY::from_color(&t1), c1, epsilon=1e-6);

        let c2 = XyY::from_channels(1.0 / 3.0, 1.0 / 3.0, 1.0);
        let t2 = Xyz::from_color(&c2);
        assert_relative_eq!(t2, Xyz::from_channels(1.0, 1.0, 1.0), epsilon=1e-6);
        assert_relative_eq!(XyY::from_color(&t2), c2, epsilon=1e-6);

        let c3 = XyY::from_channels(0.3, 0.5, 0.3);
        let t3 = Xyz::from_color(&c3);
        assert_relative_eq!(t3, Xyz::from_channels(0.18, 0.3, 0.12), epsilon=1e-6);
        assert_relative_eq!(XyY::from_color(&t3), c3, epsilon=1e-6);

        let c4 = XyY::from_channels(0.285, 0.4194, 0.583);
        let t4 = Xyz::from_color(&c4);
        assert_relative_eq!(t4, Xyz::from_channels(0.396173, 0.5830, 0.410908), epsilon=1e-6);
        assert_relative_eq!(XyY::from_color(&t4), c4, epsilon=1e-6);
    }
}
<|endoftext|>


<commit_before>use {Token, SyntaxKind};
use syntax_kinds::*;

mod ptr;
use self::ptr::Ptr;

mod classes;
use self::classes::*;

pub fn next_token(text: &str) -> Token {
    assert!(!text.is_empty());
    let mut ptr = Ptr::new(text);
    let c = ptr.bump().unwrap();
    let kind = next_token_inner(c, &mut ptr);
    let len = ptr.into_len();
    Token { kind, len }
}

fn next_token_inner(c: char, ptr: &mut Ptr) -> SyntaxKind {
    // Note: r as in r" or r#" is part of a raw string literal,
    // b as in b' is part of a byte literal.
    // They are not identifiers, and are handled further down.
    let ident_start = is_ident_start(c) && !string_literal_start(c, ptr.next(), ptr.nnext());
    if ident_start {
        ptr.bump_while(is_ident_continue);
        return IDENT;
    }

    if is_whitespace(c) {
        ptr.bump_while(is_whitespace);
        return WHITESPACE;
    }

    return ERROR
}

fn string_literal_start(c: char, c1: Option<char>, c2: Option<char>) -> bool {
    match (c, c1, c2) {
        ('r', Some('"'), _) |
        ('r', Some('#'), _) |
        ('b', Some('"'), _) |
        ('b', Some('\''), _) |
        ('b', Some('r'), Some('"')) |
        ('b', Some('r'), Some('#')) => true,
        _ => false
    }
}
<commit_msg>Style<commit_after>use {Token, SyntaxKind};
use syntax_kinds::*;

mod ptr;
use self::ptr::Ptr;

mod classes;
use self::classes::*;

pub fn next_token(text: &str) -> Token {
    assert!(!text.is_empty());
    let mut ptr = Ptr::new(text);
    let c = ptr.bump().unwrap();
    let kind = next_token_inner(c, &mut ptr);
    let len = ptr.into_len();
    Token { kind, len }
}

fn next_token_inner(c: char, ptr: &mut Ptr) -> SyntaxKind {
    // Note: r as in r" or r#" is part of a raw string literal,
    // b as in b' is part of a byte literal.
    // They are not identifiers, and are handled further down.
    let ident_start = is_ident_start(c) && !string_literal_start(c, ptr.next(), ptr.nnext());
    if ident_start {
        ptr.bump_while(is_ident_continue);
        return IDENT;
    }

    if is_whitespace(c) {
        ptr.bump_while(is_whitespace);
        return WHITESPACE;
    }

    ERROR
}

fn string_literal_start(c: char, c1: Option<char>, c2: Option<char>) -> bool {
    match (c, c1, c2) {
        ('r', Some('"'), _) |
        ('r', Some('#'), _) |
        ('b', Some('"'), _) |
        ('b', Some('\''), _) |
        ('b', Some('r'), Some('"')) |
        ('b', Some('r'), Some('#')) => true,
        _ => false
    }
}
<|endoftext|>


<commit_before>use std::path::Path;

use async_trait::async_trait;
use codecs::utf16::Utf16Ext;
use language_tags::LanguageTag;

use crate::{
    build::BuildStep,
    bundle::{
        layout::{Layout, WindowsTarget},
        KbdgenBundle, DEFAULT_DECIMAL,
    },
    util::split_keys,
};

use super::{
    klc::{
        dead_key::KlcDeadKeys,
        file::KlcFileMetadata,
        file::{KlcFile, KLC_EXT},
        key::KlcKey,
        keymap::MSKLC_KEYS,
        layout::{KlcLayer, KlcLayout, KlcLayoutRow},
        ligature::{KlcLigature, KlcLigatureRow},
    },
    layer_set::{populate_layer_set, WindowsLayerSet, WindowsLayerSetKey, SG_CAP},
};

pub struct GenerateKlc {}

#[async_trait(?Send)]
impl BuildStep for GenerateKlc {
    async fn build(&self, bundle: &KbdgenBundle, output_path: &Path) {
        // One .klc file per language with Windows primary platform
        for (language_tag, layout) in &bundle.layouts {
            if let Some(windows_target) = &layout.windows {
                let metadata =
                    generate_metadata(bundle.clone(), language_tag, layout, windows_target);

                let layers = &windows_target.primary.layers;

                let mut klc_layout_rows = Vec::new();
                let mut klc_ligature_rows = Vec::new();
                let mut dead_key_characters = Vec::new();

                let mut cursor = 0;
                for (_iso_key, klc_key) in MSKLC_KEYS.iter() {
                    let mut layer_set = WindowsLayerSet::default();

                    // Layer set to determine caps_mode, special escapes, dead keys and null keys
                    for (layer, key_map) in layers {
                        let key_map: Vec<String> = split_keys(&key_map);

                        tracing::debug!(
                            "iso len: {}; keymap len: {}",
                            MSKLC_KEYS.len(),
                            key_map.len()
                        );
                        if MSKLC_KEYS.len() > key_map.len() {
                            panic!(
                                r#"Provided layer does not have enough keys, expected {} keys but got {}, in {}:{}:{}:{:?}: \n{:?}"#,
                                MSKLC_KEYS.len(),
                                key_map.len(),
                                language_tag.to_string(),
                                "Windows",
                                "Primary",
                                layer,
                                key_map
                            );
                        }

                        populate_layer_set(
                            &mut layer_set,
                            layer,
                            key_map,
                            cursor,
                            windows_target.dead_keys.as_ref(),
                        );
                    }

                    let caps_mode = layer_set.caps_mode();

                    klc_layout_rows.push(KlcLayoutRow {
                        scancode: klc_key.scancode.clone(),
                        virtual_key: klc_key.virtual_key.clone(),
                        caps_mode: caps_mode.clone(), // Generate extra key for SGCap here?
                        default_key: convert_to_klc_key(
                            layer_set.default,
                            &klc_key.virtual_key,
                            KlcLayer::Default,
                            &mut klc_ligature_rows,
                            &mut dead_key_characters,
                        ),
                        shift_key: convert_to_klc_key(
                            layer_set.shift,
                            &klc_key.virtual_key,
                            KlcLayer::Shift,
                            &mut klc_ligature_rows,
                            &mut dead_key_characters,
                        ),
                        ctrl_key: convert_to_klc_key(
                            layer_set.ctrl,
                            &klc_key.virtual_key,
                            KlcLayer::Ctrl,
                            &mut klc_ligature_rows,
                            &mut dead_key_characters,
                        ),
                        alt_key: convert_to_klc_key(
                            layer_set.alt,
                            &klc_key.virtual_key,
                            KlcLayer::Alt,
                            &mut klc_ligature_rows,
                            &mut dead_key_characters,
                        ),
                        alt_and_shift_key: convert_to_klc_key(
                            layer_set.alt_and_shift,
                            &klc_key.virtual_key,
                            KlcLayer::AltAndShift,
                            &mut klc_ligature_rows,
                            &mut dead_key_characters,
                        ),
                    });

                    if caps_mode == SG_CAP {
                        let caps_key = convert_to_klc_key(
                            layer_set.caps,
                            &klc_key.virtual_key,
                            KlcLayer::Default,
                            &mut klc_ligature_rows,
                            &mut dead_key_characters,
                        );

                        let caps_shift_key = convert_to_klc_key(
                            layer_set.caps_and_shift,
                            &klc_key.virtual_key,
                            KlcLayer::Shift,
                            &mut klc_ligature_rows,
                            &mut dead_key_characters,
                        );

                        klc_layout_rows.push(KlcLayoutRow {
                            scancode: "-1".to_owned(),
                            virtual_key: "-1".to_owned(),
                            caps_mode: "0".to_owned(),
                            default_key: caps_key,
                            shift_key: caps_shift_key,
                            ctrl_key: KlcKey::Skip,
                            alt_key: KlcKey::Skip,
                            alt_and_shift_key: KlcKey::Skip,
                        });
                    }

                    cursor += 1;
                }

                klc_layout_rows.push(space_layout_row());
                klc_layout_rows.push(decimal_layout_row(&layout.decimal));

                let klc_file = KlcFile {
                    metadata,
                    layout: KlcLayout {
                        rows: klc_layout_rows,
                    },
                    ligature: KlcLigature {
                        rows: klc_ligature_rows,
                    },
                    dead_keys: KlcDeadKeys {
                        characters: dead_key_characters,
                        transforms: &layout.transforms,
                    },
                };

                let klc_bytes = klc_file.to_string().encode_utf16_le_bom();
                let klc_path =
                    output_path.join(format!("{}.{}", klc_file.metadata.keyboard_name, KLC_EXT));
                std::fs::write(klc_path, klc_bytes).unwrap();
            }
        }
    }
}

fn generate_metadata(
    bundle: &KbdgenBundle,
    language_tag: &LanguageTag,
    layout: &Layout,
    target: &WindowsTarget,
) -> KlcFileMetadata {
    let keyboard_name = format!(
        "kbd{}",
        target
            .config
            .as_ref()
            .and_then(|t| t.id.as_ref())
            .map(|x| x.to_string())
            .unwrap_or_else(|| language_tag.as_str().chars().take(5).collect::<String>())
    );

    let description = layout.autonym().to_owned();

    let copyright = bundle.project.copyright.clone();
    let company = bundle.project.organisation.clone();

    let autonym = layout.autonym().to_string();

    // Language Code Identifier
    // https://docs.microsoft.com/en-us/openspecs/windows_protocols/ms-lcid/70feba9f-294e-491e-b6eb-56532684c37f
    let lcid_record = iso639::lcid::get(
        language_tag.primary_language(),
        language_tag.script(),
        language_tag.region(),
    );

    let locale_id = match lcid_record {
        Some(record) => record.lcid,
        None => 0x2000,
    };

    let locale_name = target
        .config
        .as_ref()
        .and_then(|config| config.locale.as_ref())
        .map(|locale| locale.to_string())
        .unwrap_or_else(|| match lcid_record {
            Some(_) => language_tag.to_string(),
            None => format!(
                "{}-{}-{}",
                language_tag.primary_language(),
                language_tag.script().unwrap_or("Latn"),
                language_tag.region().unwrap_or("001")
            ),
        });

    KlcFileMetadata {
        keyboard_name,
        description,
        copyright,
        company,
        autonym,
        locale_id,
        locale_name,
    }
}

fn convert_to_klc_key(
    key: Option<WindowsLayerSetKey>,
    virtual_key: &str,
    klc_layer: KlcLayer,
    klc_ligature_rows: &mut Vec<KlcLigatureRow>,
    dead_key_characters: &mut Vec<char>,
) -> KlcKey {
    match key {
        Some(key) => {
            let utf16s: Vec<u16> = key.string.encode_utf16().collect::<Vec<_>>();

            if utf16s.len() == 1 {
                let character = key.string.chars().next().unwrap();

                if key.dead_key {
                    dead_key_characters.push(character);

                    KlcKey::DeadKey(character)
                } else {
                    KlcKey::Character(character)
                }
            } else {
                let ligature_row = KlcLigatureRow {
                    virtual_key: virtual_key.to_owned(),
                    shift_state: (klc_layer as u8).to_string(),
                    utf16s,
                };

                klc_ligature_rows.push(ligature_row);

                KlcKey::Ligature
            }
        }
        None => KlcKey::None,
    }
}

fn space_layout_row() -> KlcLayoutRow {
    KlcLayoutRow {
        scancode: "39".to_owned(),
        virtual_key: "SPACE".to_owned(),
        caps_mode: "0".to_owned(),
        default_key: KlcKey::Character(' '),
        shift_key: KlcKey::Character(' '),
        ctrl_key: KlcKey::Character(' '),
        alt_key: KlcKey::None,
        alt_and_shift_key: KlcKey::None,
    }
}

fn decimal_layout_row(layout_decimal: &Option<String>) -> KlcLayoutRow {
    let mut decimal = DEFAULT_DECIMAL.to_owned();
    if let Some(layout_decimal) = layout_decimal {
        decimal = layout_decimal.clone();
    }

    let decimal = decimal
        .chars()
        .next()
        .expect("Layout decimal must be a single character.");

    KlcLayoutRow {
        scancode: "53".to_owned(),
        virtual_key: "DECIMAL".to_owned(),
        caps_mode: "0".to_owned(),
        default_key: KlcKey::Character(decimal),
        shift_key: KlcKey::Character(decimal),
        ctrl_key: KlcKey::None,
        alt_key: KlcKey::None,
        alt_and_shift_key: KlcKey::None,
    }
}
<commit_msg>Add some more tracing to the klc generation step<commit_after>use std::path::Path;

use async_trait::async_trait;
use codecs::utf16::Utf16Ext;
use language_tags::LanguageTag;
use tracing::{trace, debug};

use crate::{
    build::BuildStep,
    bundle::{
        layout::{Layout, WindowsTarget},
        KbdgenBundle, DEFAULT_DECIMAL,
    },
    util::split_keys,
};

use super::{
    klc::{
        dead_key::KlcDeadKeys,
        file::KlcFileMetadata,
        file::{KlcFile, KLC_EXT},
        key::KlcKey,
        keymap::MSKLC_KEYS,
        layout::{KlcLayer, KlcLayout, KlcLayoutRow},
        ligature::{KlcLigature, KlcLigatureRow},
    },
    layer_set::{populate_layer_set, WindowsLayerSet, WindowsLayerSetKey, SG_CAP},
};

pub struct GenerateKlc {}

#[async_trait(?Send)]
impl BuildStep for GenerateKlc {
    async fn build(&self, bundle: &KbdgenBundle, output_path: &Path) {
        trace!("Generating klc files");
        // One .klc file per language with Windows primary platform
        for (language_tag, layout) in &bundle.layouts {
            trace!("Checking if we need a klc file for {}", language_tag);
            if let Some(windows_target) = &layout.windows {
                debug!("Generating klc for {}", language_tag);
                let metadata =
                    generate_metadata(bundle.clone(), language_tag, layout, windows_target);

                let layers = &windows_target.primary.layers;

                let mut klc_layout_rows = Vec::new();
                let mut klc_ligature_rows = Vec::new();
                let mut dead_key_characters = Vec::new();

                let mut cursor = 0;
                for (_iso_key, klc_key) in MSKLC_KEYS.iter() {
                    let mut layer_set = WindowsLayerSet::default();

                    // Layer set to determine caps_mode, special escapes, dead keys and null keys
                    for (layer, key_map) in layers {
                        let key_map: Vec<String> = split_keys(&key_map);

                        tracing::debug!(
                            "iso len: {}; keymap len: {}",
                            MSKLC_KEYS.len(),
                            key_map.len()
                        );
                        if MSKLC_KEYS.len() > key_map.len() {
                            panic!(
                                r#"Provided layer does not have enough keys, expected {} keys but got {}, in {}:{}:{}:{:?}: \n{:?}"#,
                                MSKLC_KEYS.len(),
                                key_map.len(),
                                language_tag.to_string(),
                                "Windows",
                                "Primary",
                                layer,
                                key_map
                            );
                        }

                        populate_layer_set(
                            &mut layer_set,
                            layer,
                            key_map,
                            cursor,
                            windows_target.dead_keys.as_ref(),
                        );
                    }

                    let caps_mode = layer_set.caps_mode();

                    klc_layout_rows.push(KlcLayoutRow {
                        scancode: klc_key.scancode.clone(),
                        virtual_key: klc_key.virtual_key.clone(),
                        caps_mode: caps_mode.clone(), // Generate extra key for SGCap here?
                        default_key: convert_to_klc_key(
                            layer_set.default,
                            &klc_key.virtual_key,
                            KlcLayer::Default,
                            &mut klc_ligature_rows,
                            &mut dead_key_characters,
                        ),
                        shift_key: convert_to_klc_key(
                            layer_set.shift,
                            &klc_key.virtual_key,
                            KlcLayer::Shift,
                            &mut klc_ligature_rows,
                            &mut dead_key_characters,
                        ),
                        ctrl_key: convert_to_klc_key(
                            layer_set.ctrl,
                            &klc_key.virtual_key,
                            KlcLayer::Ctrl,
                            &mut klc_ligature_rows,
                            &mut dead_key_characters,
                        ),
                        alt_key: convert_to_klc_key(
                            layer_set.alt,
                            &klc_key.virtual_key,
                            KlcLayer::Alt,
                            &mut klc_ligature_rows,
                            &mut dead_key_characters,
                        ),
                        alt_and_shift_key: convert_to_klc_key(
                            layer_set.alt_and_shift,
                            &klc_key.virtual_key,
                            KlcLayer::AltAndShift,
                            &mut klc_ligature_rows,
                            &mut dead_key_characters,
                        ),
                    });

                    if caps_mode == SG_CAP {
                        let caps_key = convert_to_klc_key(
                            layer_set.caps,
                            &klc_key.virtual_key,
                            KlcLayer::Default,
                            &mut klc_ligature_rows,
                            &mut dead_key_characters,
                        );

                        let caps_shift_key = convert_to_klc_key(
                            layer_set.caps_and_shift,
                            &klc_key.virtual_key,
                            KlcLayer::Shift,
                            &mut klc_ligature_rows,
                            &mut dead_key_characters,
                        );

                        klc_layout_rows.push(KlcLayoutRow {
                            scancode: "-1".to_owned(),
                            virtual_key: "-1".to_owned(),
                            caps_mode: "0".to_owned(),
                            default_key: caps_key,
                            shift_key: caps_shift_key,
                            ctrl_key: KlcKey::Skip,
                            alt_key: KlcKey::Skip,
                            alt_and_shift_key: KlcKey::Skip,
                        });
                    }

                    cursor += 1;
                }

                klc_layout_rows.push(space_layout_row());
                klc_layout_rows.push(decimal_layout_row(&layout.decimal));

                let klc_file = KlcFile {
                    metadata,
                    layout: KlcLayout {
                        rows: klc_layout_rows,
                    },
                    ligature: KlcLigature {
                        rows: klc_ligature_rows,
                    },
                    dead_keys: KlcDeadKeys {
                        characters: dead_key_characters,
                        transforms: &layout.transforms,
                    },
                };

                let klc_bytes = klc_file.to_string().encode_utf16_le_bom();
                let klc_path =
                    output_path.join(format!("{}.{}", klc_file.metadata.keyboard_name, KLC_EXT));
                std::fs::write(klc_path, klc_bytes).unwrap();
            }
        }
    }
}

fn generate_metadata(
    bundle: &KbdgenBundle,
    language_tag: &LanguageTag,
    layout: &Layout,
    target: &WindowsTarget,
) -> KlcFileMetadata {
    let keyboard_name = format!(
        "kbd{}",
        target
            .config
            .as_ref()
            .and_then(|t| t.id.as_ref())
            .map(|x| x.to_string())
            .unwrap_or_else(|| language_tag.as_str().chars().take(5).collect::<String>())
    );

    let description = layout.autonym().to_owned();

    let copyright = bundle.project.copyright.clone();
    let company = bundle.project.organisation.clone();

    let autonym = layout.autonym().to_string();

    // Language Code Identifier
    // https://docs.microsoft.com/en-us/openspecs/windows_protocols/ms-lcid/70feba9f-294e-491e-b6eb-56532684c37f
    let lcid_record = iso639::lcid::get(
        language_tag.primary_language(),
        language_tag.script(),
        language_tag.region(),
    );

    let locale_id = match lcid_record {
        Some(record) => record.lcid,
        None => 0x2000,
    };

    let locale_name = target
        .config
        .as_ref()
        .and_then(|config| config.locale.as_ref())
        .map(|locale| locale.to_string())
        .unwrap_or_else(|| match lcid_record {
            Some(_) => language_tag.to_string(),
            None => format!(
                "{}-{}-{}",
                language_tag.primary_language(),
                language_tag.script().unwrap_or("Latn"),
                language_tag.region().unwrap_or("001")
            ),
        });

    KlcFileMetadata {
        keyboard_name,
        description,
        copyright,
        company,
        autonym,
        locale_id,
        locale_name,
    }
}

fn convert_to_klc_key(
    key: Option<WindowsLayerSetKey>,
    virtual_key: &str,
    klc_layer: KlcLayer,
    klc_ligature_rows: &mut Vec<KlcLigatureRow>,
    dead_key_characters: &mut Vec<char>,
) -> KlcKey {
    match key {
        Some(key) => {
            let utf16s: Vec<u16> = key.string.encode_utf16().collect::<Vec<_>>();

            if utf16s.len() == 1 {
                let character = key.string.chars().next().unwrap();

                if key.dead_key {
                    dead_key_characters.push(character);

                    KlcKey::DeadKey(character)
                } else {
                    KlcKey::Character(character)
                }
            } else {
                let ligature_row = KlcLigatureRow {
                    virtual_key: virtual_key.to_owned(),
                    shift_state: (klc_layer as u8).to_string(),
                    utf16s,
                };

                klc_ligature_rows.push(ligature_row);

                KlcKey::Ligature
            }
        }
        None => KlcKey::None,
    }
}

fn space_layout_row() -> KlcLayoutRow {
    KlcLayoutRow {
        scancode: "39".to_owned(),
        virtual_key: "SPACE".to_owned(),
        caps_mode: "0".to_owned(),
        default_key: KlcKey::Character(' '),
        shift_key: KlcKey::Character(' '),
        ctrl_key: KlcKey::Character(' '),
        alt_key: KlcKey::None,
        alt_and_shift_key: KlcKey::None,
    }
}

fn decimal_layout_row(layout_decimal: &Option<String>) -> KlcLayoutRow {
    let mut decimal = DEFAULT_DECIMAL.to_owned();
    if let Some(layout_decimal) = layout_decimal {
        decimal = layout_decimal.clone();
    }

    let decimal = decimal
        .chars()
        .next()
        .expect("Layout decimal must be a single character.");

    KlcLayoutRow {
        scancode: "53".to_owned(),
        virtual_key: "DECIMAL".to_owned(),
        caps_mode: "0".to_owned(),
        default_key: KlcKey::Character(decimal),
        shift_key: KlcKey::Character(decimal),
        ctrl_key: KlcKey::None,
        alt_key: KlcKey::None,
        alt_and_shift_key: KlcKey::None,
    }
}
<|endoftext|>


<commit_before>mod page;
pub use self::page::Page;

use std::env;
use std::io::Read;
use std::ascii::AsciiExt;
use std::default::Default;
use std::collections::BTreeMap;

use url::Url;
use reqwest::Client as HttpClient;
use reqwest::header::{Headers, UserAgent};
use reqwest::StatusCode;
use reqwest::Response as HttpResponse;
use robotparser::RobotFileParser;
use html5ever::tokenizer::Tokenizer;
use html5ever::tokenizer::buffer_queue::BufferQueue;
use sidekiq::Client as SidekiqClient;
use sidekiq::ClientOpts as SidekiqClientOpts;
use sidekiq::RedisPool;
use encoding::{Encoding, DecoderTrap};
use encoding::all::UTF_8;
use url_serde::Serde;

const MAMAN_ENV: &'static str = "MAMAN_ENV";
const MAMAN_ENV_DEFAULT: &'static str = "development";

pub struct Spider<'a> {
    pub base_url: Url,
    pub visited_urls: Vec<Serde<Url>>,
    pub unvisited_urls: Vec<Serde<Url>>,
    pub env: String,
    pub limit: isize,
    sidekiq: SidekiqClient,
    robot_parser: RobotFileParser<'a>,
}

impl<'a> Spider<'a> {
    pub fn new(redis_pool: RedisPool, base_url: Url, limit: isize) -> Spider<'a> {
        let maman_env =
            env::var(&MAMAN_ENV.to_owned()).unwrap_or_else(|_| MAMAN_ENV_DEFAULT.to_owned());
        let robots_txt = base_url.join("/robots.txt").unwrap();
        let robot_file_parser = RobotFileParser::new(robots_txt);
        let client_opts = SidekiqClientOpts { namespace: Some(maman_env.to_string()) };
        let sidekiq = SidekiqClient::new(redis_pool, client_opts);
        Spider {
            base_url: base_url,
            visited_urls: Vec::new(),
            unvisited_urls: Vec::new(),
            sidekiq: sidekiq,
            env: maman_env,
            robot_parser: robot_file_parser,
            limit: limit,
        }
    }

    pub fn visit_page(&mut self, page: Page) {
        self.visited_urls.push(page.url.clone());
        for u in &page.urls {
            self.unvisited_urls.push(u.clone());
        }
        if let Err(err) = self.sidekiq.push(page.to_job()) {
            error!("SidekiqClient push failed: {}", err);
        }
    }

    pub fn crawl(&mut self) {
        self.robot_parser.read();
        let base_url = self.base_url.clone();
        if let Some(response) = Spider::load_url(self.base_url.as_ref()) {
            self.visit(&base_url, response);
            while let Some(url) = self.unvisited_urls.pop() {
                if self.continue_to_crawl() {
                    if !self.visited_urls.contains(&url) {
                        if let Some(response) = Spider::load_url(url.as_ref()) {
                            self.visit(&url, response);
                        }
                    }
                } else {
                    break;
                }
            }
        }
    }

    pub fn read_page(page: Page, document: &str) -> Tokenizer<Page> {
        let mut tok = Tokenizer::new(page, Default::default());
        let mut input = BufferQueue::new();
        input.push_back(String::from(document).into());
        let _ = tok.feed(&mut input);
        tok.end();
        tok
    }

    fn visit(&mut self, page_url: &Url, response: HttpResponse) {
        if self.can_visit(page_url) {
            info!("{}", page_url);
            if let Some(page) = Spider::read_response(page_url, response) {
                self.visit_page(page);
            }
        }
    }

    fn continue_to_crawl(&self) -> bool {
        self.limit == 0 || (self.visited_urls.len() as isize) < self.limit
    }

    fn can_visit(&self, page_url: &Url) -> bool {
        self.robot_parser
            .can_fetch(maman_name!(), page_url.path())
    }

    fn read_response(page_url: &Url, mut response: HttpResponse) -> Option<Page> {
        let mut headers = BTreeMap::new();
        {
            for h in response.headers().iter() {
                headers.insert(h.name().to_ascii_lowercase(), h.value_string());
            }
        }
        let mut document = vec![];
        let _ = response.read_to_end(&mut document);
        match UTF_8.decode(&document, DecoderTrap::Ignore) {
            Ok(doc) => {
                let page = Page::new(page_url.clone(),
                                     doc.to_string(),
                                     headers.clone(),
                                     response.status().to_string());
                let read = Spider::read_page(page, &doc).unwrap();
                Some(read)
            }
            Err(_) => None,
        }
    }

    fn load_url(url: &str) -> Option<HttpResponse> {
        let client = HttpClient::new().expect("HttpClient failed to construct");
        let mut headers = Headers::new();
        headers.set(UserAgent::new(maman_user_agent!().to_owned()));
        match client.get(url) {
            Err(_) => None,
            Ok(mut request) => {
                request.headers(headers);
                match request.send() {
                    Ok(response) => {
                        match response.status() {
                            StatusCode::Ok | StatusCode::NotModified => Some(response),
                            _ => None,
                        }
                    }
                    Err(_) => None,
                }
            }
        }
    }
}
<commit_msg>Fix clippy warnings<commit_after>mod page;
pub use self::page::Page;

use std::env;
use std::io::Read;
use std::ascii::AsciiExt;
use std::default::Default;
use std::collections::BTreeMap;

use url::Url;
use reqwest::Client as HttpClient;
use reqwest::header::{Headers, UserAgent};
use reqwest::StatusCode;
use reqwest::Response as HttpResponse;
use robotparser::RobotFileParser;
use html5ever::tokenizer::Tokenizer;
use html5ever::tokenizer::buffer_queue::BufferQueue;
use sidekiq::Client as SidekiqClient;
use sidekiq::ClientOpts as SidekiqClientOpts;
use sidekiq::RedisPool;
use encoding::{Encoding, DecoderTrap};
use encoding::all::UTF_8;
use url_serde::Serde;

const MAMAN_ENV: &str = "MAMAN_ENV";
const MAMAN_ENV_DEFAULT: &str = "development";

pub struct Spider<'a> {
    pub base_url: Url,
    pub visited_urls: Vec<Serde<Url>>,
    pub unvisited_urls: Vec<Serde<Url>>,
    pub env: String,
    pub limit: isize,
    sidekiq: SidekiqClient,
    robot_parser: RobotFileParser<'a>,
}

impl<'a> Spider<'a> {
    pub fn new(redis_pool: RedisPool, base_url: Url, limit: isize) -> Spider<'a> {
        let maman_env =
            env::var(&MAMAN_ENV.to_owned()).unwrap_or_else(|_| MAMAN_ENV_DEFAULT.to_owned());
        let robots_txt = base_url.join("/robots.txt").unwrap();
        let robot_file_parser = RobotFileParser::new(robots_txt);
        let client_opts = SidekiqClientOpts { namespace: Some(maman_env.to_string()) };
        let sidekiq = SidekiqClient::new(redis_pool, client_opts);
        Spider {
            base_url: base_url,
            visited_urls: Vec::new(),
            unvisited_urls: Vec::new(),
            sidekiq: sidekiq,
            env: maman_env,
            robot_parser: robot_file_parser,
            limit: limit,
        }
    }

    pub fn visit_page(&mut self, page: Page) {
        self.visited_urls.push(page.url.clone());
        for u in &page.urls {
            self.unvisited_urls.push(u.clone());
        }
        if let Err(err) = self.sidekiq.push(page.to_job()) {
            error!("SidekiqClient push failed: {}", err);
        }
    }

    pub fn crawl(&mut self) {
        self.robot_parser.read();
        let base_url = self.base_url.clone();
        if let Some(response) = Spider::load_url(self.base_url.as_ref()) {
            self.visit(&base_url, response);
            while let Some(url) = self.unvisited_urls.pop() {
                if self.continue_to_crawl() {
                    if !self.visited_urls.contains(&url) {
                        if let Some(response) = Spider::load_url(url.as_ref()) {
                            self.visit(&url, response);
                        }
                    }
                } else {
                    break;
                }
            }
        }
    }

    pub fn read_page(page: Page, document: &str) -> Tokenizer<Page> {
        let mut tok = Tokenizer::new(page, Default::default());
        let mut input = BufferQueue::new();
        input.push_back(String::from(document).into());
        let _ = tok.feed(&mut input);
        tok.end();
        tok
    }

    fn visit(&mut self, page_url: &Url, response: HttpResponse) {
        if self.can_visit(page_url) {
            info!("{}", page_url);
            if let Some(page) = Spider::read_response(page_url, response) {
                self.visit_page(page);
            }
        }
    }

    fn continue_to_crawl(&self) -> bool {
        self.limit == 0 || (self.visited_urls.len() as isize) < self.limit
    }

    fn can_visit(&self, page_url: &Url) -> bool {
        self.robot_parser
            .can_fetch(maman_name!(), page_url.path())
    }

    fn read_response(page_url: &Url, mut response: HttpResponse) -> Option<Page> {
        let mut headers = BTreeMap::new();
        {
            for h in response.headers().iter() {
                headers.insert(h.name().to_ascii_lowercase(), h.value_string());
            }
        }
        let mut document = vec![];
        let _ = response.read_to_end(&mut document);
        match UTF_8.decode(&document, DecoderTrap::Ignore) {
            Ok(doc) => {
                let page = Page::new(page_url.clone(),
                                     doc.to_string(),
                                     headers.clone(),
                                     response.status().to_string());
                let read = Spider::read_page(page, &doc).unwrap();
                Some(read)
            }
            Err(_) => None,
        }
    }

    fn load_url(url: &str) -> Option<HttpResponse> {
        let client = HttpClient::new().expect("HttpClient failed to construct");
        let mut headers = Headers::new();
        headers.set(UserAgent::new(maman_user_agent!().to_owned()));
        match client.get(url) {
            Err(_) => None,
            Ok(mut request) => {
                request.headers(headers);
                match request.send() {
                    Ok(response) => {
                        match response.status() {
                            StatusCode::Ok | StatusCode::NotModified => Some(response),
                            _ => None,
                        }
                    }
                    Err(_) => None,
                }
            }
        }
    }
}
<|endoftext|>


<commit_before>mod page;
pub use self::page::Page;

use std::collections::BTreeMap;
use std::default::Default;
use std::env;
use std::str::FromStr;
use std::time::Duration;

use html5ever::tokenizer::BufferQueue;
use html5ever::tokenizer::Tokenizer;
use mime;
use reqwest::header::{CONTENT_TYPE, USER_AGENT};
use reqwest::Client as HttpClient;
use reqwest::Response as HttpResponse;
use reqwest::StatusCode;
use robotparser::RobotFileParser;
use sidekiq::Client as SidekiqClient;
use sidekiq::ClientOpts as SidekiqClientOpts;
use sidekiq::RedisPool;
use url::Url;

const MAMAN_ENV: &str = "MAMAN_ENV";
const MAMAN_ENV_DEFAULT: &str = "development";

pub struct Spider<'a> {
    pub base_url: Url,
    pub visited_urls: Vec<Url>,
    pub unvisited_urls: Vec<Url>,
    pub env: String,
    pub limit: isize,
    pub mime_types: Vec<mime::Mime>,
    sidekiq: SidekiqClient,
    robot_parser: RobotFileParser<'a>,
}

impl<'a> Spider<'a> {
    pub fn new(
        redis_pool: RedisPool,
        base_url: Url,
        limit: isize,
        mime_types: Vec<mime::Mime>,
    ) -> Spider<'a> {
        let maman_env =
            env::var(&MAMAN_ENV.to_owned()).unwrap_or_else(|_| MAMAN_ENV_DEFAULT.to_owned());
        let robots_txt = base_url.join("/robots.txt").unwrap();
        let robot_file_parser = RobotFileParser::new(robots_txt);
        let client_opts = SidekiqClientOpts {
            namespace: Some(maman_env.to_string()),
        };
        let sidekiq = SidekiqClient::new(redis_pool, client_opts);
        Spider {
            base_url,
            visited_urls: Vec::new(),
            unvisited_urls: Vec::new(),
            sidekiq,
            env: maman_env,
            robot_parser: robot_file_parser,
            limit,
            mime_types,
        }
    }

    pub fn visit_page(&mut self, page: Page) {
        self.visited_urls.push(page.url.clone());
        for u in &page.urls {
            self.unvisited_urls.push(u.clone());
        }
        if let Err(err) = self.sidekiq.push(page.to_job()) {
            error!("SidekiqClient push failed: {}", err);
        }
    }

    pub fn crawl(&mut self) {
        self.robot_parser.read();
        let base_url = self.base_url.clone();
        if let Some(response) = Spider::load_url(self.base_url.as_ref(), &self.mime_types) {
            self.visit(&base_url, response);
            while let Some(url) = self.unvisited_urls.pop() {
                if self.continue_to_crawl() {
                    if !self.visited_urls.contains(&url) {
                        if let Some(response) = Spider::load_url(url.as_ref(), &self.mime_types) {
                            self.visit(&url, response);
                        }
                    }
                } else {
                    break;
                }
            }
        }
    }

    pub fn read_page(page: Page, document: &str) -> Tokenizer<Page> {
        let mut tok = Tokenizer::new(page, Default::default());
        let mut input = BufferQueue::new();
        input.push_back(String::from(document).into());
        let _ = tok.feed(&mut input);
        tok.end();
        tok
    }

    fn visit(&mut self, page_url: &Url, response: HttpResponse) {
        if self.can_visit(page_url) {
            info!("{}", page_url);
            if let Some(page) = Spider::read_response(page_url, response) {
                self.visit_page(page);
            }
        }
    }

    fn continue_to_crawl(&self) -> bool {
        self.limit == 0 || (self.visited_urls.len() as isize) < self.limit
    }

    fn can_visit(&self, page_url: &Url) -> bool {
        self.robot_parser.can_fetch(maman_name!(), page_url.path())
    }

    fn read_response(page_url: &Url, mut response: HttpResponse) -> Option<Page> {
        let mut headers = BTreeMap::new();
        {
            for (key, value) in response.headers().iter() {
                headers.insert(
                    key.as_str().to_string(),
                    value.to_str().unwrap_or("").to_string(),
                );
            }
        }
        match response.text() {
            Ok(content) => {
                let page = Page::new(
                    page_url.clone(),
                    content.to_string(),
                    headers,
                    response.status().to_string(),
                );
                let read = Spider::read_page(page, &content);
                Some(read.sink)
            }
            _ => None,
        }
    }

    fn load_url(url: &str, mime_types: &[mime::Mime]) -> Option<HttpResponse> {
        let client = HttpClient::builder()
            .timeout(Duration::from_secs(5))
            .build()
            .expect("HttpClient failed to construct");
        match client
            .get(url)
            .header(USER_AGENT, maman_user_agent!())
            .send()
        {
            Err(_) => None,
            Ok(response) => match response.status() {
                StatusCode::OK | StatusCode::NOT_MODIFIED => {
                    if mime_types.is_empty() {
                        Some(response)
                    } else {
                        let content_type = response
                            .headers()
                            .get(CONTENT_TYPE)
                            .and_then(|value| value.to_str().ok())
                            .and_then(|value| value.parse::<mime::Mime>().ok())
                            .and_then(|value| match (value.type_(), value.subtype()) {
                                (type_, subtype) => {
                                    let mut text = type_.to_string();
                                    text.push_str("/");
                                    text.push_str(subtype.as_ref());
                                    mime::Mime::from_str(&text).ok()
                                }
                            });
                        match content_type {
                            Some(ct) => {
                                if mime_types.contains(&ct) {
                                    Some(response)
                                } else {
                                    None
                                }
                            }
                            None => None,
                        }
                    }
                }
                _ => None,
            },
        }
    }
}
<commit_msg>Remove redundant import mime<commit_after>mod page;
pub use self::page::Page;

use std::collections::BTreeMap;
use std::default::Default;
use std::env;
use std::str::FromStr;
use std::time::Duration;

use html5ever::tokenizer::BufferQueue;
use html5ever::tokenizer::Tokenizer;
use reqwest::header::{CONTENT_TYPE, USER_AGENT};
use reqwest::Client as HttpClient;
use reqwest::Response as HttpResponse;
use reqwest::StatusCode;
use robotparser::RobotFileParser;
use sidekiq::Client as SidekiqClient;
use sidekiq::ClientOpts as SidekiqClientOpts;
use sidekiq::RedisPool;
use url::Url;

const MAMAN_ENV: &str = "MAMAN_ENV";
const MAMAN_ENV_DEFAULT: &str = "development";

pub struct Spider<'a> {
    pub base_url: Url,
    pub visited_urls: Vec<Url>,
    pub unvisited_urls: Vec<Url>,
    pub env: String,
    pub limit: isize,
    pub mime_types: Vec<mime::Mime>,
    sidekiq: SidekiqClient,
    robot_parser: RobotFileParser<'a>,
}

impl<'a> Spider<'a> {
    pub fn new(
        redis_pool: RedisPool,
        base_url: Url,
        limit: isize,
        mime_types: Vec<mime::Mime>,
    ) -> Spider<'a> {
        let maman_env =
            env::var(&MAMAN_ENV.to_owned()).unwrap_or_else(|_| MAMAN_ENV_DEFAULT.to_owned());
        let robots_txt = base_url.join("/robots.txt").unwrap();
        let robot_file_parser = RobotFileParser::new(robots_txt);
        let client_opts = SidekiqClientOpts {
            namespace: Some(maman_env.to_string()),
        };
        let sidekiq = SidekiqClient::new(redis_pool, client_opts);
        Spider {
            base_url,
            visited_urls: Vec::new(),
            unvisited_urls: Vec::new(),
            sidekiq,
            env: maman_env,
            robot_parser: robot_file_parser,
            limit,
            mime_types,
        }
    }

    pub fn visit_page(&mut self, page: Page) {
        self.visited_urls.push(page.url.clone());
        for u in &page.urls {
            self.unvisited_urls.push(u.clone());
        }
        if let Err(err) = self.sidekiq.push(page.to_job()) {
            error!("SidekiqClient push failed: {}", err);
        }
    }

    pub fn crawl(&mut self) {
        self.robot_parser.read();
        let base_url = self.base_url.clone();
        if let Some(response) = Spider::load_url(self.base_url.as_ref(), &self.mime_types) {
            self.visit(&base_url, response);
            while let Some(url) = self.unvisited_urls.pop() {
                if self.continue_to_crawl() {
                    if !self.visited_urls.contains(&url) {
                        if let Some(response) = Spider::load_url(url.as_ref(), &self.mime_types) {
                            self.visit(&url, response);
                        }
                    }
                } else {
                    break;
                }
            }
        }
    }

    pub fn read_page(page: Page, document: &str) -> Tokenizer<Page> {
        let mut tok = Tokenizer::new(page, Default::default());
        let mut input = BufferQueue::new();
        input.push_back(String::from(document).into());
        let _ = tok.feed(&mut input);
        tok.end();
        tok
    }

    fn visit(&mut self, page_url: &Url, response: HttpResponse) {
        if self.can_visit(page_url) {
            info!("{}", page_url);
            if let Some(page) = Spider::read_response(page_url, response) {
                self.visit_page(page);
            }
        }
    }

    fn continue_to_crawl(&self) -> bool {
        self.limit == 0 || (self.visited_urls.len() as isize) < self.limit
    }

    fn can_visit(&self, page_url: &Url) -> bool {
        self.robot_parser.can_fetch(maman_name!(), page_url.path())
    }

    fn read_response(page_url: &Url, mut response: HttpResponse) -> Option<Page> {
        let mut headers = BTreeMap::new();
        {
            for (key, value) in response.headers().iter() {
                headers.insert(
                    key.as_str().to_string(),
                    value.to_str().unwrap_or("").to_string(),
                );
            }
        }
        match response.text() {
            Ok(content) => {
                let page = Page::new(
                    page_url.clone(),
                    content.to_string(),
                    headers,
                    response.status().to_string(),
                );
                let read = Spider::read_page(page, &content);
                Some(read.sink)
            }
            _ => None,
        }
    }

    fn load_url(url: &str, mime_types: &[mime::Mime]) -> Option<HttpResponse> {
        let client = HttpClient::builder()
            .timeout(Duration::from_secs(5))
            .build()
            .expect("HttpClient failed to construct");
        match client
            .get(url)
            .header(USER_AGENT, maman_user_agent!())
            .send()
        {
            Err(_) => None,
            Ok(response) => match response.status() {
                StatusCode::OK | StatusCode::NOT_MODIFIED => {
                    if mime_types.is_empty() {
                        Some(response)
                    } else {
                        let content_type = response
                            .headers()
                            .get(CONTENT_TYPE)
                            .and_then(|value| value.to_str().ok())
                            .and_then(|value| value.parse::<mime::Mime>().ok())
                            .and_then(|value| match (value.type_(), value.subtype()) {
                                (type_, subtype) => {
                                    let mut text = type_.to_string();
                                    text.push_str("/");
                                    text.push_str(subtype.as_ref());
                                    mime::Mime::from_str(&text).ok()
                                }
                            });
                        match content_type {
                            Some(ct) => {
                                if mime_types.contains(&ct) {
                                    Some(response)
                                } else {
                                    None
                                }
                            }
                            None => None,
                        }
                    }
                }
                _ => None,
            },
        }
    }
}
<|endoftext|>


<commit_before>#[path = "../tests/common/mod.rs"]
mod common;

use common::{load_graph, score_graphs};
use criterion::{criterion_group, criterion_main, Benchmark, Criterion};
use graph_neighbor_matching::graph::OwnedGraph;

const ITERS: usize = 100;
const EPS: f32 = 0.01;

fn assert_similarity(
    expected_similarity_in_percent: usize,
    a: &OwnedGraph<f32>,
    b: &OwnedGraph<f32>,
) {
    let graph_score = score_graphs(a, b, ITERS, EPS, false);
    assert_eq!(
        expected_similarity_in_percent,
        (graph_score * 100.0) as usize
    );
}

fn bench_collect_distribute_a(c: &mut Criterion) {
    let g_collect_distribute_3_3 = load_graph("graphs/collect_distribute_3_3.gml");
    let g_collect_distribute_3_3a = load_graph("graphs/collect_distribute_3_3a.gml");

    c.bench_function("score_graphs/collect_distribute_3_3a/79", move |b| {
        b.iter(|| {
            // Removing one link -> 79% similarity
            assert_similarity(79, &g_collect_distribute_3_3, &g_collect_distribute_3_3a);
        })
    });
}

fn bench_collect_distribute_b(c: &mut Criterion) {
    let g_collect_distribute_3_3 = load_graph("graphs/collect_distribute_3_3.gml");
    let g_collect_distribute_3_3b = load_graph("graphs/collect_distribute_3_3b.gml");

    c.bench_function("score_graphs/collect_distribute_3_3b/64", move |b| {
        b.iter(|| {
            // Removing two links -> 64% similarity
            assert_similarity(64, &g_collect_distribute_3_3, &g_collect_distribute_3_3b);
        })
    });
}

fn bench_skorpion(c: &mut Criterion) {
    let g_skorpion = load_graph("graphs/skorpion.gml");
    let g_skorpion_approx44 = load_graph("graphs/skorpion_approx44.gml");

    c.bench_function("score_graphs/skorpion/44", move |b| {
        b.iter(|| {
            assert_similarity(44, &g_skorpion, &g_skorpion_approx44);
        })
    });
}

fn bench_isomorphic_ffl35(c: &mut Criterion) {
    let g_ffl_iso_1_35 = load_graph("graphs/ffl/ffl_iso_1_35.gml");
    let g_ffl_iso_2_35 = load_graph("graphs/ffl/ffl_iso_2_35.gml");

    c.bench(
        "score_graphs",
        Benchmark::new("isomorphic_ffl_35/100", move |b| {
            b.iter(|| {
                assert_eq!(
                    1.0,
                    score_graphs(&g_ffl_iso_1_35, &g_ffl_iso_2_35, ITERS, EPS, true)
                );
            });
        })
        .sample_size(10),
    );
}

fn bench_isomorphic_ffl100(c: &mut Criterion) {
    let g_ffl_iso_1_100 = load_graph("graphs/ffl/ffl_iso_1_100.gml");
    let g_ffl_iso_2_100 = load_graph("graphs/ffl/ffl_iso_2_100.gml");

    c.bench(
        "score_graphs",
        Benchmark::new("isomorphic_ffl_100/100", move |b| {
            b.iter(|| {
                assert_eq!(
                    1.0,
                    score_graphs(&g_ffl_iso_1_100, &g_ffl_iso_2_100, ITERS, EPS, true)
                );
            });
        })
        .sample_size(10),
    );
}

criterion_group!(
    benches,
    bench_collect_distribute_a,
    bench_collect_distribute_b,
    bench_skorpion,
    bench_isomorphic_ffl35,
    bench_isomorphic_ffl100
);
criterion_main!(benches);
<commit_msg>Group all benchmarks under "score_graphs"<commit_after>#[path = "../tests/common/mod.rs"]
mod common;

use common::{load_graph, score_graphs};
use criterion::{criterion_group, criterion_main, Benchmark, Criterion};
use graph_neighbor_matching::graph::OwnedGraph;

const ITERS: usize = 100;
const EPS: f32 = 0.01;

fn assert_similarity(
    expected_similarity_in_percent: usize,
    a: &OwnedGraph<f32>,
    b: &OwnedGraph<f32>,
) {
    let graph_score = score_graphs(a, b, ITERS, EPS, false);
    assert_eq!(
        expected_similarity_in_percent,
        (graph_score * 100.0) as usize
    );
}

fn bench_collect_distribute_a(c: &mut Criterion) {
    let g_collect_distribute_3_3 = load_graph("graphs/collect_distribute_3_3.gml");
    let g_collect_distribute_3_3a = load_graph("graphs/collect_distribute_3_3a.gml");

    c.bench(
        "score_graphs",
        Benchmark::new("collect_distribute_3_3a_79", move |b| {
            b.iter(|| {
                // Removing one link -> 79% similarity
                assert_similarity(79, &g_collect_distribute_3_3, &g_collect_distribute_3_3a);
            });
        }),
    );
}

fn bench_collect_distribute_b(c: &mut Criterion) {
    let g_collect_distribute_3_3 = load_graph("graphs/collect_distribute_3_3.gml");
    let g_collect_distribute_3_3b = load_graph("graphs/collect_distribute_3_3b.gml");

    c.bench(
        "score_graphs",
        Benchmark::new("collect_distribute_3_3b_64", move |b| {
            b.iter(|| {
                // removing two links -> 64% similarity
                assert_similarity(64, &g_collect_distribute_3_3, &g_collect_distribute_3_3b);
            });
        }),
    );
}

fn bench_skorpion(c: &mut Criterion) {
    let g_skorpion = load_graph("graphs/skorpion.gml");
    let g_skorpion_approx44 = load_graph("graphs/skorpion_approx44.gml");

    c.bench(
        "score_graphs",
        Benchmark::new("skorpion_44", move |b| {
            b.iter(|| {
                assert_similarity(44, &g_skorpion, &g_skorpion_approx44);
            });
        }),
    );
}

fn bench_isomorphic_ffl35(c: &mut Criterion) {
    let g_ffl_iso_1_35 = load_graph("graphs/ffl/ffl_iso_1_35.gml");
    let g_ffl_iso_2_35 = load_graph("graphs/ffl/ffl_iso_2_35.gml");

    c.bench(
        "score_graphs",
        Benchmark::new("isomorphic_ffl_35_100", move |b| {
            b.iter(|| {
                assert_eq!(
                    1.0,
                    score_graphs(&g_ffl_iso_1_35, &g_ffl_iso_2_35, ITERS, EPS, true)
                );
            });
        })
        .sample_size(10),
    );
}

fn bench_isomorphic_ffl100(c: &mut Criterion) {
    let g_ffl_iso_1_100 = load_graph("graphs/ffl/ffl_iso_1_100.gml");
    let g_ffl_iso_2_100 = load_graph("graphs/ffl/ffl_iso_2_100.gml");

    c.bench(
        "score_graphs",
        Benchmark::new("isomorphic_ffl_100_100", move |b| {
            b.iter(|| {
                assert_eq!(
                    1.0,
                    score_graphs(&g_ffl_iso_1_100, &g_ffl_iso_2_100, ITERS, EPS, true)
                );
            });
        })
        .sample_size(10),
    );
}

criterion_group!(
    benches,
    bench_collect_distribute_a,
    bench_collect_distribute_b,
    bench_skorpion,
    bench_isomorphic_ffl35,
    bench_isomorphic_ffl100
);
criterion_main!(benches);
<|endoftext|>


<commit_before>// Copyright 2015-2016, The Gtk-rs Project Developers.
// See the COPYRIGHT file at the top-level directory of this distribution.
// Licensed under the MIT license, see the LICENSE file or <http://opensource.org/licenses/MIT>

//! Traits and essential types inteded for blanket imports.

pub use glib::prelude::*;

pub use auto::traits::ActionableExt;
pub use auto::traits::BinExt;
pub use auto::traits::BoxExt;
pub use auto::traits::ButtonExt;
pub use auto::traits::CellAreaExt;
pub use auto::traits::CellEditableExt;
pub use auto::traits::CellLayoutExt;
pub use auto::traits::CellRendererExt;
pub use auto::traits::CellRendererTextExt;
pub use auto::traits::CheckButtonExt;
pub use auto::traits::ComboBoxExt;
pub use auto::traits::ContainerExt;
//pub use auto::traits::DialogExt;
pub use auto::traits::EditableExt;
pub use auto::traits::EntryExt;
#[cfg(feature = "v3_14")]
pub use auto::traits::EventControllerExt;
pub use auto::traits::FileChooserExt;
pub use auto::traits::FontChooserExt;
pub use auto::traits::FrameExt;
#[cfg(feature = "v3_14")]
pub use auto::traits::GestureExt;
#[cfg(feature = "v3_14")]
pub use auto::traits::GestureDragExt;
#[cfg(feature = "v3_14")]
pub use auto::traits::GestureSingleExt;
pub use auto::traits::MenuItemExt;
pub use auto::traits::MenuShellExt;
pub use auto::traits::MiscExt;
pub use auto::traits::OrientableExt;
#[cfg(feature = "v3_12")]
pub use auto::traits::PopoverExt;
pub use auto::traits::RangeExt;
pub use auto::traits::RecentChooserExt;
pub use auto::traits::ScaleButtonExt;
pub use auto::traits::ScrollableExt;
pub use auto::traits::ScrolledWindowExt;
pub use auto::traits::StyleProviderExt;
pub use auto::traits::ToggleButtonExt;
pub use auto::traits::ToolButtonExt;
pub use auto::traits::ToolItemExt;
pub use auto::traits::ToolShellExt;
pub use auto::traits::TreeModelExt;
//pub use auto::traits::TreeSortableExt;
//pub use auto::traits::WidgetExt;
pub use auto::traits::WindowExt;

pub use app_chooser::AppChooserExt;
pub use color_chooser::ColorChooserExt;
pub use dialog::DialogExt;
pub use tree_sortable::TreeSortableExt;
pub use widget::WidgetExt;

pub use signal::*;
<commit_msg>Reexport gio::prelude::*<commit_after>// Copyright 2015-2016, The Gtk-rs Project Developers.
// See the COPYRIGHT file at the top-level directory of this distribution.
// Licensed under the MIT license, see the LICENSE file or <http://opensource.org/licenses/MIT>

//! Traits and essential types inteded for blanket imports.

pub use glib::prelude::*;
pub use gio::prelude::*;

pub use auto::traits::ActionableExt;
pub use auto::traits::BinExt;
pub use auto::traits::BoxExt;
pub use auto::traits::ButtonExt;
pub use auto::traits::CellAreaExt;
pub use auto::traits::CellEditableExt;
pub use auto::traits::CellLayoutExt;
pub use auto::traits::CellRendererExt;
pub use auto::traits::CellRendererTextExt;
pub use auto::traits::CheckButtonExt;
pub use auto::traits::ComboBoxExt;
pub use auto::traits::ContainerExt;
//pub use auto::traits::DialogExt;
pub use auto::traits::EditableExt;
pub use auto::traits::EntryExt;
#[cfg(feature = "v3_14")]
pub use auto::traits::EventControllerExt;
pub use auto::traits::FileChooserExt;
pub use auto::traits::FontChooserExt;
pub use auto::traits::FrameExt;
#[cfg(feature = "v3_14")]
pub use auto::traits::GestureExt;
#[cfg(feature = "v3_14")]
pub use auto::traits::GestureDragExt;
#[cfg(feature = "v3_14")]
pub use auto::traits::GestureSingleExt;
pub use auto::traits::MenuItemExt;
pub use auto::traits::MenuShellExt;
pub use auto::traits::MiscExt;
pub use auto::traits::OrientableExt;
#[cfg(feature = "v3_12")]
pub use auto::traits::PopoverExt;
pub use auto::traits::RangeExt;
pub use auto::traits::RecentChooserExt;
pub use auto::traits::ScaleButtonExt;
pub use auto::traits::ScrollableExt;
pub use auto::traits::ScrolledWindowExt;
pub use auto::traits::StyleProviderExt;
pub use auto::traits::ToggleButtonExt;
pub use auto::traits::ToolButtonExt;
pub use auto::traits::ToolItemExt;
pub use auto::traits::ToolShellExt;
pub use auto::traits::TreeModelExt;
//pub use auto::traits::TreeSortableExt;
//pub use auto::traits::WidgetExt;
pub use auto::traits::WindowExt;

pub use app_chooser::AppChooserExt;
pub use color_chooser::ColorChooserExt;
pub use dialog::DialogExt;
pub use tree_sortable::TreeSortableExt;
pub use widget::WidgetExt;

pub use signal::*;
<|endoftext|>