<commit_before>// Copyright 2018 The Rust Project Developers. See the COPYRIGHT
// file at the top-level directory of this distribution and at
// https://rust-lang.org/COPYRIGHT.
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// https://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or https://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.
//! The triangular distribution.

use Rng;
use distributions::{Distribution, Standard};

/// The triangular distribution.
///
/// # Example
///
/// ```rust
/// use rand::distributions::{Triangular, Distribution};
///
/// let d = Triangular::new(0., 5., 2.5);
/// let v = d.sample(&mut rand::thread_rng());
/// println!("{} is from a triangular distribution", v);
/// ```
#[derive(Clone, Copy, Debug)]
pub struct Triangular {
    min: f64,
    max: f64,
    mode: f64,
}

impl Triangular {
    /// Construct a new `Triangular` with minimum `min`, maximum `max` and mode
    /// `mode`.
    ///
    /// # Panics
    ///
    /// If `max < mode`, `mode < max` or `max == min`.
    ///
    #[inline]
    pub fn new(min: f64, max: f64, mode: f64) -> Triangular {
        assert!(max >= mode);
        assert!(mode >= min);
        assert!(max != min);
        Triangular { min, max, mode }
    }
}

impl Distribution<f64> for Triangular {
    #[inline]
    fn sample<R: Rng + ?Sized>(&self, rng: &mut R) -> f64 {
        let f: f64 = rng.sample(Standard);
        let diff_mode_min = self.mode - self.min;
        let diff_max_min = self.max - self.min;
        if f < diff_mode_min / diff_max_min {
            self.min + (f * diff_max_min * diff_mode_min).sqrt()
        } else {
            self.max - ((1. - f) * diff_max_min * (self.max - self.mode)).sqrt()
        }
    }
}

#[cfg(test)]
mod test {
    use distributions::Distribution;
    use super::Triangular;

    #[test]
    fn test_new() {
        for &(min, max, mode) in &[
            (-1., 1., 0.), (1., 2., 1.), (5., 25., 25.), (1e-5, 1e5, 1e-3),
            (0., 1., 0.9), (-4., -0.5, -2.), (-13.039, 8.41, 1.17),
        ] {
            println!("{} {} {}", min, max, mode);
            let _ = Triangular::new(min, max, mode);
        }
    }

    #[test]
    fn test_sample() {
        let norm = Triangular::new(0., 1., 0.5);
        let mut rng = ::test::rng(1);
        for _ in 0..1000 {
            norm.sample(&mut rng);
        }
    }
}
<commit_msg>Triangular: Avoid division<commit_after>// Copyright 2018 The Rust Project Developers. See the COPYRIGHT
// file at the top-level directory of this distribution and at
// https://rust-lang.org/COPYRIGHT.
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// https://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or https://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.
//! The triangular distribution.

use Rng;
use distributions::{Distribution, Standard};

/// The triangular distribution.
///
/// # Example
///
/// ```rust
/// use rand::distributions::{Triangular, Distribution};
///
/// let d = Triangular::new(0., 5., 2.5);
/// let v = d.sample(&mut rand::thread_rng());
/// println!("{} is from a triangular distribution", v);
/// ```
#[derive(Clone, Copy, Debug)]
pub struct Triangular {
    min: f64,
    max: f64,
    mode: f64,
}

impl Triangular {
    /// Construct a new `Triangular` with minimum `min`, maximum `max` and mode
    /// `mode`.
    ///
    /// # Panics
    ///
    /// If `max < mode`, `mode < max` or `max == min`.
    ///
    #[inline]
    pub fn new(min: f64, max: f64, mode: f64) -> Triangular {
        assert!(max >= mode);
        assert!(mode >= min);
        assert!(max != min);
        Triangular { min, max, mode }
    }
}

impl Distribution<f64> for Triangular {
    #[inline]
    fn sample<R: Rng + ?Sized>(&self, rng: &mut R) -> f64 {
        let f: f64 = rng.sample(Standard);
        let diff_mode_min = self.mode - self.min;
        let diff_max_min = self.max - self.min;
        if f * diff_max_min < diff_mode_min {
            self.min + (f * diff_max_min * diff_mode_min).sqrt()
        } else {
            self.max - ((1. - f) * diff_max_min * (self.max - self.mode)).sqrt()
        }
    }
}

#[cfg(test)]
mod test {
    use distributions::Distribution;
    use super::Triangular;

    #[test]
    fn test_new() {
        for &(min, max, mode) in &[
            (-1., 1., 0.), (1., 2., 1.), (5., 25., 25.), (1e-5, 1e5, 1e-3),
            (0., 1., 0.9), (-4., -0.5, -2.), (-13.039, 8.41, 1.17),
        ] {
            println!("{} {} {}", min, max, mode);
            let _ = Triangular::new(min, max, mode);
        }
    }

    #[test]
    fn test_sample() {
        let norm = Triangular::new(0., 1., 0.5);
        let mut rng = ::test::rng(1);
        for _ in 0..1000 {
            norm.sample(&mut rng);
        }
    }
}
<|endoftext|>


<commit_before>// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

use devicemapper::DM;

use std::collections::BTreeMap;
use std::collections::BTreeSet;
use std::iter::FromIterator;
use std::path::Path;
use std::path::PathBuf;
use std::vec::Vec;

use time;
use uuid::Uuid;
use serde_json;

use engine::EngineError;
use engine::EngineResult;
use engine::ErrorEnum;
use engine::Filesystem;
use engine::Pool;
use engine::RenameAction;
use engine::engine::Redundancy;
use engine::strat_engine::lineardev::LinearDev;
use engine::strat_engine::thinpooldev::ThinPoolDev;

use super::super::engine::{FilesystemUuid, HasName, HasUuid};
use super::super::structures::Table;

use super::serde_structs::StratSave;
use super::blockdev::{BlockDev, initialize, resolve_devices};
use super::filesystem::StratFilesystem;
use super::metadata::MIN_MDA_SECTORS;

use types::DataBlocks;
use types::Sectors;

#[derive(Debug)]
pub struct StratPool {
    name: String,
    pool_uuid: Uuid,
    pub block_devs: BTreeMap<PathBuf, BlockDev>,
    pub filesystems: Table<StratFilesystem>,
    redundancy: Redundancy,
    thin_pool: ThinPoolDev,
}

impl StratPool {
    pub fn new(name: &str,
               dm: &DM,
               paths: &[&Path],
               redundancy: Redundancy,
               force: bool)
               -> EngineResult<StratPool> {
        let pool_uuid = Uuid::new_v4();

        let devices = try!(resolve_devices(paths));
        let bds = try!(initialize(&pool_uuid, devices, MIN_MDA_SECTORS, force));

        // TODO: We've got some temporary code in BlockDev::initialize that
        // makes sure we've got at least 2 blockdevs supplied - one for a meta
        // and one for data.  In the future, we will be able to deal with a
        // single blockdev.  When that code is added to use a single blockdev,
        // the check for 2 devs in BlockDev::initialize should be removed.
        assert!(bds.len() >= 2);
        let meta_dev;
        let data_dev;
        let length;
        {
            let mut data_devs = Vec::from_iter(bds.values());

            meta_dev = try!(LinearDev::new(&format!("stratis_{}_meta", name),
                                           dm,
                                           &vec![data_devs.remove(0)]));

            data_dev = try!(LinearDev::new(&format!("stratis_{}_data", name), dm, &data_devs));
            length = try!(data_dev.size()).sectors();

        }
        // TODO Fix hard coded data blocksize and low water mark.
        let thinpool_dev = try!(ThinPoolDev::new(&format!("stratis_{}_thinpool", name),
                                                 dm,
                                                 length,
                                                 Sectors(1024),
                                                 DataBlocks(256000),
                                                 meta_dev,
                                                 data_dev));

        let mut pool = StratPool {
            name: name.to_owned(),
            pool_uuid: pool_uuid,
            block_devs: bds,
            filesystems: Table::new(),
            redundancy: redundancy,
            thin_pool: thinpool_dev,
        };

        try!(pool.write_metadata());

        Ok(pool)
    }

    /// Return the metadata from the first blockdev with up-to-date, readable
    /// metadata.
    /// Precondition: All Blockdevs in blockdevs must belong to the same pool.
    pub fn read_metadata(blockdevs: &[&BlockDev]) -> Option<Vec<u8>> {

        let mut bds: Vec<_> = blockdevs.iter()
            .filter(|bd| bd.last_update_time().is_some())
            .collect();

        if bds.is_empty() {
            return None;
        }

        bds.sort_by_key(|k| {
            k.last_update_time().expect("devs without some last update time filtered above.")
        });

        let last_update_time = bds.last()
            .expect("There is a last bd, since bds.is_empty() was false.")
            .last_update_time()
            .expect("devs without some last update time filtered above.");

        for bd in bds.iter()
            .rev()
            .take_while(|bd| {
                bd.last_update_time()
                    .expect("devs without some last update time filtered above.") ==
                last_update_time
            }) {
            match bd.load_state() {
                Ok(Some(data)) => return Some(data),
                _ => continue,
            }
        }
        None
    }

    /// Write the given data to all blockdevs
    pub fn write_metadata(&mut self) -> EngineResult<()> {

        let data = try!(serde_json::to_string(&self.to_save()));

        // TODO: Cap # of blockdevs written to, as described in SWDD
        // TODO: Check current time against global last updated, and use
        // alternate time value if earlier, as described in SWDD

        let time = time::now().to_timespec();

        // TODO: Do something better than panic when saving to blockdev fails.
        for (_, bd) in &mut self.block_devs {
            bd.save_state(&time, data.as_bytes()).unwrap();
        }

        Ok(())
    }

    pub fn to_save(&self) -> StratSave {
        StratSave {
            name: self.name.clone(),
            id: self.pool_uuid.simple().to_string(),
            block_devs: self.block_devs
                .iter()
                .map(|(_, bd)| (bd.uuid().simple().to_string(), bd.to_save()))
                .collect(),
        }
    }
}

impl Pool for StratPool {
    fn create_filesystems<'a, 'b>(&'a mut self,
                                  specs: &[&'b str])
                                  -> EngineResult<Vec<(&'b str, FilesystemUuid)>> {
        let names = BTreeSet::from_iter(specs);
        for name in names.iter() {
            if self.filesystems.contains_name(name) {
                return Err(EngineError::Engine(ErrorEnum::AlreadyExists, name.to_string()));
            }
        }

        let mut result = Vec::new();
        for name in names.iter() {
            let uuid = Uuid::new_v4();
            let dm = try!(DM::new());
            let new_filesystem = try!(StratFilesystem::new(uuid, name, &dm, &mut self.thin_pool));
            self.filesystems.insert(new_filesystem);
            result.push((**name, uuid));
        }

        Ok(result)
    }

    fn add_blockdevs(&mut self, paths: &[&Path], force: bool) -> EngineResult<Vec<PathBuf>> {
        let devices = try!(resolve_devices(paths));
        let mut bds = try!(initialize(&self.pool_uuid, devices, MIN_MDA_SECTORS, force));
        let bdev_paths = bds.iter().map(|p| p.1.devnode.clone()).collect();
        self.block_devs.append(&mut bds);
        try!(self.write_metadata());
        Ok(bdev_paths)
    }

    fn destroy(mut self) -> EngineResult<()> {

        // TODO Do we want to create a new File each time we interact with DM?
        let dm = try!(DM::new());
        try!(self.thin_pool.teardown(&dm));

        for bd in self.block_devs.values_mut() {
            try!(bd.wipe_metadata());
        }

        Ok(())
    }

    fn destroy_filesystems<'a, 'b>(&'a mut self,
                                   fs_uuids: &[&'b FilesystemUuid])
                                   -> EngineResult<Vec<&'b FilesystemUuid>> {
        destroy_filesystems!{self; fs_uuids}
    }

    fn rename_filesystem(&mut self,
                         uuid: &FilesystemUuid,
                         new_name: &str)
                         -> EngineResult<RenameAction> {
        rename_filesystem!{self; uuid; new_name}
    }

    fn rename(&mut self, name: &str) {
        self.name = name.to_owned();
    }

    fn get_filesystem(&mut self, uuid: &FilesystemUuid) -> Option<&mut Filesystem> {
        get_filesystem!(self; uuid)
    }
}

impl HasUuid for StratPool {
    fn uuid(&self) -> &FilesystemUuid {
        &self.pool_uuid
    }
}

impl HasName for StratPool {
    fn name(&self) -> &str {
        &self.name
    }
}
<commit_msg>Change implementation of StratPool::read_metadata<commit_after>// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

use devicemapper::DM;

use std::collections::BTreeMap;
use std::collections::BTreeSet;
use std::iter::FromIterator;
use std::path::Path;
use std::path::PathBuf;
use std::vec::Vec;

use time;
use uuid::Uuid;
use serde_json;

use engine::EngineError;
use engine::EngineResult;
use engine::ErrorEnum;
use engine::Filesystem;
use engine::Pool;
use engine::RenameAction;
use engine::engine::Redundancy;
use engine::strat_engine::lineardev::LinearDev;
use engine::strat_engine::thinpooldev::ThinPoolDev;

use super::super::engine::{FilesystemUuid, HasName, HasUuid};
use super::super::structures::Table;

use super::serde_structs::StratSave;
use super::blockdev::{BlockDev, initialize, resolve_devices};
use super::filesystem::StratFilesystem;
use super::metadata::MIN_MDA_SECTORS;

use types::DataBlocks;
use types::Sectors;

#[derive(Debug)]
pub struct StratPool {
    name: String,
    pool_uuid: Uuid,
    pub block_devs: BTreeMap<PathBuf, BlockDev>,
    pub filesystems: Table<StratFilesystem>,
    redundancy: Redundancy,
    thin_pool: ThinPoolDev,
}

impl StratPool {
    pub fn new(name: &str,
               dm: &DM,
               paths: &[&Path],
               redundancy: Redundancy,
               force: bool)
               -> EngineResult<StratPool> {
        let pool_uuid = Uuid::new_v4();

        let devices = try!(resolve_devices(paths));
        let bds = try!(initialize(&pool_uuid, devices, MIN_MDA_SECTORS, force));

        // TODO: We've got some temporary code in BlockDev::initialize that
        // makes sure we've got at least 2 blockdevs supplied - one for a meta
        // and one for data.  In the future, we will be able to deal with a
        // single blockdev.  When that code is added to use a single blockdev,
        // the check for 2 devs in BlockDev::initialize should be removed.
        assert!(bds.len() >= 2);
        let meta_dev;
        let data_dev;
        let length;
        {
            let mut data_devs = Vec::from_iter(bds.values());

            meta_dev = try!(LinearDev::new(&format!("stratis_{}_meta", name),
                                           dm,
                                           &vec![data_devs.remove(0)]));

            data_dev = try!(LinearDev::new(&format!("stratis_{}_data", name), dm, &data_devs));
            length = try!(data_dev.size()).sectors();

        }
        // TODO Fix hard coded data blocksize and low water mark.
        let thinpool_dev = try!(ThinPoolDev::new(&format!("stratis_{}_thinpool", name),
                                                 dm,
                                                 length,
                                                 Sectors(1024),
                                                 DataBlocks(256000),
                                                 meta_dev,
                                                 data_dev));

        let mut pool = StratPool {
            name: name.to_owned(),
            pool_uuid: pool_uuid,
            block_devs: bds,
            filesystems: Table::new(),
            redundancy: redundancy,
            thin_pool: thinpool_dev,
        };

        try!(pool.write_metadata());

        Ok(pool)
    }

    /// Return the metadata from the first blockdev with up-to-date, readable
    /// metadata.
    /// Precondition: All Blockdevs in blockdevs must belong to the same pool.
    pub fn read_metadata(blockdevs: &[&BlockDev]) -> Option<Vec<u8>> {
        if blockdevs.is_empty() {
            return None;
        }

        let most_recent_blockdev = blockdevs.iter()
            .max_by_key(|bd| bd.last_update_time())
            .expect("must be a maximum since bds is non-empty");

        let most_recent_time = most_recent_blockdev.last_update_time();

        if most_recent_time.is_none() {
            return None;
        }

        for bd in blockdevs.iter().filter(|b| b.last_update_time() == most_recent_time) {
            match bd.load_state() {
                Ok(Some(data)) => return Some(data),
                _ => continue,
            }
        }

        None
    }

    /// Write the given data to all blockdevs
    pub fn write_metadata(&mut self) -> EngineResult<()> {

        let data = try!(serde_json::to_string(&self.to_save()));

        // TODO: Cap # of blockdevs written to, as described in SWDD
        // TODO: Check current time against global last updated, and use
        // alternate time value if earlier, as described in SWDD

        let time = time::now().to_timespec();

        // TODO: Do something better than panic when saving to blockdev fails.
        for (_, bd) in &mut self.block_devs {
            bd.save_state(&time, data.as_bytes()).unwrap();
        }

        Ok(())
    }

    pub fn to_save(&self) -> StratSave {
        StratSave {
            name: self.name.clone(),
            id: self.pool_uuid.simple().to_string(),
            block_devs: self.block_devs
                .iter()
                .map(|(_, bd)| (bd.uuid().simple().to_string(), bd.to_save()))
                .collect(),
        }
    }
}

impl Pool for StratPool {
    fn create_filesystems<'a, 'b>(&'a mut self,
                                  specs: &[&'b str])
                                  -> EngineResult<Vec<(&'b str, FilesystemUuid)>> {
        let names = BTreeSet::from_iter(specs);
        for name in names.iter() {
            if self.filesystems.contains_name(name) {
                return Err(EngineError::Engine(ErrorEnum::AlreadyExists, name.to_string()));
            }
        }

        let mut result = Vec::new();
        for name in names.iter() {
            let uuid = Uuid::new_v4();
            let dm = try!(DM::new());
            let new_filesystem = try!(StratFilesystem::new(uuid, name, &dm, &mut self.thin_pool));
            self.filesystems.insert(new_filesystem);
            result.push((**name, uuid));
        }

        Ok(result)
    }

    fn add_blockdevs(&mut self, paths: &[&Path], force: bool) -> EngineResult<Vec<PathBuf>> {
        let devices = try!(resolve_devices(paths));
        let mut bds = try!(initialize(&self.pool_uuid, devices, MIN_MDA_SECTORS, force));
        let bdev_paths = bds.iter().map(|p| p.1.devnode.clone()).collect();
        self.block_devs.append(&mut bds);
        try!(self.write_metadata());
        Ok(bdev_paths)
    }

    fn destroy(mut self) -> EngineResult<()> {

        // TODO Do we want to create a new File each time we interact with DM?
        let dm = try!(DM::new());
        try!(self.thin_pool.teardown(&dm));

        for bd in self.block_devs.values_mut() {
            try!(bd.wipe_metadata());
        }

        Ok(())
    }

    fn destroy_filesystems<'a, 'b>(&'a mut self,
                                   fs_uuids: &[&'b FilesystemUuid])
                                   -> EngineResult<Vec<&'b FilesystemUuid>> {
        destroy_filesystems!{self; fs_uuids}
    }

    fn rename_filesystem(&mut self,
                         uuid: &FilesystemUuid,
                         new_name: &str)
                         -> EngineResult<RenameAction> {
        rename_filesystem!{self; uuid; new_name}
    }

    fn rename(&mut self, name: &str) {
        self.name = name.to_owned();
    }

    fn get_filesystem(&mut self, uuid: &FilesystemUuid) -> Option<&mut Filesystem> {
        get_filesystem!(self; uuid)
    }
}

impl HasUuid for StratPool {
    fn uuid(&self) -> &FilesystemUuid {
        &self.pool_uuid
    }
}

impl HasName for StratPool {
    fn name(&self) -> &str {
        &self.name
    }
}
<|endoftext|>


<commit_before>use cssparser::Parser;
use markup5ever::{expanded_name, local_name, namespace_url, ns};

use crate::document::AcquiredNodes;
use crate::drawing_ctx::DrawingCtx;
use crate::element::{ElementResult, SetAttributes};
use crate::error::*;
use crate::node::{CascadedValues, Node};
use crate::parsers::{Parse, ParseValue};
use crate::properties::ColorInterpolationFilters;
use crate::rect::IRect;
use crate::session::Session;
use crate::surface_utils::{iterators::Pixels, shared_surface::ExclusiveImageSurface};
use crate::xml::Attributes;

use super::bounds::BoundsBuilder;
use super::context::{FilterContext, FilterOutput};
use super::{
    FilterEffect, FilterError, FilterResolveError, Input, Primitive, PrimitiveParams,
    ResolvedPrimitive,
};

/// Enumeration of the color channels the displacement map can source.
#[derive(Clone, Copy)]
enum ColorChannel {
    R,
    G,
    B,
    A,
}

enum_default!(ColorChannel, ColorChannel::A);

/// The `feDisplacementMap` filter primitive.
#[derive(Default)]
pub struct FeDisplacementMap {
    base: Primitive,
    params: DisplacementMap,
}

/// Resolved `feDisplacementMap` primitive for rendering.
#[derive(Clone, Default)]
pub struct DisplacementMap {
    in1: Input,
    in2: Input,
    scale: f64,
    x_channel_selector: ColorChannel,
    y_channel_selector: ColorChannel,
    color_interpolation_filters: ColorInterpolationFilters,
}

impl SetAttributes for FeDisplacementMap {
    fn set_attributes(&mut self, attrs: &Attributes, session: &Session) -> ElementResult {
        let (in1, in2) = self.base.parse_two_inputs(attrs, session);
        self.params.in1 = in1;
        self.params.in2 = in2;

        for (attr, value) in attrs.iter() {
            match attr.expanded() {
                expanded_name!("", "scale") => self.params.scale = attr.parse(value)?,
                expanded_name!("", "xChannelSelector") => {
                    self.params.x_channel_selector = attr.parse(value)?
                }
                expanded_name!("", "yChannelSelector") => {
                    self.params.y_channel_selector = attr.parse(value)?
                }
                _ => (),
            }
        }

        Ok(())
    }
}

impl DisplacementMap {
    pub fn render(
        &self,
        bounds_builder: BoundsBuilder,
        ctx: &FilterContext,
        acquired_nodes: &mut AcquiredNodes<'_>,
        draw_ctx: &mut DrawingCtx,
    ) -> Result<FilterOutput, FilterError> {
        // https://www.w3.org/TR/filter-effects/#feDisplacementMapElement
        // "The color-interpolation-filters property only applies to
        // the in2 source image and does not apply to the in source
        // image. The in source image must remain in its current color
        // space.

        let input_1 = ctx.get_input(
            acquired_nodes,
            draw_ctx,
            &self.in1,
            ColorInterpolationFilters::Auto,
        )?;
        let displacement_input = ctx.get_input(
            acquired_nodes,
            draw_ctx,
            &self.in2,
            self.color_interpolation_filters,
        )?;
        let bounds: IRect = bounds_builder
            .add_input(&input_1)
            .add_input(&displacement_input)
            .compute(ctx)
            .clipped
            .into();

        // Displacement map's values need to be non-premultiplied.
        let displacement_surface = displacement_input.surface().unpremultiply(bounds)?;

        let (sx, sy) = ctx.paffine().transform_distance(self.scale, self.scale);

        let mut surface = ExclusiveImageSurface::new(
            ctx.source_graphic().width(),
            ctx.source_graphic().height(),
            input_1.surface().surface_type(),
        )?;

        surface.draw::<cairo::Error>(&mut |cr| {
            for (x, y, displacement_pixel) in Pixels::within(&displacement_surface, bounds) {
                let get_value = |channel| match channel {
                    ColorChannel::R => displacement_pixel.r,
                    ColorChannel::G => displacement_pixel.g,
                    ColorChannel::B => displacement_pixel.b,
                    ColorChannel::A => displacement_pixel.a,
                };

                let process = |x| f64::from(x) / 255.0 - 0.5;

                let dx = process(get_value(self.x_channel_selector));
                let dy = process(get_value(self.y_channel_selector));

                let x = f64::from(x);
                let y = f64::from(y);
                let ox = sx * dx;
                let oy = sy * dy;

                // Doing this in a loop doesn't look too bad performance wise, and allows not to
                // manually implement bilinear or other interpolation.
                cr.rectangle(x, y, 1.0, 1.0);
                cr.reset_clip();
                cr.clip();

                input_1.surface().set_as_source_surface(&cr, -ox, -oy)?;
                cr.paint()?;
            }

            Ok(())
        })?;

        Ok(FilterOutput {
            surface: surface.share()?,
            bounds,
        })
    }
}

impl FilterEffect for FeDisplacementMap {
    fn resolve(
        &self,
        _acquired_nodes: &mut AcquiredNodes<'_>,
        node: &Node,
    ) -> Result<ResolvedPrimitive, FilterResolveError> {
        let cascaded = CascadedValues::new_from_node(node);
        let values = cascaded.get();

        let mut params = self.params.clone();
        params.color_interpolation_filters = values.color_interpolation_filters();

        Ok(ResolvedPrimitive {
            primitive: self.base.clone(),
            params: PrimitiveParams::DisplacementMap(params),
        })
    }
}

impl Parse for ColorChannel {
    fn parse<'i>(parser: &mut Parser<'i, '_>) -> Result<Self, ParseError<'i>> {
        Ok(parse_identifiers!(
            parser,
            "R" => ColorChannel::R,
            "G" => ColorChannel::G,
            "B" => ColorChannel::B,
            "A" => ColorChannel::A,
        )?)
    }
}
<commit_msg>filters/displacement_map.rs: use set_attribute()<commit_after>use cssparser::Parser;
use markup5ever::{expanded_name, local_name, namespace_url, ns};

use crate::document::AcquiredNodes;
use crate::drawing_ctx::DrawingCtx;
use crate::element::{set_attribute, ElementResult, SetAttributes};
use crate::error::*;
use crate::node::{CascadedValues, Node};
use crate::parsers::{Parse, ParseValue};
use crate::properties::ColorInterpolationFilters;
use crate::rect::IRect;
use crate::session::Session;
use crate::surface_utils::{iterators::Pixels, shared_surface::ExclusiveImageSurface};
use crate::xml::Attributes;

use super::bounds::BoundsBuilder;
use super::context::{FilterContext, FilterOutput};
use super::{
    FilterEffect, FilterError, FilterResolveError, Input, Primitive, PrimitiveParams,
    ResolvedPrimitive,
};

/// Enumeration of the color channels the displacement map can source.
#[derive(Clone, Copy)]
enum ColorChannel {
    R,
    G,
    B,
    A,
}

enum_default!(ColorChannel, ColorChannel::A);

/// The `feDisplacementMap` filter primitive.
#[derive(Default)]
pub struct FeDisplacementMap {
    base: Primitive,
    params: DisplacementMap,
}

/// Resolved `feDisplacementMap` primitive for rendering.
#[derive(Clone, Default)]
pub struct DisplacementMap {
    in1: Input,
    in2: Input,
    scale: f64,
    x_channel_selector: ColorChannel,
    y_channel_selector: ColorChannel,
    color_interpolation_filters: ColorInterpolationFilters,
}

impl SetAttributes for FeDisplacementMap {
    fn set_attributes(&mut self, attrs: &Attributes, session: &Session) -> ElementResult {
        let (in1, in2) = self.base.parse_two_inputs(attrs, session);
        self.params.in1 = in1;
        self.params.in2 = in2;

        for (attr, value) in attrs.iter() {
            match attr.expanded() {
                expanded_name!("", "scale") => {
                    set_attribute(&mut self.params.scale, attr.parse(value), session)
                }
                expanded_name!("", "xChannelSelector") => {
                    set_attribute(
                        &mut self.params.x_channel_selector,
                        attr.parse(value),
                        session,
                    );
                }
                expanded_name!("", "yChannelSelector") => {
                    set_attribute(
                        &mut self.params.y_channel_selector,
                        attr.parse(value),
                        session,
                    );
                }
                _ => (),
            }
        }

        Ok(())
    }
}

impl DisplacementMap {
    pub fn render(
        &self,
        bounds_builder: BoundsBuilder,
        ctx: &FilterContext,
        acquired_nodes: &mut AcquiredNodes<'_>,
        draw_ctx: &mut DrawingCtx,
    ) -> Result<FilterOutput, FilterError> {
        // https://www.w3.org/TR/filter-effects/#feDisplacementMapElement
        // "The color-interpolation-filters property only applies to
        // the in2 source image and does not apply to the in source
        // image. The in source image must remain in its current color
        // space.

        let input_1 = ctx.get_input(
            acquired_nodes,
            draw_ctx,
            &self.in1,
            ColorInterpolationFilters::Auto,
        )?;
        let displacement_input = ctx.get_input(
            acquired_nodes,
            draw_ctx,
            &self.in2,
            self.color_interpolation_filters,
        )?;
        let bounds: IRect = bounds_builder
            .add_input(&input_1)
            .add_input(&displacement_input)
            .compute(ctx)
            .clipped
            .into();

        // Displacement map's values need to be non-premultiplied.
        let displacement_surface = displacement_input.surface().unpremultiply(bounds)?;

        let (sx, sy) = ctx.paffine().transform_distance(self.scale, self.scale);

        let mut surface = ExclusiveImageSurface::new(
            ctx.source_graphic().width(),
            ctx.source_graphic().height(),
            input_1.surface().surface_type(),
        )?;

        surface.draw::<cairo::Error>(&mut |cr| {
            for (x, y, displacement_pixel) in Pixels::within(&displacement_surface, bounds) {
                let get_value = |channel| match channel {
                    ColorChannel::R => displacement_pixel.r,
                    ColorChannel::G => displacement_pixel.g,
                    ColorChannel::B => displacement_pixel.b,
                    ColorChannel::A => displacement_pixel.a,
                };

                let process = |x| f64::from(x) / 255.0 - 0.5;

                let dx = process(get_value(self.x_channel_selector));
                let dy = process(get_value(self.y_channel_selector));

                let x = f64::from(x);
                let y = f64::from(y);
                let ox = sx * dx;
                let oy = sy * dy;

                // Doing this in a loop doesn't look too bad performance wise, and allows not to
                // manually implement bilinear or other interpolation.
                cr.rectangle(x, y, 1.0, 1.0);
                cr.reset_clip();
                cr.clip();

                input_1.surface().set_as_source_surface(&cr, -ox, -oy)?;
                cr.paint()?;
            }

            Ok(())
        })?;

        Ok(FilterOutput {
            surface: surface.share()?,
            bounds,
        })
    }
}

impl FilterEffect for FeDisplacementMap {
    fn resolve(
        &self,
        _acquired_nodes: &mut AcquiredNodes<'_>,
        node: &Node,
    ) -> Result<ResolvedPrimitive, FilterResolveError> {
        let cascaded = CascadedValues::new_from_node(node);
        let values = cascaded.get();

        let mut params = self.params.clone();
        params.color_interpolation_filters = values.color_interpolation_filters();

        Ok(ResolvedPrimitive {
            primitive: self.base.clone(),
            params: PrimitiveParams::DisplacementMap(params),
        })
    }
}

impl Parse for ColorChannel {
    fn parse<'i>(parser: &mut Parser<'i, '_>) -> Result<Self, ParseError<'i>> {
        Ok(parse_identifiers!(
            parser,
            "R" => ColorChannel::R,
            "G" => ColorChannel::G,
            "B" => ColorChannel::B,
            "A" => ColorChannel::A,
        )?)
    }
}
<|endoftext|>


<commit_before>extern crate itertools;
use super::merge_policy::{MergePolicy, MergeCandidate};
use core::SegmentMeta;
use self::itertools::Itertools;

pub struct LogMergePolicy;
use std::f64;

const PACK_LEN: usize = 8;
const LEVEL_LOG_SIZE: f64 = 0.75;

impl MergePolicy for LogMergePolicy {
    fn compute_merge_candidates(&self, segments: &[SegmentMeta]) -> Vec<MergeCandidate> {
        // what about when segments is empty??
        // take log of size of each segment
        // log2
        let mut size_sorted_tuples = segments.iter()
            .map(|x| {x.num_docs})
            .enumerate()
            .collect::<Vec<(usize, u32)>>();

        size_sorted_tuples.sort_by_key(|x| {x.1});

        let size_sorted_log_tuples: Vec<_> = size_sorted_tuples.iter()
            .map(|x| {(x.0, (x.1 as f64).log2())}).collect();

        let (first_ind, first_score) = size_sorted_log_tuples[0];
        let mut current_max_log_size = first_score;
        let mut levels = Vec::new();
        levels.push(Vec::new());
        levels.last_mut().unwrap().push(first_ind);
        for &(ind, score) in (&size_sorted_log_tuples).iter().skip(1) {
            if score < (current_max_log_size - LEVEL_LOG_SIZE) {
                current_max_log_size = score;
                levels.push(Vec::new());
            }
            levels.last_mut().unwrap().push(ind);
        }

        // now levels contains vectors, each of which is one level

        // based on log sizes, quantize into levels
        // given quantized levels, find valid merges within each level

        // how to find a valid merge

        // quantizing into levels
        // find the least segment within LEVEL_LOG_SIZE of the starting max size. it and everything above are the top level.
        // next, find the max of all other segments. this is the new max size. repeat. so LEVEL_LOG_SIZE basically is what determines how big your segments are.
        unimplemented!();
    }
}

impl Default for LogMergePolicy {
    fn default() -> LogMergePolicy {
        LogMergePolicy
    }
}
<commit_msg>Finished but probably wrong LogMergePolicy<commit_after>extern crate itertools;
use super::merge_policy::{MergePolicy, MergeCandidate};
use core::SegmentMeta;

pub struct LogMergePolicy;
use std::f64;

const LEVEL_LOG_SIZE: f64 = 0.75;

impl MergePolicy for LogMergePolicy {
    fn compute_merge_candidates(&self, segments: &[SegmentMeta]) -> Vec<MergeCandidate> {
        // what about when segments is empty??
        // take log of size of each segment
        // log2
        let mut size_sorted_tuples = segments.iter()
            .map(|x| {x.num_docs})
            .enumerate()
            .collect::<Vec<(usize, u32)>>();

        size_sorted_tuples.sort_by_key(|x| {x.1});

        let size_sorted_log_tuples: Vec<_> = size_sorted_tuples.iter()
            .map(|x| {(x.0, (x.1 as f64).log2())}).collect();

        let (first_ind, first_score) = size_sorted_log_tuples[0];
        let mut current_max_log_size = first_score;
        let mut levels = Vec::new();
        levels.push(Vec::new());
        levels.last_mut().unwrap().push(first_ind);
        for &(ind, score) in (&size_sorted_log_tuples).iter().skip(1) {
            if score < (current_max_log_size - LEVEL_LOG_SIZE) {
                current_max_log_size = score;
                levels.push(Vec::new());
            }
            levels.last_mut().unwrap().push(ind);
        }

        let result = levels.iter()
            .map(|ind_vec| {
                MergeCandidate(ind_vec.iter()
                    .map(|&ind| {
                        segments[ind].segment_id
                    }).collect())
            }).collect();

        result
    }
}

impl Default for LogMergePolicy {
    fn default() -> LogMergePolicy {
        LogMergePolicy
    }
}
<|endoftext|>


<commit_before>use image::ImageResult;
use mli::{Backward, Forward, Graph, Train};
use mli_conv::{Conv2, Conv2n, Conv3};
use mli_ndarray::{Map2One, Map3One, Reshape3to2};
use mli_relu::Blu;
use ndarray::{array, s, Array, Array2, OwnedRepr};
use ndarray_image::{open_gray_image, save_gray_image};
use rand_core::{RngCore, SeedableRng};
use rand_distr::{Distribution, Normal};
use rand_pcg::Pcg64;
use std::path::{Path, PathBuf};
use structopt::StructOpt;

#[derive(Debug, StructOpt)]
#[structopt(name = "image", about = "Loads and convolves an image as a test")]
struct Opt {
    /// Number of epochs
    #[structopt(short = "t", default_value = "4000")]
    epochs: usize,
    /// Number of epochs per output image
    #[structopt(short = "s", default_value = "1")]
    show_every: usize,
    /// Pre-start learning rate (for first 10 epochs)
    #[structopt(short = "p", default_value = "0.1")]
    prestart_learning_rate: f32,
    /// Initial learning rate
    #[structopt(short = "i", default_value = "0.5")]
    initial_learning_rate: f32,
    /// Learning rate multiplier per epoch
    #[structopt(short = "m", default_value = "0.999")]
    learning_rate_multiplier: f32,
    /// Seed
    #[structopt(short = "z", default_value = "32")]
    seed: u32,
    /// Beta value for NAG
    #[structopt(short = "b", default_value = "0.9")]
    beta: f32,
    /// File to load
    #[structopt(parse(from_os_str))]
    file: PathBuf,
    /// Output directory
    #[structopt(parse(from_os_str))]
    output_dir: PathBuf,
}

fn sobel(image: &Array2<f32>) -> Array2<f32> {
    let down_filter = array![[-1.0, -2.0, -1.0], [0.0, 0.0, 0.0], [1.0, 2.0, 1.0],];
    let right_filter = down_filter.t().to_owned();
    let down = Conv2::new(down_filter).forward(&image).1;
    let right = Conv2::new(right_filter).forward(&image).1;
    (down.map(|f| f.powi(2)) + right.map(|f| f.powi(2))).map(|f| f.sqrt())
}

fn open_image(path: impl AsRef<Path>) -> ImageResult<Array2<f32>> {
    let image = open_gray_image(path)?;
    Ok(image.map(|&n| f32::from(n) / 255.0))
}

fn save_image(path: impl AsRef<Path>, image: &Array2<f32>) -> ImageResult<()> {
    let image = image.map(|&n| num::clamp(n * 255.0, 0.0, 255.0) as u8);
    save_gray_image(path, image.view())
}

fn main() -> ImageResult<()> {
    let opt = Opt::from_args();
    let image = open_image(opt.file)?;
    let sobel_image = sobel(&image);
    let conv_layers = 4;
    let filter_radius = 1usize;
    let filter_depth = 2usize;
    let filter_area = (filter_radius * 2 + 1).pow(2);
    let filter_volume = filter_area * filter_depth;
    let padding = (conv_layers * filter_radius - 1) as i32;
    #[allow(clippy::deref_addrof)]
    let expected = sobel_image.slice(s![padding..-padding, padding..-padding]);
    save_image(opt.output_dir.join("actual.png"), &expected.to_owned())?;
    let mut prng = make_prng(opt.seed);
    let mut prng_2filter = make_prng(prng.next_u32());
    let mut random_2filter = |mean: f32, variance: f32| -> Conv2<OwnedRepr<f32>> {
        // Xavier initialize by changing the variance to be 1/N where N is the area of the filter.
        Conv2::new(
            Array::from_iter(
                Normal::new(mean / filter_area as f32, variance / filter_area as f32)
                    .unwrap()
                    .sample_iter(&mut prng_2filter)
                    .take(filter_area),
            )
            .into_shape((filter_radius * 2 + 1, filter_radius * 2 + 1))
            .unwrap(),
        )
    };
    let mut prng_2nfilter = make_prng(prng.next_u32());
    let mut random_2nfilter = |mean: f32, variance: f32| -> Conv2n<OwnedRepr<f32>> {
        // Xavier initialize by changing the variance to be 1/N where N is the area of the filter.
        Conv2n::new(
            Array::from_iter(
                Normal::new(mean / filter_area as f32, variance / filter_area as f32)
                    .unwrap()
                    .sample_iter(&mut prng_2nfilter)
                    .take(filter_volume),
            )
            .into_shape((filter_depth, filter_radius * 2 + 1, filter_radius * 2 + 1))
            .unwrap(),
        )
    };
    let mut prng_3filter = make_prng(prng.next_u32());
    let mut random_3filter = |mean: f32, variance: f32| -> Conv3<OwnedRepr<f32>> {
        // Xavier initialize by changing the variance to be 1/N where N is the volume of the filter.
        Conv3::new(
            Array::from_iter(
                Normal::new(mean / filter_volume as f32, variance / filter_volume as f32)
                    .unwrap()
                    .sample_iter(&mut prng_3filter)
                    .take(filter_volume),
            )
            .into_shape((filter_depth, filter_radius * 2 + 1, filter_radius * 2 + 1))
            .unwrap(),
        )
    };
    let mut random_blu = |mean: f32, variance: f32| -> Blu {
        // Xavier initialize by changing the variance to be 1/N where N is the number of neurons.
        let distr = Normal::new(mean, variance).unwrap();
        Blu::new(distr.sample(&mut prng), distr.sample(&mut prng))
    };
    let mut generate_filter = || {
        random_2nfilter(0.0, 4.0)
            .chain(Map3One(random_blu(0.0, 0.5)))
            .chain(random_3filter(0.0, 4.0))
            .chain(Reshape3to2::new())
            .chain(Map2One(random_blu(0.0, 0.5)))
            .chain(random_2filter(0.0, 4.0))
            .chain(Map2One(random_blu(0.0, 0.5)))
            .chain(random_2filter(0.0, 4.0))
    };

    loop {
        let mut train_filter = generate_filter();
        let mut learn_rate = opt.initial_learning_rate;
        // Weird hack to initialize the momentum to zero without knowing the shape of the ndarray in advance.
        let mut momentum = {
            let (internal, output) = train_filter.forward(&image);
            train_filter.backward_train(&image, &internal, &output)
        };
        let mut last_loss = 0.0;
        momentum *= 0.0f32;
        for i in 0..opt.epochs {
            // Compute beta * momentum.
            momentum *= opt.beta;
            train_filter.train(&momentum);
            let (internal, output) = train_filter.forward(&image);
            // Show the image if the frame is divisible by show_every.
            if i % opt.show_every == 0 {
                save_image(opt.output_dir.join(format!("epoch{:04}.png", i)), &output)?;
            }
            if i == opt.epochs - 1 {
                eprintln!("Finished!");
                return Ok(());
            }
            let output_len = output.len() as f32;
            let local_learn_rate = if i < 10 {
                opt.prestart_learning_rate
            } else {
                let llr = learn_rate;
                learn_rate *= opt.learning_rate_multiplier;
                llr
            };
            // Compute the loss for display only (we don't actually need the loss itself for backprop, just its derivative).
            let loss = (output.clone() - expected.view()).map(|n| n.powi(2)).sum() / output_len;
            eprintln!(
                "epoch {:04} loss: {} ({:+}%), learn_rate: {}",
                i,
                loss,
                (loss - last_loss) / last_loss * 100.0,
                local_learn_rate
            );
            last_loss = loss;
            if !loss.is_normal() {
                eprintln!("abnormal loss at epoch {}; starting over", i);
                break;
            }
            // The loss function is (n - t)^2, so 2*(n - t) is dE/df where
            // `E` is loss and `f` is output.
            let delta_loss = (output - expected).map(|n| 2.0 * n / output_len);
            // Compute the output delta.
            let output_delta = -local_learn_rate * delta_loss;
            // Compute the trainable variable delta.
            let mut train_delta = train_filter.backward_train(&image, &internal, &output_delta);
            // Make the train delta a small component.
            train_delta *= 1.0 - opt.beta;
            train_filter.train(&train_delta);
            // Add the small component to the momentum.
            momentum += train_delta;
        }
    }
}

fn make_prng(seed: u32) -> Pcg64 {
    Pcg64::from_seed([
        (seed & 0xFF) as u8,
        ((seed >> 8) & 0xFF) as u8,
        ((seed >> 16) & 0xFF) as u8,
        ((seed >> 24) & 0xFF) as u8,
        0,
        0,
        0,
        0,
        0,
        0,
        0,
        0,
        0,
        0,
        0,
        0,
        0,
        0,
        0,
        0,
        0,
        0,
        0,
        0,
        0,
        0,
        0,
        0,
        0,
        0,
        0,
        0,
    ])
}
<commit_msg>looks like initial learning rate can be bumped to 1.0<commit_after>use image::ImageResult;
use mli::{Backward, Forward, Graph, Train};
use mli_conv::{Conv2, Conv2n, Conv3};
use mli_ndarray::{Map2One, Map3One, Reshape3to2};
use mli_relu::Blu;
use ndarray::{array, s, Array, Array2, OwnedRepr};
use ndarray_image::{open_gray_image, save_gray_image};
use rand_core::{RngCore, SeedableRng};
use rand_distr::{Distribution, Normal};
use rand_pcg::Pcg64;
use std::path::{Path, PathBuf};
use structopt::StructOpt;

#[derive(Debug, StructOpt)]
#[structopt(name = "image", about = "Loads and convolves an image as a test")]
struct Opt {
    /// Number of epochs
    #[structopt(short = "t", default_value = "4000")]
    epochs: usize,
    /// Number of epochs per output image
    #[structopt(short = "s", default_value = "1")]
    show_every: usize,
    /// Pre-start learning rate (for first 10 epochs)
    #[structopt(short = "p", default_value = "0.1")]
    prestart_learning_rate: f32,
    /// Initial learning rate
    #[structopt(short = "i", default_value = "1.0")]
    initial_learning_rate: f32,
    /// Learning rate multiplier per epoch
    #[structopt(short = "m", default_value = "0.999")]
    learning_rate_multiplier: f32,
    /// Seed
    #[structopt(short = "z", default_value = "32")]
    seed: u32,
    /// Beta value for NAG
    #[structopt(short = "b", default_value = "0.9")]
    beta: f32,
    /// File to load
    #[structopt(parse(from_os_str))]
    file: PathBuf,
    /// Output directory
    #[structopt(parse(from_os_str))]
    output_dir: PathBuf,
}

fn sobel(image: &Array2<f32>) -> Array2<f32> {
    let down_filter = array![[-1.0, -2.0, -1.0], [0.0, 0.0, 0.0], [1.0, 2.0, 1.0],];
    let right_filter = down_filter.t().to_owned();
    let down = Conv2::new(down_filter).forward(&image).1;
    let right = Conv2::new(right_filter).forward(&image).1;
    (down.map(|f| f.powi(2)) + right.map(|f| f.powi(2))).map(|f| f.sqrt())
}

fn open_image(path: impl AsRef<Path>) -> ImageResult<Array2<f32>> {
    let image = open_gray_image(path)?;
    Ok(image.map(|&n| f32::from(n) / 255.0))
}

fn save_image(path: impl AsRef<Path>, image: &Array2<f32>) -> ImageResult<()> {
    let image = image.map(|&n| num::clamp(n * 255.0, 0.0, 255.0) as u8);
    save_gray_image(path, image.view())
}

fn main() -> ImageResult<()> {
    let opt = Opt::from_args();
    let image = open_image(opt.file)?;
    let sobel_image = sobel(&image);
    let conv_layers = 4;
    let filter_radius = 1usize;
    let filter_depth = 2usize;
    let filter_area = (filter_radius * 2 + 1).pow(2);
    let filter_volume = filter_area * filter_depth;
    let padding = (conv_layers * filter_radius - 1) as i32;
    #[allow(clippy::deref_addrof)]
    let expected = sobel_image.slice(s![padding..-padding, padding..-padding]);
    save_image(opt.output_dir.join("actual.png"), &expected.to_owned())?;
    let mut prng = make_prng(opt.seed);
    let mut prng_2filter = make_prng(prng.next_u32());
    let mut random_2filter = |mean: f32, variance: f32| -> Conv2<OwnedRepr<f32>> {
        // Xavier initialize by changing the variance to be 1/N where N is the area of the filter.
        Conv2::new(
            Array::from_iter(
                Normal::new(mean / filter_area as f32, variance / filter_area as f32)
                    .unwrap()
                    .sample_iter(&mut prng_2filter)
                    .take(filter_area),
            )
            .into_shape((filter_radius * 2 + 1, filter_radius * 2 + 1))
            .unwrap(),
        )
    };
    let mut prng_2nfilter = make_prng(prng.next_u32());
    let mut random_2nfilter = |mean: f32, variance: f32| -> Conv2n<OwnedRepr<f32>> {
        // Xavier initialize by changing the variance to be 1/N where N is the area of the filter.
        Conv2n::new(
            Array::from_iter(
                Normal::new(mean / filter_area as f32, variance / filter_area as f32)
                    .unwrap()
                    .sample_iter(&mut prng_2nfilter)
                    .take(filter_volume),
            )
            .into_shape((filter_depth, filter_radius * 2 + 1, filter_radius * 2 + 1))
            .unwrap(),
        )
    };
    let mut prng_3filter = make_prng(prng.next_u32());
    let mut random_3filter = |mean: f32, variance: f32| -> Conv3<OwnedRepr<f32>> {
        // Xavier initialize by changing the variance to be 1/N where N is the volume of the filter.
        Conv3::new(
            Array::from_iter(
                Normal::new(mean / filter_volume as f32, variance / filter_volume as f32)
                    .unwrap()
                    .sample_iter(&mut prng_3filter)
                    .take(filter_volume),
            )
            .into_shape((filter_depth, filter_radius * 2 + 1, filter_radius * 2 + 1))
            .unwrap(),
        )
    };
    let mut random_blu = |mean: f32, variance: f32| -> Blu {
        // Xavier initialize by changing the variance to be 1/N where N is the number of neurons.
        let distr = Normal::new(mean, variance).unwrap();
        Blu::new(distr.sample(&mut prng), distr.sample(&mut prng))
    };
    let mut generate_filter = || {
        random_2nfilter(0.0, 4.0)
            .chain(Map3One(random_blu(0.0, 0.5)))
            .chain(random_3filter(0.0, 4.0))
            .chain(Reshape3to2::new())
            .chain(Map2One(random_blu(0.0, 0.5)))
            .chain(random_2filter(0.0, 4.0))
            .chain(Map2One(random_blu(0.0, 0.5)))
            .chain(random_2filter(0.0, 4.0))
    };

    loop {
        let mut train_filter = generate_filter();
        let mut learn_rate = opt.initial_learning_rate;
        // Weird hack to initialize the momentum to zero without knowing the shape of the ndarray in advance.
        let mut momentum = {
            let (internal, output) = train_filter.forward(&image);
            train_filter.backward_train(&image, &internal, &output)
        };
        let mut last_loss = 0.0;
        momentum *= 0.0f32;
        for i in 0..opt.epochs {
            // Compute beta * momentum.
            momentum *= opt.beta;
            train_filter.train(&momentum);
            let (internal, output) = train_filter.forward(&image);
            // Show the image if the frame is divisible by show_every.
            if i % opt.show_every == 0 {
                save_image(opt.output_dir.join(format!("epoch{:04}.png", i)), &output)?;
            }
            if i == opt.epochs - 1 {
                eprintln!("Finished!");
                return Ok(());
            }
            let output_len = output.len() as f32;
            let local_learn_rate = if i < 10 {
                opt.prestart_learning_rate
            } else {
                let llr = learn_rate;
                learn_rate *= opt.learning_rate_multiplier;
                llr
            };
            // Compute the loss for display only (we don't actually need the loss itself for backprop, just its derivative).
            let loss = (output.clone() - expected.view()).map(|n| n.powi(2)).sum() / output_len;
            eprintln!(
                "epoch {:04} loss: {} ({:+}%), learn_rate: {}",
                i,
                loss,
                (loss - last_loss) / last_loss * 100.0,
                local_learn_rate
            );
            last_loss = loss;
            if !loss.is_normal() {
                eprintln!("abnormal loss at epoch {}; starting over", i);
                break;
            }
            // The loss function is (n - t)^2, so 2*(n - t) is dE/df where
            // `E` is loss and `f` is output.
            let delta_loss = (output - expected).map(|n| 2.0 * n / output_len);
            // Compute the output delta.
            let output_delta = -local_learn_rate * delta_loss;
            // Compute the trainable variable delta.
            let mut train_delta = train_filter.backward_train(&image, &internal, &output_delta);
            // Make the train delta a small component.
            train_delta *= 1.0 - opt.beta;
            train_filter.train(&train_delta);
            // Add the small component to the momentum.
            momentum += train_delta;
        }
    }
}

fn make_prng(seed: u32) -> Pcg64 {
    Pcg64::from_seed([
        (seed & 0xFF) as u8,
        ((seed >> 8) & 0xFF) as u8,
        ((seed >> 16) & 0xFF) as u8,
        ((seed >> 24) & 0xFF) as u8,
        0,
        0,
        0,
        0,
        0,
        0,
        0,
        0,
        0,
        0,
        0,
        0,
        0,
        0,
        0,
        0,
        0,
        0,
        0,
        0,
        0,
        0,
        0,
        0,
        0,
        0,
        0,
        0,
    ])
}
<|endoftext|>


<commit_before>/* Copyright 2020 The TensorFlow Authors. All Rights Reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/

//! Command-line interface for the main entry point.

use clap::Clap;
use log::{debug, error, info, LevelFilter};
use std::fs::File;
use std::io::{Read, Write};
use std::net::{IpAddr, SocketAddr};
use std::path::{Path, PathBuf};
use std::str::FromStr;
use std::thread;
use std::time::{Duration, Instant};
use tokio::net::TcpListener;
use tonic::transport::Server;

use crate::commit::Commit;
use crate::logdir::LogdirLoader;
use crate::proto::tensorboard::data;
use crate::server::DataProviderHandler;

use data::tensor_board_data_provider_server::TensorBoardDataProviderServer;

#[derive(Clap, Debug)]
#[clap(name = "rustboard", version = "0.2.0-alpha.0")]
struct Opts {
    /// Log directory to load
    ///
    /// Directory to recursively scan for event files (files matching the `*tfevents*` glob). This
    /// directory, its descendants, and its event files will be periodically polled for new data.
    #[clap(long)]
    logdir: PathBuf,

    /// Bind to this IP address
    ///
    /// IP address to bind this server to. May be an IPv4 address (e.g., 127.0.0.1 or 0.0.0.0) or
    /// an IPv6 address (e.g., ::1 or ::0).
    #[clap(long, default_value = "::1")]
    host: IpAddr,

    /// Bind to this port
    ///
    /// Port to bind this server to. Use `0` to request an arbitrary free port from the OS.
    #[clap(long, default_value = "6806")]
    port: u16,

    /// Delay between reload cycles (seconds)
    ///
    /// Number of seconds to wait between finishing one load cycle and starting the next one. This
    /// does not include the time for the reload itself.
    #[clap(long, default_value = "5")]
    reload_interval: Seconds,

    /// Use verbose output (-vv for very verbose output)
    #[clap(long = "verbose", short, parse(from_occurrences))]
    verbosity: u32,

    /// Kill this server once stdin is closed
    ///
    /// While this server is running, read stdin to end of file and then kill the server. Used to
    /// portably ensure that the server exits when the parent process dies, even due to a crash.
    /// Don't set this if stdin is connected to a tty and the process will be backgrounded, since
    /// then the server will receive `SIGTTIN` and its process will be stopped (in the `SIGSTOP`
    /// sense) but not killed.
    #[clap(long)]
    die_after_stdin: bool,

    /// Write bound port to this file
    ///
    /// Once a server socket is opened, write the port on which it's listening to the file at this
    /// path. Useful with `--port 0`. Port will be written as ASCII decimal followed by a newline
    /// (e.g., "6806\n"). If the server fails to start, this file may not be written at all. If the
    /// port file is specified but cannot be written, the server will die.
    #[clap(long)]
    port_file: Option<PathBuf>,

    /// Checksum all records (negate with `--no-checksum`)
    ///
    /// With `--checksum`, every record will be checksummed before being parsed. With
    /// `--no-checksum` (the default), records are only checksummed if parsing fails. Skipping
    /// checksums for records that successfully parse can be significantly faster, but also means
    /// that some bit flips may not be detected.
    #[clap(long, multiple_occurrences = true, overrides_with = "no_checksum")]
    checksum: bool,

    /// Only checksum records that fail to parse
    ///
    /// Negates `--checksum`. This is the default.
    #[clap(
        long,
        multiple_occurrences = true,
        overrides_with = "checksum",
        hidden = true
    )]
    #[allow(unused)]
    no_checksum: bool,
}

/// A duration in seconds.
#[derive(Debug, Copy, Clone)]
struct Seconds(u64);
impl FromStr for Seconds {
    type Err = <u64 as FromStr>::Err;
    fn from_str(s: &str) -> Result<Self, Self::Err> {
        s.parse().map(Seconds)
    }
}
impl Seconds {
    fn duration(self) -> Duration {
        Duration::from_secs(self.0)
    }
}

#[tokio::main]
pub async fn main() -> Result<(), Box<dyn std::error::Error>> {
    let opts = Opts::parse();
    init_logging(match opts.verbosity {
        0 => LevelFilter::Warn,
        1 => LevelFilter::Info,
        _ => LevelFilter::max(),
    });
    debug!("Parsed options: {:?}", opts);

    if opts.die_after_stdin {
        thread::Builder::new()
            .name("StdinWatcher".to_string())
            .spawn(die_after_stdin)
            .expect("failed to spawn stdin watcher thread");
    }

    let addr = SocketAddr::new(opts.host, opts.port);
    let listener = TcpListener::bind(addr).await?;
    let bound = listener.local_addr()?;
    eprintln!("listening on {:?}", bound);

    if let Some(port_file) = opts.port_file {
        let port = bound.port();
        if let Err(e) = write_port_file(&port_file, port) {
            error!(
                "Failed to write port \"{}\" to {}: {}",
                port,
                port_file.display(),
                e
            );
            std::process::exit(1);
        }
        info!("Wrote port \"{}\" to {}", port, port_file.display());
    }

    // Leak the commit object, since the Tonic server must have only 'static references. This only
    // leaks the outer commit structure (of constant size), not the pointers to the actual data.
    let commit: &'static Commit = Box::leak(Box::new(Commit::new()));

    thread::Builder::new()
        .name("Reloader".to_string())
        .spawn({
            let logdir = opts.logdir;
            let reload_interval = opts.reload_interval;
            let mut loader = LogdirLoader::new(commit, logdir);
            // Checksum only if `--checksum` given (i.e., off by default).
            loader.checksum(opts.checksum);
            move || loop {
                info!("Starting load cycle");
                let start = Instant::now();
                loader.reload();
                let end = Instant::now();
                info!("Finished load cycle ({:?})", end - start);
                thread::sleep(reload_interval.duration());
            }
        })
        .expect("failed to spawn reloader thread");

    let handler = DataProviderHandler { commit };
    Server::builder()
        .add_service(TensorBoardDataProviderServer::new(handler))
        .serve_with_incoming(listener)
        .await?;
    Ok(())
}

/// Installs a logging handler whose behavior is determined by the `RUST_LOG` environment variable
/// (per <https://docs.rs/env_logger> semantics), or by including all logs at `default_log_level`
/// or above if `RUST_LOG_LEVEL` is not given.
fn init_logging(default_log_level: LevelFilter) {
    use env_logger::{Builder, Env};
    Builder::from_env(Env::default().default_filter_or(default_log_level.to_string())).init();
}

/// Locks stdin and reads it to EOF, then exits the process.
fn die_after_stdin() {
    let stdin = std::io::stdin();
    let stdin_lock = stdin.lock();
    for _ in stdin_lock.bytes() {}
    info!("Stdin closed; exiting");
    std::process::exit(0);
}

/// Writes `port` to file `path` as an ASCII decimal followed by newline.
fn write_port_file(path: &Path, port: u16) -> std::io::Result<()> {
    let mut f = File::create(path)?;
    writeln!(f, "{}", port)?;
    f.sync_all()?;
    Ok(())
}
<commit_msg>rust: skip “listening on…” if `--port-file` given (#4576)<commit_after>/* Copyright 2020 The TensorFlow Authors. All Rights Reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/

//! Command-line interface for the main entry point.

use clap::Clap;
use log::{debug, error, info, LevelFilter};
use std::fs::File;
use std::io::{Read, Write};
use std::net::{IpAddr, SocketAddr};
use std::path::{Path, PathBuf};
use std::str::FromStr;
use std::thread;
use std::time::{Duration, Instant};
use tokio::net::TcpListener;
use tonic::transport::Server;

use crate::commit::Commit;
use crate::logdir::LogdirLoader;
use crate::proto::tensorboard::data;
use crate::server::DataProviderHandler;

use data::tensor_board_data_provider_server::TensorBoardDataProviderServer;

#[derive(Clap, Debug)]
#[clap(name = "rustboard", version = "0.2.0-alpha.0")]
struct Opts {
    /// Log directory to load
    ///
    /// Directory to recursively scan for event files (files matching the `*tfevents*` glob). This
    /// directory, its descendants, and its event files will be periodically polled for new data.
    #[clap(long)]
    logdir: PathBuf,

    /// Bind to this IP address
    ///
    /// IP address to bind this server to. May be an IPv4 address (e.g., 127.0.0.1 or 0.0.0.0) or
    /// an IPv6 address (e.g., ::1 or ::0).
    #[clap(long, default_value = "::1")]
    host: IpAddr,

    /// Bind to this port
    ///
    /// Port to bind this server to. Use `0` to request an arbitrary free port from the OS.
    #[clap(long, default_value = "6806")]
    port: u16,

    /// Delay between reload cycles (seconds)
    ///
    /// Number of seconds to wait between finishing one load cycle and starting the next one. This
    /// does not include the time for the reload itself.
    #[clap(long, default_value = "5")]
    reload_interval: Seconds,

    /// Use verbose output (-vv for very verbose output)
    #[clap(long = "verbose", short, parse(from_occurrences))]
    verbosity: u32,

    /// Kill this server once stdin is closed
    ///
    /// While this server is running, read stdin to end of file and then kill the server. Used to
    /// portably ensure that the server exits when the parent process dies, even due to a crash.
    /// Don't set this if stdin is connected to a tty and the process will be backgrounded, since
    /// then the server will receive `SIGTTIN` and its process will be stopped (in the `SIGSTOP`
    /// sense) but not killed.
    #[clap(long)]
    die_after_stdin: bool,

    /// Write bound port to this file
    ///
    /// Once a server socket is opened, write the port on which it's listening to the file at this
    /// path. Useful with `--port 0`. Port will be written as ASCII decimal followed by a newline
    /// (e.g., "6806\n"). If the server fails to start, this file may not be written at all. If the
    /// port file is specified but cannot be written, the server will die.
    ///
    /// This also suppresses the "listening on HOST:PORT" line that is otherwise written to stderr
    /// when the server starts.
    #[clap(long)]
    port_file: Option<PathBuf>,

    /// Checksum all records (negate with `--no-checksum`)
    ///
    /// With `--checksum`, every record will be checksummed before being parsed. With
    /// `--no-checksum` (the default), records are only checksummed if parsing fails. Skipping
    /// checksums for records that successfully parse can be significantly faster, but also means
    /// that some bit flips may not be detected.
    #[clap(long, multiple_occurrences = true, overrides_with = "no_checksum")]
    checksum: bool,

    /// Only checksum records that fail to parse
    ///
    /// Negates `--checksum`. This is the default.
    #[clap(
        long,
        multiple_occurrences = true,
        overrides_with = "checksum",
        hidden = true
    )]
    #[allow(unused)]
    no_checksum: bool,
}

/// A duration in seconds.
#[derive(Debug, Copy, Clone)]
struct Seconds(u64);
impl FromStr for Seconds {
    type Err = <u64 as FromStr>::Err;
    fn from_str(s: &str) -> Result<Self, Self::Err> {
        s.parse().map(Seconds)
    }
}
impl Seconds {
    fn duration(self) -> Duration {
        Duration::from_secs(self.0)
    }
}

#[tokio::main]
pub async fn main() -> Result<(), Box<dyn std::error::Error>> {
    let opts = Opts::parse();
    init_logging(match opts.verbosity {
        0 => LevelFilter::Warn,
        1 => LevelFilter::Info,
        _ => LevelFilter::max(),
    });
    debug!("Parsed options: {:?}", opts);

    if opts.die_after_stdin {
        thread::Builder::new()
            .name("StdinWatcher".to_string())
            .spawn(die_after_stdin)
            .expect("failed to spawn stdin watcher thread");
    }

    let addr = SocketAddr::new(opts.host, opts.port);
    let listener = TcpListener::bind(addr).await?;
    let bound = listener.local_addr()?;

    if let Some(port_file) = opts.port_file {
        let port = bound.port();
        if let Err(e) = write_port_file(&port_file, port) {
            error!(
                "Failed to write port \"{}\" to {}: {}",
                port,
                port_file.display(),
                e
            );
            std::process::exit(1);
        }
        info!("Wrote port \"{}\" to {}", port, port_file.display());
    } else {
        eprintln!("listening on {:?}", bound);
    }

    // Leak the commit object, since the Tonic server must have only 'static references. This only
    // leaks the outer commit structure (of constant size), not the pointers to the actual data.
    let commit: &'static Commit = Box::leak(Box::new(Commit::new()));

    thread::Builder::new()
        .name("Reloader".to_string())
        .spawn({
            let logdir = opts.logdir;
            let reload_interval = opts.reload_interval;
            let mut loader = LogdirLoader::new(commit, logdir);
            // Checksum only if `--checksum` given (i.e., off by default).
            loader.checksum(opts.checksum);
            move || loop {
                info!("Starting load cycle");
                let start = Instant::now();
                loader.reload();
                let end = Instant::now();
                info!("Finished load cycle ({:?})", end - start);
                thread::sleep(reload_interval.duration());
            }
        })
        .expect("failed to spawn reloader thread");

    let handler = DataProviderHandler { commit };
    Server::builder()
        .add_service(TensorBoardDataProviderServer::new(handler))
        .serve_with_incoming(listener)
        .await?;
    Ok(())
}

/// Installs a logging handler whose behavior is determined by the `RUST_LOG` environment variable
/// (per <https://docs.rs/env_logger> semantics), or by including all logs at `default_log_level`
/// or above if `RUST_LOG_LEVEL` is not given.
fn init_logging(default_log_level: LevelFilter) {
    use env_logger::{Builder, Env};
    Builder::from_env(Env::default().default_filter_or(default_log_level.to_string())).init();
}

/// Locks stdin and reads it to EOF, then exits the process.
fn die_after_stdin() {
    let stdin = std::io::stdin();
    let stdin_lock = stdin.lock();
    for _ in stdin_lock.bytes() {}
    info!("Stdin closed; exiting");
    std::process::exit(0);
}

/// Writes `port` to file `path` as an ASCII decimal followed by newline.
fn write_port_file(path: &Path, port: u16) -> std::io::Result<()> {
    let mut f = File::create(path)?;
    writeln!(f, "{}", port)?;
    f.sync_all()?;
    Ok(())
}
<|endoftext|>


<commit_before>/// HTTP methods
#[deriving(Eq)]
pub enum Method {
    Get,
    Head,
    Post,
    Put,
    Delete,
    Trace,
    Options,
    Connect,
    Patch,
    UnregisteredMethod(~str),
}

impl ToStr for Method {
    /// Get the proper name of a method, e.g. `Get.to_str() == ~"GET"`
    fn to_str(&self) -> ~str {
        match *self {
            Get                       => ~"GET",
            Head                      => ~"HEAD",
            Post                      => ~"POST",
            Put                       => ~"PUT",
            Delete                    => ~"DELETE",
            Trace                     => ~"TRACE",
            Options                   => ~"OPTIONS",
            Connect                   => ~"CONNECT",
            Patch                     => ~"PATCH",
            UnregisteredMethod(ref s) => (*s).clone(),
        }
    }
}

/**
 * Normalise an HTTP method name into capital letters.
 *
 * IMPORTANT: ensure that method.is_ascii() is true before calling this,
 * or things will probably blow up in an uncomfortable sort of way
 */
fn normalise_method_name(method: &str) -> ~str {
    unsafe { method.to_ascii_nocheck() }.to_upper().to_str_ascii()
}

impl FromStr for Method {
    /**
     * Get a *known* `Method` from an *ASCII* string, regardless of case.
     *
     * If you want to support unregistered methods, use `from_str_or_new` instead.
     *
     * (If the string isn't ASCII, this will at present fail: TODO fix that.)
     */
    pub fn from_str(method: &str) -> Option<Method> {
        if (!method.is_ascii()) {
            return None;
        }
        match normalise_method_name(method) {
            ~"GET"     => Some(Get),
            ~"HEAD"    => Some(Head),
            ~"POST"    => Some(Post),
            ~"PUT"     => Some(Put),
            ~"DELETE"  => Some(Delete),
            ~"TRACE"   => Some(Trace),
            ~"OPTIONS" => Some(Options),
            ~"CONNECT" => Some(Connect),
            ~"PATCH"   => Some(Patch),
            _          => None
        }
    }
}

impl Method {
    /**
     * Get a `Method` from an *ASCII* string.
     *
     * (If the string isn't ASCII, this will at present fail.)
     */
    pub fn from_str_or_new(method: &str) -> Method {
        assert!(method.is_ascii());
        match normalise_method_name(method) {
            ~"GET"     => Get,
            ~"HEAD"    => Head,
            ~"POST"    => Post,
            ~"PUT"     => Put,
            ~"DELETE"  => Delete,
            ~"TRACE"   => Trace,
            ~"OPTIONS" => Options,
            ~"CONNECT" => Connect,
            ~"PATCH"   => Patch,
            _          => UnregisteredMethod(method.to_owned()),
        }
    }
}
<commit_msg>Make methods consistent with order/terminology of the specs.<commit_after>/// HTTP methods, as defined in RFC 2616, §5.1.1.
///
/// Method names are case-sensitive.
#[deriving(Eq)]
pub enum Method {
    Options,
    Get,
    Head,
    Post,
    Put,
    Delete,
    Trace,
    Connect,
    Patch,  // RFC 5789
    ExtensionMethod(~str),
}

impl ToStr for Method {
    /// Get the proper name of a method, e.g. `Get.to_str() == ~"GET"`
    fn to_str(&self) -> ~str {
        match *self {
            Options                => ~"OPTIONS",
            Get                    => ~"GET",
            Head                   => ~"HEAD",
            Post                   => ~"POST",
            Put                    => ~"PUT",
            Delete                 => ~"DELETE",
            Trace                  => ~"TRACE",
            Connect                => ~"CONNECT",
            Patch                  => ~"PATCH",
            ExtensionMethod(ref s) => (*s).clone(),
        }
    }
}

impl FromStr for Method {
    /**
     * Get a *known* `Method` from an *ASCII* string, regardless of case.
     *
     * If you want to support unregistered methods, use `from_str_or_new` instead.
     *
     * (If the string isn't ASCII, this will at present fail: TODO fix that.)
     */
    pub fn from_str(method: &str) -> Option<Method> {
        if (!method.is_ascii()) {
            return None;
        }
        match method {
            "OPTIONS" => Some(Options),
            "GET"     => Some(Get),
            "HEAD"    => Some(Head),
            "POST"    => Some(Post),
            "PUT"     => Some(Put),
            "DELETE"  => Some(Delete),
            "TRACE"   => Some(Trace),
            "CONNECT" => Some(Connect),
            "PATCH"   => Some(Patch),
            _         => None
        }
    }
}

impl Method {
    /**
     * Get a `Method` from an *ASCII* string.
     *
     * (If the string isn't ASCII, this will at present fail.)
     */
    pub fn from_str_or_new(method: &str) -> Method {
        assert!(method.is_ascii());
        match method {
            "OPTIONS" => Options,
            "GET"     => Get,
            "HEAD"    => Head,
            "POST"    => Post,
            "PUT"     => Put,
            "DELETE"  => Delete,
            "TRACE"   => Trace,
            "CONNECT" => Connect,
            "PATCH"   => Patch,
            _         => ExtensionMethod(method.to_owned()),
        }
    }
}
<|endoftext|>