<commit_before>// Copyright 2014 Google Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

#![crate_id="hat#0.1"]
#![crate_type="bin"]
#![license = "ALv2"]

// #![warn(missing_doc)]
#![warn(non_uppercase_statics)]
#![warn(non_camel_case_types)]
#![warn(managed_heap_memory)]
#![warn(unnecessary_qualification)]

#![feature(globs)]

// Standard Rust imports
extern crate collections;
extern crate libc;
extern crate rand;
extern crate serialize;
extern crate sync;
extern crate test;
extern crate time;

// Rust bindings
extern crate sodiumoxide;
extern crate sqlite3;
extern crate snappy;

// Testing
#[cfg(test)]
extern crate quickcheck;

use std::os;
use std::io::stdio::{println};

mod callback_container;
mod cumulative_counter;
mod ordered_collection;
mod periodic_timer;
mod unique_priority_queue;

mod hat;
mod listdir;
mod process;

mod hash_index;
mod hash_tree;
mod hash_store;

mod blob_index;
mod blob_store;

mod key_index;
mod key_store;

static MAX_BLOB_SIZE: uint = 4 * 1024 * 1024;

fn blob_dir() -> Path { Path::new("blobs") }

fn usage() {
  println(format!("Usage: {} [snapshot|checkout] name path", os::args().get(0)));
}

fn license() {
  println(include_str!("../../COPYING"));
}

#[main]
fn main() {
  // Initialize sodium (must only be called once)
  sodiumoxide::init();

  let args = os::args();
  if args.len() == 2 {
    let flag = args.get(1);
    if flag == &"--license".to_owned() {
        license();
    }
    else if flag == &"--help".to_owned() {
      usage();
      license();
    }
    return;
  }

  if args.len() != 4 {
    return usage();
  }

  let cmd = args.get(1);

  if cmd == &"snapshot".to_owned() {
    let name = args.get(2);  // used for naming the key index
    let path = args.get(3);

    {
      let backend = blob_store::FileBackend::new(blob_dir());
      let hat_opt = hat::Hat::openRepository(
        &Path::new("repo"), backend, MAX_BLOB_SIZE);
      let hat = hat_opt.expect(format!("Could not open repository in {}.", path));

      let family_opt = hat.openFamily(name.clone());
      let family = family_opt.expect(format!("Could not open family '{}'", name));

      family.snapshotDir(Path::new(path.clone()));
      family.flush();
    }

    println("Waiting for final flush...");
    return;
  }
  else if cmd == &"checkout".to_owned() {
    let name = args.get(2);  // used for naming the key index
    let path = args.get(3);

    let backend = blob_store::FileBackend::new(blob_dir());
    let hat_opt = hat::Hat::openRepository(
      &Path::new("repo"), backend, MAX_BLOB_SIZE);
    let hat = hat_opt.expect(format!("Could not open repository in {}.", path));

    let family_opt = hat.openFamily(name.clone());
    let family = family_opt.expect(format!("Could not open family '{}'", name));

    family.checkoutInDir(&mut Path::new(path.clone()), None);
    return;
  }

  usage();

}<commit_msg>Minor fix: COPYING => LICENSE<commit_after>// Copyright 2014 Google Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

#![crate_id="hat#0.1"]
#![crate_type="bin"]
#![license = "ALv2"]

// #![warn(missing_doc)]
#![warn(non_uppercase_statics)]
#![warn(non_camel_case_types)]
#![warn(managed_heap_memory)]
#![warn(unnecessary_qualification)]

#![feature(globs)]

// Standard Rust imports
extern crate collections;
extern crate libc;
extern crate rand;
extern crate serialize;
extern crate sync;
extern crate test;
extern crate time;

// Rust bindings
extern crate sodiumoxide;
extern crate sqlite3;
extern crate snappy;

// Testing
#[cfg(test)]
extern crate quickcheck;

use std::os;
use std::io::stdio::{println};

mod callback_container;
mod cumulative_counter;
mod ordered_collection;
mod periodic_timer;
mod unique_priority_queue;

mod hat;
mod listdir;
mod process;

mod hash_index;
mod hash_tree;
mod hash_store;

mod blob_index;
mod blob_store;

mod key_index;
mod key_store;

static MAX_BLOB_SIZE: uint = 4 * 1024 * 1024;

fn blob_dir() -> Path { Path::new("blobs") }

fn usage() {
  println(format!("Usage: {} [snapshot|checkout] name path", os::args().get(0)));
}

fn license() {
  println(include_str!("../../LICENSE"));
}

#[main]
fn main() {
  // Initialize sodium (must only be called once)
  sodiumoxide::init();

  let args = os::args();
  if args.len() == 2 {
    let flag = args.get(1);
    if flag == &"--license".to_owned() {
        license();
    }
    else if flag == &"--help".to_owned() {
      usage();
      license();
    }
    return;
  }

  if args.len() != 4 {
    return usage();
  }

  let cmd = args.get(1);

  if cmd == &"snapshot".to_owned() {
    let name = args.get(2);  // used for naming the key index
    let path = args.get(3);

    {
      let backend = blob_store::FileBackend::new(blob_dir());
      let hat_opt = hat::Hat::openRepository(
        &Path::new("repo"), backend, MAX_BLOB_SIZE);
      let hat = hat_opt.expect(format!("Could not open repository in {}.", path));

      let family_opt = hat.openFamily(name.clone());
      let family = family_opt.expect(format!("Could not open family '{}'", name));

      family.snapshotDir(Path::new(path.clone()));
      family.flush();
    }

    println("Waiting for final flush...");
    return;
  }
  else if cmd == &"checkout".to_owned() {
    let name = args.get(2);  // used for naming the key index
    let path = args.get(3);

    let backend = blob_store::FileBackend::new(blob_dir());
    let hat_opt = hat::Hat::openRepository(
      &Path::new("repo"), backend, MAX_BLOB_SIZE);
    let hat = hat_opt.expect(format!("Could not open repository in {}.", path));

    let family_opt = hat.openFamily(name.clone());
    let family = family_opt.expect(format!("Could not open family '{}'", name));

    family.checkoutInDir(&mut Path::new(path.clone()), None);
    return;
  }

  usage();

}<|endoftext|>


<commit_before>use std::thread;
use std::path::PathBuf;
use std::process::Child;
use std::process::Command;

pub struct App {
    pub port: u16,
    #[allow(dead_code)]
    process: Child,
}

impl App {
    pub fn new(name: &str, port: u16, app_home: &str) -> App {
        let mut path_buf = PathBuf::from(app_home);
        path_buf.push(&name);

        let child_process = Command::new("foreman")
            .arg("start")
            .current_dir(path_buf.as_path())
            .env("PORT", port.to_string())
            .spawn().unwrap();

        sleep_until_port_open(port);

        App{
            port: port,
            process: child_process,
        }
    }
}

fn sleep_until_port_open(port: u16) {
    while !Command::new("nc").arg("-z").arg("localhost").arg(format!("{}", port)).status().unwrap().success() {
        thread::sleep_ms(300);
    }
}
<commit_msg>feature: Display error if child process cannot be spawned<commit_after>use std::thread;
use std::path::PathBuf;
use std::process::Child;
use std::process::Command;

pub struct App {
    pub port: u16,
    #[allow(dead_code)]
    process: Child,
}

impl App {
    pub fn new(name: &str, port: u16, app_home: &str) -> App {
        let mut path_buf = PathBuf::from(app_home);
        path_buf.push(&name);

        let child_process = match Command::new("foreman")
            .arg("start")
            .current_dir(path_buf.as_path())
            .env("PORT", port.to_string())
            .spawn() {
                Ok(value) => value,
                Err(e) => panic!("{}", e),
            };

        sleep_until_port_open(port);

        App{
            port: port,
            process: child_process,
        }
    }
}

fn sleep_until_port_open(port: u16) {
    while !Command::new("nc").arg("-z").arg("localhost").arg(format!("{}", port)).status().unwrap().success() {
        thread::sleep_ms(300);
    }
}
<|endoftext|>


<commit_before>use ::libc::{self, c_int, O_EXCL, O_RDWR, O_CREAT};
use ::libc::types::os::arch::posix01::stat as stat_t;
use std::io;
use std::os::unix::io::{RawFd, FromRawFd};
use std::fs::{self, File, OpenOptions};
use std::path::Path;
use std::ffi::CString;
use ::util::{tmpname, cstr};
use super::unix_common::O_CLOEXEC;
pub use super::unix_common::create;

unsafe fn stat(fd: RawFd) -> io::Result<stat_t> {
    let mut meta: stat_t = ::std::mem::zeroed();
    if libc::fstat(fd, &mut meta as *mut stat_t) != 0 {
        Err(io::Error::last_os_error())
    } else {
        Ok(meta)
    }
}

// Helper for ensuring that the temporary file gets deleted.
struct DeleteGuard<'a>(&'a Path);

impl<'a> Drop for DeleteGuard<'a> {
    fn drop(&mut self) {
        let _ = fs::remove_file(self.0);
    }
}

pub fn create_shared(dir: &Path, count: usize) -> io::Result<Vec<File>> {
    let mut opts = OpenOptions::new();
    opts.read(true).write(true).create(false);

    if count == 0 {
        return Ok(vec![]);
    }
    'outer: loop {
        let tmp_path = dir.join(&tmpname());
        return match unsafe {
            libc::open(try!(cstr(&tmp_path)).as_ptr(), O_CLOEXEC | O_EXCL | O_RDWR | O_CREAT, 0o600)
        } {
            -1 => {
                let err = io::Error::last_os_error();
                if err.kind() == io::ErrorKind::AlreadyExists {
                    continue;
                } else {
                    Err(err)
                }
            },
            fd => unsafe {
                let first = FromRawFd::from_raw_fd(fd);
                let dg = DeleteGuard(&tmp_path);

                let target_meta = try!(stat(fd));
                let mut files: Vec<File> = try!((1..count).map(|_| opts.open(&tmp_path)).collect());
                for f in &files {
                    let meta = try!(stat(fd));
                    if meta.st_dev != target_meta.st_dev ||
                       meta.st_ino != target_meta.st_ino ||
                       // Even if the device information get's reused, the owner should actually be
                       // sufficient.
                       meta.st_uid != target_meta.st_uid ||
                       meta.st_gid != target_meta.st_gid {

                        // Error? Panic? If we hit this, we're likely under attack (or a hardware
                        // glitch/reconfiguration?.
                        continue 'outer;
                    }

                }
                files.push(first);
                Ok(files)
            },
        }
    }
}
<commit_msg>Actually check that the re-opened files are the same on unix.<commit_after>use ::libc::{self, c_int, O_EXCL, O_RDWR, O_CREAT};
use ::libc::types::os::arch::posix01::stat as stat_t;
use std::io;
use std::os::unix::io::{RawFd, FromRawFd};
use std::fs::{self, File, OpenOptions};
use std::path::Path;
use std::ffi::CString;
use ::util::{tmpname, cstr};
use super::unix_common::O_CLOEXEC;
pub use super::unix_common::create;

unsafe fn stat(fd: RawFd) -> io::Result<stat_t> {
    let mut meta: stat_t = ::std::mem::zeroed();
    if libc::fstat(fd, &mut meta as *mut stat_t) != 0 {
        Err(io::Error::last_os_error())
    } else {
        Ok(meta)
    }
}

// Helper for ensuring that the temporary file gets deleted.
struct DeleteGuard<'a>(&'a Path);

impl<'a> Drop for DeleteGuard<'a> {
    fn drop(&mut self) {
        let _ = fs::remove_file(self.0);
    }
}

pub fn create_shared(dir: &Path, count: usize) -> io::Result<Vec<File>> {
    let mut opts = OpenOptions::new();
    opts.read(true).write(true).create(false);

    if count == 0 {
        return Ok(vec![]);
    }
    'outer: loop {
        let tmp_path = dir.join(&tmpname());
        return match unsafe {
            libc::open(try!(cstr(&tmp_path)).as_ptr(), O_CLOEXEC | O_EXCL | O_RDWR | O_CREAT, 0o600)
        } {
            -1 => {
                let err = io::Error::last_os_error();
                if err.kind() == io::ErrorKind::AlreadyExists {
                    continue;
                } else {
                    Err(err)
                }
            },
            fd => unsafe {
                let first = FromRawFd::from_raw_fd(fd);
                let dg = DeleteGuard(&tmp_path);

                let target_meta = try!(stat(fd));
                let mut files: Vec<File> = try!((1..count).map(|_| opts.open(&tmp_path)).collect());
                for file in &files {
                    let meta = try!(stat(file.as_raw_fd()));
                    if meta.st_dev != target_meta.st_dev ||
                       meta.st_ino != target_meta.st_ino ||
                       // Even if the device information get's reused, the owner should actually be
                       // sufficient.
                       meta.st_uid != target_meta.st_uid ||
                       meta.st_gid != target_meta.st_gid {

                        // Error? Panic? If we hit this, we're likely under attack (or a hardware
                        // glitch/reconfiguration?.
                        continue 'outer;
                    }

                }
                files.push(first);
                Ok(files)
            },
        }
    }
}
<|endoftext|>


<commit_before>// Copyright 2014-2016 bluss and ndarray developers.
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.

use std::any::Any;
use libnum::Complex;

/// Elements that can be used as direct operands in arithmetic with arrays.
///
/// For example, `f64` is a `ScalarOperand` which means that for an array `a`,
/// arithmetic like `a + 1.0`, and, `a * 2.`, and `a += 3.` are allowed.
///
/// In the description below, let `A` be an array or array view,
/// let `B` be an array with owned data,
/// and let `C` be an array with mutable data.
///
/// `ScalarOperand` determines for which scalars `K` operations `&A @ K`, and `B @ K`,
/// and `C @= K` are defined, as ***right hand side operands***, for applicable
/// arithmetic operators (denoted `@`).
///
/// ***Left hand side*** scalar operands are not related to this trait.
/// (They need one `impl` per concrete scalar type); they are
/// implemented for the default `ScalarOperand` types, allowing
/// operations `K @ &A`, and `K @ B`.
///
/// This trait ***does not*** limit which elements can be stored in an array in general.
/// Non-`ScalarOperand` types can still participate in arithmetic as array elements in
/// in array-array operations.
pub trait ScalarOperand : Any + Clone { }
impl ScalarOperand for bool { }
impl ScalarOperand for i8 { }
impl ScalarOperand for u8 { }
impl ScalarOperand for i16 { }
impl ScalarOperand for u16 { }
impl ScalarOperand for i32 { }
impl ScalarOperand for u32 { }
impl ScalarOperand for i64 { }
impl ScalarOperand for u64 { }
impl ScalarOperand for f32 { }
impl ScalarOperand for f64 { }
impl ScalarOperand for Complex<f32> { }
impl ScalarOperand for Complex<f64> { }

macro_rules! impl_binary_op(
    ($trt:ident, $mth:ident, $imth:ident, $imth_scalar:ident, $doc:expr) => (
/// Perform elementwise
#[doc=$doc]
/// between `self` and `rhs`,
/// and return the result (based on `self`).
///
/// `self` must be an `OwnedArray` or `RcArray`.
///
/// If their shapes disagree, `rhs` is broadcast to the shape of `self`.
///
/// **Panics** if broadcasting isn’t possible.
impl<A, S, S2, D, E> $trt<ArrayBase<S2, E>> for ArrayBase<S, D>
    where A: Clone + $trt<A, Output=A>,
          S: DataOwned<Elem=A> + DataMut,
          S2: Data<Elem=A>,
          D: Dimension,
          E: Dimension,
{
    type Output = ArrayBase<S, D>;
    fn $mth(self, rhs: ArrayBase<S2, E>) -> ArrayBase<S, D>
    {
        self.$mth(&rhs)
    }
}

/// Perform elementwise
#[doc=$doc]
/// between `self` and reference `rhs`,
/// and return the result (based on `self`).
///
/// If their shapes disagree, `rhs` is broadcast to the shape of `self`.
///
/// **Panics** if broadcasting isn’t possible.
impl<'a, A, S, S2, D, E> $trt<&'a ArrayBase<S2, E>> for ArrayBase<S, D>
    where A: Clone + $trt<A, Output=A>,
          S: DataMut<Elem=A>,
          S2: Data<Elem=A>,
          D: Dimension,
          E: Dimension,
{
    type Output = ArrayBase<S, D>;
    fn $mth (mut self, rhs: &ArrayBase<S2, E>) -> ArrayBase<S, D>
    {
        self.$imth(rhs);
        self
    }
}

/// Perform elementwise
#[doc=$doc]
/// between references `self` and `rhs`,
/// and return the result as a new `OwnedArray`.
///
/// If their shapes disagree, `rhs` is broadcast to the shape of `self`.
///
/// **Panics** if broadcasting isn’t possible.
impl<'a, 'b, A, S, S2, D, E> $trt<&'a ArrayBase<S2, E>> for &'b ArrayBase<S, D>
    where A: Clone + $trt<A, Output=A>,
          S: Data<Elem=A>,
          S2: Data<Elem=A>,
          D: Dimension,
          E: Dimension,
{
    type Output = OwnedArray<A, D>;
    fn $mth (self, rhs: &'a ArrayBase<S2, E>) -> OwnedArray<A, D>
    {
        // FIXME: Can we co-broadcast arrays here? And how?
        self.to_owned().$mth(rhs)
    }
}

/// Perform elementwise
#[doc=$doc]
/// between `self` and the scalar `x`,
/// and return the result (based on `self`).
///
/// `self` must be an `OwnedArray` or `RcArray`.
impl<A, S, D, B> $trt<B> for ArrayBase<S, D>
    where A: Clone + $trt<B, Output=A>,
          S: DataOwned<Elem=A> + DataMut,
          D: Dimension,
          B: ScalarOperand,
{
    type Output = ArrayBase<S, D>;
    fn $mth (mut self, x: B) -> ArrayBase<S, D>
    {
        self.unordered_foreach_mut(move |elt| {
            *elt = elt.clone().$mth(x.clone());
        });
        self
    }
}

/// Perform elementwise
#[doc=$doc]
/// between the reference `self` and the scalar `x`,
/// and return the result as a new `OwnedArray`.
impl<'a, A, S, D, B> $trt<B> for &'a ArrayBase<S, D>
    where A: Clone + $trt<B, Output=A>,
          S: Data<Elem=A>,
          D: Dimension,
          B: ScalarOperand,
{
    type Output = OwnedArray<A, D>;
    fn $mth(self, x: B) -> OwnedArray<A, D>
    {
        self.to_owned().$mth(x)
    }
}
    );
);

macro_rules! impl_scalar_op {
    ($scalar:ty, $trt:ident, $mth:ident, $doc:expr) => (
// these have no doc -- they are not visible in rustdoc
// Perform elementwise
// between the scalar `self` and array `rhs`,
// and return the result (based on `self`).
impl<S, D> $trt<ArrayBase<S, D>> for $scalar
    where S: DataMut<Elem=$scalar>,
          D: Dimension,
{
    type Output = ArrayBase<S, D>;
    fn $mth (self, mut rhs: ArrayBase<S, D>) -> ArrayBase<S, D>
    {
        rhs.unordered_foreach_mut(move |elt| {
            *elt = self.$mth(*elt);
        });
        rhs
    }
}

// Perform elementwise
// between the scalar `self` and array `rhs`,
// and return the result as a new `OwnedArray`.
impl<'a, S, D> $trt<&'a ArrayBase<S, D>> for $scalar
    where S: Data<Elem=$scalar>,
          D: Dimension,
{
    type Output = OwnedArray<$scalar, D>;
    fn $mth (self, rhs: &ArrayBase<S, D>) -> OwnedArray<$scalar, D>
    {
        self.$mth(rhs.to_owned())
    }
}
    );
}

mod arithmetic_ops {
    use super::*;
    use imp_prelude::*;

    use std::ops::*;
    use libnum::Complex;

    impl_binary_op!(Add, add, iadd, iadd_scalar, "addition");
    impl_binary_op!(Sub, sub, isub, isub_scalar, "subtraction");
    impl_binary_op!(Mul, mul, imul, imul_scalar, "multiplication");
    impl_binary_op!(Div, div, idiv, idiv_scalar, "division");
    impl_binary_op!(Rem, rem, irem, irem_scalar, "remainder");
    impl_binary_op!(BitAnd, bitand, ibitand, ibitand_scalar, "bit and");
    impl_binary_op!(BitOr, bitor, ibitor, ibitor_scalar, "bit or");
    impl_binary_op!(BitXor, bitxor, ibitxor, ibitxor_scalar, "bit xor");
    impl_binary_op!(Shl, shl, ishl, ishl_scalar, "left shift");
    impl_binary_op!(Shr, shr, ishr, ishr_scalar, "right shift");

    macro_rules! all_scalar_ops {
        ($int_scalar:ty) => (
            impl_scalar_op!($int_scalar, Add, add, "addition");
            impl_scalar_op!($int_scalar, Sub, sub, "subtraction");
            impl_scalar_op!($int_scalar, Mul, mul, "multiplication");
            impl_scalar_op!($int_scalar, Div, div, "division");
            impl_scalar_op!($int_scalar, Rem, rem, "remainder");
            impl_scalar_op!($int_scalar, BitAnd, bitand, "bit and");
            impl_scalar_op!($int_scalar, BitOr, bitor, "bit or");
            impl_scalar_op!($int_scalar, BitXor, bitxor, "bit xor");
            impl_scalar_op!($int_scalar, Shl, shl, "left shift");
            impl_scalar_op!($int_scalar, Shr, shr, "right shift");
        );
    }
    all_scalar_ops!(i8);
    all_scalar_ops!(u8);
    all_scalar_ops!(i16);
    all_scalar_ops!(u16);
    all_scalar_ops!(i32);
    all_scalar_ops!(u32);
    all_scalar_ops!(i64);
    all_scalar_ops!(u64);

    impl_scalar_op!(bool, BitAnd, bitand, "bit and");
    impl_scalar_op!(bool, BitOr, bitor, "bit or");
    impl_scalar_op!(bool, BitXor, bitxor, "bit xor");

    impl_scalar_op!(f32, Add, add, "addition");
    impl_scalar_op!(f32, Sub, sub, "subtraction");
    impl_scalar_op!(f32, Mul, mul, "multiplication");
    impl_scalar_op!(f32, Div, div, "division");
    impl_scalar_op!(f32, Rem, rem, "remainder");

    impl_scalar_op!(f64, Add, add, "addition");
    impl_scalar_op!(f64, Sub, sub, "subtraction");
    impl_scalar_op!(f64, Mul, mul, "multiplication");
    impl_scalar_op!(f64, Div, div, "division");
    impl_scalar_op!(f64, Rem, rem, "remainder");

    impl_scalar_op!(Complex<f32>, Add, add, "addition");
    impl_scalar_op!(Complex<f32>, Sub, sub, "subtraction");
    impl_scalar_op!(Complex<f32>, Mul, mul, "multiplication");
    impl_scalar_op!(Complex<f32>, Div, div, "division");

    impl_scalar_op!(Complex<f64>, Add, add, "addition");
    impl_scalar_op!(Complex<f64>, Sub, sub, "subtraction");
    impl_scalar_op!(Complex<f64>, Mul, mul, "multiplication");
    impl_scalar_op!(Complex<f64>, Div, div, "division");

    impl<A, S, D> Neg for ArrayBase<S, D>
        where A: Clone + Neg<Output=A>,
              S: DataMut<Elem=A>,
              D: Dimension
    {
        type Output = Self;
        /// Perform an elementwise negation of `self` and return the result.
        fn neg(mut self) -> Self {
            self.ineg();
            self
        }
    }

    impl<A, S, D> Not for ArrayBase<S, D>
        where A: Clone + Not<Output=A>,
              S: DataMut<Elem=A>,
              D: Dimension
    {
        type Output = Self;
        /// Perform an elementwise unary not of `self` and return the result.
        fn not(mut self) -> Self {
            self.inot();
            self
        }
    }
}

#[cfg(feature = "assign_ops")]
mod assign_ops {
    use super::*;
    use imp_prelude::*;

    macro_rules! impl_assign_op {
        ($trt:ident, $method:ident, $doc:expr) => {
    use std::ops::$trt;

    #[doc=$doc]
    /// If their shapes disagree, `rhs` is broadcast to the shape of `self`.
    ///
    /// **Panics** if broadcasting isn’t possible.
    ///
    /// **Requires crate feature `"assign_ops"`**
    impl<'a, A, S, S2, D, E> $trt<&'a ArrayBase<S2, E>> for ArrayBase<S, D>
        where A: Clone + $trt<A>,
              S: DataMut<Elem=A>,
              S2: Data<Elem=A>,
              D: Dimension,
              E: Dimension,
    {
        fn $method(&mut self, rhs: &ArrayBase<S2, E>) {
            self.zip_mut_with(rhs, |x, y| {
                x.$method(y.clone());
            });
        }
    }

    #[doc=$doc]
    /// **Requires crate feature `"assign_ops"`**
    impl<A, S, D> $trt<A> for ArrayBase<S, D>
        where A: ScalarOperand + $trt<A>,
              S: DataMut<Elem=A>,
              D: Dimension,
    {
        fn $method(&mut self, rhs: A) {
            self.unordered_foreach_mut(move |elt| {
                elt.$method(rhs.clone());
            });
        }
    }

        };
    }

    impl_assign_op!(AddAssign, add_assign,
                    "Perform `self += rhs` as elementwise addition (in place).\n");
    impl_assign_op!(SubAssign, sub_assign,
                    "Perform `self -= rhs` as elementwise subtraction (in place).\n");
    impl_assign_op!(MulAssign, mul_assign,
                    "Perform `self *= rhs` as elementwise multiplication (in place).\n");
    impl_assign_op!(DivAssign, div_assign,
                    "Perform `self /= rhs` as elementwise division (in place).\n");
    impl_assign_op!(RemAssign, rem_assign,
                    "Perform `self %= rhs` as elementwise remainder (in place).\n");
    impl_assign_op!(BitAndAssign, bitand_assign,
                    "Perform `self &= rhs` as elementwise bit and (in place).\n");
    impl_assign_op!(BitOrAssign, bitor_assign,
                    "Perform `self |= rhs` as elementwise bit or (in place).\n");
    impl_assign_op!(BitXorAssign, bitxor_assign,
                    "Perform `self ^= rhs` as elementwise bit xor (in place).\n");
    impl_assign_op!(ShlAssign, shl_assign,
                    "Perform `self <<= rhs` as elementwise left shift (in place).\n");
    impl_assign_op!(ShrAssign, shr_assign,
                    "Perform `self >>= rhs` as elementwise right shift (in place).\n");
}
<commit_msg>Edit docs<commit_after>// Copyright 2014-2016 bluss and ndarray developers.
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.

use std::any::Any;
use libnum::Complex;

/// Elements that can be used as direct operands in arithmetic with arrays.
///
/// For example, `f64` is a `ScalarOperand` which means that for an array `a`,
/// arithmetic like `a + 1.0`, and, `a * 2.`, and `a += 3.` are allowed.
///
/// In the description below, let `A` be an array or array view,
/// let `B` be an array with owned data,
/// and let `C` be an array with mutable data.
///
/// `ScalarOperand` determines for which scalars `K` operations `&A @ K`, and `B @ K`,
/// and `C @= K` are defined, as ***right hand side operands***, for applicable
/// arithmetic operators (denoted `@`).
///
/// ***Left hand side*** scalar operands are not related to this trait
/// (they need one `impl` per concrete scalar type); but they are still
/// implemented for the same types, allowing operations
/// `K @ &A`, and `K @ B` for primitive numeric types `K`.
///
/// This trait ***does not*** limit which elements can be stored in an array in general.
/// Non-`ScalarOperand` types can still participate in arithmetic as array elements in
/// in array-array operations.
pub trait ScalarOperand : Any + Clone { }
impl ScalarOperand for bool { }
impl ScalarOperand for i8 { }
impl ScalarOperand for u8 { }
impl ScalarOperand for i16 { }
impl ScalarOperand for u16 { }
impl ScalarOperand for i32 { }
impl ScalarOperand for u32 { }
impl ScalarOperand for i64 { }
impl ScalarOperand for u64 { }
impl ScalarOperand for f32 { }
impl ScalarOperand for f64 { }
impl ScalarOperand for Complex<f32> { }
impl ScalarOperand for Complex<f64> { }

macro_rules! impl_binary_op(
    ($trt:ident, $mth:ident, $imth:ident, $imth_scalar:ident, $doc:expr) => (
/// Perform elementwise
#[doc=$doc]
/// between `self` and `rhs`,
/// and return the result (based on `self`).
///
/// `self` must be an `OwnedArray` or `RcArray`.
///
/// If their shapes disagree, `rhs` is broadcast to the shape of `self`.
///
/// **Panics** if broadcasting isn’t possible.
impl<A, S, S2, D, E> $trt<ArrayBase<S2, E>> for ArrayBase<S, D>
    where A: Clone + $trt<A, Output=A>,
          S: DataOwned<Elem=A> + DataMut,
          S2: Data<Elem=A>,
          D: Dimension,
          E: Dimension,
{
    type Output = ArrayBase<S, D>;
    fn $mth(self, rhs: ArrayBase<S2, E>) -> ArrayBase<S, D>
    {
        self.$mth(&rhs)
    }
}

/// Perform elementwise
#[doc=$doc]
/// between `self` and reference `rhs`,
/// and return the result (based on `self`).
///
/// If their shapes disagree, `rhs` is broadcast to the shape of `self`.
///
/// **Panics** if broadcasting isn’t possible.
impl<'a, A, S, S2, D, E> $trt<&'a ArrayBase<S2, E>> for ArrayBase<S, D>
    where A: Clone + $trt<A, Output=A>,
          S: DataMut<Elem=A>,
          S2: Data<Elem=A>,
          D: Dimension,
          E: Dimension,
{
    type Output = ArrayBase<S, D>;
    fn $mth (mut self, rhs: &ArrayBase<S2, E>) -> ArrayBase<S, D>
    {
        self.$imth(rhs);
        self
    }
}

/// Perform elementwise
#[doc=$doc]
/// between references `self` and `rhs`,
/// and return the result as a new `OwnedArray`.
///
/// If their shapes disagree, `rhs` is broadcast to the shape of `self`.
///
/// **Panics** if broadcasting isn’t possible.
impl<'a, 'b, A, S, S2, D, E> $trt<&'a ArrayBase<S2, E>> for &'b ArrayBase<S, D>
    where A: Clone + $trt<A, Output=A>,
          S: Data<Elem=A>,
          S2: Data<Elem=A>,
          D: Dimension,
          E: Dimension,
{
    type Output = OwnedArray<A, D>;
    fn $mth (self, rhs: &'a ArrayBase<S2, E>) -> OwnedArray<A, D>
    {
        // FIXME: Can we co-broadcast arrays here? And how?
        self.to_owned().$mth(rhs)
    }
}

/// Perform elementwise
#[doc=$doc]
/// between `self` and the scalar `x`,
/// and return the result (based on `self`).
///
/// `self` must be an `OwnedArray` or `RcArray`.
impl<A, S, D, B> $trt<B> for ArrayBase<S, D>
    where A: Clone + $trt<B, Output=A>,
          S: DataOwned<Elem=A> + DataMut,
          D: Dimension,
          B: ScalarOperand,
{
    type Output = ArrayBase<S, D>;
    fn $mth (mut self, x: B) -> ArrayBase<S, D>
    {
        self.unordered_foreach_mut(move |elt| {
            *elt = elt.clone().$mth(x.clone());
        });
        self
    }
}

/// Perform elementwise
#[doc=$doc]
/// between the reference `self` and the scalar `x`,
/// and return the result as a new `OwnedArray`.
impl<'a, A, S, D, B> $trt<B> for &'a ArrayBase<S, D>
    where A: Clone + $trt<B, Output=A>,
          S: Data<Elem=A>,
          D: Dimension,
          B: ScalarOperand,
{
    type Output = OwnedArray<A, D>;
    fn $mth(self, x: B) -> OwnedArray<A, D>
    {
        self.to_owned().$mth(x)
    }
}
    );
);

macro_rules! impl_scalar_op {
    ($scalar:ty, $trt:ident, $mth:ident, $doc:expr) => (
// these have no doc -- they are not visible in rustdoc
// Perform elementwise
// between the scalar `self` and array `rhs`,
// and return the result (based on `self`).
impl<S, D> $trt<ArrayBase<S, D>> for $scalar
    where S: DataMut<Elem=$scalar>,
          D: Dimension,
{
    type Output = ArrayBase<S, D>;
    fn $mth (self, mut rhs: ArrayBase<S, D>) -> ArrayBase<S, D>
    {
        rhs.unordered_foreach_mut(move |elt| {
            *elt = self.$mth(*elt);
        });
        rhs
    }
}

// Perform elementwise
// between the scalar `self` and array `rhs`,
// and return the result as a new `OwnedArray`.
impl<'a, S, D> $trt<&'a ArrayBase<S, D>> for $scalar
    where S: Data<Elem=$scalar>,
          D: Dimension,
{
    type Output = OwnedArray<$scalar, D>;
    fn $mth (self, rhs: &ArrayBase<S, D>) -> OwnedArray<$scalar, D>
    {
        self.$mth(rhs.to_owned())
    }
}
    );
}

mod arithmetic_ops {
    use super::*;
    use imp_prelude::*;

    use std::ops::*;
    use libnum::Complex;

    impl_binary_op!(Add, add, iadd, iadd_scalar, "addition");
    impl_binary_op!(Sub, sub, isub, isub_scalar, "subtraction");
    impl_binary_op!(Mul, mul, imul, imul_scalar, "multiplication");
    impl_binary_op!(Div, div, idiv, idiv_scalar, "division");
    impl_binary_op!(Rem, rem, irem, irem_scalar, "remainder");
    impl_binary_op!(BitAnd, bitand, ibitand, ibitand_scalar, "bit and");
    impl_binary_op!(BitOr, bitor, ibitor, ibitor_scalar, "bit or");
    impl_binary_op!(BitXor, bitxor, ibitxor, ibitxor_scalar, "bit xor");
    impl_binary_op!(Shl, shl, ishl, ishl_scalar, "left shift");
    impl_binary_op!(Shr, shr, ishr, ishr_scalar, "right shift");

    macro_rules! all_scalar_ops {
        ($int_scalar:ty) => (
            impl_scalar_op!($int_scalar, Add, add, "addition");
            impl_scalar_op!($int_scalar, Sub, sub, "subtraction");
            impl_scalar_op!($int_scalar, Mul, mul, "multiplication");
            impl_scalar_op!($int_scalar, Div, div, "division");
            impl_scalar_op!($int_scalar, Rem, rem, "remainder");
            impl_scalar_op!($int_scalar, BitAnd, bitand, "bit and");
            impl_scalar_op!($int_scalar, BitOr, bitor, "bit or");
            impl_scalar_op!($int_scalar, BitXor, bitxor, "bit xor");
            impl_scalar_op!($int_scalar, Shl, shl, "left shift");
            impl_scalar_op!($int_scalar, Shr, shr, "right shift");
        );
    }
    all_scalar_ops!(i8);
    all_scalar_ops!(u8);
    all_scalar_ops!(i16);
    all_scalar_ops!(u16);
    all_scalar_ops!(i32);
    all_scalar_ops!(u32);
    all_scalar_ops!(i64);
    all_scalar_ops!(u64);

    impl_scalar_op!(bool, BitAnd, bitand, "bit and");
    impl_scalar_op!(bool, BitOr, bitor, "bit or");
    impl_scalar_op!(bool, BitXor, bitxor, "bit xor");

    impl_scalar_op!(f32, Add, add, "addition");
    impl_scalar_op!(f32, Sub, sub, "subtraction");
    impl_scalar_op!(f32, Mul, mul, "multiplication");
    impl_scalar_op!(f32, Div, div, "division");
    impl_scalar_op!(f32, Rem, rem, "remainder");

    impl_scalar_op!(f64, Add, add, "addition");
    impl_scalar_op!(f64, Sub, sub, "subtraction");
    impl_scalar_op!(f64, Mul, mul, "multiplication");
    impl_scalar_op!(f64, Div, div, "division");
    impl_scalar_op!(f64, Rem, rem, "remainder");

    impl_scalar_op!(Complex<f32>, Add, add, "addition");
    impl_scalar_op!(Complex<f32>, Sub, sub, "subtraction");
    impl_scalar_op!(Complex<f32>, Mul, mul, "multiplication");
    impl_scalar_op!(Complex<f32>, Div, div, "division");

    impl_scalar_op!(Complex<f64>, Add, add, "addition");
    impl_scalar_op!(Complex<f64>, Sub, sub, "subtraction");
    impl_scalar_op!(Complex<f64>, Mul, mul, "multiplication");
    impl_scalar_op!(Complex<f64>, Div, div, "division");

    impl<A, S, D> Neg for ArrayBase<S, D>
        where A: Clone + Neg<Output=A>,
              S: DataMut<Elem=A>,
              D: Dimension
    {
        type Output = Self;
        /// Perform an elementwise negation of `self` and return the result.
        fn neg(mut self) -> Self {
            self.ineg();
            self
        }
    }

    impl<A, S, D> Not for ArrayBase<S, D>
        where A: Clone + Not<Output=A>,
              S: DataMut<Elem=A>,
              D: Dimension
    {
        type Output = Self;
        /// Perform an elementwise unary not of `self` and return the result.
        fn not(mut self) -> Self {
            self.inot();
            self
        }
    }
}

#[cfg(feature = "assign_ops")]
mod assign_ops {
    use super::*;
    use imp_prelude::*;

    macro_rules! impl_assign_op {
        ($trt:ident, $method:ident, $doc:expr) => {
    use std::ops::$trt;

    #[doc=$doc]
    /// If their shapes disagree, `rhs` is broadcast to the shape of `self`.
    ///
    /// **Panics** if broadcasting isn’t possible.
    ///
    /// **Requires crate feature `"assign_ops"`**
    impl<'a, A, S, S2, D, E> $trt<&'a ArrayBase<S2, E>> for ArrayBase<S, D>
        where A: Clone + $trt<A>,
              S: DataMut<Elem=A>,
              S2: Data<Elem=A>,
              D: Dimension,
              E: Dimension,
    {
        fn $method(&mut self, rhs: &ArrayBase<S2, E>) {
            self.zip_mut_with(rhs, |x, y| {
                x.$method(y.clone());
            });
        }
    }

    #[doc=$doc]
    /// **Requires crate feature `"assign_ops"`**
    impl<A, S, D> $trt<A> for ArrayBase<S, D>
        where A: ScalarOperand + $trt<A>,
              S: DataMut<Elem=A>,
              D: Dimension,
    {
        fn $method(&mut self, rhs: A) {
            self.unordered_foreach_mut(move |elt| {
                elt.$method(rhs.clone());
            });
        }
    }

        };
    }

    impl_assign_op!(AddAssign, add_assign,
                    "Perform `self += rhs` as elementwise addition (in place).\n");
    impl_assign_op!(SubAssign, sub_assign,
                    "Perform `self -= rhs` as elementwise subtraction (in place).\n");
    impl_assign_op!(MulAssign, mul_assign,
                    "Perform `self *= rhs` as elementwise multiplication (in place).\n");
    impl_assign_op!(DivAssign, div_assign,
                    "Perform `self /= rhs` as elementwise division (in place).\n");
    impl_assign_op!(RemAssign, rem_assign,
                    "Perform `self %= rhs` as elementwise remainder (in place).\n");
    impl_assign_op!(BitAndAssign, bitand_assign,
                    "Perform `self &= rhs` as elementwise bit and (in place).\n");
    impl_assign_op!(BitOrAssign, bitor_assign,
                    "Perform `self |= rhs` as elementwise bit or (in place).\n");
    impl_assign_op!(BitXorAssign, bitxor_assign,
                    "Perform `self ^= rhs` as elementwise bit xor (in place).\n");
    impl_assign_op!(ShlAssign, shl_assign,
                    "Perform `self <<= rhs` as elementwise left shift (in place).\n");
    impl_assign_op!(ShrAssign, shr_assign,
                    "Perform `self >>= rhs` as elementwise right shift (in place).\n");
}
<|endoftext|>


<commit_before>use std::rc::Rc;
use std::cell::RefCell;
use std::sync::{Arc, Mutex};
use std::any::Any;
use std::sync::mpsc::{Sender, Receiver, channel};

use core::marker::PhantomData;

use columnar::{Columnar, ColumnarStack};
use communication::{Observer, Pushable, Pullable};
use networking::networking::MessageHeader;
use std::default::Default;

// The Communicator trait presents the interface a worker has to the outside world.
// The worker can see its index, the total number of peers, and acquire channels to and from the other workers.
// There is an assumption that each worker performs the same channel allocation logic; things go wrong otherwise.
// TODO : Commented out for now, due to boxing issues (new_channel has generic params, making Box<Communicator> bad for Rust).
// pub trait Communicator : 'static {
//     fn index(&self) -> u64;     // number out of peers
//     fn peers(&self) -> u64;     // number of peers
//     fn new_channel<T:Send>(&mut self) -> (Vec<Box<Pushable<T>>>, Box<Pullable<T>>);
// }

pub enum CommunicatorEnum {
    Thread(Box<ThreadCommunicator>),     // same thread communicator (mostly no-ops)
    Process(Box<ProcessCommunicator>),   // same process communicator (typed channels)
    Binary(Box<BinaryCommunicator>),     // remote communicators (serialized channels)
}

impl Communicator {
    pub fn index(&self) -> u64 {
        match self {
            &Communicator::Thread(ref t)  => t.index(),
            &Communicator::Process(ref p) => p.index(),
            &Communicator::Binary(ref b)  => b.index(),
        }
    }
    pub fn peers(&self) -> u64 {
        match self {
            &Communicator::Thread(ref t)  => t.peers(),
            &Communicator::Process(ref p) => p.peers(),
            &Communicator::Binary(ref b)  => b.peers(),
        }
    }
    pub fn new_channel<T:Send+Columnar+'static>(&mut self) -> (Vec<Box<Pushable<T>>>, Box<Pullable<T>>) {
        match self {
            &mut Communicator::Thread(ref mut t)  => t.new_channel(),
            &mut Communicator::Process(ref mut p) => p.new_channel(),
            &mut Communicator::Binary(ref mut b)  => b.new_channel(),
        }
    }
}

// The simplest communicator remains worker-local and just queues sent messages.
pub struct ThreadCommunicator;
impl ThreadCommunicator {
    fn index(&self) -> u64 { 0 }
    fn peers(&self) -> u64 { 1 }
    fn new_channel<T:Send+'static>(&mut self) -> (Vec<Box<Pushable<T>>>, Box<Pullable<T>>) {
        let shared = Rc::new(RefCell::new(Vec::new()));
        return (vec![Box::new(shared.clone()) as Box<Pushable<T>>], Box::new(shared.clone()) as Box<Pullable<T>>)
    }
}

// A specific Communicator for inter-thread intra-process communication
pub struct ProcessCommunicator {
    inner:      Communicator,                   // inner ThreadCommunicator
    index:      u64,                            // number out of peers
    peers:      u64,                            // number of peer allocators (for typed channel allocation).
    allocated:  u64,                            // indicates how many have been allocated (locally).
    channels:   Arc<Mutex<Vec<Box<Any+Send>>>>, // Box<Any+Send> -> Box<Vec<Option<(Vec<Sender<T>>, Receiver<T>)>>>
}

impl ProcessCommunicator {
    fn index(&self) -> u64 { self.index }
    fn peers(&self) -> u64 { self.peers }
    fn inner<'a>(&'a mut self) -> &'a Communicator { &mut self.inner }
    fn new_channel<T:Send+'static>(&mut self) -> (Vec<Box<Pushable<T>>>, Box<Pullable<T>>) {
        let mut channels = self.channels.lock().ok().expect("mutex error?");
        if self.allocated == channels.len() as u64 {  // we need a new channel ...
            let mut senders = Vec::new();
            let mut receivers = Vec::new();
            for _ in range(0, self.peers) {
                let (s, r): (Sender<T>, Receiver<T>) = channel();
                senders.push(s);
                receivers.push(r);
            }

            let mut to_box = Vec::new();
            for recv in receivers.drain() {
                to_box.push(Some((senders.clone(), recv)));
            }

            channels.push(Box::new(to_box));
        }

        match channels[self.allocated as usize].downcast_mut::<(Vec<Option<(Vec<Sender<T>>, Receiver<T>)>>)>() {
            Some(ref mut vector) => {
                self.allocated += 1;
                let (mut send, recv) = vector[self.index as usize].take().unwrap();
                let mut temp = Vec::new();
                for s in send.drain() { temp.push(Box::new(s) as Box<Pushable<T>>); }
                return (temp, Box::new(recv) as Box<Pullable<T>>)
            }
            _ => { panic!("unable to cast channel correctly"); }
        }
    }

    pub fn new_vector(count: u64) -> Vec<Communicator> {
        let channels = Arc::new(Mutex::new(Vec::new()));
        return (0 .. count).map(|index| Communicator::Process(Box::new(ProcessCommunicator {
            inner:      Communicator::Thread(Box::new(ThreadCommunicator)),
            index:      index,
            peers:      count,
            allocated:  0,
            channels:   channels.clone(),
        }))).collect();
    }
}

// A communicator intended for binary channels (networking, pipes, shared memory)
pub struct BinaryCommunicator {
    pub inner:      Communicator,    // inner ProcessCommunicator (use for process-local channels)
    pub index:      u64,             // index of this worker
    pub peers:      u64,             // number of peer workers
    pub graph:      u64,             // identifier for the current graph
    pub allocated:  u64,             // indicates how many channels have been allocated (locally).

    // for loading up state in the networking threads.
    pub writers:    Vec<Sender<((u64, u64, u64), Sender<Vec<u8>>)>>,                     // (index, back-to-worker)
    pub readers:    Vec<Sender<((u64, u64, u64), Sender<Vec<u8>>, Receiver<Vec<u8>>)>>,  // (index, data-to-worker, back-from-worker)
    pub senders:    Vec<Sender<(MessageHeader, Vec<u8>)>>                                // for sending bytes!
}

// A Communicator backed by Sender<Vec<u8>>/Receiver<Vec<u8>> pairs (e.g. networking, shared memory, files, pipes)
impl BinaryCommunicator {
    fn index(&self) -> u64 { self.index }
    fn peers(&self) -> u64 { self.peers }
    fn inner<'a>(&'a mut self) -> &'a Communicator { &mut self.inner }
    fn new_channel<T:Send+Columnar+'static>(&mut self) -> (Vec<Box<Pushable<T>>>, Box<Pullable<T>>) {
        let mut pushers: Vec<Box<Pushable<T>>> = Vec::new(); // built-up vector of Box<Pushable<T>> to return

        // we'll need process-local channels as well (no self-loop binary connection in this design; perhaps should allow)
        let inner_peers = self.inner.peers();
        let (inner_sends, inner_recv) = self.inner.new_channel();

        // prep a pushable for each endpoint, multiplied by inner_peers
        for (index, writer) in self.writers.iter().enumerate() {
            for _ in (0..inner_peers) {
                let (s,r) = channel();  // generate a binary (Vec<u8>) channel pair of (back_to_worker, back_from_net)
                let target_index = if index as u64 >= (self.index * inner_peers) { index as u64 + inner_peers } else { index as u64 };
                println!("init'ing send channel: ({} {} {})", self.index, self.graph, self.allocated);
                writer.send(((self.index, self.graph, self.allocated), s)).unwrap();
                let header = MessageHeader {
                    graph:      self.graph,
                    channel:    self.allocated,
                    source:     self.index,
                    target:     target_index,
                    length:     0,
                };
                pushers.push(Box::new(BinaryPushable::new(header, self.senders[index].clone(), r)));
            }
        }

        // splice inner_sends into the vector of pushables
        for (index, writer) in inner_sends.into_iter().enumerate() {
            pushers.insert((self.index * inner_peers) as usize + index, writer);
        }

        // prep a Box<Pullable<T>> using inner_recv and fresh registered pullables
        let (send,recv) = channel();    // binary channel from binary listener to BinaryPullable<T>
        let mut pullsends = Vec::new();
        for reader in self.readers.iter() {
            let (s,r) = channel();
            pullsends.push(s);
            println!("init'ing recv channel: ({} {} {})", self.index, self.graph, self.allocated);
            reader.send(((self.index, self.graph, self.allocated), send.clone(), r)).unwrap();
        }

        let pullable = Box::new(BinaryPullable {
            inner:      inner_recv,
            senders:    pullsends,
            receiver:   recv,
            stack:      Default::default(),
        });

        self.allocated += 1;

        return (pushers, pullable);
    }
}

struct BinaryPushable<T: Columnar> {
    header:     MessageHeader,
    sender:     Sender<(MessageHeader, Vec<u8>)>,   // targets for each remote destination
    receiver:   Receiver<Vec<u8>>,                  // source of empty binary vectors
    phantom:    PhantomData<T>,
    buffer:     Vec<u8>,
    stack:      <T as Columnar>::Stack,
}

impl<T: Columnar> BinaryPushable<T> {
    pub fn new(header: MessageHeader, sender: Sender<(MessageHeader, Vec<u8>)>, receiver: Receiver<Vec<u8>>) -> BinaryPushable<T> {
        BinaryPushable {
            header:     header,
            sender:     sender,
            receiver:   receiver,
            phantom:    PhantomData,
            buffer:     Vec::new(),
            stack:      Default::default(),
        }
    }
}

impl<T:Columnar+'static> Pushable<T> for BinaryPushable<T> {
    #[inline]
    fn push(&mut self, data: T) {
        let mut bytes = if let Some(buffer) = self.receiver.try_recv().ok() { buffer } else { Vec::new() };
        bytes.clear();

        self.stack.push(data);
        self.stack.encode(&mut bytes).unwrap();

        let mut header = self.header;
        header.length = bytes.len() as u64;

        self.sender.send((header, bytes)).ok();
    }
}

struct BinaryPullable<T: Columnar> {
    inner:      Box<Pullable<T>>,       // inner pullable (e.g. intra-process typed queue)
    senders:    Vec<Sender<Vec<u8>>>,   // places to put used binary vectors
    receiver:   Receiver<Vec<u8>>,      // source of serialized buffers
    stack:      <T as Columnar>::Stack,
}

impl<T:Columnar+'static> Pullable<T> for BinaryPullable<T> {
    #[inline]
    fn pull(&mut self) -> Option<T> {
        if let Some(data) = self.inner.pull() { Some(data) }
        else if let Some(bytes) = self.receiver.try_recv().ok() {
            self.stack.decode(&mut bytes.as_slice()).unwrap();
            self.senders[0].send(bytes).unwrap();                   // TODO : Not clear where bytes came from; find out!
            self.stack.pop()
        }
        else { None }
    }
}
<commit_msg>more logic moved to unary<commit_after>use std::rc::Rc;
use std::cell::RefCell;
use std::sync::{Arc, Mutex};
use std::any::Any;
use std::sync::mpsc::{Sender, Receiver, channel};

use core::marker::PhantomData;

use columnar::{Columnar, ColumnarStack};
use communication::{Observer, Pushable, Pullable};
use networking::networking::MessageHeader;
use std::default::Default;

// The Communicator trait presents the interface a worker has to the outside world.
// The worker can see its index, the total number of peers, and acquire channels to and from the other workers.
// There is an assumption that each worker performs the same channel allocation logic; things go wrong otherwise.
// TODO : Commented out for now, due to boxing issues (new_channel has generic params, making Box<Communicator> bad for Rust).
// pub trait Communicator : 'static {
//     fn index(&self) -> u64;     // number out of peers
//     fn peers(&self) -> u64;     // number of peers
//     fn new_channel<T:Send>(&mut self) -> (Vec<Box<Pushable<T>>>, Box<Pullable<T>>);
// }

pub enum Communicator {
    Thread(Box<ThreadCommunicator>),     // same thread communicator (mostly no-ops)
    Process(Box<ProcessCommunicator>),   // same process communicator (typed channels)
    Binary(Box<BinaryCommunicator>),     // remote communicators (serialized channels)
}

impl Communicator {
    pub fn index(&self) -> u64 {
        match self {
            &Communicator::Thread(ref t)  => t.index(),
            &Communicator::Process(ref p) => p.index(),
            &Communicator::Binary(ref b)  => b.index(),
        }
    }
    pub fn peers(&self) -> u64 {
        match self {
            &Communicator::Thread(ref t)  => t.peers(),
            &Communicator::Process(ref p) => p.peers(),
            &Communicator::Binary(ref b)  => b.peers(),
        }
    }
    pub fn new_channel<T:Send+Columnar+'static>(&mut self) -> (Vec<Box<Pushable<T>>>, Box<Pullable<T>>) {
        match self {
            &mut Communicator::Thread(ref mut t)  => t.new_channel(),
            &mut Communicator::Process(ref mut p) => p.new_channel(),
            &mut Communicator::Binary(ref mut b)  => b.new_channel(),
        }
    }
}

// The simplest communicator remains worker-local and just queues sent messages.
pub struct ThreadCommunicator;
impl ThreadCommunicator {
    fn index(&self) -> u64 { 0 }
    fn peers(&self) -> u64 { 1 }
    fn new_channel<T:Send+'static>(&mut self) -> (Vec<Box<Pushable<T>>>, Box<Pullable<T>>) {
        let shared = Rc::new(RefCell::new(Vec::new()));
        return (vec![Box::new(shared.clone()) as Box<Pushable<T>>], Box::new(shared.clone()) as Box<Pullable<T>>)
    }
}

// A specific Communicator for inter-thread intra-process communication
pub struct ProcessCommunicator {
    inner:      Communicator,                   // inner ThreadCommunicator
    index:      u64,                            // number out of peers
    peers:      u64,                            // number of peer allocators (for typed channel allocation).
    allocated:  u64,                            // indicates how many have been allocated (locally).
    channels:   Arc<Mutex<Vec<Box<Any+Send>>>>, // Box<Any+Send> -> Box<Vec<Option<(Vec<Sender<T>>, Receiver<T>)>>>
}

impl ProcessCommunicator {
    fn index(&self) -> u64 { self.index }
    fn peers(&self) -> u64 { self.peers }
    fn inner<'a>(&'a mut self) -> &'a Communicator { &mut self.inner }
    fn new_channel<T:Send+'static>(&mut self) -> (Vec<Box<Pushable<T>>>, Box<Pullable<T>>) {
        let mut channels = self.channels.lock().ok().expect("mutex error?");
        if self.allocated == channels.len() as u64 {  // we need a new channel ...
            let mut senders = Vec::new();
            let mut receivers = Vec::new();
            for _ in range(0, self.peers) {
                let (s, r): (Sender<T>, Receiver<T>) = channel();
                senders.push(s);
                receivers.push(r);
            }

            let mut to_box = Vec::new();
            for recv in receivers.drain() {
                to_box.push(Some((senders.clone(), recv)));
            }

            channels.push(Box::new(to_box));
        }

        match channels[self.allocated as usize].downcast_mut::<(Vec<Option<(Vec<Sender<T>>, Receiver<T>)>>)>() {
            Some(ref mut vector) => {
                self.allocated += 1;
                let (mut send, recv) = vector[self.index as usize].take().unwrap();
                let mut temp = Vec::new();
                for s in send.drain() { temp.push(Box::new(s) as Box<Pushable<T>>); }
                return (temp, Box::new(recv) as Box<Pullable<T>>)
            }
            _ => { panic!("unable to cast channel correctly"); }
        }
    }

    pub fn new_vector(count: u64) -> Vec<Communicator> {
        let channels = Arc::new(Mutex::new(Vec::new()));
        return (0 .. count).map(|index| Communicator::Process(Box::new(ProcessCommunicator {
            inner:      Communicator::Thread(Box::new(ThreadCommunicator)),
            index:      index,
            peers:      count,
            allocated:  0,
            channels:   channels.clone(),
        }))).collect();
    }
}

// A communicator intended for binary channels (networking, pipes, shared memory)
pub struct BinaryCommunicator {
    pub inner:      Communicator,    // inner ProcessCommunicator (use for process-local channels)
    pub index:      u64,             // index of this worker
    pub peers:      u64,             // number of peer workers
    pub graph:      u64,             // identifier for the current graph
    pub allocated:  u64,             // indicates how many channels have been allocated (locally).

    // for loading up state in the networking threads.
    pub writers:    Vec<Sender<((u64, u64, u64), Sender<Vec<u8>>)>>,                     // (index, back-to-worker)
    pub readers:    Vec<Sender<((u64, u64, u64), Sender<Vec<u8>>, Receiver<Vec<u8>>)>>,  // (index, data-to-worker, back-from-worker)
    pub senders:    Vec<Sender<(MessageHeader, Vec<u8>)>>                                // for sending bytes!
}

// A Communicator backed by Sender<Vec<u8>>/Receiver<Vec<u8>> pairs (e.g. networking, shared memory, files, pipes)
impl BinaryCommunicator {
    fn index(&self) -> u64 { self.index }
    fn peers(&self) -> u64 { self.peers }
    fn inner<'a>(&'a mut self) -> &'a Communicator { &mut self.inner }
    fn new_channel<T:Send+Columnar+'static>(&mut self) -> (Vec<Box<Pushable<T>>>, Box<Pullable<T>>) {
        let mut pushers: Vec<Box<Pushable<T>>> = Vec::new(); // built-up vector of Box<Pushable<T>> to return

        // we'll need process-local channels as well (no self-loop binary connection in this design; perhaps should allow)
        let inner_peers = self.inner.peers();
        let (inner_sends, inner_recv) = self.inner.new_channel();

        // prep a pushable for each endpoint, multiplied by inner_peers
        for (index, writer) in self.writers.iter().enumerate() {
            for _ in (0..inner_peers) {
                let (s,r) = channel();  // generate a binary (Vec<u8>) channel pair of (back_to_worker, back_from_net)
                let target_index = if index as u64 >= (self.index * inner_peers) { index as u64 + inner_peers } else { index as u64 };
                println!("init'ing send channel: ({} {} {})", self.index, self.graph, self.allocated);
                writer.send(((self.index, self.graph, self.allocated), s)).unwrap();
                let header = MessageHeader {
                    graph:      self.graph,
                    channel:    self.allocated,
                    source:     self.index,
                    target:     target_index,
                    length:     0,
                };
                pushers.push(Box::new(BinaryPushable::new(header, self.senders[index].clone(), r)));
            }
        }

        // splice inner_sends into the vector of pushables
        for (index, writer) in inner_sends.into_iter().enumerate() {
            pushers.insert((self.index * inner_peers) as usize + index, writer);
        }

        // prep a Box<Pullable<T>> using inner_recv and fresh registered pullables
        let (send,recv) = channel();    // binary channel from binary listener to BinaryPullable<T>
        let mut pullsends = Vec::new();
        for reader in self.readers.iter() {
            let (s,r) = channel();
            pullsends.push(s);
            println!("init'ing recv channel: ({} {} {})", self.index, self.graph, self.allocated);
            reader.send(((self.index, self.graph, self.allocated), send.clone(), r)).unwrap();
        }

        let pullable = Box::new(BinaryPullable {
            inner:      inner_recv,
            senders:    pullsends,
            receiver:   recv,
            stack:      Default::default(),
        });

        self.allocated += 1;

        return (pushers, pullable);
    }
}

struct BinaryPushable<T: Columnar> {
    header:     MessageHeader,
    sender:     Sender<(MessageHeader, Vec<u8>)>,   // targets for each remote destination
    receiver:   Receiver<Vec<u8>>,                  // source of empty binary vectors
    phantom:    PhantomData<T>,
    buffer:     Vec<u8>,
    stack:      <T as Columnar>::Stack,
}

impl<T: Columnar> BinaryPushable<T> {
    pub fn new(header: MessageHeader, sender: Sender<(MessageHeader, Vec<u8>)>, receiver: Receiver<Vec<u8>>) -> BinaryPushable<T> {
        BinaryPushable {
            header:     header,
            sender:     sender,
            receiver:   receiver,
            phantom:    PhantomData,
            buffer:     Vec::new(),
            stack:      Default::default(),
        }
    }
}

impl<T:Columnar+'static> Pushable<T> for BinaryPushable<T> {
    #[inline]
    fn push(&mut self, data: T) {
        let mut bytes = if let Some(buffer) = self.receiver.try_recv().ok() { buffer } else { Vec::new() };
        bytes.clear();

        self.stack.push(data);
        self.stack.encode(&mut bytes).unwrap();

        let mut header = self.header;
        header.length = bytes.len() as u64;

        self.sender.send((header, bytes)).ok();
    }
}

struct BinaryPullable<T: Columnar> {
    inner:      Box<Pullable<T>>,       // inner pullable (e.g. intra-process typed queue)
    senders:    Vec<Sender<Vec<u8>>>,   // places to put used binary vectors
    receiver:   Receiver<Vec<u8>>,      // source of serialized buffers
    stack:      <T as Columnar>::Stack,
}

impl<T:Columnar+'static> Pullable<T> for BinaryPullable<T> {
    #[inline]
    fn pull(&mut self) -> Option<T> {
        if let Some(data) = self.inner.pull() { Some(data) }
        else if let Some(bytes) = self.receiver.try_recv().ok() {
            self.stack.decode(&mut bytes.as_slice()).unwrap();
            self.senders[0].send(bytes).unwrap();                   // TODO : Not clear where bytes came from; find out!
            self.stack.pop()
        }
        else { None }
    }
}
<|endoftext|>


<commit_before>use component::Component;
use tokio_core::reactor::Handle;
use error::Error;
use xcb::{self, Connection};
use xcb_event_stream;
use futures::stream::Stream;

const ATOM_FOCUS_CHANGE: xcb::AtomEnum = 334;

/// This struct is used for creation and storing the refresh rate.
pub struct WindowTitle {}

// Get the title of the window that currently is focused
fn get_window_title() -> String {
    // Connect to Xorg
    let (conn, _) = Connection::connect(None).unwrap();

    // Get the window that is currently in focus
    let input_focus_cookie = xcb::get_input_focus(&conn);
    let window = input_focus_cookie.get_reply().unwrap().focus();

    // Use ewmh to poll for the _NET_WM_NAME of the focused window
    let property_cookie = xcb::get_property(
        &conn,
        false,
        window,
        xcb::ATOM_WM_NAME,
        xcb::ATOM_STRING,
        0,
        32,
    );
    let property_reply = property_cookie.get_reply().unwrap();
    let name = String::from_utf8_lossy(property_reply.value());

    name.into()
}

// Poll the window title on a timer
impl Component for WindowTitle {
    type Error = Error;
    type Stream = Box<Stream<Item = String, Error = Error>>;

    fn stream(self, _: Handle) -> Self::Stream {
        let (conn, screen_num) = Connection::connect(None).unwrap();

        {
            let setup = conn.get_setup();
            let screen = setup.roots().nth(screen_num as usize).unwrap();

            xcb::change_window_attributes(
                &conn,
                screen.root(),
                &[(xcb::CW_EVENT_MASK, xcb::EVENT_MASK_PROPERTY_CHANGE)],
            );
        }

        conn.flush();
        let stream = xcb_event_stream::XcbEventStream::new(conn)
            .filter(|event| unsafe {
                let property_event: &xcb::PropertyNotifyEvent = xcb::cast_event(&event);
                let property_atom = property_event.atom();
                property_atom == ATOM_FOCUS_CHANGE
            })
            .and_then(move |_| Ok(get_window_title()));

        Box::new(stream)
    }
}
<commit_msg>Add support for EWMH window title<commit_after>use component::Component;
use tokio_core::reactor::Handle;
use error::{Error, Result, ErrorKind};
use xcb::{self, Connection};
use xcb_event_stream;
use futures::stream::Stream;

const ATOM_FOCUS_CHANGE: xcb::AtomEnum = 334;

/// This struct is used for creation and storing the refresh rate.
pub struct WindowTitle {}

// Get the title of the window that currently is focused
fn get_window_title() -> Result<String> {
    // Connect to Xorg
    let (conn, _) = Connection::connect(None)
        .map_err(|e| ErrorKind::XcbConnection(e))?;

    // Get the window that is currently in focus
    let input_focus_cookie = xcb::get_input_focus(&conn);
    let window = input_focus_cookie.get_reply()?.focus();

    // Use ewmh to poll for the _NET_WM_NAME of the focused window
    let atom = xcb::intern_atom(&conn, true, "_NET_WM_NAME")
        .get_reply()?
        .atom();

    let property_reply = xcb::get_property(&conn, false, window, atom, xcb::ATOM_ANY, 0, 32)
        .get_reply();
    let reply = match property_reply {
        Ok(reply) => reply,
        _ => {
            xcb::get_property(
                &conn,
                false,
                window,
                xcb::ATOM_WM_NAME,
                xcb::ATOM_STRING,
                0,
                32,
            ).get_reply()?
        }
    };

    let name = String::from_utf8_lossy(reply.value());
    Ok(name.into())
}

// Poll the window title on a timer
impl Component for WindowTitle {
    type Error = Error;
    type Stream = Box<Stream<Item = String, Error = Error>>;

    fn stream(self, _: Handle) -> Self::Stream {
        let (conn, screen_num) = Connection::connect(None).unwrap();

        {
            let setup = conn.get_setup();
            let screen = setup.roots().nth(screen_num as usize).unwrap();

            xcb::change_window_attributes(
                &conn,
                screen.root(),
                &[(xcb::CW_EVENT_MASK, xcb::EVENT_MASK_PROPERTY_CHANGE)],
            );
        }

        conn.flush();
        let stream = xcb_event_stream::XcbEventStream::new(conn)
            .filter(|event| unsafe {
                let property_event: &xcb::PropertyNotifyEvent = xcb::cast_event(&event);
                let property_atom = property_event.atom();
                property_atom == ATOM_FOCUS_CHANGE
            })
            .and_then(move |_| get_window_title());

        Box::new(stream)
    }
}
<|endoftext|>


<commit_before>use component::Component;
use tokio_core::reactor::Handle;
use futures::{Stream, Future};
use tokio_timer::Timer;
use std::time::Duration;
use utils::LoopFn;
use error::Error;
use xcb::{self, Connection};

/// This struct is used for creation and storing the refresh rate.
pub struct WindowTitle {
    pub refresh_rate: Duration,
}

// Set the default refresh rate used for this component to one second.
impl Default for WindowTitle {
    fn default() -> WindowTitle {
        WindowTitle { refresh_rate: Duration::from_secs(1) }
    }
}

// Get the title of the window that currently is focused
fn get_window_title() -> String {
    // Connect to Xorg
    let (conn, _) = Connection::connect(None).unwrap();

    // Get the window that is currently in focus
    let input_focus_cookie = xcb::get_input_focus(&conn);
    let window = input_focus_cookie.get_reply().unwrap().focus();

    // Use ewmh to poll for the _NET_WM_NAME of the focused window
    let property_cookie = xcb::get_property(
        &conn,
        false,
        window,
        xcb::ATOM_WM_NAME,
        xcb::ATOM_STRING,
        0,
        32,
    );
    let property_reply = property_cookie.get_reply().unwrap();
    let name = String::from_utf8_lossy(property_reply.value());

    name.into()
}

// Poll the window title on a timer
impl Component for WindowTitle {
    type Error = Error;
    type Stream = Box<Stream<Item = String, Error = Error>>;

    fn stream(self, _: Handle) -> Self::Stream {
        LoopFn::new(move || {
            let timer = Timer::default();
            timer
                .sleep(self.refresh_rate)
                .and_then(|_| Ok(get_window_title()))
        }).map_err(|_| "timer error".into())
            .boxed()
    }
}
<commit_msg>Use proper futures to implement event polling<commit_after>use component::Component;
use tokio_core::reactor::Handle;
use std::time::Duration;
use error::Error;
use xcb::{self, Connection};
use xcb_event_stream;
use futures::stream::Stream;

/// This struct is used for creation and storing the refresh rate.
pub struct WindowTitle {
    pub refresh_rate: Duration,
}

// Set the default refresh rate used for this component to one second.
impl Default for WindowTitle {
    fn default() -> WindowTitle {
        WindowTitle { refresh_rate: Duration::from_secs(1) }
    }
}

// Get the title of the window that currently is focused
fn get_window_title() -> String {
    // Connect to Xorg
    let (conn, _) = Connection::connect(None).unwrap();

    // Get the window that is currently in focus
    let input_focus_cookie = xcb::get_input_focus(&conn);
    let window = input_focus_cookie.get_reply().unwrap().focus();

    // Use ewmh to poll for the _NET_WM_NAME of the focused window
    let property_cookie = xcb::get_property(
        &conn,
        false,
        window,
        xcb::ATOM_WM_NAME,
        xcb::ATOM_STRING,
        0,
        32,
    );
    let property_reply = property_cookie.get_reply().unwrap();
    let name = String::from_utf8_lossy(property_reply.value());

    name.into()
}

// Poll the window title on a timer
impl Component for WindowTitle {
    type Error = Error;
    type Stream = Box<Stream<Item = String, Error = Error>>;

    fn stream(self, _: Handle) -> Self::Stream {
        let (conn, screen_num) = Connection::connect(None).unwrap();

        {
            let setup = conn.get_setup();
            let screen = setup.roots().nth(screen_num as usize).unwrap();

            xcb::change_window_attributes(
                &conn,
                screen.root(),
                &[(xcb::CW_EVENT_MASK, xcb::EVENT_MASK_PROPERTY_CHANGE)],
            );
        }

        conn.flush();
        let stream = xcb_event_stream::XcbEventStream::new(conn)
            .and_then(move |_| Ok(get_window_title()));
        return Box::new(stream);
    }
}
<|endoftext|>


<commit_before><commit_msg>Add eval with literal, quote, begin, and if<commit_after><|endoftext|>


<commit_before>// Benchmark our hashing function.

#![feature(test)]

extern crate rsure;
extern crate tempdir;
extern crate test;

use rsure::{Progress, SureHash};
use tempdir::TempDir;
use std::fs::File;
use std::io::Write;
use test::Bencher;

// To compute hashing speed, use 1 over the benchmark time in seconds.  For
// example, if the benchmark runs in 1,863,225 ns/iter, that would be about
// 536 MiB/sec hash performance.
#[bench]
fn tree_mb_bench(b: &mut Bencher) {
    let tmp = TempDir::new("rsure-bench").unwrap();
    {
        let mut fd = File::create(tmp.path().join("large")).unwrap();
        let buf = vec![0; 1024];
        for _ in 0 .. 1024 {
            fd.write_all(&buf).unwrap();
        }
    }

    b.iter(|| {
        let mut tree = rsure::scan_fs(tmp.path()).unwrap();
        let estimate = tree.hash_estimate();
        let mut progress = Progress::new(estimate.files, estimate.bytes);
        tree.hash_update(tmp.path(), &mut progress);
        // progress.flush();
    })
}
<commit_msg>Increase the size of the hash benchmark<commit_after>// Benchmark our hashing function.

#![feature(test)]

extern crate rsure;
extern crate tempdir;
extern crate test;

use rsure::{Progress, SureHash};
use tempdir::TempDir;
use std::fs::File;
use std::io::Write;
use test::Bencher;

// To compute hashing speed, use 1 over the benchmark time in seconds, and
// then multiply the result by the number of iterations in the 'for i'
// loop.  For example, if the benchmark runs in 29,924,583 ns/iter, and the
// count is 16, that would be about 534 MiB/sec hash performance.
//
// The loop count should be large enough to overflow the CPU's largest
// cache, with the value 16 (16MiB) overflowing the 8MiB cache on the Core
// i7-950 I wrote this on.
#[bench]
fn tree_mb_bench(b: &mut Bencher) {
    let tmp = TempDir::new("rsure-bench").unwrap();
    for i in 0 .. 16 {
        let name = format!("large-{}", i);
        let mut fd = File::create(tmp.path().join(&name)).unwrap();
        let buf = vec![0; 1024];
        for _ in 0 .. 1024 {
            fd.write_all(&buf).unwrap();
        }
    }

    b.iter(|| {
        let mut tree = rsure::scan_fs(tmp.path()).unwrap();
        let estimate = tree.hash_estimate();
        let mut progress = Progress::new(estimate.files, estimate.bytes);
        tree.hash_update(tmp.path(), &mut progress);
        // progress.flush();
    })
}
<|endoftext|>


<commit_before>use tokio_tungstenite as tungstenite;

use atomic_counter::AtomicCounter;
use atomic_counter::RelaxedCounter;

use serde::Deserialize;
use serde_json::json;

use std::result::Result;
use std::sync::Arc;
use std::sync::atomic::AtomicBool;
use std::sync::atomic::Ordering;
use std::sync::RwLock;
use std::time::Instant;

use tokio::sync::mpsc;
use tokio::sync::oneshot;

use crate::database::{RunQuery, ResQuery};
use crate::parser::santizer;

// Type alias for msg_id
pub type MsgId = Arc<RelaxedCounter>;

#[derive(Debug)]
enum Event {
    UserEvent(UserEvent),
    SystemControl(SystemControl),
    MessageControl(MessageControl),
}

#[derive(Debug, Deserialize)]
#[serde(tag = "type")]
#[serde(rename_all = "snake_case")]
pub enum UserEvent {
    // These events are what plugins will care for:
    //
    // Slack messages
    // TODO: more involved here for now basics
    Message {
        subtype: Option<String>,
        hidden: Option<bool>,
        #[serde(rename = "channel")]
        channel_id: Option<String>,
        #[serde(rename = "user")]
        user_id: String,
        text: String,
        ts: String,
        thread_ts: Option<String>,
    },

    // TODO: consider looking at slack_api for types to reuse here
    ReactionAdded {
        #[serde(rename = "user")]
        user_id: String,
        reaction: String,
        item_user: Option<String>,
        // item: ListResponseItem
        event_ts: String,
    },
    ReactionRemoved {
        #[serde(rename = "user")]
        user_id: String,
        reaction: String,
        item_user: Option<String>,
        // item: ListResponseItem
        event_ts: String,
    },
}

#[derive(Debug, Deserialize)]
#[serde(tag = "type")]
#[serde(rename_all = "snake_case")]
enum SystemControl {
    // First message upon successful connection establishment
    Hello,

    // Client should reconnect after getting such message
    Goodbye,

    // Reply to Ping = { type: ping, id: num }
    Pong { reply_to: usize },
}

#[derive(Debug, Deserialize)]
#[serde(untagged)]
enum MessageControl {
    // Reply to message sent
    MessageSent {
        ok: bool,
        reply_to: usize,
        text: String,
        ts: String
    },

    // Reply to failed message sent
    MessageError {
        ok: bool,
        reply_to: usize,
        error: ErrorDetail,
    },
}

#[derive(Debug, Deserialize)]
struct ErrorDetail {
    code: usize,
    msg: String,
}

fn parse_event(s: String) -> Option<Event> {
    // Return User Events first, then the other two
    serde_json::from_str(&s).and_then(|ue| {
        println!("Inbound - \tUser Event = {:?}", ue);
        Ok(Event::UserEvent(ue))
    }).or_else(|_| {
        serde_json::from_str(&s).and_then(|sc| {
            println!("Inbound - \tSystem Control = {:?}", sc);
            Ok(Event::SystemControl(sc))
        }).or_else(|_| {
            serde_json::from_str(&s).and_then(|mc| {
                println!("Inbound - \tMessage Control = {:?}", mc);
                Ok(Event::MessageControl(mc))
            }).or_else(|x| {
                // TODO: for now print to stderr what didn't get deserialized
                // later can have config option to log to file the error or not
                //eprintln!("Inbound - \tFail parse = {:?}", s);
                Err(x)
            })
        })
    }).ok()
}

pub async fn send_simple_message(
    msg_id: MsgId,
    tx: &mut mpsc::Sender<tungstenite::tungstenite::Message>,
    channel: String,
    thread_ts: Option<String>,
    text: String
) -> Result<(), &'static str> {
    // TODO: track if it got santized or not
    let text = santizer::santize_output(&text);

    // TODO: register this message send to be tracked later
    let ws_msg = match thread_ts {
        Some(ts) => json!({
            "id": msg_id.inc(),
            "type": "message",
            "channel": channel,
            "text": text,
            "thread_ts": ts,
        }).to_string(),
        None => json!({
            "id": msg_id.inc(),
            "type": "message",
            "channel": channel,
            "text": text,
        }).to_string(),
    };
    tx.send(tungstenite::tungstenite::Message::from(ws_msg)).await.map_err(|_| "Error sending")
}

pub async fn send_slack_ping(
    msg_id: MsgId,
    tx: &mut mpsc::Sender<tungstenite::tungstenite::Message>,
    last_ping_sent: Arc<RwLock<Instant>>,
) -> Result<(), &'static str> {
    {
        let mut timer = last_ping_sent.write().unwrap();
        *timer = Instant::now();
    }

    // TODO: optionally include timestamp for the pong func to track the time delta later
    let ws_msg = json!({
	"id": msg_id.inc(),
	"type": "ping",
    }).to_string();
    tx.send(tungstenite::tungstenite::Message::from(ws_msg)).await.map_err(|_| "Error sending")
}

pub async fn send_query(
    sql_tx: &mut mpsc::Sender<(RunQuery, Option<oneshot::Sender<ResQuery>>)>,
    query: RunQuery,
) -> Result<ResQuery, &'static str> {
    let (tx, rx) = oneshot::channel();
    sql_tx.send((query, Some(tx))).await.map_err(|_| "Error sending")?;
    rx.await.map_err(|_| "Error recieving")
}

pub async fn process_control_message(
    mut tx: mpsc::Sender<tungstenite::tungstenite::Message>,
    can_send: Arc<AtomicBool>,
    reconnect: Arc<AtomicBool>,
    reconnect_count: Arc<RelaxedCounter>,
    last_message_recieved: Arc<RwLock<Instant>>,
    msg: tungstenite::tungstenite::Message,
) -> Result<Option<UserEvent>, Box<dyn std::error::Error>>
{
    // Parse incoming message
    let raw_msg = match msg {
        tungstenite::tungstenite::Message::Text(x) => {
            {
                let mut timer = last_message_recieved.write().unwrap();
                *timer = Instant::now();
            }
            Some(x)
        },

        tungstenite::tungstenite::Message::Ping(x) => {
            // We got a ping, send a websocket pong
            println!("Inbound - Ping");
            let _ = tx.send(tungstenite::tungstenite::Message::Pong(x)).await;
            None
        },

        tungstenite::tungstenite::Message::Close(reason) => {
            // We got a Close, reconnect
            println!("Inbound - Close: {:?}", reason);

            reconnect.store(true, Ordering::Relaxed);
            can_send.store(false, Ordering::Relaxed);
            None
        },

        _ => {
            println!("Inbound - \tUnsupported ws message type - {:?}", msg);
            None
        },
    };
    //println!("Inbound - raw event = {:?}", raw_msg);

    if let Some(e) = raw_msg.and_then(parse_event) {
        match e {
            Event::UserEvent(event)    => Ok(Some(event)),
            Event::MessageControl(_mc) => Ok(None),
            Event::SystemControl(sc)   => {
                match sc {
                    SystemControl::Hello => {
                        // Hold on sending messages till this is recieved.
                        can_send.store(true, Ordering::Relaxed);
                        reconnect_count.reset();
                    },
                    SystemControl::Goodbye => {
                        // When this is recieved, reconnect
                        reconnect.store(true, Ordering::Relaxed);
                        can_send.store(false, Ordering::Relaxed);
                    },
                    SystemControl::Pong{..} => {
                        // TODO: Employ this to keep the connection alive/inspect latency
                        // For now we should be ok with just sending out pings
                    },
                }
                Ok(None)
            },
        }
    } else {
        Ok(None)
    }
}
<commit_msg>Track ping-pong delta in ms<commit_after>use tokio_tungstenite as tungstenite;

use atomic_counter::AtomicCounter;
use atomic_counter::RelaxedCounter;

use serde::Deserialize;
use serde_json::json;

use std::result::Result;
use std::sync::Arc;
use std::sync::atomic::AtomicBool;
use std::sync::atomic::Ordering;
use std::sync::RwLock;
use std::time::Instant;
use chrono::prelude::Utc;

use tokio::sync::mpsc;
use tokio::sync::oneshot;

use crate::database::{RunQuery, ResQuery};
use crate::parser::santizer;

// Type alias for msg_id
pub type MsgId = Arc<RelaxedCounter>;

#[derive(Debug)]
enum Event {
    UserEvent(UserEvent),
    SystemControl(SystemControl),
    MessageControl(MessageControl),
}

#[derive(Debug, Deserialize)]
#[serde(tag = "type")]
#[serde(rename_all = "snake_case")]
pub enum UserEvent {
    // These events are what plugins will care for:
    //
    // Slack messages
    // TODO: more involved here for now basics
    Message {
        subtype: Option<String>,
        hidden: Option<bool>,
        #[serde(rename = "channel")]
        channel_id: Option<String>,
        #[serde(rename = "user")]
        user_id: String,
        text: String,
        ts: String,
        thread_ts: Option<String>,
    },

    // TODO: consider looking at slack_api for types to reuse here
    ReactionAdded {
        #[serde(rename = "user")]
        user_id: String,
        reaction: String,
        item_user: Option<String>,
        // item: ListResponseItem
        event_ts: String,
    },
    ReactionRemoved {
        #[serde(rename = "user")]
        user_id: String,
        reaction: String,
        item_user: Option<String>,
        // item: ListResponseItem
        event_ts: String,
    },
}

#[derive(Debug, Deserialize)]
#[serde(tag = "type")]
#[serde(rename_all = "snake_case")]
enum SystemControl {
    // First message upon successful connection establishment
    Hello,

    // Client should reconnect after getting such message
    Goodbye,

    // Reply to Ping = { type: ping, id: num }
    Pong { reply_to: usize, timestamp: Option<i64> },
}

#[derive(Debug, Deserialize)]
#[serde(untagged)]
enum MessageControl {
    // Reply to message sent
    MessageSent {
        ok: bool,
        reply_to: usize,
        text: String,
        ts: String
    },

    // Reply to failed message sent
    MessageError {
        ok: bool,
        reply_to: usize,
        error: ErrorDetail,
    },
}

#[derive(Debug, Deserialize)]
struct ErrorDetail {
    code: usize,
    msg: String,
}

fn parse_event(s: String) -> Option<Event> {
    // Return User Events first, then the other two
    serde_json::from_str(&s).and_then(|ue| {
        println!("Inbound - \tUser Event = {:?}", ue);
        Ok(Event::UserEvent(ue))
    }).or_else(|_| {
        serde_json::from_str(&s).and_then(|sc| {
            println!("Inbound - \tSystem Control = {:?}", sc);
            Ok(Event::SystemControl(sc))
        }).or_else(|_| {
            serde_json::from_str(&s).and_then(|mc| {
                println!("Inbound - \tMessage Control = {:?}", mc);
                Ok(Event::MessageControl(mc))
            }).or_else(|x| {
                // TODO: for now print to stderr what didn't get deserialized
                // later can have config option to log to file the error or not
                //eprintln!("Inbound - \tFail parse = {:?}", s);
                Err(x)
            })
        })
    }).ok()
}

pub async fn send_simple_message(
    msg_id: MsgId,
    tx: &mut mpsc::Sender<tungstenite::tungstenite::Message>,
    channel: String,
    thread_ts: Option<String>,
    text: String
) -> Result<(), &'static str> {
    // TODO: track if it got santized or not
    let text = santizer::santize_output(&text);

    // TODO: register this message send to be tracked later
    let ws_msg = match thread_ts {
        Some(ts) => json!({
            "id": msg_id.inc(),
            "type": "message",
            "channel": channel,
            "text": text,
            "thread_ts": ts,
        }).to_string(),
        None => json!({
            "id": msg_id.inc(),
            "type": "message",
            "channel": channel,
            "text": text,
        }).to_string(),
    };
    tx.send(tungstenite::tungstenite::Message::from(ws_msg)).await.map_err(|_| "Error sending")
}

pub async fn send_slack_ping(
    msg_id: MsgId,
    tx: &mut mpsc::Sender<tungstenite::tungstenite::Message>,
    last_ping_sent: Arc<RwLock<Instant>>,
) -> Result<(), &'static str> {
    {
        let mut timer = last_ping_sent.write().unwrap();
        *timer = Instant::now();
    }
    let ws_msg = json!({
	"id": msg_id.inc(),
	"type": "ping",
        "timestamp": Utc::now().timestamp_millis(),
    }).to_string();
    tx.send(tungstenite::tungstenite::Message::from(ws_msg)).await.map_err(|_| "Error sending")
}

pub async fn send_query(
    sql_tx: &mut mpsc::Sender<(RunQuery, Option<oneshot::Sender<ResQuery>>)>,
    query: RunQuery,
) -> Result<ResQuery, &'static str> {
    let (tx, rx) = oneshot::channel();
    sql_tx.send((query, Some(tx))).await.map_err(|_| "Error sending")?;
    rx.await.map_err(|_| "Error recieving")
}

pub async fn process_control_message(
    mut tx: mpsc::Sender<tungstenite::tungstenite::Message>,
    can_send: Arc<AtomicBool>,
    reconnect: Arc<AtomicBool>,
    reconnect_count: Arc<RelaxedCounter>,
    last_message_recieved: Arc<RwLock<Instant>>,
    msg: tungstenite::tungstenite::Message,
) -> Result<Option<UserEvent>, Box<dyn std::error::Error>>
{
    // Parse incoming message
    let raw_msg = match msg {
        tungstenite::tungstenite::Message::Text(x) => {
            {
                let mut timer = last_message_recieved.write().unwrap();
                *timer = Instant::now();
            }
            Some(x)
        },

        tungstenite::tungstenite::Message::Ping(x) => {
            // We got a ping, send a websocket pong
            println!("Inbound - Ping");
            let _ = tx.send(tungstenite::tungstenite::Message::Pong(x)).await;
            None
        },

        tungstenite::tungstenite::Message::Close(reason) => {
            // We got a Close, reconnect
            println!("Inbound - Close: {:?}", reason);

            reconnect.store(true, Ordering::Relaxed);
            can_send.store(false, Ordering::Relaxed);
            None
        },

        _ => {
            println!("Inbound - \tUnsupported ws message type - {:?}", msg);
            None
        },
    };
    //println!("Inbound - raw event = {:?}", raw_msg);

    if let Some(e) = raw_msg.and_then(parse_event) {
        match e {
            Event::UserEvent(event)    => Ok(Some(event)),
            Event::MessageControl(_mc) => Ok(None),
            Event::SystemControl(sc)   => {
                match sc {
                    SystemControl::Hello => {
                        // Hold on sending messages till this is recieved.
                        can_send.store(true, Ordering::Relaxed);
                        reconnect_count.reset();
                    },
                    SystemControl::Goodbye => {
                        // When this is recieved, reconnect
                        reconnect.store(true, Ordering::Relaxed);
                        can_send.store(false, Ordering::Relaxed);
                    },
                    SystemControl::Pong{reply_to: _, timestamp: ts} => {
                        if let Some(old) = ts {
                            let now = Utc::now().timestamp_millis();
                            println!("Pong - Delta: {:?}ms", now - old);
                        }
                    },
                }
                Ok(None)
            },
        }
    } else {
        Ok(None)
    }
}
<|endoftext|>


<commit_before>// Copyright (C) 2017, 2018 O.S. Systems Sofware LTDA
//
// SPDX-License-Identifier: MPL-2.0
//

use core::ops::Index;
use std::collections::btree_map::{Entry, Keys};
use std::collections::BTreeMap;
use std::io;
use std::str::FromStr;

#[derive(Debug, Serialize, PartialEq, Default)]
pub struct MetadataValue(BTreeMap<String, Vec<String>>);

impl FromStr for MetadataValue {
    type Err = io::Error;

    fn from_str(s: &str) -> Result<MetadataValue, io::Error> {
        let mut values = Vec::new();
        for line in s.lines() {
            let v: Vec<_> = line.splitn(2, '=').map(|v| v.trim().to_string()).collect();
            if v.len() != 2 {
                return Err(io::Error::new(
                    io::ErrorKind::InvalidInput,
                    format!(
                        "Invalid format for value '{:?}'. \
                         An <key>=<value> output is \
                         expected",
                        v
                    ),
                ));
            }

            values.push((v[0].clone(), v[1].clone()));
        }
        values.sort();

        let mut mv = MetadataValue::default();
        for (k, v) in values {
            // replace to use .and_modify once #44733 is closed. Code
            // is below:
            //   mv.entry(k)
            //     .and_modify(|e| e.push(v.clone()))
            //     .or_insert(vec![v]);
            match mv.entry(k) {
                Entry::Occupied(entry) => {
                    entry.into_mut().push(v.clone());
                }
                Entry::Vacant(entry) => {
                    entry.insert(vec![v]);
                }
            }
        }

        Ok(mv)
    }
}

impl MetadataValue {
    pub fn entry(&mut self, key: String) -> Entry<String, Vec<String>> {
        self.0.entry(key)
    }

    pub fn keys(&self) -> Keys<String, Vec<String>> {
        self.0.keys()
    }

    pub fn is_empty(&self) -> bool {
        self.0.len() == 0
    }

    pub fn len(&self) -> usize {
        self.0.len()
    }
}

impl<'a> Index<&'a str> for MetadataValue {
    type Output = Vec<String>;

    #[inline]
    fn index(&self, key: &str) -> &Vec<String> {
        self.0.get(key).expect("no entry found for key")
    }
}

#[test]
fn valid() {
    let v = MetadataValue::from_str("key1=v1\nkey=b\nnv=\nkey=a").unwrap();

    assert_eq!(v.keys().len(), 3);
    assert_eq!(v.keys().collect::<Vec<_>>(), ["key", "key1", "nv"]);
    assert_eq!(v["key1"], ["v1"]);
    assert_eq!(v["key"], ["a", "b"]);
    assert_eq!(v["nv"], [""]);
}

#[test]
fn invalid() {
    assert!(MetadataValue::from_str("\n").is_err());
    assert!(MetadataValue::from_str("key").is_err());
}
<commit_msg>firmware: replace to use .and_modify<commit_after>// Copyright (C) 2017, 2018 O.S. Systems Sofware LTDA
//
// SPDX-License-Identifier: MPL-2.0
//

use core::ops::Index;
use std::collections::btree_map::{Entry, Keys};
use std::collections::BTreeMap;
use std::io;
use std::str::FromStr;

#[derive(Debug, Serialize, PartialEq, Default)]
pub struct MetadataValue(BTreeMap<String, Vec<String>>);

impl FromStr for MetadataValue {
    type Err = io::Error;

    fn from_str(s: &str) -> Result<MetadataValue, io::Error> {
        let mut values = Vec::new();
        for line in s.lines() {
            let v: Vec<_> = line.splitn(2, '=').map(|v| v.trim().to_string()).collect();
            if v.len() != 2 {
                return Err(io::Error::new(
                    io::ErrorKind::InvalidInput,
                    format!(
                        "Invalid format for value '{:?}'. \
                         An <key>=<value> output is \
                         expected",
                        v
                    ),
                ));
            }

            values.push((v[0].clone(), v[1].clone()));
        }
        values.sort();

        let mut mv = MetadataValue::default();
        for (k, v) in values {
            mv.entry(k)
                .and_modify(|e| e.push(v.clone()))
                .or_insert(vec![v]);
        }

        Ok(mv)
    }
}

impl MetadataValue {
    pub fn entry(&mut self, key: String) -> Entry<String, Vec<String>> {
        self.0.entry(key)
    }

    pub fn keys(&self) -> Keys<String, Vec<String>> {
        self.0.keys()
    }

    pub fn is_empty(&self) -> bool {
        self.0.len() == 0
    }

    pub fn len(&self) -> usize {
        self.0.len()
    }
}

impl<'a> Index<&'a str> for MetadataValue {
    type Output = Vec<String>;

    #[inline]
    fn index(&self, key: &str) -> &Vec<String> {
        self.0.get(key).expect("no entry found for key")
    }
}

#[test]
fn valid() {
    let v = MetadataValue::from_str("key1=v1\nkey=b\nnv=\nkey=a").unwrap();

    assert_eq!(v.keys().len(), 3);
    assert_eq!(v.keys().collect::<Vec<_>>(), ["key", "key1", "nv"]);
    assert_eq!(v["key1"], ["v1"]);
    assert_eq!(v["key"], ["a", "b"]);
    assert_eq!(v["nv"], [""]);
}

#[test]
fn invalid() {
    assert!(MetadataValue::from_str("\n").is_err());
    assert!(MetadataValue::from_str("key").is_err());
}
<|endoftext|>