<commit_before>use byteorder::{LittleEndian, ReadBytesExt};
use internal::prelude::*;
use serde_json;
use std::{
    ffi::OsStr,
    fs::File,
    io::{
        BufReader,
        ErrorKind as IoErrorKind,
        Read,
        Result as IoResult
    },
    process::{
        Child,
        Command,
        Stdio
    },
    result::Result as StdResult
};
use super::{
    AudioSource,
    AudioType,
    DcaError,
    DcaMetadata,
    VoiceError
};

struct ChildContainer(Child);

impl Read for ChildContainer {
    fn read(&mut self, buffer: &mut [u8]) -> IoResult<usize> {
        self.0.stdout.as_mut().unwrap().read(buffer)
    }
}

impl Drop for ChildContainer {
    fn drop (&mut self) {
        if let Err(e) = self.0.kill() {
            debug!("[Voice] Error awaiting child process: {:?}", e);
        }
    }
}

struct InputSource<R: Read + Send + 'static> {
    stereo: bool,
    reader: R,
    kind: AudioType,
}

impl<R: Read + Send> AudioSource for InputSource<R> {
    fn is_stereo(&mut self) -> bool { self.stereo }

    fn get_type(&self) -> AudioType { self.kind }

    fn read_pcm_frame(&mut self, buffer: &mut [i16]) -> Option<usize> {
        for (i, v) in buffer.iter_mut().enumerate() {
            *v = match self.reader.read_i16::<LittleEndian>() {
                Ok(v) => v,
                Err(ref e) => {
                    return if e.kind() == IoErrorKind::UnexpectedEof {
                        Some(i)
                    } else {
                        None
                    }
                },
            }
        }

        Some(buffer.len())
    }

    fn read_opus_frame(&mut self) -> Option<Vec<u8>> {
        match self.reader.read_i16::<LittleEndian>() {
            Ok(size) => {
                if size <= 0 {
                    warn!("Invalid opus frame size: {}", size);
                    return None;
                }

                let mut frame = Vec::with_capacity(size as usize);

                {
                    let reader = self.reader.by_ref();

                    if reader.take(size as u64).read_to_end(&mut frame).is_err() {
                        return None;
                    }
                }

                Some(frame)
            },
            Err(ref e) => if e.kind() == IoErrorKind::UnexpectedEof {
                Some(Vec::new())
            } else {
                None
            },
        }
    }
}

/// Opens an audio file through `ffmpeg` and creates an audio source.
pub fn ffmpeg<P: AsRef<OsStr>>(path: P) -> Result<Box<AudioSource>> {
    _ffmpeg(path.as_ref())
}

fn _ffmpeg(path: &OsStr) -> Result<Box<AudioSource>> {
    // Will fail if the path is not to a file on the fs. Likely a YouTube URI.
    let is_stereo = is_stereo(path).unwrap_or(false);
    let stereo_val = if is_stereo { "2" } else { "1" };

    ffmpeg_optioned(path, &[
        "-f",
        "s16le",
        "-ac",
        stereo_val,
        "-ar",
        "48000",
        "-acodec",
        "pcm_s16le",
        "-",
    ])
}

/// Opens an audio file through `ffmpeg` and creates an audio source, with
/// user-specified arguments to pass to ffmpeg.
///
/// Note that this does _not_ build on the arguments passed by the [`ffmpeg`]
/// function.
///
/// # Examples
///
/// Pass options to create a custom ffmpeg streamer:
///
/// ```rust,no_run
/// use serenity::voice;
///
/// let stereo_val = "2";
///
/// let streamer = voice::ffmpeg_optioned("./some_file.mp3", &[
///     "-f",
///     "s16le",
///     "-ac",
///     stereo_val,
///     "-ar",
///     "48000",
///     "-acodec",
///     "pcm_s16le",
///     "-",
/// ]);
pub fn ffmpeg_optioned<P: AsRef<OsStr>>(
    path: P,
    args: &[&str],
) -> Result<Box<AudioSource>> {
    _ffmpeg_optioned(path.as_ref(), args)
}

fn _ffmpeg_optioned(path: &OsStr, args: &[&str]) -> Result<Box<AudioSource>> {
    let is_stereo = is_stereo(path).unwrap_or(false);

    let command = Command::new("ffmpeg")
        .arg("-i")
        .arg(path)
        .args(args)
        .stderr(Stdio::null())
        .stdin(Stdio::null())
        .stdout(Stdio::piped())
        .spawn()?;

    Ok(pcm(is_stereo, ChildContainer(command)))
}

/// Creates a streamed audio source from a DCA file.
/// Currently only accepts the DCA1 format.
pub fn dca<P: AsRef<OsStr>>(path: P) -> StdResult<Box<AudioSource>, DcaError> {
    _dca(path.as_ref())
}

fn _dca(path: &OsStr) -> StdResult<Box<AudioSource>, DcaError> {
    let file = File::open(path).map_err(DcaError::IoError)?;

    let mut reader = BufReader::new(file);

    let mut header = [0u8; 4];

    // Read in the magic number to verify it's a DCA file.
    reader.read_exact(&mut header).map_err(DcaError::IoError)?;

    if header != b"DCA1"[..] {
        return Err(DcaError::InvalidHeader);
    }

    reader.read_exact(&mut header).map_err(DcaError::IoError)?;

    let size = (&header[..]).read_i32::<LittleEndian>().unwrap();

    // Sanity check
    if size < 2 {
        return Err(DcaError::InvalidSize(size));
    }

    let mut raw_json = Vec::with_capacity(size as usize);

    {
        let json_reader = reader.by_ref();
        json_reader
            .take(size as u64)
            .read_to_end(&mut raw_json)
            .map_err(DcaError::IoError)?;
    }

    let metadata = serde_json::from_slice::<DcaMetadata>(raw_json.as_slice())
        .map_err(DcaError::InvalidMetadata)?;

    Ok(opus(metadata.is_stereo(), reader))
}

/// Creates an Opus audio source. This makes certain assumptions: namely, that the input stream
/// is composed ONLY of opus frames of the variety that Discord expects.
///
/// If you want to decode a `.opus` file, use [`ffmpeg`]
///
/// [`ffmpeg`]: fn.ffmpeg.html
pub fn opus<R: Read + Send + 'static>(is_stereo: bool, reader: R) -> Box<AudioSource> {
    Box::new(InputSource {
        stereo: is_stereo,
        reader,
        kind: AudioType::Opus,
    })
}

/// Creates a PCM audio source.
pub fn pcm<R: Read + Send + 'static>(is_stereo: bool, reader: R) -> Box<AudioSource> {
    Box::new(InputSource {
        stereo: is_stereo,
        reader,
        kind: AudioType::Pcm,
    })
}

/// Creates a streamed audio source with `youtube-dl` and `ffmpeg`.
pub fn ytdl(uri: &str) -> Result<Box<AudioSource>> {
    let args = [
        "-f",
        "webm[abr>0]/bestaudio/best",
        "--no-playlist",
        "--print-json",
        "--skip-download",
        uri,
    ];

    let out = Command::new("youtube-dl")
        .args(&args)
        .stdin(Stdio::null())
        .output()?;

    if !out.status.success() {
        return Err(Error::Voice(VoiceError::YouTubeDLRun(out)));
    }

    let value = serde_json::from_reader(&out.stdout[..])?;
    let mut obj = match value {
        Value::Object(obj) => obj,
        other => return Err(Error::Voice(VoiceError::YouTubeDLProcessing(other))),
    };

    let uri = match obj.remove("url") {
        Some(v) => match v {
            Value::String(uri) => uri,
            other => return Err(Error::Voice(VoiceError::YouTubeDLUrl(other))),
        },
        None => return Err(Error::Voice(VoiceError::YouTubeDLUrl(Value::Object(obj)))),
    };

    ffmpeg(&uri)
}

fn is_stereo(path: &OsStr) -> Result<bool> {
    let args = ["-v", "quiet", "-of", "json", "-show-streams", "-i"];

    let out = Command::new("ffprobe")
        .args(&args)
        .arg(path)
        .stdin(Stdio::null())
        .output()?;

    let value: Value = serde_json::from_reader(&out.stdout[..])?;

    let streams = value
        .as_object()
        .and_then(|m| m.get("streams"))
        .and_then(|v| v.as_array())
        .ok_or(Error::Voice(VoiceError::Streams))?;

    let check = streams.iter().any(|stream| {
        let channels = stream
            .as_object()
            .and_then(|m| m.get("channels").and_then(|v| v.as_i64()));

        channels == Some(2)
    });

    Ok(check)
}
<commit_msg>Fewer ffprobe calls when playing audio through ffmpeg.<commit_after>use byteorder::{LittleEndian, ReadBytesExt};
use internal::prelude::*;
use serde_json;
use std::{
    ffi::OsStr,
    fs::File,
    io::{
        BufReader,
        ErrorKind as IoErrorKind,
        Read,
        Result as IoResult
    },
    process::{
        Child,
        Command,
        Stdio
    },
    result::Result as StdResult
};
use super::{
    AudioSource,
    AudioType,
    DcaError,
    DcaMetadata,
    VoiceError
};

struct ChildContainer(Child);

impl Read for ChildContainer {
    fn read(&mut self, buffer: &mut [u8]) -> IoResult<usize> {
        self.0.stdout.as_mut().unwrap().read(buffer)
    }
}

impl Drop for ChildContainer {
    fn drop (&mut self) {
        if let Err(e) = self.0.kill() {
            debug!("[Voice] Error awaiting child process: {:?}", e);
        }
    }
}

struct InputSource<R: Read + Send + 'static> {
    stereo: bool,
    reader: R,
    kind: AudioType,
}

impl<R: Read + Send> AudioSource for InputSource<R> {
    fn is_stereo(&mut self) -> bool { self.stereo }

    fn get_type(&self) -> AudioType { self.kind }

    fn read_pcm_frame(&mut self, buffer: &mut [i16]) -> Option<usize> {
        for (i, v) in buffer.iter_mut().enumerate() {
            *v = match self.reader.read_i16::<LittleEndian>() {
                Ok(v) => v,
                Err(ref e) => {
                    return if e.kind() == IoErrorKind::UnexpectedEof {
                        Some(i)
                    } else {
                        None
                    }
                },
            }
        }

        Some(buffer.len())
    }

    fn read_opus_frame(&mut self) -> Option<Vec<u8>> {
        match self.reader.read_i16::<LittleEndian>() {
            Ok(size) => {
                if size <= 0 {
                    warn!("Invalid opus frame size: {}", size);
                    return None;
                }

                let mut frame = Vec::with_capacity(size as usize);

                {
                    let reader = self.reader.by_ref();

                    if reader.take(size as u64).read_to_end(&mut frame).is_err() {
                        return None;
                    }
                }

                Some(frame)
            },
            Err(ref e) => if e.kind() == IoErrorKind::UnexpectedEof {
                Some(Vec::new())
            } else {
                None
            },
        }
    }
}

/// Opens an audio file through `ffmpeg` and creates an audio source.
pub fn ffmpeg<P: AsRef<OsStr>>(path: P) -> Result<Box<AudioSource>> {
    _ffmpeg(path.as_ref())
}

fn _ffmpeg(path: &OsStr) -> Result<Box<AudioSource>> {
    // Will fail if the path is not to a file on the fs. Likely a YouTube URI.
    let is_stereo = is_stereo(path).unwrap_or(false);
    let stereo_val = if is_stereo { "2" } else { "1" };

    _ffmpeg_optioned(path, &[
        "-f",
        "s16le",
        "-ac",
        stereo_val,
        "-ar",
        "48000",
        "-acodec",
        "pcm_s16le",
        "-",
    ], Some(is_stereo))
}

/// Opens an audio file through `ffmpeg` and creates an audio source, with
/// user-specified arguments to pass to ffmpeg.
///
/// Note that this does _not_ build on the arguments passed by the [`ffmpeg`]
/// function.
///
/// # Examples
///
/// Pass options to create a custom ffmpeg streamer:
///
/// ```rust,no_run
/// use serenity::voice;
///
/// let stereo_val = "2";
///
/// let streamer = voice::ffmpeg_optioned("./some_file.mp3", &[
///     "-f",
///     "s16le",
///     "-ac",
///     stereo_val,
///     "-ar",
///     "48000",
///     "-acodec",
///     "pcm_s16le",
///     "-",
/// ]);
pub fn ffmpeg_optioned<P: AsRef<OsStr>>(
    path: P,
    args: &[&str],
) -> Result<Box<AudioSource>> {
    _ffmpeg_optioned(path.as_ref(), args, None)
}

fn _ffmpeg_optioned(path: &OsStr, args: &[&str], is_stereo_known: Option<bool>) -> Result<Box<AudioSource>> {
    let is_stereo = is_stereo_known
        .or_else(|| is_stereo(path).ok())
        .unwrap_or(false);

    let command = Command::new("ffmpeg")
        .arg("-i")
        .arg(path)
        .args(args)
        .stderr(Stdio::null())
        .stdin(Stdio::null())
        .stdout(Stdio::piped())
        .spawn()?;

    Ok(pcm(is_stereo, ChildContainer(command)))
}

/// Creates a streamed audio source from a DCA file.
/// Currently only accepts the DCA1 format.
pub fn dca<P: AsRef<OsStr>>(path: P) -> StdResult<Box<AudioSource>, DcaError> {
    _dca(path.as_ref())
}

fn _dca(path: &OsStr) -> StdResult<Box<AudioSource>, DcaError> {
    let file = File::open(path).map_err(DcaError::IoError)?;

    let mut reader = BufReader::new(file);

    let mut header = [0u8; 4];

    // Read in the magic number to verify it's a DCA file.
    reader.read_exact(&mut header).map_err(DcaError::IoError)?;

    if header != b"DCA1"[..] {
        return Err(DcaError::InvalidHeader);
    }

    reader.read_exact(&mut header).map_err(DcaError::IoError)?;

    let size = (&header[..]).read_i32::<LittleEndian>().unwrap();

    // Sanity check
    if size < 2 {
        return Err(DcaError::InvalidSize(size));
    }

    let mut raw_json = Vec::with_capacity(size as usize);

    {
        let json_reader = reader.by_ref();
        json_reader
            .take(size as u64)
            .read_to_end(&mut raw_json)
            .map_err(DcaError::IoError)?;
    }

    let metadata = serde_json::from_slice::<DcaMetadata>(raw_json.as_slice())
        .map_err(DcaError::InvalidMetadata)?;

    Ok(opus(metadata.is_stereo(), reader))
}

/// Creates an Opus audio source. This makes certain assumptions: namely, that the input stream
/// is composed ONLY of opus frames of the variety that Discord expects.
///
/// If you want to decode a `.opus` file, use [`ffmpeg`]
///
/// [`ffmpeg`]: fn.ffmpeg.html
pub fn opus<R: Read + Send + 'static>(is_stereo: bool, reader: R) -> Box<AudioSource> {
    Box::new(InputSource {
        stereo: is_stereo,
        reader,
        kind: AudioType::Opus,
    })
}

/// Creates a PCM audio source.
pub fn pcm<R: Read + Send + 'static>(is_stereo: bool, reader: R) -> Box<AudioSource> {
    Box::new(InputSource {
        stereo: is_stereo,
        reader,
        kind: AudioType::Pcm,
    })
}

/// Creates a streamed audio source with `youtube-dl` and `ffmpeg`.
pub fn ytdl(uri: &str) -> Result<Box<AudioSource>> {
    let args = [
        "-f",
        "webm[abr>0]/bestaudio/best",
        "--no-playlist",
        "--print-json",
        "--skip-download",
        uri,
    ];

    let out = Command::new("youtube-dl")
        .args(&args)
        .stdin(Stdio::null())
        .output()?;

    if !out.status.success() {
        return Err(Error::Voice(VoiceError::YouTubeDLRun(out)));
    }

    let value = serde_json::from_reader(&out.stdout[..])?;
    let mut obj = match value {
        Value::Object(obj) => obj,
        other => return Err(Error::Voice(VoiceError::YouTubeDLProcessing(other))),
    };

    let uri = match obj.remove("url") {
        Some(v) => match v {
            Value::String(uri) => uri,
            other => return Err(Error::Voice(VoiceError::YouTubeDLUrl(other))),
        },
        None => return Err(Error::Voice(VoiceError::YouTubeDLUrl(Value::Object(obj)))),
    };

    ffmpeg(&uri)
}

fn is_stereo(path: &OsStr) -> Result<bool> {
    let args = ["-v", "quiet", "-of", "json", "-show-streams", "-i"];

    let out = Command::new("ffprobe")
        .args(&args)
        .arg(path)
        .stdin(Stdio::null())
        .output()?;

    let value: Value = serde_json::from_reader(&out.stdout[..])?;

    let streams = value
        .as_object()
        .and_then(|m| m.get("streams"))
        .and_then(|v| v.as_array())
        .ok_or(Error::Voice(VoiceError::Streams))?;

    let check = streams.iter().any(|stream| {
        let channels = stream
            .as_object()
            .and_then(|m| m.get("channels").and_then(|v| v.as_i64()));

        channels == Some(2)
    });

    Ok(check)
}
<|endoftext|>


<commit_before>#![macro_escape]

use libc::c_ulonglong;

#[doc(hidden)]
pub fn marshal<T>(buf: &[u8],
                  padbefore: uint,
                  bytestodrop: uint,
                  f: |*mut u8, *const u8, c_ulonglong| -> T
                 ) -> (Vec<u8>, T) {
    let mut dst = Vec::with_capacity(buf.len() + padbefore);
    for _ in range(0, padbefore) {
        dst.push(0);
    }
    dst.push_all(buf);
    let pdst = dst.as_mut_ptr();
    let psrc = dst.as_ptr();
    let res = f(pdst, psrc, dst.len() as c_ulonglong);
    (dst.into_iter().skip(bytestodrop).collect(), res)
}

macro_rules! newtype_clone (($newtype:ident) => (
        impl Clone for $newtype {
            fn clone(&self) -> $newtype {
                let &$newtype(v) = self;
                $newtype(v)
            }
        }

        ));

macro_rules! newtype_drop (($newtype:ident) => (
        impl Drop for $newtype {
            fn drop(&mut self) {
                let &$newtype(ref mut v) = self;
                unsafe {
                    volatile_set_memory(v.as_mut_ptr(), 0, v.len());
                }
            }
        }
        ));

macro_rules! newtype_impl (($newtype:ident, $len:expr) => (
    impl $newtype {
        /**
         * `from_slice()` creates an object from a byte slice
         *
         * This function will fail and return None if the length of
         * the byte-slice isn't equal to the length of the object
         */
        pub fn from_slice(bs: &[u8]) -> Option<$newtype> {
            if bs.len() != $len {
                return None
            }
            let mut n = $newtype([0; $len]);
            {
                let $newtype(ref mut b) = n;
                for (bi, &bsi) in b.iter_mut().zip(bs.iter()) {
                    *bi = bsi
                }
            }
            Some(n)
        }
        /**
         * `as_slice()` returns a byte slice containing the object contents
         *
         * WARNING: it might be tempting to do comparisons on objects by
         * using `x.as_slice() == y.as_slice()`. This will open up for
         * timing attacks when comparing for example authenticator
         * tags. Because of this only use the comparison functions
         * exposed by the sodiumoxide API.
         */
        pub fn as_slice(&self) -> &[u8] {
            let &$newtype(ref bs) = self;
            bs.as_slice()
        }
        /**
         * `as_mut_slice()` returns a mutable byte slice containing the object
         * contents
         *
         * WARNING: it might be tempting to do comparisons on objects by
         * using `x.as_mut_slice() == y.as_mut_slice()`. This will
         * open up for timing attacks when comparing for example
         * authenticator tags. Because of this only use the comparison
         * functions exposed by the sodiumoxide API.
         */
        pub fn as_mut_slice(&mut self) -> &mut [u8] {
            let &$newtype(ref mut bs) = self;
            bs.as_mut_slice()
        }
    }
    ));
<commit_msg>Updated to new closure syntax<commit_after>#![macro_escape]

use libc::c_ulonglong;

#[doc(hidden)]
pub fn marshal<T, F>(buf: &[u8],
                     padbefore: uint,
                     bytestodrop: uint,
                     f: F
                     ) -> (Vec<u8>, T)
    where F: Fn(*mut u8, *const u8, c_ulonglong) -> T {
    let mut dst = Vec::with_capacity(buf.len() + padbefore);
    for _ in range(0, padbefore) {
        dst.push(0);
    }
    dst.push_all(buf);
    let pdst = dst.as_mut_ptr();
    let psrc = dst.as_ptr();
    let res = f(pdst, psrc, dst.len() as c_ulonglong);
    (dst.into_iter().skip(bytestodrop).collect(), res)
}

macro_rules! newtype_clone (($newtype:ident) => (
        impl Clone for $newtype {
            fn clone(&self) -> $newtype {
                let &$newtype(v) = self;
                $newtype(v)
            }
        }

        ));

macro_rules! newtype_drop (($newtype:ident) => (
        impl Drop for $newtype {
            fn drop(&mut self) {
                let &$newtype(ref mut v) = self;
                unsafe {
                    volatile_set_memory(v.as_mut_ptr(), 0, v.len());
                }
            }
        }
        ));

macro_rules! newtype_impl (($newtype:ident, $len:expr) => (
    impl $newtype {
        /**
         * `from_slice()` creates an object from a byte slice
         *
         * This function will fail and return None if the length of
         * the byte-slice isn't equal to the length of the object
         */
        pub fn from_slice(bs: &[u8]) -> Option<$newtype> {
            if bs.len() != $len {
                return None
            }
            let mut n = $newtype([0; $len]);
            {
                let $newtype(ref mut b) = n;
                for (bi, &bsi) in b.iter_mut().zip(bs.iter()) {
                    *bi = bsi
                }
            }
            Some(n)
        }
        /**
         * `as_slice()` returns a byte slice containing the object contents
         *
         * WARNING: it might be tempting to do comparisons on objects by
         * using `x.as_slice() == y.as_slice()`. This will open up for
         * timing attacks when comparing for example authenticator
         * tags. Because of this only use the comparison functions
         * exposed by the sodiumoxide API.
         */
        pub fn as_slice(&self) -> &[u8] {
            let &$newtype(ref bs) = self;
            bs.as_slice()
        }
        /**
         * `as_mut_slice()` returns a mutable byte slice containing the object
         * contents
         *
         * WARNING: it might be tempting to do comparisons on objects by
         * using `x.as_mut_slice() == y.as_mut_slice()`. This will
         * open up for timing attacks when comparing for example
         * authenticator tags. Because of this only use the comparison
         * functions exposed by the sodiumoxide API.
         */
        pub fn as_mut_slice(&mut self) -> &mut [u8] {
            let &$newtype(ref mut bs) = self;
            bs.as_mut_slice()
        }
    }
    ));
<|endoftext|>


<commit_before>pub static WIDTH: uint = 64;
pub static REPWIDTH: uint = WIDTH / 8;
pub static HEIGHT: uint = 32;

/// CHIP-8 gliphs, see: mattmik.com/chip8.html
pub static GLYPHS: [u8,..16*5] =  [
    0xF0, 0x90, 0x90, 0x90, 0xF0, // 0
    0x20, 0x60, 0x20, 0x20, 0x70, // 1
    0xF0, 0x10, 0xF0, 0x80, 0xF0, // 2
    0xF0, 0x10, 0xF0, 0x10, 0xF0, // 3
    0x90, 0x90, 0xF0, 0x10, 0x10, // 4
    0xF0, 0x80, 0xF0, 0x10, 0xF0, // 5
    0xF0, 0x80, 0xF0, 0x90, 0xF0, // 6
    0xF0, 0x10, 0x20, 0x40, 0x40, // 7
    0xF0, 0x90, 0xF0, 0x90, 0xF0, // 8
    0xF0, 0x90, 0xF0, 0x10, 0xF0, // 9
    0xF0, 0x90, 0xF0, 0x90, 0x90, // A
    0xE0, 0x90, 0xE0, 0x90, 0xE0, // B
    0xF0, 0x80, 0x80, 0x80, 0xF0, // C
    0xE0, 0x90, 0x90, 0x90, 0xE0, // D
    0xF0, 0x80, 0xF0, 0x80, 0xF0, // E
    0xF0, 0x80, 0xF0, 0x80, 0x80  // F
];

pub struct Video {
    repr: [u8,..REPWIDTH*HEIGHT]
}

impl Video {
    pub fn new() -> Video {
        Video { repr: [0x00, ..REPWIDTH*HEIGHT] }
    }

    pub fn clear(&mut self) {
        for px in self.repr.mut_iter() {
            *px = 0x0;
        }
    }

    pub fn draw(&mut self, x: u8, y: u8, val: u8) -> u8 {
        if (x as uint) >= WIDTH || (y as uint) >= HEIGHT {
            return 0x1;
        }

        let i = x / 8 + y * REPWIDTH as u8;
        let shift = x % 8;

        if shift != 0 {
            let lval = val >> shift;
            let rval = val << (8 - shift);

            let lold = self.repr[i];
            self.repr[i] ^= lval;
            let rold = self.repr[i+1];
            self.repr[i+1] ^= rval;

            if flipped(lold, self.repr[i]) || flipped(rold, self.repr[i+1]) { 0x0 } else { 0x1 }
        }
        else {
            let old = self.repr[i];
            self.repr[i] ^= val;

            if flipped(old, self.repr[i]) { 0x0 } else { 0x1 }
        }
    }
}

/// Returns true if any of the bits have been fliped from set to unset
fn flipped(v1: u8, v2: u8) -> bool {
    v1 & !v2 != 0x0
}

#[test]
fn test_flipped() {
    assert!(flipped(0b_0000_0000, 0b_0001_0000) == false);
    assert!(flipped(0b_0001_0000, 0b_0000_0000) == true);
    assert!(flipped(0b_1010_0101, 0b_1111_1111) == false);
    assert!(flipped(0b_1010_0101, 0b_0000_0000) == true);
    assert!(flipped(0b_1010_0101, 0b_1010_0100) == true);
    assert!(flipped(0b_1111_0000, 0b_1111_1111) == false);
    assert!(flipped(0b_1111_0000, 0b_0000_1111) == true);
}
<commit_msg>Mini-fix<commit_after>pub static WIDTH: uint = 64;
pub static REPWIDTH: uint = WIDTH / 8;
pub static HEIGHT: uint = 32;

/// CHIP-8 gliphs, see: mattmik.com/chip8.html
pub static GLYPHS: [u8,..16*5] =  [
    0xF0, 0x90, 0x90, 0x90, 0xF0, // 0
    0x20, 0x60, 0x20, 0x20, 0x70, // 1
    0xF0, 0x10, 0xF0, 0x80, 0xF0, // 2
    0xF0, 0x10, 0xF0, 0x10, 0xF0, // 3
    0x90, 0x90, 0xF0, 0x10, 0x10, // 4
    0xF0, 0x80, 0xF0, 0x10, 0xF0, // 5
    0xF0, 0x80, 0xF0, 0x90, 0xF0, // 6
    0xF0, 0x10, 0x20, 0x40, 0x40, // 7
    0xF0, 0x90, 0xF0, 0x90, 0xF0, // 8
    0xF0, 0x90, 0xF0, 0x10, 0xF0, // 9
    0xF0, 0x90, 0xF0, 0x90, 0x90, // A
    0xE0, 0x90, 0xE0, 0x90, 0xE0, // B
    0xF0, 0x80, 0x80, 0x80, 0xF0, // C
    0xE0, 0x90, 0x90, 0x90, 0xE0, // D
    0xF0, 0x80, 0xF0, 0x80, 0xF0, // E
    0xF0, 0x80, 0xF0, 0x80, 0x80  // F
];

pub struct Video {
    repr: [u8,..REPWIDTH*HEIGHT]
}

impl Video {
    pub fn new() -> Video {
        Video { repr: [0x00, ..REPWIDTH*HEIGHT] }
    }

    pub fn clear(&mut self) {
        for px in self.repr.mut_iter() {
            *px = 0x0;
        }
    }

    pub fn draw(&mut self, x: u8, y: u8, val: u8) -> u8 {
        if (x as uint + 8) >= WIDTH || (y as uint) >= HEIGHT {
            return 0x1;
        }

        let i = x / 8 + y * REPWIDTH as u8;
        let shift = x % 8;

        if shift != 0 {
            let lval = val >> shift;
            let rval = val << (8 - shift);

            let lold = self.repr[i];
            self.repr[i] ^= lval;
            let rold = self.repr[i+1];
            self.repr[i+1] ^= rval;

            if flipped(lold, self.repr[i]) || flipped(rold, self.repr[i+1]) { 0x0 } else { 0x1 }
        }
        else {
            let old = self.repr[i];
            self.repr[i] ^= val;

            if flipped(old, self.repr[i]) { 0x0 } else { 0x1 }
        }
    }
}

/// Returns true if any of the bits have been fliped from set to unset
fn flipped(v1: u8, v2: u8) -> bool {
    v1 & !v2 != 0x0
}

#[test]
fn test_flipped() {
    assert!(flipped(0b_0000_0000, 0b_0001_0000) == false);
    assert!(flipped(0b_0001_0000, 0b_0000_0000) == true);
    assert!(flipped(0b_1010_0101, 0b_1111_1111) == false);
    assert!(flipped(0b_1010_0101, 0b_0000_0000) == true);
    assert!(flipped(0b_1010_0101, 0b_1010_0100) == true);
    assert!(flipped(0b_1111_0000, 0b_1111_1111) == false);
    assert!(flipped(0b_1111_0000, 0b_0000_1111) == true);
}
<|endoftext|>


<commit_before>use std::collections::HashMap;
use std::mem;

use sha1::Sha1;

use wasm_context::{
    FnTy,
    FunctionKind,
    FunctionIndex,
    TypeIndex,
    Import,
    Export,
    Context,
    Opcode
};

fn hash( string: &str ) -> String {
    let mut hasher = Sha1::new();
    hasher.update( string.as_bytes() );
    format!( "{}", hasher.digest() )
}

pub struct JsSnippet {
    pub name: String,
    pub code: String,
    pub ty: FnTy
}

impl JsSnippet {
    pub fn arg_count( &self ) -> usize {
        self.ty.params.len()
    }
}

struct Snippet {
    name: String,
    code: String,
    ty: FnTy,
    function_index: FunctionIndex
}

pub fn process_and_extract( ctx: &mut Context ) -> Vec< JsSnippet > {
    let mut shim_map: HashMap< FunctionIndex, TypeIndex > = HashMap::new();
    let mut snippet_offset_to_type_index = HashMap::new();
    let mut snippet_index_by_offset = HashMap::new();
    let mut snippet_index_by_hash: HashMap< String, usize > = HashMap::new();
    let mut snippets = Vec::new();

    for (&function_index, function) in &ctx.functions {
        if let &FunctionKind::Import { type_index, ref import, .. } = function {
            if import.module == "env" && import.field.starts_with( "__js_" ) {
                shim_map.insert( function_index, type_index );
            }
        }
    }

    for (_, function) in &ctx.functions {
        if let &FunctionKind::Definition { ref opcodes, .. } = function {
            for (index, opcode) in opcodes.iter().enumerate() {
                match opcode {
                    &Opcode::Call( function_index ) => {
                        if let Some( &type_index ) = shim_map.get( &function_index ) {
                            match opcodes[ index - 1 ] {
                                Opcode::I32Const( offset ) => {
                                    if let Some( previous_ty ) = snippet_offset_to_type_index.get( &offset ).cloned() {
                                        if type_index != previous_ty {
                                            panic!( "internal error: same snippet of JS (by offset) is used with two different shims; please report this!" );
                                        }
                                    }

                                    snippet_offset_to_type_index.insert( offset, type_index );
                                },
                                _ => panic!( "internal error: unexpected way of calling JS shims; please report this!" )
                            }
                        }
                    },
                    _ => {}
                }
            }
        }
    }

    let mut data_entries = Vec::new();
    mem::swap( &mut data_entries, &mut ctx.data );

    fn add_js_snippet( ctx: &mut Context, value_slice: &[u8], snippet_index_by_hash: &mut HashMap< String, usize >, snippet_index_by_offset: &mut HashMap< i32, usize >, snippets: &mut Vec< Snippet >, offset: i32, type_index: u32 ) {
        let code = match String::from_utf8( value_slice.to_owned() ) {
            Ok( code ) => code,
            Err( error ) => {
                panic!( "You have invalid UTF-8 in one of your `js!` snippets! (offset = {}, length = {})", offset, value_slice.len() );
            }
        };

        let code_hash = hash( &code );

        let shim_ty = ctx.fn_ty_by_index( type_index ).unwrap();
        let ty = FnTy {
            params: shim_ty.params.iter().cloned().take( shim_ty.params.len() - 1 ).collect(),
            return_type: shim_ty.return_type.clone()
        };

        if let Some( &snippet_index ) = snippet_index_by_hash.get( &code_hash ) {
            let snippet: &Snippet = &snippets[ snippet_index ];
            if snippet.ty != ty {
                panic!( "internal error: same snippet of JS (by value) is used with two different shims; please report this!" );
            }

            snippet_index_by_offset.insert( offset, snippet_index );
        } else {
            let snippet = Snippet {
                name: format!( "__extjs_{}", code_hash ),
                code,
                ty,
                function_index: 0xFFFFFFFF
            };

            snippet_index_by_offset.insert( offset, snippets.len() );
            snippet_index_by_hash.insert( code_hash, snippets.len() );
            snippets.push( snippet );
        }
    }

    // For older Rust nightlies.
    data_entries.retain( |data| {
        if data.offset.len() != 2 {
            return true;
        }

        let (offset, type_index) = match data.constant_offset() {
            Some( offset ) => {
                if let Some( &type_index ) = snippet_offset_to_type_index.get( &offset ) {
                    (offset, type_index)
                } else {
                    return true;
                }
            },
            _ => return true
        };

        let mut value_slice = data.value.as_slice();
        // Strip a trailing null if present.
        if value_slice.last().map( |&last_byte| last_byte == 0 ).unwrap_or( false ) {
            let len = value_slice.len();
            value_slice = &value_slice[ 0..len - 1 ];
        }

        add_js_snippet( ctx, value_slice, &mut snippet_index_by_hash, &mut snippet_index_by_offset, &mut snippets, offset, type_index );
        return false;
    });

    // For newer Rust nightlies.
    for (offset, type_index) in snippet_offset_to_type_index {
        if snippet_index_by_offset.contains_key( &offset ) {
            continue; // Already done.
        }

        let (data, data_offset) = data_entries.iter()
            .filter_map( |data| {
                if let Some( offset ) = data.constant_offset() {
                    Some( (data, offset) )
                } else {
                    None
                }
            })
            .find( |&(data, data_offset)| offset >= data_offset && offset < (data_offset + data.value.len() as i32) )
            .expect( "js! snippet not found in data section" );

        let relative_offset = offset - data_offset;
        let slice = &data.value[ relative_offset as usize.. ];
        let slice = &slice[ 0..slice.iter().position( |&byte| byte == 0 ).unwrap_or( slice.len() ) ];

        // TODO: Purge this with the help of the new "linking" WASM section?
        add_js_snippet( ctx, slice, &mut snippet_index_by_hash, &mut snippet_index_by_offset, &mut snippets, offset, type_index );
    }

    ctx.data = data_entries;

    for snippet in &mut snippets {
        let type_index = ctx.get_or_add_fn_type( snippet.ty.clone() );
        let function_index = ctx.add_function( FunctionKind::Import {
            export: Export::none(),
            import: Import { module: "env".to_owned(), field: snippet.name.clone() },
            name: Some( snippet.name.clone() ),
            type_index
        });

        snippet.function_index = function_index;
    }

    for function_index in shim_map.keys() {
        ctx.functions.remove( function_index );
    }

    ctx.patch_code( |opcodes| {
        let should_process = opcodes.iter().any( |opcode| {
            match opcode {
                &Opcode::Call( function_index ) => shim_map.contains_key( &function_index ),
                _ => false
            }
        });

        if !should_process {
            return;
        }

        let mut new_opcodes = Vec::with_capacity( opcodes.len() );
        let mut old_opcodes = Vec::new();
        mem::swap( opcodes, &mut old_opcodes );

        for opcode in old_opcodes {
            match opcode {
                Opcode::Call( function_index ) if shim_map.contains_key( &function_index ) => {
                    // Pop the last argument which was a pointer to the code.
                    let offset = match new_opcodes.pop().unwrap() {
                        Opcode::I32Const( offset ) => offset,
                        _ => panic!()
                    };
                    let &snippet_index = snippet_index_by_offset.get( &offset ).unwrap();
                    let snippet = &snippets[ snippet_index ];
                    new_opcodes.push( Opcode::Call( snippet.function_index ) );
                }
                opcode => {
                    new_opcodes.push( opcode );
                }
            }
        }

        *opcodes = new_opcodes;
    });

    snippets.into_iter().map( |snippet| {
        JsSnippet {
            name: snippet.name,
            code: snippet.code,
            ty: snippet.ty
        }
    }).collect()
}
<commit_msg>Further nightly Rust fixes<commit_after>use std::collections::HashMap;
use std::mem;

use sha1::Sha1;

use wasm_context::{
    FnTy,
    FunctionKind,
    FunctionIndex,
    TypeIndex,
    Import,
    Export,
    Context,
    Opcode
};

fn hash( string: &str ) -> String {
    let mut hasher = Sha1::new();
    hasher.update( string.as_bytes() );
    format!( "{}", hasher.digest() )
}

pub struct JsSnippet {
    pub name: String,
    pub code: String,
    pub ty: FnTy
}

impl JsSnippet {
    pub fn arg_count( &self ) -> usize {
        self.ty.params.len()
    }
}

struct Snippet {
    name: String,
    code: String,
    ty: FnTy,
    function_index: FunctionIndex
}

pub fn process_and_extract( ctx: &mut Context ) -> Vec< JsSnippet > {
    let mut shim_map: HashMap< FunctionIndex, TypeIndex > = HashMap::new();
    let mut snippet_offset_to_type_index = HashMap::new();
    let mut snippet_index_by_offset = HashMap::new();
    let mut snippet_index_by_hash: HashMap< String, usize > = HashMap::new();
    let mut snippets = Vec::new();

    for (&function_index, function) in &ctx.functions {
        if let &FunctionKind::Import { type_index, ref import, .. } = function {
            if import.module == "env" && import.field.starts_with( "__js_" ) {
                shim_map.insert( function_index, type_index );
            }
        }
    }

    for (_, function) in &ctx.functions {
        if let &FunctionKind::Definition { ref opcodes, .. } = function {
            for (index, opcode) in opcodes.iter().enumerate() {
                match opcode {
                    &Opcode::Call( function_index ) => {
                        if let Some( &type_index ) = shim_map.get( &function_index ) {
                            match opcodes[ index - 1 ] {
                                Opcode::I32Const( offset ) => {
                                    if let Some( previous_ty ) = snippet_offset_to_type_index.get( &offset ).cloned() {
                                        if type_index != previous_ty {
                                            panic!( "internal error: same snippet of JS (by offset) is used with two different shims; please report this!" );
                                        }
                                    }

                                    snippet_offset_to_type_index.insert( offset, type_index );
                                },
                                _ => panic!( "internal error: unexpected way of calling JS shims; please report this!" )
                            }
                        }
                    },
                    _ => {}
                }
            }
        }
    }

    let mut data_entries = Vec::new();
    mem::swap( &mut data_entries, &mut ctx.data );

    fn add_js_snippet( ctx: &mut Context, value_slice: &[u8], snippet_index_by_hash: &mut HashMap< String, usize >, snippet_index_by_offset: &mut HashMap< i32, usize >, snippets: &mut Vec< Snippet >, offset: i32, type_index: u32 ) {
        let code = match String::from_utf8( value_slice.to_owned() ) {
            Ok( code ) => code,
            Err( error ) => {
                panic!( "You have invalid UTF-8 in one of your `js!` snippets! (offset = {}, length = {})", offset, value_slice.len() );
            }
        };

        let code_hash = hash( &code );

        let shim_ty = ctx.fn_ty_by_index( type_index ).unwrap();
        let ty = FnTy {
            params: shim_ty.params.iter().cloned().take( shim_ty.params.len() - 1 ).collect(),
            return_type: shim_ty.return_type.clone()
        };

        if let Some( &snippet_index ) = snippet_index_by_hash.get( &code_hash ) {
            let snippet: &Snippet = &snippets[ snippet_index ];
            if snippet.ty != ty {
                panic!( "internal error: same snippet of JS (by value) is used with two different shims; please report this!" );
            }

            snippet_index_by_offset.insert( offset, snippet_index );
        } else {
            let snippet = Snippet {
                name: format!( "__extjs_{}", code_hash ),
                code,
                ty,
                function_index: 0xFFFFFFFF
            };

            snippet_index_by_offset.insert( offset, snippets.len() );
            snippet_index_by_hash.insert( code_hash, snippets.len() );
            snippets.push( snippet );
        }
    }

    // For older Rust nightlies.
    data_entries.retain( |data| {
        if data.offset.len() != 2 {
            return true;
        }

        let (offset, type_index) = match data.constant_offset() {
            Some( offset ) => {
                if let Some( &type_index ) = snippet_offset_to_type_index.get( &offset ) {
                    (offset, type_index)
                } else {
                    return true;
                }
            },
            _ => return true
        };

        let mut slice = data.value.as_slice();
        let slice = &slice[ 0..slice.iter().position( |&byte| byte == 0 ).unwrap_or( slice.len() ) ];
        if slice.len() + 1 < data.value.len() {
            return true;
        }

        add_js_snippet( ctx, slice, &mut snippet_index_by_hash, &mut snippet_index_by_offset, &mut snippets, offset, type_index );
        return false;
    });

    // For newer Rust nightlies.
    for (offset, type_index) in snippet_offset_to_type_index {
        if snippet_index_by_offset.contains_key( &offset ) {
            continue; // Already done.
        }

        let (data, data_offset) = data_entries.iter()
            .filter_map( |data| {
                if let Some( offset ) = data.constant_offset() {
                    Some( (data, offset) )
                } else {
                    None
                }
            })
            .find( |&(data, data_offset)| offset >= data_offset && offset < (data_offset + data.value.len() as i32) )
            .expect( "js! snippet not found in data section" );

        let relative_offset = offset - data_offset;
        let slice = &data.value[ relative_offset as usize.. ];
        let slice = &slice[ 0..slice.iter().position( |&byte| byte == 0 ).unwrap_or( slice.len() ) ];

        // TODO: Purge this with the help of the new "linking" WASM section?
        add_js_snippet( ctx, slice, &mut snippet_index_by_hash, &mut snippet_index_by_offset, &mut snippets, offset, type_index );
    }

    ctx.data = data_entries;

    for snippet in &mut snippets {
        let type_index = ctx.get_or_add_fn_type( snippet.ty.clone() );
        let function_index = ctx.add_function( FunctionKind::Import {
            export: Export::none(),
            import: Import { module: "env".to_owned(), field: snippet.name.clone() },
            name: Some( snippet.name.clone() ),
            type_index
        });

        snippet.function_index = function_index;
    }

    for function_index in shim_map.keys() {
        ctx.functions.remove( function_index );
    }

    ctx.patch_code( |opcodes| {
        let should_process = opcodes.iter().any( |opcode| {
            match opcode {
                &Opcode::Call( function_index ) => shim_map.contains_key( &function_index ),
                _ => false
            }
        });

        if !should_process {
            return;
        }

        let mut new_opcodes = Vec::with_capacity( opcodes.len() );
        let mut old_opcodes = Vec::new();
        mem::swap( opcodes, &mut old_opcodes );

        for opcode in old_opcodes {
            match opcode {
                Opcode::Call( function_index ) if shim_map.contains_key( &function_index ) => {
                    // Pop the last argument which was a pointer to the code.
                    let offset = match new_opcodes.pop().unwrap() {
                        Opcode::I32Const( offset ) => offset,
                        _ => panic!()
                    };
                    let &snippet_index = snippet_index_by_offset.get( &offset ).unwrap();
                    let snippet = &snippets[ snippet_index ];
                    new_opcodes.push( Opcode::Call( snippet.function_index ) );
                }
                opcode => {
                    new_opcodes.push( opcode );
                }
            }
        }

        *opcodes = new_opcodes;
    });

    snippets.into_iter().map( |snippet| {
        JsSnippet {
            name: snippet.name,
            code: snippet.code,
            ty: snippet.ty
        }
    }).collect()
}
<|endoftext|>


<commit_before>/// Actual version of `qt_ritual_build` crate used as build dependency
pub const QT_RITUAL_BUILD_VERSION: &str = "0.3.0";

/// Version of all generated Qt crates
pub const QT_OUTPUT_CRATES_VERSION: &str = "0.4.0-alpha1";
<commit_msg>bump qt crates version<commit_after>/// Actual version of `qt_ritual_build` crate used as build dependency
pub const QT_RITUAL_BUILD_VERSION: &str = "0.3.0";

/// Version of all generated Qt crates
pub const QT_OUTPUT_CRATES_VERSION: &str = "0.4.0";
<|endoftext|>


<commit_before>// Copyright 2018 Developers of the Rand project.
// Copyright 2016-2017 The Rust Project Developers.
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// https://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or https://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.

//! The Poisson distribution.

use rand::Rng;
use crate::{Distribution, Cauchy};
use crate::utils::log_gamma;

/// The Poisson distribution `Poisson(lambda)`.
///
/// This distribution has a density function:
/// `f(k) = lambda^k * exp(-lambda) / k!` for `k >= 0`.
///
/// # Example
///
/// ```
/// use rand_distr::{Poisson, Distribution};
///
/// let poi = Poisson::new(2.0).unwrap();
/// let v = poi.sample(&mut rand::thread_rng());
/// println!("{} is from a Poisson(2) distribution", v);
/// ```
#[derive(Clone, Copy, Debug)]
pub struct Poisson {
    lambda: f64,
    // precalculated values
    exp_lambda: f64,
    log_lambda: f64,
    sqrt_2lambda: f64,
    magic_val: f64,
}

/// Error type returned from `Poisson::new`.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum Error {
    /// `lambda <= 0` or `nan`.
    ShapeTooSmall,
}

impl Poisson {
    /// Construct a new `Poisson` with the given shape parameter
    /// `lambda`.
    pub fn new(lambda: f64) -> Result<Poisson, Error> {
        if !(lambda > 0.0) {
            return Err(Error::ShapeTooSmall);
        }
        let log_lambda = lambda.ln();
        Ok(Poisson {
            lambda,
            exp_lambda: (-lambda).exp(),
            log_lambda,
            sqrt_2lambda: (2.0 * lambda).sqrt(),
            magic_val: lambda * log_lambda - log_gamma(1.0 + lambda),
        })
    }
}

impl Distribution<u64> for Poisson {
    fn sample<R: Rng + ?Sized>(&self, rng: &mut R) -> u64 {
        // using the algorithm from Numerical Recipes in C

        // for low expected values use the Knuth method
        if self.lambda < 12.0 {
            let mut result = 0;
            let mut p = 1.0;
            while p > self.exp_lambda {
                p *= rng.gen::<f64>();
                result += 1;
            }
            result - 1
        }
        // high expected values - rejection method
        else {
            let mut int_result: u64;

            // we use the Cauchy distribution as the comparison distribution
            // f(x) ~ 1/(1+x^2)
            let cauchy = Cauchy::new(0.0, 1.0).unwrap();

            loop {
                let mut result;
                let mut comp_dev;

                loop {
                    // draw from the Cauchy distribution
                    comp_dev = rng.sample(cauchy);
                    // shift the peak of the comparison ditribution
                    result = self.sqrt_2lambda * comp_dev + self.lambda;
                    // repeat the drawing until we are in the range of possible values
                    if result >= 0.0 {
                        break;
                    }
                }
                // now the result is a random variable greater than 0 with Cauchy distribution
                // the result should be an integer value
                result = result.floor();
                assert!(result >= 0.);
                assert!(result <= ::core::u64::MAX as f64);
                int_result = result as u64;

                // this is the ratio of the Poisson distribution to the comparison distribution
                // the magic value scales the distribution function to a range of approximately 0-1
                // since it is not exact, we multiply the ratio by 0.9 to avoid ratios greater than 1
                // this doesn't change the resulting distribution, only increases the rate of failed drawings
                let check = 0.9 * (1.0 + comp_dev * comp_dev)
                    * (result * self.log_lambda - log_gamma(1.0 + result) - self.magic_val).exp();

                // check with uniform random value - if below the threshold, we are within the target distribution
                if rng.gen::<f64>() <= check {
                    break;
                }
            }
            int_result
        }
    }
}

#[cfg(test)]
mod test {
    use crate::Distribution;
    use super::Poisson;

    #[test]
    fn test_poisson_10() {
        let poisson = Poisson::new(10.0).unwrap();
        let mut rng = crate::test::rng(123);
        let mut sum = 0;
        for _ in 0..1000 {
            sum += poisson.sample(&mut rng);
        }
        let avg = (sum as f64) / 1000.0;
        println!("Poisson average: {}", avg);
        assert!((avg - 10.0).abs() < 0.5); // not 100% certain, but probable enough
    }

    #[test]
    fn test_poisson_15() {
        // Take the 'high expected values' path
        let poisson = Poisson::new(15.0).unwrap();
        let mut rng = crate::test::rng(123);
        let mut sum = 0;
        for _ in 0..1000 {
            sum += poisson.sample(&mut rng);
        }
        let avg = (sum as f64) / 1000.0;
        println!("Poisson average: {}", avg);
        assert!((avg - 15.0).abs() < 0.5); // not 100% certain, but probable enough
    }

    #[test]
    #[should_panic]
    fn test_poisson_invalid_lambda_zero() {
        Poisson::new(0.0).unwrap();
    }

    #[test]
    #[should_panic]
    fn test_poisson_invalid_lambda_neg() {
        Poisson::new(-10.0).unwrap();
    }
}
<commit_msg>rand_distr: Add support for `f64` to `Poisson`<commit_after>// Copyright 2018 Developers of the Rand project.
// Copyright 2016-2017 The Rust Project Developers.
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// https://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or https://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.

//! The Poisson distribution.

use rand::Rng;
use crate::{Distribution, Cauchy};
use crate::utils::log_gamma;

/// The Poisson distribution `Poisson(lambda)`.
///
/// This distribution has a density function:
/// `f(k) = lambda^k * exp(-lambda) / k!` for `k >= 0`.
///
/// # Example
///
/// ```
/// use rand_distr::{Poisson, Distribution};
///
/// let poi = Poisson::new(2.0).unwrap();
/// let v: u64 = poi.sample(&mut rand::thread_rng());
/// println!("{} is from a Poisson(2) distribution", v);
/// ```
#[derive(Clone, Copy, Debug)]
pub struct Poisson {
    lambda: f64,
    // precalculated values
    exp_lambda: f64,
    log_lambda: f64,
    sqrt_2lambda: f64,
    magic_val: f64,
}

/// Error type returned from `Poisson::new`.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum Error {
    /// `lambda <= 0` or `nan`.
    ShapeTooSmall,
}

impl Poisson {
    /// Construct a new `Poisson` with the given shape parameter
    /// `lambda`.
    pub fn new(lambda: f64) -> Result<Poisson, Error> {
        if !(lambda > 0.0) {
            return Err(Error::ShapeTooSmall);
        }
        let log_lambda = lambda.ln();
        Ok(Poisson {
            lambda,
            exp_lambda: (-lambda).exp(),
            log_lambda,
            sqrt_2lambda: (2.0 * lambda).sqrt(),
            magic_val: lambda * log_lambda - log_gamma(1.0 + lambda),
        })
    }
}

impl Distribution<f64> for Poisson {
    fn sample<R: Rng + ?Sized>(&self, rng: &mut R) -> f64 {
        // using the algorithm from Numerical Recipes in C

        // for low expected values use the Knuth method
        if self.lambda < 12.0 {
            let mut result = 0.;
            let mut p = 1.0;
            while p > self.exp_lambda {
                p *= rng.gen::<f64>();
                result += 1.;
            }
            result - 1.
        }
        // high expected values - rejection method
        else {
            // we use the Cauchy distribution as the comparison distribution
            // f(x) ~ 1/(1+x^2)
            let cauchy = Cauchy::new(0.0, 1.0).unwrap();
            let mut result;

            loop {
                let mut comp_dev;

                loop {
                    // draw from the Cauchy distribution
                    comp_dev = rng.sample(cauchy);
                    // shift the peak of the comparison ditribution
                    result = self.sqrt_2lambda * comp_dev + self.lambda;
                    // repeat the drawing until we are in the range of possible values
                    if result >= 0.0 {
                        break;
                    }
                }
                // now the result is a random variable greater than 0 with Cauchy distribution
                // the result should be an integer value
                result = result.floor();

                // this is the ratio of the Poisson distribution to the comparison distribution
                // the magic value scales the distribution function to a range of approximately 0-1
                // since it is not exact, we multiply the ratio by 0.9 to avoid ratios greater than 1
                // this doesn't change the resulting distribution, only increases the rate of failed drawings
                let check = 0.9 * (1.0 + comp_dev * comp_dev)
                    * (result * self.log_lambda - log_gamma(1.0 + result) - self.magic_val).exp();

                // check with uniform random value - if below the threshold, we are within the target distribution
                if rng.gen::<f64>() <= check {
                    break;
                }
            }
            result
        }
    }
}

impl Distribution<u64> for Poisson {
    fn sample<R: Rng + ?Sized>(&self, rng: &mut R) -> u64 {
        let result: f64 = self.sample(rng);
        assert!(result >= 0.);
        assert!(result <= ::core::u64::MAX as f64);
        result as u64
    }
}

#[cfg(test)]
mod test {
    use crate::Distribution;
    use super::Poisson;

    #[test]
    fn test_poisson_10() {
        let poisson = Poisson::new(10.0).unwrap();
        let mut rng = crate::test::rng(123);
        let mut sum = 0;
        for _ in 0..1000 {
            let s: u64 = poisson.sample(&mut rng);
            sum += s;
        }
        let avg = (sum as f64) / 1000.0;
        println!("Poisson average: {}", avg);
        assert!((avg - 10.0).abs() < 0.5); // not 100% certain, but probable enough
    }

    #[test]
    fn test_poisson_15() {
        // Take the 'high expected values' path
        let poisson = Poisson::new(15.0).unwrap();
        let mut rng = crate::test::rng(123);
        let mut sum = 0;
        for _ in 0..1000 {
            let s: u64 = poisson.sample(&mut rng);
            sum += s;
        }
        let avg = (sum as f64) / 1000.0;
        println!("Poisson average: {}", avg);
        assert!((avg - 15.0).abs() < 0.5); // not 100% certain, but probable enough
    }

    #[test]
    #[should_panic]
    fn test_poisson_invalid_lambda_zero() {
        Poisson::new(0.0).unwrap();
    }

    #[test]
    #[should_panic]
    fn test_poisson_invalid_lambda_neg() {
        Poisson::new(-10.0).unwrap();
    }
}
<|endoftext|>


<commit_before>#[macro_use]
extern crate chan;
extern crate crossbeam;

const MESSAGES: usize = 5_000_000;
const THREADS: usize = 4;

fn new<T>(cap: Option<usize>) -> (chan::Sender<T>, chan::Receiver<T>) {
    match cap {
        None => chan::async(),
        Some(cap) => chan::sync(cap)
    }
}

fn seq(cap: Option<usize>) {
    let (tx, rx) = new::<i32>(cap);

    for i in 0..MESSAGES {
        tx.send(i as i32);
    }

    for _ in 0..MESSAGES {
        rx.recv().unwrap();
    }
}

fn spsc(cap: Option<usize>) {
    let (tx, rx) = new::<i32>(cap);

    crossbeam::scope(|s| {
        s.spawn(|| {
            for i in 0..MESSAGES {
                tx.send(i as i32);
            }
        });

        for _ in 0..MESSAGES {
            rx.recv().unwrap();
        }
    });
}

fn mpsc(cap: Option<usize>) {
    let (tx, rx) = new::<i32>(cap);

    crossbeam::scope(|s| {
        for _ in 0..THREADS {
            s.spawn(|| {
                for i in 0..MESSAGES / THREADS {
                    tx.send(i as i32);
                }
            });
        }

        for _ in 0..MESSAGES {
            rx.recv().unwrap();
        }
    });
}

fn mpmc(cap: Option<usize>) {
    let (tx, rx) = new::<i32>(cap);

    crossbeam::scope(|s| {
        for _ in 0..THREADS {
            s.spawn(|| {
                for i in 0..MESSAGES / THREADS {
                    tx.send(i as i32);
                }
            });
        }

        for _ in 0..THREADS {
            s.spawn(|| {
                for _ in 0..MESSAGES / THREADS {
                    rx.recv().unwrap();
                }
            });
        }
    });
}

fn select_rx(cap: Option<usize>) {
    assert_eq!(THREADS, 4);
    let chans = (0..THREADS).map(|_| new::<i32>(cap)).collect::<Vec<_>>();

    crossbeam::scope(|s| {
        for (tx, _) in &chans {
            let tx = tx.clone();
            s.spawn(move || {
                for i in 0..MESSAGES / THREADS {
                    tx.send(i as i32);
                }
            });
        }

        let rx0 = &chans[0].1;
        let rx1 = &chans[1].1;
        let rx2 = &chans[2].1;
        let rx3 = &chans[3].1;

        for _ in 0..MESSAGES {
            chan_select! {
                rx0.recv() -> m => assert!(m.is_some()),
                rx1.recv() -> m => assert!(m.is_some()),
                rx2.recv() -> m => assert!(m.is_some()),
                rx3.recv() -> m => assert!(m.is_some()),
            }
        }
    });
}

fn select_both(cap: Option<usize>) {
    assert_eq!(THREADS, 4);
    let chans = (0..THREADS).map(|_| new::<i32>(cap)).collect::<Vec<_>>();

    crossbeam::scope(|s| {
        for _ in 0..THREADS {
            let chans = chans.clone();
            s.spawn(move || {
                let tx0 = &chans[0].0;
                let tx1 = &chans[1].0;
                let tx2 = &chans[2].0;
                let tx3 = &chans[3].0;
                for i in 0..MESSAGES / THREADS {
                    chan_select! {
                        tx0.send(i as i32) => {},
                        tx1.send(i as i32) => {},
                        tx2.send(i as i32) => {},
                        tx3.send(i as i32) => {},
                    }
                }
            });
        }

        for _ in 0..THREADS {
            let chans = chans.clone();
            s.spawn(move || {
                let rx0 = &chans[0].1;
                let rx1 = &chans[1].1;
                let rx2 = &chans[2].1;
                let rx3 = &chans[3].1;
                for _ in 0..MESSAGES / THREADS {
                    chan_select! {
                        rx0.recv() -> m => assert!(m.is_some()),
                        rx1.recv() -> m => assert!(m.is_some()),
                        rx2.recv() -> m => assert!(m.is_some()),
                        rx3.recv() -> m => assert!(m.is_some()),
                    }
                }
            });
        }
    });
}

fn main() {
    macro_rules! run {
        ($name:expr, $f:expr) => {
            let now = ::std::time::Instant::now();
            $f;
            let elapsed = now.elapsed();
            println!(
                "{:25} {:15} {:7.3} sec",
                $name,
                "Rust chan",
                elapsed.as_secs() as f64 + elapsed.subsec_nanos() as f64 / 1e9
            );
        }
    }

    run!("bounded0_mpmc", mpmc(Some(0)));
    run!("bounded0_mpsc", mpsc(Some(0)));
    run!("bounded0_select_rx", select_rx(Some(0)));
    run!("bounded0_spsc", spsc(Some(0)));

    run!("bounded1_mpmc", mpmc(Some(1)));
    run!("bounded1_mpsc", mpsc(Some(1)));
    run!("bounded1_select_both", select_both(Some(1)));
    run!("bounded1_select_rx", select_rx(Some(1)));
    run!("bounded1_spsc", spsc(Some(1)));

    run!("bounded_mpmc", mpmc(Some(MESSAGES)));
    run!("bounded_mpsc", mpsc(Some(MESSAGES)));
    run!("bounded_select_both", select_both(Some(MESSAGES)));
    run!("bounded_select_rx", select_rx(Some(MESSAGES)));
    run!("bounded_seq", seq(Some(MESSAGES)));
    run!("bounded_spsc", spsc(Some(MESSAGES)));

    run!("unbounded_mpmc", mpmc(None));
    run!("unbounded_mpsc", mpsc(None));
    run!("unbounded_select_both", select_both(None));
    run!("unbounded_select_rx", select_rx(None));
    run!("unbounded_seq", seq(None));
    run!("unbounded_spsc", spsc(None));
}
<commit_msg>Cleanup<commit_after>#[macro_use]
extern crate chan;
extern crate crossbeam;

const MESSAGES: usize = 5_000_000;
const THREADS: usize = 4;

fn new<T>(cap: Option<usize>) -> (chan::Sender<T>, chan::Receiver<T>) {
    match cap {
        None => chan::async(),
        Some(cap) => chan::sync(cap)
    }
}

fn seq(cap: Option<usize>) {
    let (tx, rx) = new::<i32>(cap);

    for i in 0..MESSAGES {
        tx.send(i as i32);
    }

    for _ in 0..MESSAGES {
        rx.recv().unwrap();
    }
}

fn spsc(cap: Option<usize>) {
    let (tx, rx) = new::<i32>(cap);

    crossbeam::scope(|s| {
        s.spawn(|| {
            for i in 0..MESSAGES {
                tx.send(i as i32);
            }
        });

        for _ in 0..MESSAGES {
            rx.recv().unwrap();
        }
    });
}

fn mpsc(cap: Option<usize>) {
    let (tx, rx) = new::<i32>(cap);

    crossbeam::scope(|s| {
        for _ in 0..THREADS {
            s.spawn(|| {
                for i in 0..MESSAGES / THREADS {
                    tx.send(i as i32);
                }
            });
        }

        for _ in 0..MESSAGES {
            rx.recv().unwrap();
        }
    });
}

fn mpmc(cap: Option<usize>) {
    let (tx, rx) = new::<i32>(cap);

    crossbeam::scope(|s| {
        for _ in 0..THREADS {
            s.spawn(|| {
                for i in 0..MESSAGES / THREADS {
                    tx.send(i as i32);
                }
            });
        }

        for _ in 0..THREADS {
            s.spawn(|| {
                for _ in 0..MESSAGES / THREADS {
                    rx.recv().unwrap();
                }
            });
        }
    });
}

fn select_rx(cap: Option<usize>) {
    assert_eq!(THREADS, 4);
    let chans = (0..THREADS).map(|_| new::<i32>(cap)).collect::<Vec<_>>();

    crossbeam::scope(|s| {
        for (tx, _) in &chans {
            let tx = tx.clone();
            s.spawn(move || {
                for i in 0..MESSAGES / THREADS {
                    tx.send(i as i32);
                }
            });
        }

        let rx0 = &chans[0].1;
        let rx1 = &chans[1].1;
        let rx2 = &chans[2].1;
        let rx3 = &chans[3].1;
        for _ in 0..MESSAGES {
            chan_select! {
                rx0.recv() -> m => assert!(m.is_some()),
                rx1.recv() -> m => assert!(m.is_some()),
                rx2.recv() -> m => assert!(m.is_some()),
                rx3.recv() -> m => assert!(m.is_some()),
            }
        }
    });
}

fn select_both(cap: Option<usize>) {
    assert_eq!(THREADS, 4);
    let chans = (0..THREADS).map(|_| new::<i32>(cap)).collect::<Vec<_>>();

    crossbeam::scope(|s| {
        for _ in 0..THREADS {
            let chans = chans.clone();
            s.spawn(move || {
                let tx0 = &chans[0].0;
                let tx1 = &chans[1].0;
                let tx2 = &chans[2].0;
                let tx3 = &chans[3].0;

                for i in 0..MESSAGES / THREADS {
                    chan_select! {
                        tx0.send(i as i32) => {},
                        tx1.send(i as i32) => {},
                        tx2.send(i as i32) => {},
                        tx3.send(i as i32) => {},
                    }
                }
            });
        }

        for _ in 0..THREADS {
            let chans = chans.clone();
            s.spawn(move || {
                let rx0 = &chans[0].1;
                let rx1 = &chans[1].1;
                let rx2 = &chans[2].1;
                let rx3 = &chans[3].1;

                for _ in 0..MESSAGES / THREADS {
                    chan_select! {
                        rx0.recv() -> m => assert!(m.is_some()),
                        rx1.recv() -> m => assert!(m.is_some()),
                        rx2.recv() -> m => assert!(m.is_some()),
                        rx3.recv() -> m => assert!(m.is_some()),
                    }
                }
            });
        }
    });
}

fn main() {
    macro_rules! run {
        ($name:expr, $f:expr) => {
            let now = ::std::time::Instant::now();
            $f;
            let elapsed = now.elapsed();
            println!(
                "{:25} {:15} {:7.3} sec",
                $name,
                "Rust chan",
                elapsed.as_secs() as f64 + elapsed.subsec_nanos() as f64 / 1e9
            );
        }
    }

    run!("bounded0_mpmc", mpmc(Some(0)));
    run!("bounded0_mpsc", mpsc(Some(0)));
    run!("bounded0_select_rx", select_rx(Some(0)));
    run!("bounded0_spsc", spsc(Some(0)));

    run!("bounded1_mpmc", mpmc(Some(1)));
    run!("bounded1_mpsc", mpsc(Some(1)));
    run!("bounded1_select_both", select_both(Some(1)));
    run!("bounded1_select_rx", select_rx(Some(1)));
    run!("bounded1_spsc", spsc(Some(1)));

    run!("bounded_mpmc", mpmc(Some(MESSAGES)));
    run!("bounded_mpsc", mpsc(Some(MESSAGES)));
    run!("bounded_select_both", select_both(Some(MESSAGES)));
    run!("bounded_select_rx", select_rx(Some(MESSAGES)));
    run!("bounded_seq", seq(Some(MESSAGES)));
    run!("bounded_spsc", spsc(Some(MESSAGES)));

    run!("unbounded_mpmc", mpmc(None));
    run!("unbounded_mpsc", mpsc(None));
    run!("unbounded_select_both", select_both(None));
    run!("unbounded_select_rx", select_rx(None));
    run!("unbounded_seq", seq(None));
    run!("unbounded_spsc", spsc(None));
}
<|endoftext|>


<commit_before>//! Multi-way equijoin expression plan.
//!
//! This plan provides us the opportunity to map out a non-trivial differential
//! implementation for a complex join query. In particular, we are able to invoke
//! delta-query and worst-case optimal join plans, which avoid any intermediate
//! materialization.
//!
//! Each `MultiwayJoin` indicates several source collections, equality constraints
//! among their attributes, and then the set of attributes to produce as results.
//!
//! One naive implementation would take each input collection in order, and develop
//! the join restricted to the prefix of relations so far. Ideally the order would
//! be such that joined collections have equality constraints and prevent Cartesian
//! explosion. At each step, a new collection picks out some of the attributes and
//! instantiates a primitive binary join between the accumulated collection and the
//! next collection.
//!
//! A more sophisticated implementation establishes delta queries for each input
//! collection, which responds to changes in that input collection against the
//! current other input collections. For each input collection we may choose very
//! different join orders, as the order must follow equality constraints.
//!
//! A further implementation could develop the results attribute-by-attribute, as
//! opposed to collection-by-collection, which gives us the ability to use column
//! indices rather than whole-collection indices.

use std::hash::Hash;

use timely::dataflow::Scope;

use differential_dataflow::operators::Consolidate;
use differential_dataflow::operators::arrange::{ArrangeBySelf, ArrangeByKey};

use differential_dataflow::{Collection, ExchangeData};
use plan::{Plan, Render};
use {TraceManager, Time, Diff};

/// A multiway join of muliple relations.
///
/// By expressing multiple relations and required equivalances between their attributes,
/// we can more efficiently design incremental update strategies without materializing
/// and indexing intermediate relations.
#[derive(Serialize, Deserialize, Clone, Debug, Eq, PartialEq, Ord, PartialOrd, Hash)]
pub struct MultiwayJoin<Value> {
    /// A list of (attribute index, input) pairs to extract.
    pub results: Vec<(usize, usize)>,
    /// A list of source collections.
    pub sources: Vec<Plan<Value>>,
    /// Equality constraints.
    ///
    /// Equality constraints are presented as lists of `(attr, input)` equivalence classes.
    /// This means that each `(attr, input)` pair can exist in at most one list; if it would
    /// appear in more than one list, those two lists should be merged.
    pub equalities: Vec<Vec<(usize, usize)>>,
}

// TODO: This logic fails to perform restrictions in cases where a join does not
//       occur. One example could be:
//
//              result(a,b,c) := R1(a,b), R2(b,c,c)
//
//       In this case, the requirement that the 2nd and 3rd columns of R2 be equal
//       is not surfaced in any join, and is instead a filter that should be applied
//       directly to R2 (before or after the join with R1; either could be best).

impl<V: ExchangeData+Hash> Render for MultiwayJoin<V> {

    type Value = V;

    fn render<S: Scope<Timestamp = Time>>(
        &self,
        scope: &mut S,
        arrangements: &mut TraceManager<Self::Value>) -> Collection<S, Vec<Self::Value>, Diff>
    {
        // The idea here is the following:
        //
        // For each stream, we will determine a streaming delta query, in which changes
        // are joined against indexed forms of the other relations using `dogsdogsdogs`
        // stateless `propose` operators.
        //
        // For a query Q(x,y,z) := A(x,y), B(y,z), C(x,z) we might write dataflows like:
        //
        //   dQdA := dA(x,y), B(y,z), C(x,z)
        //   dQdB := dB(y,z), A(x,y), C(x,z)
        //   dQdC := dC(x,y), A(x,y), B(y,z)
        //
        // where each line is read from left to right as a sequence of `propose` joins,
        // which respond to timestamped delta changes by joining with the maintained
        // relation at that corresponding time.
        //
        // We take some care to make sure that when these joins are performed, a delta
        // interacts with relations as if we updated A, then B, then C, as if in sequence.
        // That is, when a dB delta joins A it observes all updates to A at times less or
        // equal to the delta's timestamp, but when a dB delta joins C it observes only
        // updates to C at times strictly less than the delta's timestamp.
        //
        // This is done to avoid double counting updates; any concurrent changes will be
        // accounted for by the last relation for which there is a concurrent update.

        println!("{:?}", self);

        // Attributes we may need from any and all relations.
        let mut relevant_attributes = Vec::new();
        relevant_attributes.extend(self.results.iter().cloned());
        relevant_attributes.extend(self.equalities.iter().flat_map(|list| list.iter().cloned()));
        relevant_attributes.sort();
        relevant_attributes.dedup();

        // println!("Relevant attributes: {:?}", relevant_attributes);

        // Into which we accumulate change streams.
        let mut accumulated_changes = Vec::new();

        // For each participating relation, we build a delta query dataflow.
        for (index, plan) in self.sources.iter().enumerate() {

            // println!("building dataflow for relation {}", index);

            // Restrict down to relevant attributes.
            let mut attributes: Vec<(usize, usize)> =
            relevant_attributes
                .iter()
                .filter(|(_attr, input)| input == &index)
                .cloned()
                .collect::<Vec<_>>();

            let attributes_init = attributes.clone();
            // println!("\tinitial attributes: {:?}", attributes);

            // Ensure the plan is rendered and cached.
            if arrangements.get_unkeyed(&plan).is_none() {
                // println!("\tbuilding/caching source plan");
                let collection = plan.render(scope, arrangements);
                arrangements.set_unkeyed(plan, &collection.arrange_by_self().trace);
            }
            else {
                // println!("\tsource plan found");
            }
            let changes =
            arrangements
                .get_unkeyed(&plan)
                .expect("Surely we just ensured this")
                .import(scope)
                .as_collection(|val,&()| val.clone())
                .map(move |tuple| attributes_init.iter().map(|&(attr,_)|
                    tuple[attr].clone()).collect::<Vec<_>>()
                );

            // Before constructing the dataflow, which takes a borrow on `scope`,
            // we'll want to ensure that we have all of the necessary data assets
            // in place. This requires a bit of planning first, then the building.

            // Acquire a sane sequence in which to join the relations:
            //
            // This is a sequence of relation identifiers, starting with `index`,
            // such that each has at least one attribute in common with a prior
            // relation, and so can be effectively joined.
            let join_order = plan_join_order(index, &self.equalities);
            let mut join_plan = Vec::new();

            // println!("\tjoin order: {:?}", join_order);

            // Skipping `index`, join in each relation in sequence.
            for join_idx in join_order.into_iter().skip(1) {

                // To join a relation, we need to determine any constraints on
                // attributes in common with prior relations. Any other values
                // should be appended to tuples in `changes` with care taken to
                // update `attributes`.
                let (keys, priors) = determine_keys_priors(join_idx, &self.equalities, &attributes[..]);

                // The fields in `sources[join_idx]` that should be values are those
                // that are required output or participate in an equality constraint,
                // but *WHICH ARE NOT* in `keys`.
                let vals =
                relevant_attributes
                    .iter()
                    .filter(|&(attr,index)| index == &join_idx && !keys.contains(&attr))
                    .cloned()
                    .collect::<Vec<_>>();

                // println!("\tkeys: {:?}, priors: {:?}, vals: {:?}", keys, priors, vals);

                let mut projection = Vec::new();
                for &attr in keys.iter() {
                    projection.push(attr);
                }
                for &(attr, _index) in vals.iter() {
                    projection.push(attr);
                }
                // TODO: Sort, to improve chances of re-use opportunities.
                //       Requires understanding how attributes move to get the right
                //       key selectors out though.
                // projection.sort();
                // projection.dedup(); // Should already be deduplicated, probably?

                // Get a plan for the projection on to these few attributes.
                let plan = self.sources[join_idx].clone().project(projection);

                if arrangements.get_keyed(&plan, &keys[..]).is_none() {
                    // println!("\tbuilding key: {:?}, plan: {:?}", keys, plan);
                    let keys_clone = keys.clone();
                    let arrangement =
                    plan.render(scope, arrangements)
                        .map(move |tuple| (keys_clone.iter().map(|&i| tuple[i].clone()).collect::<Vec<_>>(), tuple))
                        .arrange_by_key();

                    arrangements.set_keyed(&plan, &keys[..], &arrangement.trace);
                }
                else {
                    // println!("\tplan found: {:?}, {:?}", keys, plan);
                }

                let arrangement =
                arrangements
                    .get_keyed(&plan, &keys[..])
                    .expect("Surely we just ensured this");

                let key_selector = std::rc::Rc::new(move |change: &Vec<V>|
                    priors.iter().map(|&p| change[p].clone()).collect::<Vec<_>>()
                );

                join_plan.push((join_idx, key_selector, arrangement));

                attributes.extend(keys.into_iter().map(|x| (x, join_idx)));
                attributes.extend(vals.into_iter());
                // println!("\tattributes: {:?}", attributes);
            }

            // Build the dataflow.
            use dogsdogsdogs::altneu::AltNeu;

            let scope_name = format!("DeltaRule: {}/{}", index, self.sources.len());
            let changes = scope.clone().scoped::<AltNeu<_>,_,_>(&scope_name, |inner| {

                // This should default to an `AltNeu::Alt` timestamp.
                let mut changes =
                changes
                    .enter(inner)
                    ;

                for (join_idx, key_selector, mut trace) in join_plan.into_iter() {

                    // Use alt or neu timestamps based on relative indices.
                    // Must have an `if` statement here as the two arrangement have different
                    // types, and we would to determine `alt` v `neu` once, rather than per
                    // tuple in the cursor.
                    changes =
                    if join_idx < index {
                        let arrangement = trace.import(scope).enter_at(inner, |_,_,t| AltNeu::alt(t.clone()));
                        dogsdogsdogs::operators::propose(&changes, arrangement, key_selector)
                    }
                    else {
                        let arrangement = trace.import(scope).enter_at(inner, |_,_,t| AltNeu::neu(t.clone()));
                        dogsdogsdogs::operators::propose(&changes, arrangement, key_selector)
                    }
                    .map(|(mut prefix, extensions)| { prefix.extend(extensions.into_iter()); prefix })
                    ;

                    // TODO: Equality constraints strictly within a relation have the effect
                    //       of "filtering" data, but they are ignored at the moment. We should
                    //       check for these and do something about it.
                }

                // Extract `self.results` in order, using `attributes`.
                //
                // The specific attribute requested in `self.results` may not be present in
                // `attributes` when it is equal to another present attribute. So, we should
                // look around in `self.equalities` also.
                let mut extract_map = Vec::new();
                for result in self.results.iter() {
                    if let Some(position) = attributes.iter().position(|i| i == result) {
                        extract_map.push(position);
                    }
                    else {
                        for constraint in self.equalities.iter() {
                            if constraint.contains(result) {
                                if let Some(position) = constraint.iter().flat_map(|x| attributes.iter().position(|i| i == x)).next() {
                                    extract_map.push(position);
                                }
                                else {
                                    println!("WTF NOTHING FOUND NOOOOO!!!");
                                }
                            }
                        }
                    }
                }

                changes
                    .map(move |tuple| extract_map.iter().map(|&i| tuple[i].clone()).collect::<Vec<_>>())
                    .leave()
            });

            accumulated_changes.push(changes);
        }

        differential_dataflow::collection::concatenate(scope, accumulated_changes.into_iter())
            .consolidate()
    }
}

/// Sequences relations in `constraints`.
///
/// Relations become available for sequencing as soon as they share a constraint with
/// either `source` or another sequenced relation.
fn plan_join_order(source: usize, constraints: &[Vec<(usize, usize)>]) -> Vec<usize> {

    let mut result = vec![source];
    let mut active = true;
    while active {
        active = false;
        for constraint in constraints.iter() {
            // Check to see if the constraint contains a sequenced relation.
            if constraint.iter().any(|(_,index)| result.contains(index)) {
                // If so, sequence any unsequenced relations.
                for (_, index) in constraint.iter() {
                    if !result.contains(index) {
                        result.push(*index);
                        active = true;
                    }
                }
            }
        }
    }

    result
}

/// Identifies keys and values for a join.
///
/// The result is a sequence, for each
fn determine_keys_priors(
    relation: usize,
    constraints: &[Vec<(usize, usize)>],
    current_attributes: &[(usize, usize)],
)
-> (Vec<usize>, Vec<usize>)
{
    // The fields in `sources[join_idx]` that should be keys are those
    // that share an equality constraint with an element of `attributes`.
    // For each key, we should capture the associated `attributes` entry
    // so that we can easily prepare the keys of the `delta` stream.
    let mut keys = Vec::new();
    let mut priors = Vec::new();
    for constraint in constraints.iter() {

        // If there is an intersection between `constraint` and `current_attributes`,
        // we should capture the position in `current_attributes` and emit all of the
        // attributes for `relation`.
        if let Some(prior) = current_attributes.iter().position(|x| constraint.contains(x)) {
            for &(attr, index) in constraint.iter() {
                if index == relation {
                    keys.push(attr);
                    priors.push(prior);
                }
            }
        }
    }

    (keys, priors)
}
<commit_msg>fewer bugs; triangle example<commit_after>//! Multi-way equijoin expression plan.
//!
//! This plan provides us the opportunity to map out a non-trivial differential
//! implementation for a complex join query. In particular, we are able to invoke
//! delta-query and worst-case optimal join plans, which avoid any intermediate
//! materialization.
//!
//! Each `MultiwayJoin` indicates several source collections, equality constraints
//! among their attributes, and then the set of attributes to produce as results.
//!
//! One naive implementation would take each input collection in order, and develop
//! the join restricted to the prefix of relations so far. Ideally the order would
//! be such that joined collections have equality constraints and prevent Cartesian
//! explosion. At each step, a new collection picks out some of the attributes and
//! instantiates a primitive binary join between the accumulated collection and the
//! next collection.
//!
//! A more sophisticated implementation establishes delta queries for each input
//! collection, which responds to changes in that input collection against the
//! current other input collections. For each input collection we may choose very
//! different join orders, as the order must follow equality constraints.
//!
//! A further implementation could develop the results attribute-by-attribute, as
//! opposed to collection-by-collection, which gives us the ability to use column
//! indices rather than whole-collection indices.

use std::hash::Hash;

use timely::dataflow::Scope;

use differential_dataflow::operators::Consolidate;
use differential_dataflow::operators::arrange::{ArrangeBySelf, ArrangeByKey};

use differential_dataflow::{Collection, ExchangeData};
use plan::{Plan, Render};
use {TraceManager, Time, Diff};

/// A multiway join of muliple relations.
///
/// By expressing multiple relations and required equivalances between their attributes,
/// we can more efficiently design incremental update strategies without materializing
/// and indexing intermediate relations.
#[derive(Serialize, Deserialize, Clone, Debug, Eq, PartialEq, Ord, PartialOrd, Hash)]
pub struct MultiwayJoin<Value> {
    /// A list of (attribute index, input) pairs to extract.
    pub results: Vec<(usize, usize)>,
    /// A list of source collections.
    pub sources: Vec<Plan<Value>>,
    /// Equality constraints.
    ///
    /// Equality constraints are presented as lists of `(attr, input)` equivalence classes.
    /// This means that each `(attr, input)` pair can exist in at most one list; if it would
    /// appear in more than one list, those two lists should be merged.
    pub equalities: Vec<Vec<(usize, usize)>>,
}

// TODO: This logic fails to perform restrictions in cases where a join does not
//       occur. One example could be:
//
//              result(a,b,c) := R1(a,b), R2(b,c,c)
//
//       In this case, the requirement that the 2nd and 3rd columns of R2 be equal
//       is not surfaced in any join, and is instead a filter that should be applied
//       directly to R2 (before or after the join with R1; either could be best).

impl<V: ExchangeData+Hash> Render for MultiwayJoin<V> {

    type Value = V;

    fn render<S: Scope<Timestamp = Time>>(
        &self,
        scope: &mut S,
        arrangements: &mut TraceManager<Self::Value>) -> Collection<S, Vec<Self::Value>, Diff>
    {
        // The idea here is the following:
        //
        // For each stream, we will determine a streaming delta query, in which changes
        // are joined against indexed forms of the other relations using `dogsdogsdogs`
        // stateless `propose` operators.
        //
        // For a query Q(x,y,z) := A(x,y), B(y,z), C(x,z) we might write dataflows like:
        //
        //   dQdA := dA(x,y), B(y,z), C(x,z)
        //   dQdB := dB(y,z), A(x,y), C(x,z)
        //   dQdC := dC(x,y), A(x,y), B(y,z)
        //
        // where each line is read from left to right as a sequence of `propose` joins,
        // which respond to timestamped delta changes by joining with the maintained
        // relation at that corresponding time.
        //
        // We take some care to make sure that when these joins are performed, a delta
        // interacts with relations as if we updated A, then B, then C, as if in sequence.
        // That is, when a dB delta joins A it observes all updates to A at times less or
        // equal to the delta's timestamp, but when a dB delta joins C it observes only
        // updates to C at times strictly less than the delta's timestamp.
        //
        // This is done to avoid double counting updates; any concurrent changes will be
        // accounted for by the last relation for which there is a concurrent update.

        // println!("{:?}", self);

        // Attributes we may need from any and all relations.
        let mut relevant_attributes = Vec::new();
        relevant_attributes.extend(self.results.iter().cloned());
        relevant_attributes.extend(self.equalities.iter().flat_map(|list| list.iter().cloned()));
        relevant_attributes.sort();
        relevant_attributes.dedup();

        // println!("Relevant attributes: {:?}", relevant_attributes);

        // Into which we accumulate change streams.
        let mut accumulated_changes = Vec::new();

        // For each participating relation, we build a delta query dataflow.
        for (index, plan) in self.sources.iter().enumerate() {

            // println!("building dataflow for relation {}", index);

            // Restrict down to relevant attributes.
            let mut attributes: Vec<(usize, usize)> =
            relevant_attributes
                .iter()
                .filter(|(_attr, input)| input == &index)
                .cloned()
                .collect::<Vec<_>>();

            let attributes_init = attributes.clone();
            // println!("\tinitial attributes: {:?}", attributes);

            // Ensure the plan is rendered and cached.
            if arrangements.get_unkeyed(&plan).is_none() {
                // println!("\tbuilding/caching source plan");
                let collection = plan.render(scope, arrangements);
                arrangements.set_unkeyed(plan, &collection.arrange_by_self().trace);
            }
            else {
                // println!("\tsource plan found");
            }
            let changes =
            arrangements
                .get_unkeyed(&plan)
                .expect("Surely we just ensured this")
                .import(scope)
                .as_collection(|val,&()| val.clone())
                .map(move |tuple| attributes_init.iter().map(|&(attr,_)|
                    tuple[attr].clone()).collect::<Vec<_>>()
                );

            // Before constructing the dataflow, which takes a borrow on `scope`,
            // we'll want to ensure that we have all of the necessary data assets
            // in place. This requires a bit of planning first, then the building.

            // Acquire a sane sequence in which to join the relations:
            //
            // This is a sequence of relation identifiers, starting with `index`,
            // such that each has at least one attribute in common with a prior
            // relation, and so can be effectively joined.
            let join_order = plan_join_order(index, &self.equalities);
            let mut join_plan = Vec::new();

            // println!("\tjoin order: {:?}", join_order);

            // Skipping `index`, join in each relation in sequence.
            for join_idx in join_order.into_iter().skip(1) {

                // To join a relation, we need to determine any constraints on
                // attributes in common with prior relations. Any other values
                // should be appended to tuples in `changes` with care taken to
                // update `attributes`.
                let (keys, priors) = determine_keys_priors(join_idx, &self.equalities, &attributes[..]);

                // The fields in `sources[join_idx]` that should be values are those
                // that are required output or participate in an equality constraint,
                // but *WHICH ARE NOT* in `keys`.
                let vals =
                relevant_attributes
                    .iter()
                    .filter(|&(attr,index)| index == &join_idx && !keys.contains(&attr))
                    .cloned()
                    .collect::<Vec<_>>();

                // println!("\tkeys: {:?}, priors: {:?}, vals: {:?}", keys, priors, vals);

                let mut projection = Vec::new();
                for &attr in keys.iter() {
                    projection.push(attr);
                }
                for &(attr, _index) in vals.iter() {
                    projection.push(attr);
                }
                // TODO: Sort, to improve chances of re-use opportunities.
                //       Requires understanding how attributes move to get the right
                //       key selectors out though.
                // projection.sort();
                // projection.dedup(); // Should already be deduplicated, probably?

                // Get a plan for the projection on to these few attributes.
                let plan = self.sources[join_idx].clone().project(projection);

                if arrangements.get_keyed(&plan, &keys[..]).is_none() {
                    // println!("\tbuilding key: {:?}, plan: {:?}", keys, plan);
                    let keys_clone = keys.clone();
                    let arrangement =
                    plan.render(scope, arrangements)
                        .map(move |tuple| (keys_clone.iter().map(|&i| tuple[i].clone()).collect::<Vec<_>>(), tuple))
                        .arrange_by_key();

                    arrangements.set_keyed(&plan, &keys[..], &arrangement.trace);
                }
                else {
                    // println!("\tplan found: {:?}, {:?}", keys, plan);
                }

                let arrangement =
                arrangements
                    .get_keyed(&plan, &keys[..])
                    .expect("Surely we just ensured this");

                let key_selector = std::rc::Rc::new(move |change: &Vec<V>|
                    priors.iter().map(|&p| change[p].clone()).collect::<Vec<_>>()
                );

                join_plan.push((join_idx, key_selector, arrangement));

                attributes.extend(keys.into_iter().map(|x| (x, join_idx)));
                attributes.extend(vals.into_iter());
                // println!("\tattributes: {:?}", attributes);
            }

            // Build the dataflow.
            use dogsdogsdogs::altneu::AltNeu;

            let scope_name = format!("DeltaRule: {}/{}", index, self.sources.len());
            let changes = scope.clone().scoped::<AltNeu<_>,_,_>(&scope_name, |inner| {

                // This should default to an `AltNeu::Alt` timestamp.
                let mut changes =
                changes
                    .enter(inner)
                    ;

                for (join_idx, key_selector, mut trace) in join_plan.into_iter() {

                    // Use alt or neu timestamps based on relative indices.
                    // Must have an `if` statement here as the two arrangement have different
                    // types, and we would to determine `alt` v `neu` once, rather than per
                    // tuple in the cursor.
                    changes =
                    if join_idx < index {
                        let arrangement = trace.import(scope).enter_at(inner, |_,_,t| AltNeu::alt(t.clone()));
                        dogsdogsdogs::operators::propose(&changes, arrangement, key_selector)
                    }
                    else {
                        let arrangement = trace.import(scope).enter_at(inner, |_,_,t| AltNeu::neu(t.clone()));
                        dogsdogsdogs::operators::propose(&changes, arrangement, key_selector)
                    }
                    .map(|(mut prefix, extensions)| { prefix.extend(extensions.into_iter()); prefix })
                    ;

                    // TODO: Equality constraints strictly within a relation have the effect
                    //       of "filtering" data, but they are ignored at the moment. We should
                    //       check for these and do something about it.
                }

                // Extract `self.results` in order, using `attributes`.
                //
                // The specific attribute requested in `self.results` may not be present in
                // `attributes` when it is equal to another present attribute. So, we should
                // look around in `self.equalities` also.
                let mut extract_map = Vec::new();
                for result in self.results.iter() {
                    if let Some(position) = attributes.iter().position(|i| i == result) {
                        extract_map.push(position);
                    }
                    else {
                        for constraint in self.equalities.iter() {
                            if constraint.contains(result) {
                                if let Some(position) = constraint.iter().flat_map(|x| attributes.iter().position(|i| i == x)).next() {
                                    extract_map.push(position);
                                }
                                else {
                                    println!("WTF NOTHING FOUND NOOOOO!!!");
                                }
                            }
                        }
                    }
                }

                changes
                    .map(move |tuple| extract_map.iter().map(|&i| tuple[i].clone()).collect::<Vec<_>>())
                    .leave()
            });

            accumulated_changes.push(changes);
        }

        differential_dataflow::collection::concatenate(scope, accumulated_changes.into_iter())
            .consolidate()
    }
}

/// Sequences relations in `constraints`.
///
/// Relations become available for sequencing as soon as they share a constraint with
/// either `source` or another sequenced relation.
fn plan_join_order(source: usize, constraints: &[Vec<(usize, usize)>]) -> Vec<usize> {

    let mut result = vec![source];
    let mut active = true;
    while active {
        active = false;
        for constraint in constraints.iter() {
            // Check to see if the constraint contains a sequenced relation.
            if constraint.iter().any(|(_,index)| result.contains(index)) {
                // If so, sequence any unsequenced relations.
                for (_, index) in constraint.iter() {
                    if !result.contains(index) {
                        result.push(*index);
                        active = true;
                    }
                }
            }
        }
    }

    result
}

/// Identifies keys and values for a join.
///
/// The result is a sequence, for each
fn determine_keys_priors(
    relation: usize,
    constraints: &[Vec<(usize, usize)>],
    current_attributes: &[(usize, usize)],
)
-> (Vec<usize>, Vec<usize>)
{
    // The fields in `sources[join_idx]` that should be keys are those
    // that share an equality constraint with an element of `attributes`.
    // For each key, we should capture the associated `attributes` entry
    // so that we can easily prepare the keys of the `delta` stream.
    let mut keys = Vec::new();
    let mut priors = Vec::new();
    for constraint in constraints.iter() {

        // If there is an intersection between `constraint` and `current_attributes`,
        // we should capture the position in `current_attributes` and emit all of the
        // attributes for `relation`.
        if let Some(prior) = current_attributes.iter().position(|x| constraint.contains(x)) {
            for &(attr, index) in constraint.iter() {
                if index == relation {
                    keys.push(attr);
                    priors.push(prior);
                }
            }
        }
    }

    (keys, priors)
}
<|endoftext|>