<commit_before>use std::slice;
use std::io;

use image;
use image::ImageResult;
use image::ImageDecoder;
use color;

use super::lzw::LZWReader;

const IMAGEDESCRIPTOR: u8 = 0x2C;
const EXTENSION: u8 = 0x21;
const APPLICATION: u8 = 0xFF;
const GRAPHICCONTROL: u8 = 0xF9;
const COMMENT: u8 = 0xFE;
const TRAILER: u8 = 0x3B;

/// The Representation of a GIF decoder
pub struct GIFDecoder <R> {
    r: R,

    width: u16,
    height: u16,

    global_table: [(u8, u8, u8), ..256],
    local_table: Option<Vec<(u8, u8, u8)>>,

    delay: u16,
    image: Vec<u8>,

    global_backgroud_index: Option<u8>,
    local_transparent_index: Option<u8>,

    have_header: bool,
    decoded_rows: u32,
}

impl<R: Reader> GIFDecoder<R> {
    /// Create a new GIFDecoder from the Reader ```r```.
    /// This function takes ownership of the Reader.
    pub fn new(r: R) -> GIFDecoder<R> {
        GIFDecoder {
            r: r,

            width: 0,
            height: 0,

            global_table: [(0u8, 0u8, 0u8), ..256],
            local_table: None,

            delay: 0,
            image: Vec::new(),

            global_backgroud_index: None,
            local_transparent_index: None,

            have_header: false,
            decoded_rows: 0,
        }
    }

    ///Returns the display delay in 100th's of a second for the currently
    ///decoded image.
    pub fn delay(&mut self) -> ImageResult<u16> {

        let _ = try!(self.read_metadata());

        Ok(self.delay)
    }

    fn read_header(&mut self) -> ImageResult<()> {
        let signature = try!(self.r.read_exact(3));
        let version   = try!(self.r.read_exact(3));

        if signature.as_slice() != "GIF".as_bytes() {
            Err(image::FormatError("GIF signature not found.".to_string()))
        } else if version.as_slice() != "87a".as_bytes() &&
                  version.as_slice() != "89a".as_bytes() {
            Err(image::UnsupportedError(
                format!("GIF version {} is not supported.", version)
            ))
        } else {
            Ok(())
        }
    }

    fn read_block(&mut self) -> io::IoResult<Vec<u8>> {
        let size = try!(self.r.read_u8());
        Ok(try!(self.r.read_exact(size as uint)))
    }

    fn read_image_data(&mut self) -> ImageResult<Vec<u8>> {
        let minimum_code_size = try!(self.r.read_u8());

        if minimum_code_size > 8 {
            return Err(image::FormatError(format!("Invalid code size {}.", minimum_code_size)))
        }

        let mut data = Vec::new();
        loop {
            let b = try!(self.read_block());

            if b.len() == 0 {
                break
            }

            data = data + b;
        }

        let m = io::MemReader::new(data);
        let mut lzw = LZWReader::new(m, minimum_code_size);
        let b = lzw.read_to_end().unwrap();

        Ok(b)
    }

    fn read_image_descriptor(&mut self) -> ImageResult<()> {
        let image_left   = try!(self.r.read_le_u16());
        let image_top    = try!(self.r.read_le_u16());
        let image_width  = try!(self.r.read_le_u16());
        let image_height = try!(self.r.read_le_u16());

        let fields = try!(self.r.read_u8());

        let local_table = fields & 80 != 0;
        let interlace   = fields & 40 != 0;
        let table_size  = fields & 7;

        if interlace {
            return Err(image::UnsupportedError("Interlaced images are not supported.".to_string()))
        }

        if local_table {
            let n   = 1 << (table_size + 1) as uint;
            let buf = try!(self.r.read_exact(3 * n));
            let mut b = Vec::from_elem(n, (0u8, 0u8, 0u8));

            for (i, rgb) in buf.as_slice().chunks(3).enumerate() {
                b.as_mut_slice()[i] = (rgb[0], rgb[1], rgb[2]);
            }

            self.local_table = Some(b);
        }

        let indices = try!(self.read_image_data());

        {

            let trans_index = if self.local_transparent_index.is_some() {
                self.local_transparent_index
            } else {
                self.global_backgroud_index
            };

            let table = if self.local_table.is_some() {
                self.local_table.as_ref().unwrap().as_slice()
            } else {
                self.global_table.as_slice()
            };

            expand_image(
                table,
                indices.as_slice(),
                image_top as uint,
                image_left as uint,
                image_width as uint,
                image_height as uint,
                self.width as uint * 3,
                trans_index,
                self.image.as_mut_slice()
            );
        }

        self.local_table = None;
        self.local_transparent_index = None;

        Ok(())
    }

    fn read_extension(&mut self) -> ImageResult<()> {
        let identifier = try!(self.r.read_u8());

        match identifier {
            APPLICATION    => try!(self.read_application_extension()),
            GRAPHICCONTROL => try!(self.read_graphic_control_extension()),
            COMMENT 	   => try!(self.read_comment_extension()),
            _              => return Err(image::UnsupportedError(
                                  format!("Identifier {} is not supported.", identifier))
                              )
        }

        Ok(())
    }

    fn read_comment_extension(&mut self) -> ImageResult<()> {
        loop {
            let b = try!(self.read_block());

            if b.len() == 0 {
                break
            }
        }

        Ok(())
    }

    fn read_graphic_control_extension(&mut self) -> ImageResult<()> {
        let size   = try!(self.r.read_u8());
        assert!(size == 4);

        let fields = try!(self.r.read_u8());
        self.delay = try!(self.r.read_le_u16());
        let trans  = try!(self.r.read_u8());

        if fields & 1 != 0 {
            self.local_transparent_index = Some(trans);
        }

        let _disposal = (fields & 0x1C) >> 2;
        let _term = try!(self.r.read_u8());

        Ok(())
    }

    fn read_application_extension(&mut self) -> ImageResult<()> {
        let size = try!(self.r.read_u8());
        let _ = try!(self.r.read_exact(size as uint));

        loop {
            let b = try!(self.read_block());

            if b.len() == 0 {
                break
            }
        }

        Ok(())
    }

    fn read_logical_screen_descriptor(&mut self) -> ImageResult<()> {
        self.width  = try!(self.r.read_le_u16());
        self.height = try!(self.r.read_le_u16());
        self.image  = Vec::from_elem(self.width as uint * self.height as uint * 3, 0u8);

        let fields = try!(self.r.read_u8());

        let global_table = fields & 0x80 != 0;

        let entries = if global_table {
            1 << ((fields & 7) + 1) as uint
        } else {
            0u
        };

        if global_table {
        let b = try!(self.r.read_u8());
            self.global_backgroud_index = Some(b);
        }

        let _aspect_ratio = try!(self.r.read_u8());

        let buf = try!(self.r.read_exact(3 * entries));

        for (i, rgb) in buf.as_slice().chunks(3).enumerate() {
            self.global_table.as_mut_slice()[i] = (rgb[0], rgb[1], rgb[2]);
        }

        Ok(())
    }

    fn read_metadata(&mut self) -> ImageResult<()> {
        if !self.have_header {
            let _ = try!(self.read_header());
            let _ = try!(self.read_logical_screen_descriptor());
            self.have_header = true;
        }

        Ok(())
    }
}

impl<R: Reader> ImageDecoder for GIFDecoder<R> {
    fn dimensions(&mut self) -> ImageResult<(u32, u32)> {
        let _ = try!(self.read_metadata());
        Ok((self.width as u32, self.height as u32))
    }

    fn colortype(&mut self) -> ImageResult<color::ColorType> {
        let _ = try!(self.read_metadata());
        Ok(color::RGB(8))
    }

    fn row_len(&mut self) -> ImageResult<uint> {
        let _ = try!(self.read_metadata());
        Ok(3 * self.width as uint)
    }

    fn read_scanline(&mut self, buf: &mut [u8]) -> ImageResult<u32> {
        let _ = try!(self.read_metadata());
        if self.decoded_rows == self.height as u32 || self.image.len() == 0 {
            let _ = try!(self.read_image());
        }

        let rlen  = buf.len();
        let slice = self.image.slice(self.decoded_rows as uint * rlen,
        self.decoded_rows as uint * rlen + rlen);

        slice::bytes::copy_memory(buf, slice);
        self.decoded_rows += 1;

        Ok(self.decoded_rows)
    }

    fn read_image(&mut self) -> ImageResult<Vec<u8>> {
        let _ = try!(self.read_metadata());
        loop {
            let block = try!(self.r.read_u8());

            match block {
                EXTENSION => try!(self.read_extension()),
                IMAGEDESCRIPTOR => {
                    let _ = try!(self.read_image_descriptor());
                    return Ok(self.image.clone())
                }
                TRAILER => break,
                _       => return Err(image::UnsupportedError(
                            format!("Block type {} is not supported.", block))
                        )
            }
        }

        Err(image::ImageEnd)
    }
}

fn expand_image(palette: &[(u8, u8, u8)],
                indices: &[u8],
                y0: uint,
                x0: uint,
                width: uint,
                height: uint,
                stride: uint,
                trans_index: Option<u8>,
                image: &mut [u8]) {

    for y in range(0, height) {
        for x in range(0, width) {
            let index = indices[y * width + x];

            if trans_index == Some(index) {
                continue
            }

            let (r, g, b) = palette[index as uint];

            image[(y0 + y) * stride + x0 * 3 + x * 3 + 0] = r;
            image[(y0 + y) * stride + x0 * 3 + x * 3 + 1] = g;
            image[(y0 + y) * stride + x0 * 3 + x * 3 + 2] = b;
        }
    }
}
<commit_msg>Always read the background field.<commit_after>use std::slice;
use std::io;

use image;
use image::ImageResult;
use image::ImageDecoder;
use color;

use super::lzw::LZWReader;

const IMAGEDESCRIPTOR: u8 = 0x2C;
const EXTENSION: u8 = 0x21;
const APPLICATION: u8 = 0xFF;
const GRAPHICCONTROL: u8 = 0xF9;
const COMMENT: u8 = 0xFE;
const TRAILER: u8 = 0x3B;

/// The Representation of a GIF decoder
pub struct GIFDecoder <R> {
    r: R,

    width: u16,
    height: u16,

    global_table: [(u8, u8, u8), ..256],
    local_table: Option<Vec<(u8, u8, u8)>>,

    delay: u16,
    image: Vec<u8>,

    global_backgroud_index: Option<u8>,
    local_transparent_index: Option<u8>,

    have_header: bool,
    decoded_rows: u32,
}

impl<R: Reader> GIFDecoder<R> {
    /// Create a new GIFDecoder from the Reader ```r```.
    /// This function takes ownership of the Reader.
    pub fn new(r: R) -> GIFDecoder<R> {
        GIFDecoder {
            r: r,

            width: 0,
            height: 0,

            global_table: [(0u8, 0u8, 0u8), ..256],
            local_table: None,

            delay: 0,
            image: Vec::new(),

            global_backgroud_index: None,
            local_transparent_index: None,

            have_header: false,
            decoded_rows: 0,
        }
    }

    ///Returns the display delay in 100th's of a second for the currently
    ///decoded image.
    pub fn delay(&mut self) -> ImageResult<u16> {

        let _ = try!(self.read_metadata());

        Ok(self.delay)
    }

    fn read_header(&mut self) -> ImageResult<()> {
        let signature = try!(self.r.read_exact(3));
        let version   = try!(self.r.read_exact(3));

        if signature.as_slice() != "GIF".as_bytes() {
            Err(image::FormatError("GIF signature not found.".to_string()))
        } else if version.as_slice() != "87a".as_bytes() &&
                  version.as_slice() != "89a".as_bytes() {
            Err(image::UnsupportedError(
                format!("GIF version {} is not supported.", version)
            ))
        } else {
            Ok(())
        }
    }

    fn read_block(&mut self) -> io::IoResult<Vec<u8>> {
        let size = try!(self.r.read_u8());
        Ok(try!(self.r.read_exact(size as uint)))
    }

    fn read_image_data(&mut self) -> ImageResult<Vec<u8>> {
        let minimum_code_size = try!(self.r.read_u8());

        if minimum_code_size > 8 {
            return Err(image::FormatError(format!("Invalid code size {}.", minimum_code_size)))
        }

        let mut data = Vec::new();
        loop {
            let b = try!(self.read_block());

            if b.len() == 0 {
                break
            }

            data = data + b;
        }

        let m = io::MemReader::new(data);
        let mut lzw = LZWReader::new(m, minimum_code_size);
        let b = lzw.read_to_end().unwrap();

        Ok(b)
    }

    fn read_image_descriptor(&mut self) -> ImageResult<()> {
        let image_left   = try!(self.r.read_le_u16());
        let image_top    = try!(self.r.read_le_u16());
        let image_width  = try!(self.r.read_le_u16());
        let image_height = try!(self.r.read_le_u16());

        let fields = try!(self.r.read_u8());

        let local_table = fields & 80 != 0;
        let interlace   = fields & 40 != 0;
        let table_size  = fields & 7;

        if interlace {
            return Err(image::UnsupportedError("Interlaced images are not supported.".to_string()))
        }

        if local_table {
            let n   = 1 << (table_size + 1) as uint;
            let buf = try!(self.r.read_exact(3 * n));
            let mut b = Vec::from_elem(n, (0u8, 0u8, 0u8));

            for (i, rgb) in buf.as_slice().chunks(3).enumerate() {
                b.as_mut_slice()[i] = (rgb[0], rgb[1], rgb[2]);
            }

            self.local_table = Some(b);
        }

        let indices = try!(self.read_image_data());

        {

            let trans_index = if self.local_transparent_index.is_some() {
                self.local_transparent_index
            } else {
                self.global_backgroud_index
            };

            let table = if self.local_table.is_some() {
                self.local_table.as_ref().unwrap().as_slice()
            } else {
                self.global_table.as_slice()
            };

            expand_image(
                table,
                indices.as_slice(),
                image_top as uint,
                image_left as uint,
                image_width as uint,
                image_height as uint,
                self.width as uint * 3,
                trans_index,
                self.image.as_mut_slice()
            );
        }

        self.local_table = None;
        self.local_transparent_index = None;

        Ok(())
    }

    fn read_extension(&mut self) -> ImageResult<()> {
        let identifier = try!(self.r.read_u8());

        match identifier {
            APPLICATION    => try!(self.read_application_extension()),
            GRAPHICCONTROL => try!(self.read_graphic_control_extension()),
            COMMENT 	   => try!(self.read_comment_extension()),
            _              => return Err(image::UnsupportedError(
                                  format!("Identifier {} is not supported.", identifier))
                              )
        }

        Ok(())
    }

    fn read_comment_extension(&mut self) -> ImageResult<()> {
        loop {
            let b = try!(self.read_block());

            if b.len() == 0 {
                break
            }
        }

        Ok(())
    }

    fn read_graphic_control_extension(&mut self) -> ImageResult<()> {
        let size   = try!(self.r.read_u8());
        assert!(size == 4);

        let fields = try!(self.r.read_u8());
        self.delay = try!(self.r.read_le_u16());
        let trans  = try!(self.r.read_u8());

        if fields & 1 != 0 {
            self.local_transparent_index = Some(trans);
        }

        let _disposal = (fields & 0x1C) >> 2;
        let _term = try!(self.r.read_u8());

        Ok(())
    }

    fn read_application_extension(&mut self) -> ImageResult<()> {
        let size = try!(self.r.read_u8());
        let _ = try!(self.r.read_exact(size as uint));

        loop {
            let b = try!(self.read_block());

            if b.len() == 0 {
                break
            }
        }

        Ok(())
    }

    fn read_logical_screen_descriptor(&mut self) -> ImageResult<()> {
        self.width  = try!(self.r.read_le_u16());
        self.height = try!(self.r.read_le_u16());
        self.image  = Vec::from_elem(self.width as uint * self.height as uint * 3, 0u8);

        let fields = try!(self.r.read_u8());

        let global_table = fields & 0x80 != 0;

        let entries = if global_table {
            1 << ((fields & 7) + 1) as uint
        } else {
            0u
        };

        let b = try!(self.r.read_u8());
        if global_table {
            self.global_backgroud_index = Some(b);
        }

        let _aspect_ratio = try!(self.r.read_u8());

        let buf = try!(self.r.read_exact(3 * entries));

        for (i, rgb) in buf.as_slice().chunks(3).enumerate() {
            self.global_table.as_mut_slice()[i] = (rgb[0], rgb[1], rgb[2]);
        }

        Ok(())
    }

    fn read_metadata(&mut self) -> ImageResult<()> {
        if !self.have_header {
            let _ = try!(self.read_header());
            let _ = try!(self.read_logical_screen_descriptor());
            self.have_header = true;
        }

        Ok(())
    }
}

impl<R: Reader> ImageDecoder for GIFDecoder<R> {
    fn dimensions(&mut self) -> ImageResult<(u32, u32)> {
        let _ = try!(self.read_metadata());
        Ok((self.width as u32, self.height as u32))
    }

    fn colortype(&mut self) -> ImageResult<color::ColorType> {
        let _ = try!(self.read_metadata());
        Ok(color::RGB(8))
    }

    fn row_len(&mut self) -> ImageResult<uint> {
        let _ = try!(self.read_metadata());
        Ok(3 * self.width as uint)
    }

    fn read_scanline(&mut self, buf: &mut [u8]) -> ImageResult<u32> {
        let _ = try!(self.read_metadata());
        if self.decoded_rows == self.height as u32 || self.image.len() == 0 {
            let _ = try!(self.read_image());
        }

        let rlen  = buf.len();
        let slice = self.image.slice(self.decoded_rows as uint * rlen,
        self.decoded_rows as uint * rlen + rlen);

        slice::bytes::copy_memory(buf, slice);
        self.decoded_rows += 1;

        Ok(self.decoded_rows)
    }

    fn read_image(&mut self) -> ImageResult<Vec<u8>> {
        let _ = try!(self.read_metadata());
        loop {
            let block = try!(self.r.read_u8());

            match block {
                EXTENSION => try!(self.read_extension()),
                IMAGEDESCRIPTOR => {
                    let _ = try!(self.read_image_descriptor());
                    return Ok(self.image.clone())
                }
                TRAILER => break,
                _       => return Err(image::UnsupportedError(
                            format!("Block type {} is not supported.", block))
                        )
            }
        }

        Err(image::ImageEnd)
    }
}

fn expand_image(palette: &[(u8, u8, u8)],
                indices: &[u8],
                y0: uint,
                x0: uint,
                width: uint,
                height: uint,
                stride: uint,
                trans_index: Option<u8>,
                image: &mut [u8]) {

    for y in range(0, height) {
        for x in range(0, width) {
            let index = indices[y * width + x];

            if trans_index == Some(index) {
                continue
            }

            let (r, g, b) = palette[index as uint];

            image[(y0 + y) * stride + x0 * 3 + x * 3 + 0] = r;
            image[(y0 + y) * stride + x0 * 3 + x * 3 + 1] = g;
            image[(y0 + y) * stride + x0 * 3 + x * 3 + 2] = b;
        }
    }
}
<|endoftext|>


<commit_before>pub use self::Variable::*;
pub use self::Stmt::*;
pub use self::Expr::*;
pub use self::Node::*;

use self::Literal::*;
use self::UnOp::*;
use self::BinOp::*;

/// Literal constants
#[derive(Debug, PartialEq, Clone)]
pub enum Literal {
    TInt(i64),
    TFloat(f64),
    TStr(String),
    TBool(bool),
    TNil,
}

/// Unary operators
#[derive(Copy, Clone, PartialEq, Debug)]
pub enum UnOp {
    Negate, // -
    LNot,   // !
    BNot,   // ~
    Len,    // #
}

/// Binary operators
#[derive(Copy, Clone, PartialEq, Debug)]
pub enum BinOp {
    Add,
    Sub,
    Mul,
    Div,
    Mod,

    Eq,
    NEq,
    LEq,
    GEq,
    Less,
    Greater,

    LAnd,
    LOr,
    BAnd,
    BOr,
    BXor,
    ShiftL,
    ShiftR,
}

/// A block containing any number of statements. All blocks carry a scope in which local variables
/// can be declared.
#[derive(Clone, Debug, PartialEq)]
pub struct Block {
    stmts: Vec<Stmt>,

    /// List of locals. Collected when resolving.
    locals: Vec<String>,
}

impl Block {
    pub fn new(stmts: Vec<Stmt>) -> Block {
        Block {
            stmts: stmts,
            locals: vec![],
        }
    }
}

#[derive(Clone, PartialEq, Debug)]
pub struct Call(pub Variable, pub Vec<Expr>);

#[derive(Clone, PartialEq, Debug)]
pub struct Function(pub Vec<String>, pub Block);

/// Something that can be assigned to a value
#[derive(Clone, PartialEq, Debug)]
pub enum Variable {
    /// References a named variable; later resolved to local or global
    VNamed(String),

    /// References the local variable in the current scope with the given name
    VLocal(String),

    /// References a named global
    VGlobal(String),

    /// References an indexed variable (a field)
    VIndex(Box<Variable>, Box<Expr>),
}

/// Statement nodes
#[derive(Clone, Debug, PartialEq)]
pub enum Stmt {
    /// Declare a list of locals and assign initial values.
    ///
    /// Initial values are optional and default to `nil` (the second vector can have less elements
    /// than the first).
    SDecl(Vec<String>, Vec<Expr>),

    /// Assigns a list of expressions to a list of variables.
    ///
    /// Contains at least one pair. If the last expression is a function or varargs, all return
    /// values are considered for assignment to the leftover variables.
    ///
    /// Might contain less variables than expressions or less expressions than variables. In the
    /// latter case, the leftover variables are assigned to nil.
    SAssign(Vec<Variable>, Vec<Expr>),

    /// Execute a block in a new scope
    SDo(Block),

    /// Abort the current loop
    SBreak,

    /// Return a possibly empty list of values to the caller
    SReturn(Vec<Expr>),

    /// Function call as statement
    SCall(Call),

    /// Assign function to named variable (`function XY(...) ... end`)
    SFunc(Variable, Function),

    /// Assign function to named local (`local function XY(...) ... end`)
    SLFunc(String, Function),

    /// Executes `body` if `cond` is true and `el` if not
    SIf {
        cond: Expr,
        body: Block,
        el: Block,
    },

    /// Loops a block while `cond` is true
    SWhile {
        cond: Expr,
        body: Block,
    },

    /// Loops a block until `abort_on` is true
    SRepeat {
        abort_on: Expr,
        body: Block,
    },

    /// Numeric for loop
    SFor {
        var: String,    // named local
        start: Expr,
        step: Expr,
        end: Expr,
        body: Block,
    },

    /// Generic for loop
    SForIn {
        /// The loop variables, returned by iterator
        vars: Vec<String>,
        /// Expression list: Iterator function, invariant state, start value, [ignored ...]
        iter: Vec<Expr>,
        body: Block,
    }
}

/// Expression nodes
#[derive(Clone, PartialEq, Debug)]
pub enum Expr {
    ELit(Literal),
    EBinOp(Box<Expr>, BinOp, Box<Expr>),
    EUnOp(UnOp, Box<Expr>),

    EVar(Variable),
    ECall(Call),
    EFunc(Function),
}

/// AST nodes
#[derive(Clone)]
pub enum Node {
    ExprNode(Expr),
    StmtNode(Stmt),
}
<commit_msg>Removed obsolete "Node" enum<commit_after>pub use self::Variable::*;
pub use self::Stmt::*;
pub use self::Expr::*;

use self::Literal::*;
use self::UnOp::*;
use self::BinOp::*;

/// Literal constants
#[derive(Debug, PartialEq, Clone)]
pub enum Literal {
    TInt(i64),
    TFloat(f64),
    TStr(String),
    TBool(bool),
    TNil,
}

/// Unary operators
#[derive(Copy, Clone, PartialEq, Debug)]
pub enum UnOp {
    Negate, // -
    LNot,   // !
    BNot,   // ~
    Len,    // #
}

/// Binary operators
#[derive(Copy, Clone, PartialEq, Debug)]
pub enum BinOp {
    Add,
    Sub,
    Mul,
    Div,
    Mod,

    Eq,
    NEq,
    LEq,
    GEq,
    Less,
    Greater,

    LAnd,
    LOr,
    BAnd,
    BOr,
    BXor,
    ShiftL,
    ShiftR,
}

/// A block containing any number of statements. All blocks carry a scope in which local variables
/// can be declared.
#[derive(Clone, Debug, PartialEq)]
pub struct Block {
    stmts: Vec<Stmt>,

    /// List of locals. Collected when resolving.
    locals: Vec<String>,
}

impl Block {
    pub fn new(stmts: Vec<Stmt>) -> Block {
        Block {
            stmts: stmts,
            locals: vec![],
        }
    }
}

#[derive(Clone, PartialEq, Debug)]
pub struct Call(pub Variable, pub Vec<Expr>);

#[derive(Clone, PartialEq, Debug)]
pub struct Function(pub Vec<String>, pub Block);

/// Something that can be assigned to a value
#[derive(Clone, PartialEq, Debug)]
pub enum Variable {
    /// References a named variable; later resolved to local or global
    VNamed(String),

    /// References the local variable in the current scope with the given name
    VLocal(String),

    /// References a named global
    VGlobal(String),

    /// References an indexed variable (a field)
    VIndex(Box<Variable>, Box<Expr>),
}

/// Statement nodes
#[derive(Clone, Debug, PartialEq)]
pub enum Stmt {
    /// Declare a list of locals and assign initial values.
    ///
    /// Initial values are optional and default to `nil` (the second vector can have less elements
    /// than the first).
    SDecl(Vec<String>, Vec<Expr>),

    /// Assigns a list of expressions to a list of variables.
    ///
    /// Contains at least one pair. If the last expression is a function or varargs, all return
    /// values are considered for assignment to the leftover variables.
    ///
    /// Might contain less variables than expressions or less expressions than variables. In the
    /// latter case, the leftover variables are assigned to nil.
    SAssign(Vec<Variable>, Vec<Expr>),

    /// Execute a block in a new scope
    SDo(Block),

    /// Abort the current loop
    SBreak,

    /// Return a possibly empty list of values to the caller
    SReturn(Vec<Expr>),

    /// Function call as statement
    SCall(Call),

    /// Assign function to named variable (`function XY(...) ... end`)
    SFunc(Variable, Function),

    /// Assign function to named local (`local function XY(...) ... end`)
    SLFunc(String, Function),

    /// Executes `body` if `cond` is true and `el` if not
    SIf {
        cond: Expr,
        body: Block,
        el: Block,
    },

    /// Loops a block while `cond` is true
    SWhile {
        cond: Expr,
        body: Block,
    },

    /// Loops a block until `abort_on` is true
    SRepeat {
        abort_on: Expr,
        body: Block,
    },

    /// Numeric for loop
    SFor {
        var: String,    // named local
        start: Expr,
        step: Expr,
        end: Expr,
        body: Block,
    },

    /// Generic for loop
    SForIn {
        /// The loop variables, returned by iterator
        vars: Vec<String>,
        /// Expression list: Iterator function, invariant state, start value, [ignored ...]
        iter: Vec<Expr>,
        body: Block,
    }
}

/// Expression nodes
#[derive(Clone, PartialEq, Debug)]
pub enum Expr {
    ELit(Literal),
    EBinOp(Box<Expr>, BinOp, Box<Expr>),
    EUnOp(UnOp, Box<Expr>),

    EVar(Variable),
    ECall(Call),
    EFunc(Function),
}
<|endoftext|>


<commit_before>use std::fmt::{Display, Error, Formatter};

#[derive(Debug, PartialEq, Eq)]
pub enum BinOp {
    Star,
    Plus,
    Minus,
}

#[derive(PartialEq, Eq, PartialOrd, Ord)]
pub enum Precedence {
    Const = 100,
    Mult = 60,
    Add = 50,
}

impl BinOp {
    pub fn precedence(&self) -> Precedence {
        match self {
            &BinOp::Star => Precedence::Mult,
            &BinOp::Plus | &BinOp::Minus => Precedence::Add,
        }
    }

    pub fn is_commutative(&self) -> bool {
        match self {
            &BinOp::Minus => false,
            _ => true
        }
    }
}

impl Display for BinOp {
    fn fmt(&self, fmt: &mut Formatter) -> Result<(), Error> {
        match self {
            &BinOp::Star => write!(fmt, "*"),
            &BinOp::Plus => write!(fmt, "+"),
            &BinOp::Minus => write!(fmt, "-"),
        }
    }
}

#[derive(Debug)]
pub enum Exp {
    Int(i64),
    Float(f64),
    BinExp(Box<Exp>, BinOp, Box<Exp>),
}

impl Exp {
    pub fn precedence(&self) -> Precedence {
        match self {
            &Exp::Int(_) | &Exp::Float(_) => Precedence::Const,
            &Exp::BinExp(_, ref o, _) => o.precedence(),
        }
    }
}

impl Display for Exp {
    fn fmt(&self, fmt: &mut Formatter) -> Result<(), Error> {
        match self {
            &Exp::Int(i) => write!(fmt, "{}", i),
            &Exp::Float(f) => write!(fmt, "{}", f),
            &Exp::BinExp(ref e1, ref o, ref e2) => {
                let prec = self.precedence();

                // Put grouping parentheses if the left subexpression has a lower-precedence
                // operator, e.g. (1 + 2) * 3
                let left = if prec > e1.precedence() {
                    format!("({})", e1)
                } else {
                    format!("{}", e1)
                };

                let right_prec = e2.precedence();

                // Put grouping parentheses around the right subexpression if it has a
                // lower-precedence operator,  __OR__ if `o` is not commutative and the precedence
                // is the same, e.g. (1 + 2) * 3 __OR__ 1 - (2 + 3)
                let right = if prec > right_prec || (!o.is_commutative() && prec == right_prec) {
                    format!("({})", e2)
                } else {
                    format!("{}", e2)
                };

                write!(fmt, "{} {} {}", left, o, right)
            }
        }
    }
}
<commit_msg>fixed match on pointers to use better style<commit_after>use std::fmt::{Display, Error, Formatter};

#[derive(Debug, PartialEq, Eq)]
pub enum BinOp {
    Star,
    Plus,
    Minus,
}

#[derive(PartialEq, Eq, PartialOrd, Ord)]
pub enum Precedence {
    Const = 100,
    Mult = 60,
    Add = 50,
}

impl BinOp {
    pub fn precedence(&self) -> Precedence {
        match *self {
            BinOp::Star => Precedence::Mult,
            BinOp::Plus | BinOp::Minus => Precedence::Add,
        }
    }

    pub fn is_commutative(&self) -> bool {
        match *self {
            BinOp::Minus => false,
            _ => true
        }
    }
}

impl Display for BinOp {
    fn fmt(&self, fmt: &mut Formatter) -> Result<(), Error> {
        match *self {
            BinOp::Star => write!(fmt, "*"),
            BinOp::Plus => write!(fmt, "+"),
            BinOp::Minus => write!(fmt, "-"),
        }
    }
}

#[derive(Debug)]
pub enum Exp {
    Int(i64),
    Float(f64),
    BinExp(Box<Exp>, BinOp, Box<Exp>),
}

impl Exp {
    pub fn precedence(&self) -> Precedence {
        match *self {
            Exp::Int(_) | Exp::Float(_) => Precedence::Const,
            Exp::BinExp(_, ref o, _) => o.precedence(),
        }
    }
}

impl Display for Exp {
    fn fmt(&self, fmt: &mut Formatter) -> Result<(), Error> {
        match *self {
            Exp::Int(i) => write!(fmt, "{}", i),
            Exp::Float(f) => write!(fmt, "{}", f),
            Exp::BinExp(ref e1, ref o, ref e2) => {
                let prec = self.precedence();

                // Put grouping parentheses if the left subexpression has a lower-precedence
                // operator, e.g. (1 + 2) * 3
                let left = if prec > e1.precedence() {
                    format!("({})", e1)
                } else {
                    format!("{}", e1)
                };

                let right_prec = e2.precedence();

                // Put grouping parentheses around the right subexpression if it has a
                // lower-precedence operator,  __OR__ if `o` is not commutative and the precedence
                // is the same, e.g. (1 + 2) * 3 __OR__ 1 - (2 + 3)
                let right = if prec > right_prec || (!o.is_commutative() && prec == right_prec) {
                    format!("({})", e2)
                } else {
                    format!("{}", e2)
                };

                write!(fmt, "{} {} {}", left, o, right)
            }
        }
    }
}
<|endoftext|>


<commit_before>use std::fmt;
use crate::token::Statements;

pub trait Node {
    fn token_literal(&self) -> String;
}

#[derive(Debug)]
pub struct Program {
    pub statements: Vec<Statements>,
}

impl Node for Program {
    fn token_literal(&self) -> String {
        if !self.statements.is_empty() {
            self.statements[0].token_literal()
        } else {
            "".to_string()
        }
    }
}

impl fmt::Display for Program {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        for statement in self.statements.iter() {
            fmt::Display::fmt(&statement, f)?;
            writeln!(f, "{}", "")?
        }

        Ok(())
    }
}
<commit_msg>[ast] small refactoring<commit_after>use std::fmt;
use crate::token::Statements;

pub trait Node {
    fn token_literal(&self) -> String;
}

#[derive(Debug)]
pub struct Program {
    pub statements: Vec<Statements>,
}

impl Node for Program {
    fn token_literal(&self) -> String {
        match self.statements.first() {
            Some(statement) => statement.token_literal(),
            None => "".to_string()
        }
    }
}

impl fmt::Display for Program {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        for statement in self.statements.iter() {
            fmt::Display::fmt(&statement, f)?;
            writeln!(f, "{}", "")?
        }

        Ok(())
    }
}
<|endoftext|>


<commit_before>use std::path::PathBuf;

use xshell::pushd;

use crate::{cmd, Metadata, Result};

mod spec_links;

use spec_links::check_spec_links;

const MSRV: &str = "1.55";

/// Task to run CI tests.
pub struct CiTask {
    /// Which version of Rust to test against.
    version: Option<String>,

    /// The root of the workspace.
    project_root: PathBuf,
}

impl CiTask {
    pub(crate) fn new(version: Option<String>) -> Result<Self> {
        let project_root = Metadata::load()?.workspace_root;
        Ok(Self { version, project_root })
    }

    pub(crate) fn run(self) -> Result<()> {
        let _p = pushd(&self.project_root)?;

        match self.version.as_deref() {
            Some("msrv") => self.build_msrv()?,
            Some("stable") => self.build_stable()?,
            Some("nightly") => self.build_nightly()?,
            Some(_) => return Err("Wrong Rust version specified.".into()),
            None => {
                self.build_msrv().and(self.build_stable()).and(self.build_nightly())?;
            }
        }

        Ok(())
    }

    fn build_msrv(&self) -> Result<()> {
        // Check all crates with all features except
        // * ruma (would pull in ruma-signatures)
        // * ruma-client (tested only with client-api feature due to most / all optional HTTP client
        //   deps having less strict MSRV)
        // * ruma-signatures (MSRV exception)
        // * xtask (no real reason to enforce an MSRV for it)
        cmd!(
            "rustup run {MSRV} cargo check --workspace --all-features
                --exclude ruma
                --exclude ruma-client
                --exclude ruma-signatures
                --exclude xtask"
        )
        .run()?;

        // Check ruma-client crate with default features
        cmd!("rustup run {MSRV} cargo check -p ruma-client --features client-api").run()?;

        // Check ruma crate with default features
        cmd!("rustup run {MSRV} cargo check -p ruma").run().map_err(Into::into)
    }

    fn build_stable(&self) -> Result<()> {
        // 1. Make sure everything compiles
        cmd!("rustup run stable cargo check --workspace --all-features").run()?;
        cmd!("rustup run stable cargo check -p ruma-client --no-default-features --features client-api").run()?;
        cmd!("rustup run stable cargo check -p ruma-identifiers --no-default-features").run()?;

        // 2. Run tests
        let workspace_res =
            cmd!("rustup run stable cargo test --workspace --features full,unstable-pre-spec")
                .run();
        let events_compat_res =
            cmd!("rustup run stable cargo test -p ruma-events --features compat compat").run();

        workspace_res.and(events_compat_res).map_err(Into::into)
    }

    fn build_nightly(&self) -> Result<()> {
        // Check formatting
        let fmt_res = cmd!("rustup run nightly cargo fmt -- --check").run();
        // Check `ruma` crate with `full` feature (sometimes things only compile with an unstable
        // flag)
        let check_full_res = cmd!("rustup run nightly cargo check -p ruma --features full").run();
        // Check everything with default features with clippy
        let clippy_default_res = cmd!(
            "
            rustup run nightly cargo clippy
                --workspace --all-targets --features=full -- -D warnings
            "
        )
        .run();
        // Check everything with almost all features with clippy
        let clippy_all_res = cmd!(
            "
            rustup run nightly cargo clippy
                --workspace --all-targets --features=full,compat,unstable-pre-spec -- -D warnings
            "
        )
        .run();
        // Check dependencies being sorted
        let sort_res = cmd!(
            "
            rustup run nightly cargo sort
                --workspace --grouped --check
                --order package,lib,features,dependencies,dev-dependencies,build-dependencies
            "
        )
        .run();
        // Check that all links point to the same version of the spec
        let spec_links = check_spec_links(&self.project_root.join("crates"));

        fmt_res
            .and(check_full_res)
            .and(clippy_default_res)
            .and(clippy_all_res)
            .and(sort_res)
            .map_err(Into::into)
            .and(spec_links)
    }
}
<commit_msg>ci: Ensure ruma-client builds without any features active<commit_after>use std::path::PathBuf;

use xshell::pushd;

use crate::{cmd, Metadata, Result};

mod spec_links;

use spec_links::check_spec_links;

const MSRV: &str = "1.55";

/// Task to run CI tests.
pub struct CiTask {
    /// Which version of Rust to test against.
    version: Option<String>,

    /// The root of the workspace.
    project_root: PathBuf,
}

impl CiTask {
    pub(crate) fn new(version: Option<String>) -> Result<Self> {
        let project_root = Metadata::load()?.workspace_root;
        Ok(Self { version, project_root })
    }

    pub(crate) fn run(self) -> Result<()> {
        let _p = pushd(&self.project_root)?;

        match self.version.as_deref() {
            Some("msrv") => self.build_msrv()?,
            Some("stable") => self.build_stable()?,
            Some("nightly") => self.build_nightly()?,
            Some(_) => return Err("Wrong Rust version specified.".into()),
            None => {
                self.build_msrv().and(self.build_stable()).and(self.build_nightly())?;
            }
        }

        Ok(())
    }

    fn build_msrv(&self) -> Result<()> {
        // Check all crates with all features except
        // * ruma (would pull in ruma-signatures)
        // * ruma-client (tested only with client-api feature due to most / all optional HTTP client
        //   deps having less strict MSRV)
        // * ruma-signatures (MSRV exception)
        // * xtask (no real reason to enforce an MSRV for it)
        cmd!(
            "rustup run {MSRV} cargo check --workspace --all-features
                --exclude ruma
                --exclude ruma-client
                --exclude ruma-signatures
                --exclude xtask"
        )
        .run()?;

        // Check ruma-client crate with default features
        cmd!("rustup run {MSRV} cargo check -p ruma-client --features client-api").run()?;

        // Check ruma crate with default features
        cmd!("rustup run {MSRV} cargo check -p ruma").run().map_err(Into::into)
    }

    fn build_stable(&self) -> Result<()> {
        // 1. Make sure everything compiles
        cmd!("rustup run stable cargo check --workspace --all-features").run()?;
        cmd!("rustup run stable cargo check -p ruma-client --no-default-features").run()?;
        cmd!("rustup run stable cargo check -p ruma-identifiers --no-default-features").run()?;

        // 2. Run tests
        let workspace_res =
            cmd!("rustup run stable cargo test --workspace --features full,unstable-pre-spec")
                .run();
        let events_compat_res =
            cmd!("rustup run stable cargo test -p ruma-events --features compat compat").run();

        workspace_res.and(events_compat_res).map_err(Into::into)
    }

    fn build_nightly(&self) -> Result<()> {
        // Check formatting
        let fmt_res = cmd!("rustup run nightly cargo fmt -- --check").run();
        // Check `ruma` crate with `full` feature (sometimes things only compile with an unstable
        // flag)
        let check_full_res = cmd!("rustup run nightly cargo check -p ruma --features full").run();
        // Check everything with default features with clippy
        let clippy_default_res = cmd!(
            "
            rustup run nightly cargo clippy
                --workspace --all-targets --features=full -- -D warnings
            "
        )
        .run();
        // Check everything with almost all features with clippy
        let clippy_all_res = cmd!(
            "
            rustup run nightly cargo clippy
                --workspace --all-targets --features=full,compat,unstable-pre-spec -- -D warnings
            "
        )
        .run();
        // Check dependencies being sorted
        let sort_res = cmd!(
            "
            rustup run nightly cargo sort
                --workspace --grouped --check
                --order package,lib,features,dependencies,dev-dependencies,build-dependencies
            "
        )
        .run();
        // Check that all links point to the same version of the spec
        let spec_links = check_spec_links(&self.project_root.join("crates"));

        fmt_res
            .and(check_full_res)
            .and(clippy_default_res)
            .and(clippy_all_res)
            .and(sort_res)
            .map_err(Into::into)
            .and(spec_links)
    }
}
<|endoftext|>


<commit_before>// Copyright 2018 Google Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

extern crate colored;
extern crate dirs;
extern crate evcxr;
extern crate regex;
extern crate rustyline;

use colored::*;
use evcxr::{CommandContext, CompilationError, Error};
use rustyline::{error::ReadlineError, Editor};
use std::fs;
use std::io;
use std::sync::mpsc;

const PROMPT: &str = "# ";

struct Repl {
    command_context: CommandContext,
}

fn send_output<T: io::Write + Send + 'static>(channel: mpsc::Receiver<String>, mut output: T) {
    std::thread::spawn(move || {
        while let Ok(line) = channel.recv() {
            if writeln!(output, "{}", line).is_err() {
                break;
            }
        }
    });
}

impl Repl {
    fn new() -> Result<Repl, Error> {
        let (command_context, outputs) = CommandContext::new()?;
        send_output(outputs.stdout, io::stdout());
        send_output(outputs.stderr, io::stderr());
        Ok(Repl { command_context })
    }

    fn execute(&mut self, to_run: &str) {
        match self.command_context.execute(to_run) {
            Ok(output) => {
                if let Some(text) = output.get("text/plain") {
                    println!("{}", text);
                }
            }
            Err(evcxr::Error::CompilationErrors(errors)) => {
                self.display_errors(errors);
            }
            Err(evcxr::Error::ChildProcessTerminated(err))
            | Err(evcxr::Error::JustMessage(err)) => eprintln!("{}", err.bright_red()),
        }
    }

    fn display_errors(&mut self, errors: Vec<CompilationError>) {
        for error in errors {
            if error.is_from_user_code() {
                for spanned_message in error.spanned_messages() {
                    if let Some(span) = &spanned_message.span {
                        for _ in 1..span.start_column + PROMPT.len() {
                            print!(" ");
                        }
                        let mut carrots = String::new();
                        for _ in span.start_column..span.end_column {
                            carrots.push('^');
                        }
                        print!("{}", carrots.bright_red());
                        println!(" {}", spanned_message.label.bright_blue());
                    } else {
                        // Our error originates from both user-code and generated
                        // code.
                        println!("{}", spanned_message.label.bright_blue());
                    }
                }
                println!("{}", error.message().bright_red());
                for help in error.help() {
                    println!("{} {}", "help:".bold(), help);
                }
                if let Some(extra_hint) = error.evcxr_extra_hint() {
                    println!("{}", extra_hint);
                }
            } else {
                println!(
                    "A compilation error was found in code we generated.\n\
                     Ideally this should't happen. Type :last_error_json to see details.\n{}",
                    error.rendered()
                );
            }
        }
    }
}

fn main() {
    evcxr::runtime_hook();
    println!("Welcome to evcxr. For help, type :help");
    let mut repl = match Repl::new() {
        Ok(c) => c,
        Err(error) => {
            eprintln!("{}", error);
            return;
        }
    };

    let mut editor = Editor::<()>::new();
    let mut opt_history_file = None;
    let config_dir = dirs::config_dir().map(|h| h.join("evcxr"));
    if let Some(config_dir) = &config_dir {
        fs::create_dir_all(config_dir).ok();
        let history_file = config_dir.join("history.txt");
        editor.load_history(&history_file).ok();
        opt_history_file = Some(history_file);
    }
    loop {
        let readline = editor.readline(&PROMPT.yellow());
        match readline {
            Ok(line) => {
                editor.add_history_entry(&line);
                repl.execute(&line);
            }
            Err(ReadlineError::Interrupted) | Err(ReadlineError::Eof) => break,
            Err(err) => {
                eprintln!("Error: {:?}", err);
                break;
            }
        }
    }
    if let Some(history_file) = &opt_history_file {
        editor.save_history(&history_file).ok();
    }
}
<commit_msg>Change prompt to be slightly less minimalist. Fixes #12<commit_after>// Copyright 2018 Google Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

extern crate colored;
extern crate dirs;
extern crate evcxr;
extern crate regex;
extern crate rustyline;

use colored::*;
use evcxr::{CommandContext, CompilationError, Error};
use rustyline::{error::ReadlineError, Editor};
use std::fs;
use std::io;
use std::sync::mpsc;

const PROMPT: &str = ">> ";

struct Repl {
    command_context: CommandContext,
}

fn send_output<T: io::Write + Send + 'static>(channel: mpsc::Receiver<String>, mut output: T) {
    std::thread::spawn(move || {
        while let Ok(line) = channel.recv() {
            if writeln!(output, "{}", line).is_err() {
                break;
            }
        }
    });
}

impl Repl {
    fn new() -> Result<Repl, Error> {
        let (command_context, outputs) = CommandContext::new()?;
        send_output(outputs.stdout, io::stdout());
        send_output(outputs.stderr, io::stderr());
        Ok(Repl { command_context })
    }

    fn execute(&mut self, to_run: &str) {
        match self.command_context.execute(to_run) {
            Ok(output) => {
                if let Some(text) = output.get("text/plain") {
                    println!("{}", text);
                }
            }
            Err(evcxr::Error::CompilationErrors(errors)) => {
                self.display_errors(errors);
            }
            Err(evcxr::Error::ChildProcessTerminated(err))
            | Err(evcxr::Error::JustMessage(err)) => eprintln!("{}", err.bright_red()),
        }
    }

    fn display_errors(&mut self, errors: Vec<CompilationError>) {
        for error in errors {
            if error.is_from_user_code() {
                for spanned_message in error.spanned_messages() {
                    if let Some(span) = &spanned_message.span {
                        for _ in 1..span.start_column + PROMPT.len() {
                            print!(" ");
                        }
                        let mut carrots = String::new();
                        for _ in span.start_column..span.end_column {
                            carrots.push('^');
                        }
                        print!("{}", carrots.bright_red());
                        println!(" {}", spanned_message.label.bright_blue());
                    } else {
                        // Our error originates from both user-code and generated
                        // code.
                        println!("{}", spanned_message.label.bright_blue());
                    }
                }
                println!("{}", error.message().bright_red());
                for help in error.help() {
                    println!("{} {}", "help:".bold(), help);
                }
                if let Some(extra_hint) = error.evcxr_extra_hint() {
                    println!("{}", extra_hint);
                }
            } else {
                println!(
                    "A compilation error was found in code we generated.\n\
                     Ideally this should't happen. Type :last_error_json to see details.\n{}",
                    error.rendered()
                );
            }
        }
    }
}

fn main() {
    evcxr::runtime_hook();
    println!("Welcome to evcxr. For help, type :help");
    let mut repl = match Repl::new() {
        Ok(c) => c,
        Err(error) => {
            eprintln!("{}", error);
            return;
        }
    };

    let mut editor = Editor::<()>::new();
    let mut opt_history_file = None;
    let config_dir = dirs::config_dir().map(|h| h.join("evcxr"));
    if let Some(config_dir) = &config_dir {
        fs::create_dir_all(config_dir).ok();
        let history_file = config_dir.join("history.txt");
        editor.load_history(&history_file).ok();
        opt_history_file = Some(history_file);
    }
    loop {
        let readline = editor.readline(&PROMPT.yellow());
        match readline {
            Ok(line) => {
                editor.add_history_entry(&line);
                repl.execute(&line);
            }
            Err(ReadlineError::Interrupted) | Err(ReadlineError::Eof) => break,
            Err(err) => {
                eprintln!("Error: {:?}", err);
                break;
            }
        }
    }
    if let Some(history_file) = &opt_history_file {
        editor.save_history(&history_file).ok();
    }
}
<|endoftext|>


<commit_before>use std::io::{IoResult, IoError, IoErrorKind, TcpStream, ConnectionRefused,
              ConnectionFailed, OtherIoError};
use std::io::net::ip::{ IpAddr, Ipv4Addr };

pub struct Socks4<'a> {
    socks_host: &'a str,
    socks_port: u16,
}

impl<'a> Socks4<'a> {
    pub fn new(host: &'a str, port: u16) -> Socks4 {
        Socks4 { socks_host: host, socks_port: port }
    }

    pub fn connect(&mut self, host: IpAddr, port: u16) -> IoResult<TcpStream> {
        let mut stream = try!(TcpStream::connect(self.socks_host, self.socks_port));
        try!(stream.write([0x04, 0x01]));
        try!(stream.write_be_u16(port));
        match host {
            Ipv4Addr(oct1, oct2, oct3, oct4) => {
                try!(stream.write([oct1, oct2, oct3, oct4]));
            },
            _ => { /* Error */ }
        };
        try!(stream.write([0x00]));

        // read null byte
        if 0 != try!(stream.read_u8()) {
            return io_err(OtherIoError, "Expected null byte not found");
        }

        // read status
        match try!(stream.read_u8()) {
            // request granted
            0x5a => {
                let _port = try!(stream.read_be_u16());
                let _ip = try!(stream.read_be_u32());

                Ok(stream)
            }
            // request rejected or failed
            0x5b => io_err(ConnectionRefused, "Request rejected or failed"),
            // request failed because client is not running identd (or unreachable)
            0x5c => io_err(ConnectionFailed, "Client unreachable"),
            // request failed because client's identd could not confirm the user ID
            // string in the request
            0x5d => io_err(ConnectionRefused, "Unknown user"),
            x => fail!("Unexpected status byte: {}", x)
        }
    }
}

fn io_err<T>(kind: IoErrorKind, desc: &'static str) -> IoResult<T> {
    Err(IoError { kind: kind, desc: desc, detail: None })
}

<commit_msg>Update socks4.rs<commit_after>use std::io::{IoResult, IoError, IoErrorKind, TcpStream, ConnectionRefused,
              ConnectionFailed, OtherIoError, InvalidInput};
use std::io::net::ip::{ IpAddr, Ipv4Addr };

pub struct Socks4<'a> {
    socks_host: &'a str,
    socks_port: u16,
}

impl<'a> Socks4<'a> {
    pub fn new(host: &'a str, port: u16) -> Socks4 {
        Socks4 { socks_host: host, socks_port: port }
    }

    pub fn connect(&mut self, host: IpAddr, port: u16) -> IoResult<TcpStream> {
        let mut stream = try!(TcpStream::connect(self.socks_host, self.socks_port));
        try!(stream.write([0x04, 0x01]));
        try!(stream.write_be_u16(port));
        match host {
            Ipv4Addr(oct1, oct2, oct3, oct4) => {
                try!(stream.write([oct1, oct2, oct3, oct4]));
            },
            _ => { return io_err(InvalidInput, "Must be IPv4 address"); }
        };
        try!(stream.write([0x00]));

        // read null byte
        if 0 != try!(stream.read_u8()) {
            return io_err(OtherIoError, "Expected null byte not found");
        }

        // read status
        match try!(stream.read_u8()) {
            // request granted
            0x5a => {
                let _port = try!(stream.read_be_u16());
                let _ip = try!(stream.read_be_u32());

                Ok(stream)
            }
            // request rejected or failed
            0x5b => io_err(ConnectionRefused, "Request rejected or failed"),
            // request failed because client is not running identd (or unreachable)
            0x5c => io_err(ConnectionFailed, "Client unreachable"),
            // request failed because client's identd could not confirm the user ID
            // string in the request
            0x5d => io_err(ConnectionRefused, "Unknown user"),
            x => fail!("Unexpected status byte: {}", x)
        }
    }
}

fn io_err<T>(kind: IoErrorKind, desc: &'static str) -> IoResult<T> {
    Err(IoError { kind: kind, desc: desc, detail: None })
}

<|endoftext|>


<commit_before>use rustc::lint::{Context, LintPass, LintArray};
use syntax::ast::{Mod, Item_, ViewPath_};
use syntax::codemap::Span;
use syntax::print::pprust::path_to_string;

declare_lint!(SORTED_DECLARATIONS, Warn,
              "Warn when the declarations of crates or modules are not in alphabetical order");

pub struct Sorter;

impl LintPass for Sorter {
    fn get_lints(&self) -> LintArray {
        lint_array!(SORTED_DECLARATIONS)
    }

    fn check_mod(&mut self, cx: &Context, module: &Mod, _span: Span, _id: u32) {
        let session_codemap = cx.tcx.sess.codemap();
        let mut extern_crates = Vec::new();
        let mut uses = Vec::new();
        let mut mods = Vec::new();
        for item in &module.items {
            let item_name = item.ident.name.as_str().to_owned();
            let item_span = item.span;
            match item.node.clone() {
                Item_::ItemExternCrate(_) if item_name != "std" => {
                    extern_crates.push((item_name, item_span));
                },
                Item_::ItemMod(module) => {
                    let mod_invoked_file = session_codemap.span_to_filename(item.span);
                    let mod_declared_file = session_codemap.span_to_filename(module.inner);
                    if mod_declared_file != mod_invoked_file {      // this ignores inline modules
                        mods.push((item_name, item_span));
                    }
                },
                Item_::ItemUse(spanned) => {
                    match spanned.node {
                        ViewPath_::ViewPathSimple(_, ref path) | ViewPath_::ViewPathList(ref path, _) => {
                            uses.push((path_to_string(&path), item_span));
                        },
                        ViewPath_::ViewPathGlob(ref path) => {
                            let path_str = path_to_string(&path);
                            // we don't have any use statements like `use std::prelude::*`
                            // since it's done only by rustc, we can safely neglect those here
                            match path_str.starts_with("std::") {
                                true => continue,
                                false => uses.push((path_str, item_span)),
                            }
                        },
                    }
                },
                _ => {},
            }
        }

        check_sort(&extern_crates, cx);
        check_sort(&mods, cx);
        check_sort(&uses, cx);

        fn check_sort(old_slice: &Vec<(String, Span)>, cx: &Context) {
            let mut new_slice = old_slice
                                .iter()
                                .map(|&(ref string, _span)| string.clone())
                                .collect::<Vec<String>>();
            new_slice.sort();
            for i in 0..old_slice.len() {
                let (declaration, span) = (old_slice[i].0.clone(), old_slice[i].1);
                if declaration != new_slice[i] {
                    let suggestion = format!("\n\texpected: {}\n\tfound: {}", new_slice[i], declaration);
                    cx.span_lint(SORTED_DECLARATIONS, span, &suggestion);
                }
            }
        }
    }
}
<commit_msg>lists of uses...<commit_after>use rustc::lint::{Context, LintPass, LintArray};
use std::cmp::Ordering;
use std::fmt::Display;
use syntax::ast::{Mod, Item_, PathListItem_, ViewPath_};
use syntax::codemap::{Pos, BytePos, Span};
use syntax::print::pprust::path_to_string;

declare_lint!(UNSORTED_DECLARATIONS, Deny,
              "Warn when the declarations of crates or modules are not in alphabetical order");

pub struct Sorter;

impl LintPass for Sorter {
    fn get_lints(&self) -> LintArray {
        lint_array!(UNSORTED_DECLARATIONS)
    }

    fn check_mod(&mut self, cx: &Context, module: &Mod, _span: Span, _id: u32) {
        let session_codemap = cx.tcx.sess.codemap();
        let mut extern_crates = Vec::new();
        let mut uses = Vec::new();
        let mut mods = Vec::new();
        for item in &module.items {
            let item_name = item.ident.name.as_str();
            let item_span = item.span;
            match item.node.clone() {
                Item_::ItemExternCrate(_) if item_name != "std" => {
                    extern_crates.push((item_name, item_span));
                },
                Item_::ItemMod(module) => {
                    let mod_invoked_file = session_codemap.span_to_filename(item.span);
                    let mod_declared_file = session_codemap.span_to_filename(module.inner);
                    if mod_declared_file != mod_invoked_file {      // this ignores inline modules
                        mods.push((item_name, item_span));
                    }
                },
                Item_::ItemUse(spanned) => {
                    match spanned.node {
                        ViewPath_::ViewPathSimple(_, ref path) => {
                            uses.push((path_to_string(&path), item_span));
                        },
                        ViewPath_::ViewPathList(ref path, ref list) => {
                            let old_list = list
                                           .iter()
                                           .filter_map(|&list_item| {
                                                match list_item.node {
                                                    PathListItem_::PathListIdent { name, .. } => {
                                                        let interned = name.name.as_str();
                                                        let string = &*interned;
                                                        Some(string.to_owned())
                                                    },
                                                    _ => None,
                                                }
                                            }).collect::<Vec<String>>();
                            let mut new_list = old_list.clone();
                            new_list.sort_by(|a, b| {
                                match *a == "self" {
                                    true => Ordering::Less,
                                    false => a.cmp(b),
                                }
                            });
                            for i in 0..old_list.len() {
                                if old_list[i] != new_list[i] {
                                    let suggestion = format!("use lists should be in alphabetical order!\
                                                              \n\texpected order of list: {:?}",
                                                              new_list);
                                    cx.span_lint(UNSORTED_DECLARATIONS, path.span, &suggestion);
                                    break;
                                }
                            }
                        },
                        ViewPath_::ViewPathGlob(ref path) => {
                            let path_str = path_to_string(&path);
                            // we don't have any use statements like `use std::prelude::*`
                            // since it's done only by rustc, we can safely neglect those here
                            if !path_str.starts_with("std::") {
                                uses.push((path_str, item_span));
                            }
                        },
                    }
                },
                _ => (),
            }
        }

        check_sort(&extern_crates, cx, "crate declarations", "extern crate");
        check_sort(&mods, cx, "module declarations", "mod");
        check_sort(&uses, cx, "use statements", "use");

        fn check_sort<T: Ord + PartialEq + Clone + Display>(old_slice: &[(T, Span)],
                                                            cx: &Context,
                                                            kind: &str,
                                                            syntax: &str) {
            let mut new_slice = old_slice
                                .iter()
                                .map(|&(ref string, _span)| string.clone())
                                .collect::<Vec<T>>();
            new_slice.sort();
            for i in 0..old_slice.len() {
                let (name, mut span) = (old_slice[i].0.clone(), old_slice[i].1);
                span.lo = span.lo + BytePos::from_usize(syntax.len() + 1);
                if name != new_slice[i] {
                    let suggestion = format!("{} should be in alphabetical order!\n\t\
                                              expected: \"{} {};\"",
                                              kind, syntax, new_slice[i]);
                    cx.span_lint(UNSORTED_DECLARATIONS, span, &suggestion);
                }
            }
        }
    }
}
<|endoftext|>


<commit_before>use std::ops::Deref;
use std::slice;

/// A sparse set used for representing ordered NFA states.
///
/// This supports constant time addition and membership testing. Clearing an
/// entire set can also be done in constant time. Iteration yields elements
/// in the order in which they were inserted.
///
/// The data structure is based on: http://research.swtch.com/sparse
/// Note though that we don't actually use uninitialized memory. We generally
/// reuse allocations, so the initial allocation cost is bareable. However,
/// its other properties listed above are extremely useful.
#[derive(Clone, Debug)]
pub struct SparseSet {
    /// Dense contains the instruction pointers in the order in which they
    /// were inserted.
    dense: Vec<usize>,
    /// Sparse maps instruction pointers to their location in dense.
    ///
    /// An instruction pointer is in the set if and only if
    /// sparse[ip] < dense.len() && ip == dense[sparse[ip]].
    sparse: Box<[usize]>,
}

impl SparseSet {
    pub fn new(size: usize) -> SparseSet {
        SparseSet {
            dense: Vec::with_capacity(size),
            sparse: vec![0; size].into_boxed_slice(),
        }
    }

    pub fn len(&self) -> usize {
        self.dense.len()
    }

    pub fn is_empty(&self) -> bool {
        self.dense.is_empty()
    }

    pub fn capacity(&self) -> usize {
        self.dense.capacity()
    }

    pub fn insert(&mut self, value: usize) {
        let i = self.len();
        assert!(i < self.capacity());
        self.dense.push(value);
        self.sparse[value] = i;
    }

    pub fn contains(&self, value: usize) -> bool {
        let i = self.sparse[value];
        self.dense.get(i) == Some(&value)
    }

    pub fn clear(&mut self) {
        self.dense.clear();
    }
}

impl Deref for SparseSet {
    type Target = [usize];

    fn deref(&self) -> &Self::Target {
        &self.dense
    }
}

impl<'a> IntoIterator for &'a SparseSet {
    type Item = &'a usize;
    type IntoIter = slice::Iter<'a, usize>;
    fn into_iter(self) -> Self::IntoIter {
        self.iter()
    }
}
<commit_msg>sparse: add better Debug impl<commit_after>use std::fmt;
use std::ops::Deref;
use std::slice;

/// A sparse set used for representing ordered NFA states.
///
/// This supports constant time addition and membership testing. Clearing an
/// entire set can also be done in constant time. Iteration yields elements
/// in the order in which they were inserted.
///
/// The data structure is based on: http://research.swtch.com/sparse
/// Note though that we don't actually use uninitialized memory. We generally
/// reuse allocations, so the initial allocation cost is bareable. However,
/// its other properties listed above are extremely useful.
#[derive(Clone)]
pub struct SparseSet {
    /// Dense contains the instruction pointers in the order in which they
    /// were inserted.
    dense: Vec<usize>,
    /// Sparse maps instruction pointers to their location in dense.
    ///
    /// An instruction pointer is in the set if and only if
    /// sparse[ip] < dense.len() && ip == dense[sparse[ip]].
    sparse: Box<[usize]>,
}

impl SparseSet {
    pub fn new(size: usize) -> SparseSet {
        SparseSet {
            dense: Vec::with_capacity(size),
            sparse: vec![0; size].into_boxed_slice(),
        }
    }

    pub fn len(&self) -> usize {
        self.dense.len()
    }

    pub fn is_empty(&self) -> bool {
        self.dense.is_empty()
    }

    pub fn capacity(&self) -> usize {
        self.dense.capacity()
    }

    pub fn insert(&mut self, value: usize) {
        let i = self.len();
        assert!(i < self.capacity());
        self.dense.push(value);
        self.sparse[value] = i;
    }

    pub fn contains(&self, value: usize) -> bool {
        let i = self.sparse[value];
        self.dense.get(i) == Some(&value)
    }

    pub fn clear(&mut self) {
        self.dense.clear();
    }
}

impl fmt::Debug for SparseSet {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        write!(f, "SparseSet({:?})", self.dense)
    }
}

impl Deref for SparseSet {
    type Target = [usize];

    fn deref(&self) -> &Self::Target {
        &self.dense
    }
}

impl<'a> IntoIterator for &'a SparseSet {
    type Item = &'a usize;
    type IntoIter = slice::Iter<'a, usize>;
    fn into_iter(self) -> Self::IntoIter {
        self.iter()
    }
}
<|endoftext|>


<commit_before>// Copyright 2016 Johannes Köster.
// Licensed under the MIT license (http://opensource.org/licenses/MIT)
// This file may not be copied, modified, or distributed
// except according to those terms.

#![allow(non_snake_case)]

use std;
use std::collections;
use std::io::prelude::*;
use std::error::Error;

use csv;
use itertools::Itertools;
use cue;
use regex::Regex;

use bio::stats::logprobs::Prob;
use bio::stats::logprobs;

use io;
use model;
use model::foldchange::LogFC;
use model::cv::CV;
use codebook;

pub struct Selection {
    pub expmnt: String,
    pub cell: String
}

#[derive(Debug)]
struct Counts {
    exact: u32,
    corrected: u32
}

/*pub fn stats(N: u8, m: u8, p0: Prob, p1: Prob, dist: u8, codebook_path: &str) {
    let codebook = io::codebook::Reader::from_file(codebook_path, dist).unwrap().codebook();
    let model = model::readout::new_model(N, m, p0, p1, dist, codebook);
    let mut reader = io::merfishdata::Reader::from_reader(std::io::stdin());

    let mut total_counts = collections::HashMap::new();
    for res in reader.records() {
        let record = res.unwrap();
        let total_count = total_counts.entry(record.cell_id).or_insert(0);
        *total_count += 1;
    }

    let mut writer = csv::Writer::from_writer(std::io::stdout()).delimiter(b'\t');
    writer.write(["cell", "total_expr_ev"].iter()).unwrap();

    for (cell, total_count) in total_counts {
        writer.write([cell, format!("{}", model.expected_total(total_count))].iter()).unwrap();
    }
}*/

/// Estimate expressions.
pub fn expression(N: u8, m: u8, p0: Prob, p1: Prob, dist: u8, codebook_path: &str, estimate_path: Option<&str>, threads: usize, cells: &str, window_width: u32, print_naive: bool) {
    let codebook = io::codebook::Reader::from_file(codebook_path, dist).unwrap().codebook();
    let mut reader = io::merfishdata::Reader::from_reader(std::io::stdin());
    let mut cdf_writer = io::cdf::expression::Writer::from_writer(std::io::stdout());
    let mut est_writer = estimate_path.map(|path| io::estimation::expression::Writer::from_file(path, print_naive));

    let cells = Regex::new(cells).ok().expect("Invalid regular expression for cells.");

    let mut counts = collections::HashMap::new();
    for record in reader.records().filter_map(|res| {
            let rec = res.unwrap();
            // consider record if it is contained in the codebook and has a valid cell id
            if cells.is_match(&rec.cell_id) && codebook.contains(&rec.feature) {
                Some(rec)
            }
            else {
                None
            }
        }
    ) {
        let cell_counts = counts.entry(record.cell_id).or_insert_with(collections::HashMap::new);
        let feature_counts = cell_counts.entry(record.feature).or_insert(Counts{ exact: 0, corrected: 0});
        if record.hamming_dist == 0 {
            feature_counts.exact += 1;
        }
        else if record.hamming_dist == 1 {
            feature_counts.corrected += 1;
        }
        else {
            panic!("Hamming distance of greater than 1 is unsupported at the moment.")
        }
    }
    // add missing features from codebook
    for (_, cell_counts) in counts.iter_mut() {
        for feature in codebook.features() {
            cell_counts.entry(feature.clone()).or_insert(Counts{ exact: 0, corrected: 0});
        }
    }

    let model = model::readout::new_model(N, m, p0, p1, dist, codebook);

    cue::pipeline(
        "exp",
        threads,
        counts.into_iter(),
        |(cell, counts)| {
            let cdfs = counts.into_iter().map(|(feature, count)| {
                let (cdf, naive_estimate) = model::expression::cdf(&feature, count.exact + count.corrected, count.exact, &model, window_width);

                (feature, cdf, naive_estimate)
            }).collect_vec();
            (cell, cdfs)
        },
        |(cell, cdfs)| {
            for (feature, cdf, naive_estimate) in cdfs {
                cdf_writer.write(&cell, &feature, &cdf);

                if let Some(ref mut est_writer) = est_writer {
                    est_writer.write(
                        &cell,
                        &feature,
                        cdf.expected_value(),
                        cdf.standard_deviation(),
                        *cdf.map(),
                        cdf.credible_interval(),
                        naive_estimate
                    );
                }
            }
        }
    );
}

/// Estimate differential expression over two conditions via fold changes.
pub fn differential_expression(group1_path: &str, group2_path: &str, pmf_path: Option<&str>, max_fc: LogFC, pseudocounts: f64, threads: usize) {
    assert!(pseudocounts > 0.0, "Pseudocounts must be > 0.0 for calculating fold changes.");

    let mut reader1 = io::cdf::expression::Reader::from_file(group1_path).expect("Invalid input for group 1.");
    let mut reader2 = io::cdf::expression::Reader::from_file(group2_path).expect("Invalid input for group 2.");
    let mut cdf_writer = pmf_path.map(|path| io::cdf::diffexp::Writer::from_file(path, "log2fc"));
    let mut est_writer = io::estimation::differential_expression::Writer::from_writer(std::io::stdout(), "log2fc");

    let group1 = reader1.cdfs();
    let group2 = reader2.cdfs();

    let features = group1.features().filter(|f| group2.contains_feature(f)).cloned().collect_vec();

    let mut estimates = Vec::new();

    cue::pipeline(
        "diffexp",
        threads,
        features.iter(),
        |feature| {
            info!("Calculating {}.", feature);
            let g1 = group1.get(&feature).unwrap();
            let g2 = group2.get(&feature).unwrap();
            let cdf = model::foldchange::cdf(
                &model::expressionset::cdf(&g1, pseudocounts),
                &model::expressionset::cdf(&g2, pseudocounts)
            );
            (feature, cdf)
        },
        |(feature, cdf)| {
            if cdf.iter().any(|&(_, prob)| prob.is_nan()) {
                warn!("A CDF probability for feature {} cannot be estimated because counts are too high. It will be reported as NaN.", feature);
            }

            if let Some(ref mut cdf_writer) = cdf_writer {
                cdf_writer.write(&feature, &cdf);
            }
            estimates.push((feature, cdf.estimate(max_fc)));
        }
    );

    // calculate FDR and write estimates to STDOUT
    estimates.sort_by(|&(_, ref a), &(_, ref b)| a.differential_expression_pep.partial_cmp(&b.differential_expression_pep).unwrap());
    let expected_fds = logprobs::cumsum(estimates.iter().map(|&(_, ref a)| a.differential_expression_pep)).collect_vec();
    for (i, ((feature, estimate), fd)) in estimates.into_iter().zip(expected_fds).enumerate() {
        let fdr = fd - (i as f64 + 1.0).ln();
        est_writer.write(
            &feature,
            estimate.differential_expression_pep,
            fdr,
            estimate.differential_expression_bf,
            estimate.expected_value,
            estimate.standard_deviation,
            estimate.map,
            estimate.credible_interval
        );
    }
}

/// Estimate differential expression over multiple conditions via the coefficient of variation.
pub fn multi_differential_expression(group_paths: &[&str], pmf_path: Option<&str>, max_cv: CV, pseudocounts: f64, threads: usize) {
    let groups = group_paths.iter().enumerate().map(|(i, path)| {
        io::cdf::expression::Reader::from_file(path).expect(&format!("Invalid input for group {}.", i))
                                                    .cdfs()
    }).collect_vec();
    let mut cdf_writer = pmf_path.map(|path| io::cdf::diffexp::Writer::from_file(path, "cv"));
    let mut est_writer = io::estimation::differential_expression::Writer::from_writer(std::io::stdout(), "cv");

    // TODO take feature intersection or warn if features are not the same
    let mut features = Vec::new();
    for feature in groups[0].features() {
        let mut common = true;
        for group in groups[1..].iter() {
            common &= group.contains_feature(feature);
        }
        if common {
            features.push(feature.clone());
        }
    }

    let mut estimates = Vec::new();

    cue::pipeline(
        "multidiffexp",
        threads,
        features.iter(),
        |feature| {
            info!("Calculating {}.", feature);
            let cdfs = groups.iter().map(|group| {
                model::expressionset::cdf(
                    group.get(&feature).expect("Missing feature."),
                    pseudocounts
                )
            }).collect_vec();
            let cdf = model::cv::cdf(&cdfs);
            (feature, cdf)
        },
        |(feature, cdf)| {
            if cdf.iter().any(|&(_, prob)| prob.is_nan()) {
                warn!("A CDF probability for feature {} cannot be estimated due to numerical problems. It will be reported as NaN.", feature);
            }

            if let Some(ref mut cdf_writer) = cdf_writer {
                cdf_writer.write(&feature, &cdf);
            }
            estimates.push((feature, cdf.estimate(max_cv)));
        }
    );

    // calculate FDR and write estimates to STDOUT
    estimates.sort_by(|&(_, ref a), &(_, ref b)| a.differential_expression_pep.partial_cmp(&b.differential_expression_pep).unwrap());
    let expected_fds = logprobs::cumsum(estimates.iter().map(|&(_, ref a)| a.differential_expression_pep)).collect_vec();
    for (i, ((feature, estimate), fd)) in estimates.into_iter().zip(expected_fds).enumerate() {
        let fdr = fd - (i as f64 + 1.0).ln();
        est_writer.write(
            &feature,
            estimate.differential_expression_pep,
            fdr,
            estimate.differential_expression_bf,
            estimate.expected_value,
            estimate.standard_deviation,
            estimate.map,
            estimate.credible_interval
        );
    }
}

pub fn gen_codebook(words: &[codebook::Word]) -> Result<(), Box<Error>> {
    let stdin = std::io::stdin();
    let mut reader = stdin.lock().lines();
    let mut writer = csv::Writer::from_writer(std::io::stdout()).delimiter(b'\t');
    let mut words = words.iter();
    writer.write(["feat", "codeword", "expressed"].iter()).unwrap();

    for i in 1.. {
        match (reader.next(), words.next()) {
            (Some(feature), Some(w)) => {
                let feature = try!(feature);
                if feature.len() == 0 {
                    // TODO proper error handling
                    panic!("Empty feature found. All features provided at STDIN have to be non-empty.");
                }
                writer.write([feature, format!("{:?}", w), "1".to_owned()].into_iter()).unwrap();
            },
            (None, Some(_)) => break,
            (Some(_), None) => {
                error!("Not enough code words. Generated codebook for the first {} transcripts.", i);
                break
            },
            (None, None) => break
        }
    }
    Ok(())
}
<commit_msg>Adapt to new codebook format.<commit_after>// Copyright 2016 Johannes Köster.
// Licensed under the MIT license (http://opensource.org/licenses/MIT)
// This file may not be copied, modified, or distributed
// except according to those terms.

#![allow(non_snake_case)]

use std;
use std::collections;
use std::io::prelude::*;
use std::error::Error;

use csv;
use itertools::Itertools;
use cue;
use regex::Regex;

use bio::stats::logprobs::Prob;
use bio::stats::logprobs;

use io;
use model;
use model::foldchange::LogFC;
use model::cv::CV;
use codebook;

pub struct Selection {
    pub expmnt: String,
    pub cell: String
}

#[derive(Debug)]
struct Counts {
    exact: u32,
    corrected: u32
}

/*pub fn stats(N: u8, m: u8, p0: Prob, p1: Prob, dist: u8, codebook_path: &str) {
    let codebook = io::codebook::Reader::from_file(codebook_path, dist).unwrap().codebook();
    let model = model::readout::new_model(N, m, p0, p1, dist, codebook);
    let mut reader = io::merfishdata::Reader::from_reader(std::io::stdin());

    let mut total_counts = collections::HashMap::new();
    for res in reader.records() {
        let record = res.unwrap();
        let total_count = total_counts.entry(record.cell_id).or_insert(0);
        *total_count += 1;
    }

    let mut writer = csv::Writer::from_writer(std::io::stdout()).delimiter(b'\t');
    writer.write(["cell", "total_expr_ev"].iter()).unwrap();

    for (cell, total_count) in total_counts {
        writer.write([cell, format!("{}", model.expected_total(total_count))].iter()).unwrap();
    }
}*/

/// Estimate expressions.
pub fn expression(N: u8, m: u8, p0: Prob, p1: Prob, dist: u8, codebook_path: &str, estimate_path: Option<&str>, threads: usize, cells: &str, window_width: u32, print_naive: bool) {
    let codebook = io::codebook::Reader::from_file(codebook_path, dist).unwrap().codebook();
    let mut reader = io::merfishdata::Reader::from_reader(std::io::stdin());
    let mut cdf_writer = io::cdf::expression::Writer::from_writer(std::io::stdout());
    let mut est_writer = estimate_path.map(|path| io::estimation::expression::Writer::from_file(path, print_naive));

    let cells = Regex::new(cells).ok().expect("Invalid regular expression for cells.");

    let mut counts = collections::HashMap::new();
    for record in reader.records().filter_map(|res| {
            let rec = res.unwrap();
            // consider record if it is contained in the codebook and has a valid cell id
            if cells.is_match(&rec.cell_id) && codebook.contains(&rec.feature) {
                Some(rec)
            }
            else {
                None
            }
        }
    ) {
        let cell_counts = counts.entry(record.cell_id).or_insert_with(collections::HashMap::new);
        let feature_counts = cell_counts.entry(record.feature).or_insert(Counts{ exact: 0, corrected: 0});
        if record.hamming_dist == 0 {
            feature_counts.exact += 1;
        }
        else if record.hamming_dist == 1 {
            feature_counts.corrected += 1;
        }
        else {
            panic!("Hamming distance of greater than 1 is unsupported at the moment.")
        }
    }
    // add missing features from codebook
    for (_, cell_counts) in counts.iter_mut() {
        for feature in codebook.features() {
            cell_counts.entry(feature.clone()).or_insert(Counts{ exact: 0, corrected: 0});
        }
    }

    let model = model::readout::new_model(N, m, p0, p1, dist, codebook);

    cue::pipeline(
        "exp",
        threads,
        counts.into_iter(),
        |(cell, counts)| {
            let cdfs = counts.into_iter().map(|(feature, count)| {
                let (cdf, naive_estimate) = model::expression::cdf(&feature, count.exact + count.corrected, count.exact, &model, window_width);

                (feature, cdf, naive_estimate)
            }).collect_vec();
            (cell, cdfs)
        },
        |(cell, cdfs)| {
            for (feature, cdf, naive_estimate) in cdfs {
                cdf_writer.write(&cell, &feature, &cdf);

                if let Some(ref mut est_writer) = est_writer {
                    est_writer.write(
                        &cell,
                        &feature,
                        cdf.expected_value(),
                        cdf.standard_deviation(),
                        *cdf.map(),
                        cdf.credible_interval(),
                        naive_estimate
                    );
                }
            }
        }
    );
}

/// Estimate differential expression over two conditions via fold changes.
pub fn differential_expression(group1_path: &str, group2_path: &str, pmf_path: Option<&str>, max_fc: LogFC, pseudocounts: f64, threads: usize) {
    assert!(pseudocounts > 0.0, "Pseudocounts must be > 0.0 for calculating fold changes.");

    let mut reader1 = io::cdf::expression::Reader::from_file(group1_path).expect("Invalid input for group 1.");
    let mut reader2 = io::cdf::expression::Reader::from_file(group2_path).expect("Invalid input for group 2.");
    let mut cdf_writer = pmf_path.map(|path| io::cdf::diffexp::Writer::from_file(path, "log2fc"));
    let mut est_writer = io::estimation::differential_expression::Writer::from_writer(std::io::stdout(), "log2fc");

    let group1 = reader1.cdfs();
    let group2 = reader2.cdfs();

    let features = group1.features().filter(|f| group2.contains_feature(f)).cloned().collect_vec();

    let mut estimates = Vec::new();

    cue::pipeline(
        "diffexp",
        threads,
        features.iter(),
        |feature| {
            info!("Calculating {}.", feature);
            let g1 = group1.get(&feature).unwrap();
            let g2 = group2.get(&feature).unwrap();
            let cdf = model::foldchange::cdf(
                &model::expressionset::cdf(&g1, pseudocounts),
                &model::expressionset::cdf(&g2, pseudocounts)
            );
            (feature, cdf)
        },
        |(feature, cdf)| {
            if cdf.iter().any(|&(_, prob)| prob.is_nan()) {
                warn!("A CDF probability for feature {} cannot be estimated because counts are too high. It will be reported as NaN.", feature);
            }

            if let Some(ref mut cdf_writer) = cdf_writer {
                cdf_writer.write(&feature, &cdf);
            }
            estimates.push((feature, cdf.estimate(max_fc)));
        }
    );

    // calculate FDR and write estimates to STDOUT
    estimates.sort_by(|&(_, ref a), &(_, ref b)| a.differential_expression_pep.partial_cmp(&b.differential_expression_pep).unwrap());
    let expected_fds = logprobs::cumsum(estimates.iter().map(|&(_, ref a)| a.differential_expression_pep)).collect_vec();
    for (i, ((feature, estimate), fd)) in estimates.into_iter().zip(expected_fds).enumerate() {
        let fdr = fd - (i as f64 + 1.0).ln();
        est_writer.write(
            &feature,
            estimate.differential_expression_pep,
            fdr,
            estimate.differential_expression_bf,
            estimate.expected_value,
            estimate.standard_deviation,
            estimate.map,
            estimate.credible_interval
        );
    }
}

/// Estimate differential expression over multiple conditions via the coefficient of variation.
pub fn multi_differential_expression(group_paths: &[&str], pmf_path: Option<&str>, max_cv: CV, pseudocounts: f64, threads: usize) {
    let groups = group_paths.iter().enumerate().map(|(i, path)| {
        io::cdf::expression::Reader::from_file(path).expect(&format!("Invalid input for group {}.", i))
                                                    .cdfs()
    }).collect_vec();
    let mut cdf_writer = pmf_path.map(|path| io::cdf::diffexp::Writer::from_file(path, "cv"));
    let mut est_writer = io::estimation::differential_expression::Writer::from_writer(std::io::stdout(), "cv");

    // TODO take feature intersection or warn if features are not the same
    let mut features = Vec::new();
    for feature in groups[0].features() {
        let mut common = true;
        for group in groups[1..].iter() {
            common &= group.contains_feature(feature);
        }
        if common {
            features.push(feature.clone());
        }
    }

    let mut estimates = Vec::new();

    cue::pipeline(
        "multidiffexp",
        threads,
        features.iter(),
        |feature| {
            info!("Calculating {}.", feature);
            let cdfs = groups.iter().map(|group| {
                model::expressionset::cdf(
                    group.get(&feature).expect("Missing feature."),
                    pseudocounts
                )
            }).collect_vec();
            let cdf = model::cv::cdf(&cdfs);
            (feature, cdf)
        },
        |(feature, cdf)| {
            if cdf.iter().any(|&(_, prob)| prob.is_nan()) {
                warn!("A CDF probability for feature {} cannot be estimated due to numerical problems. It will be reported as NaN.", feature);
            }

            if let Some(ref mut cdf_writer) = cdf_writer {
                cdf_writer.write(&feature, &cdf);
            }
            estimates.push((feature, cdf.estimate(max_cv)));
        }
    );

    // calculate FDR and write estimates to STDOUT
    estimates.sort_by(|&(_, ref a), &(_, ref b)| a.differential_expression_pep.partial_cmp(&b.differential_expression_pep).unwrap());
    let expected_fds = logprobs::cumsum(estimates.iter().map(|&(_, ref a)| a.differential_expression_pep)).collect_vec();
    for (i, ((feature, estimate), fd)) in estimates.into_iter().zip(expected_fds).enumerate() {
        let fdr = fd - (i as f64 + 1.0).ln();
        est_writer.write(
            &feature,
            estimate.differential_expression_pep,
            fdr,
            estimate.differential_expression_bf,
            estimate.expected_value,
            estimate.standard_deviation,
            estimate.map,
            estimate.credible_interval
        );
    }
}

pub fn gen_codebook(words: &[codebook::Word]) -> Result<(), Box<Error>> {
    let stdin = std::io::stdin();
    let mut reader = stdin.lock().lines();
    let mut writer = csv::Writer::from_writer(std::io::stdout()).delimiter(b'\t');
    let mut words = words.iter();
    writer.write(["feat", "codeword"].iter()).unwrap();

    for i in 1.. {
        match (reader.next(), words.next()) {
            (Some(feature), Some(w)) => {
                let feature = try!(feature);
                if feature.len() == 0 {
                    // TODO proper error handling
                    panic!("Empty feature found. All features provided at STDIN have to be non-empty.");
                }
                writer.write([feature, format!("{:?}", w)].into_iter()).unwrap();
            },
            (None, Some(_)) => break,
            (Some(_), None) => {
                error!("Not enough code words. Generated codebook for the first {} transcripts.", i);
                break
            },
            (None, None) => break
        }
    }
    Ok(())
}
<|endoftext|>


<commit_before>// Copyright (c) 2013-2015 Sandstorm Development Group, Inc. and contributors
// Licensed under the MIT License:
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

#![crate_name="calculator"]
#![crate_type="bin"]
#![feature(collections, core, os)]

extern crate capnp;
extern crate "capnp-rpc" as capnp_rpc;

pub mod calculator_capnp {
  include!(concat!(env!("OUT_DIR"), "/calculator_capnp.rs"));
}

pub mod client;
pub mod server;

pub fn main() {
    let args = std::os::args();
    if args.len() >= 2 {
        match &args[1][] {
            "client" => return client::main(),
            "server" => return server::main(),
            _ => (),
        }
    }

    println!("usage: {} [client | server] ADDRESS", args[0]);
}
<commit_msg>no more feature(collections)<commit_after>// Copyright (c) 2013-2015 Sandstorm Development Group, Inc. and contributors
// Licensed under the MIT License:
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

#![crate_name="calculator"]
#![crate_type="bin"]
#![feature(core, os)]

extern crate capnp;
extern crate "capnp-rpc" as capnp_rpc;

pub mod calculator_capnp {
  include!(concat!(env!("OUT_DIR"), "/calculator_capnp.rs"));
}

pub mod client;
pub mod server;

pub fn main() {
    let args = std::os::args();
    if args.len() >= 2 {
        match &args[1][] {
            "client" => return client::main(),
            "server" => return server::main(),
            _ => (),
        }
    }

    println!("usage: {} [client | server] ADDRESS", args[0]);
}
<|endoftext|>