<commit_before>#![feature(conservative_impl_trait)]

extern crate zip;
extern crate quick_xml;
extern crate encoding;
extern crate byteorder;
#[macro_use]
extern crate error_chain;

#[macro_use]
extern crate log;

mod errors;
pub mod vba;

use std::path::Path;
use std::fs::File;
use std::io::BufReader;
use std::collections::HashMap;
use std::slice::Chunks;

pub use errors::*;
use vba::VbaProject;

use zip::read::{ZipFile, ZipArchive};
use zip::result::ZipError;
use quick_xml::{XmlReader, Event, AsStr};

macro_rules! unexp {
    ($pat: expr) => {
        {
            return Err($pat.into());
        }
    };
    ($pat: expr, $($args: expr)* ) => {
        {
            return Err(format!($pat, $($args)*).into());
        }
    };
}

#[derive(Debug, Clone, PartialEq)]
pub enum DataType {
    Int(i64),
    Float(f64),
    String(String),
    Empty,
}

enum FileType {
    /// Compound File Binary Format [MS-CFB]
    CFB(File),
    Zip(ZipArchive<File>),
}

pub struct Excel {
    zip: FileType,
    strings: Vec<String>,
    relationships: HashMap<Vec<u8>, String>,
    /// Map of sheet names/sheet path within zip archive
    sheets: HashMap<String, String>,
}

#[derive(Debug, Default)]
pub struct Range {
    position: (u32, u32),
    size: (usize, usize),
    inner: Vec<DataType>,
}

/// An iterator to read `Range` struct row by row
pub struct Rows<'a> {
    inner: Chunks<'a, DataType>,
}

impl Excel {

    /// Opens a new workbook
    pub fn open<P: AsRef<Path>>(path: P) -> Result<Excel> {
        let f = try!(File::open(&path));
        let zip = match path.as_ref().extension().and_then(|s| s.to_str()) {
            Some("xls") | Some("xla") => FileType::CFB(f),
            Some("xlsx") | Some("xlsb") | Some("xlsm") |
                Some("xlam") => FileType::Zip(try!(ZipArchive::new(f))),
            Some(e) => return Err(format!("unrecognized extension: {:?}", e).into()),
            None => return Err("expecting a file with an extension".into()),
        };
        Ok(Excel {
            zip: zip,
            strings: vec![],
            relationships: HashMap::new(),
            sheets: HashMap::new(),
        })
    }

    /// Does the workbook contain a vba project
    pub fn has_vba(&mut self) -> bool {
        match self.zip {
            FileType::CFB(_) => true,
            FileType::Zip(ref mut z) => z.by_name("xl/vbaProject.bin").is_ok()
        }
    }

    /// Gets vba project
    pub fn vba_project(&mut self) -> Result<VbaProject> {
        match self.zip {
            FileType::CFB(ref mut f) => {
                let len = try!(f.metadata()).len() as usize;
                VbaProject::new(f, len)
            },
            FileType::Zip(ref mut z) => {
                let f = try!(z.by_name("xl/vbaProject.bin"));
                let len = f.size() as usize;
                VbaProject::new(f, len)
            }
        }
    }

    /// Get all data from `Worksheet`
    pub fn worksheet_range(&mut self, name: &str) -> Result<Range> {
        try!(self.read_shared_strings());
        try!(self.read_relationships());
        try!(self.read_sheets_names());
        let strings = &self.strings;
        let z = match self.zip {
            FileType::CFB(_) => return Err("worksheet_range not implemented for CFB files".into()),
            FileType::Zip(ref mut z) => z
        };
        let ws = match self.sheets.get(name) {
            Some(p) => {
                println!("get sheet {} at path {}", name, p);
                try!(z.by_name(p))
            },
            None => unexp!("Sheet '{}' does not exist", name),
        };
        Range::from_worksheet(ws, strings)
    }

    /// Read sheets from workbook.xml and get their corresponding path from relationships
    fn read_sheets_names(&mut self) -> Result<()> {
        if self.sheets.is_empty() {
            let z = match self.zip {
                FileType::CFB(_) => return Err("read_sheet_names not implemented for CFB files".into()),
                FileType::Zip(ref mut z) => z
            };

            match z.by_name("xl/workbook.xml") {
                Ok(f) => {
                    let mut xml = XmlReader::from_reader(BufReader::new(f))
                        .with_check(false)
                        .trim_text(false);

                    while let Some(res_event) = xml.next() {
                        match res_event {
                            Ok(Event::Start(ref e)) if e.name() == b"sheet" => {
                                let mut name = String::new();
                                let mut path = String::new();
                                for a in e.attributes() {
                                    match try!(a) {
                                        (b"name", v) => name = try!(v.as_str()).to_string(),
                                        (b"r:id", v) => path = format!("xl/{}", self.relationships[v]),
                                        _ => (),
                                    }
                                }
                                self.sheets.insert(name, path);
                            }
                            Err(e) => return Err(e.into()),
                            _ => (),
                        }
                    }
                },
                Err(ZipError::FileNotFound) => (),
                Err(e) => return Err(e.into()),
            }
        }
        Ok(())
    }

    /// Read shared string list
    fn read_shared_strings(&mut self) -> Result<()> {
        if self.strings.is_empty() {
            let z = match self.zip {
                FileType::CFB(_) => return Err("read_shared_strings not implemented for CFB files".into()),
                FileType::Zip(ref mut z) => z
            };
            match z.by_name("xl/sharedStrings.xml") {
                Ok(f) => {
                    let mut xml = XmlReader::from_reader(BufReader::new(f))
                        .with_check(false)
                        .trim_text(false);

                    let mut strings = Vec::new();
                    while let Some(res_event) = xml.next() {
                        match res_event {
                            Ok(Event::Start(ref e)) if e.name() == b"t" => {
                                strings.push(try!(xml.read_text(b"t")));
                            }
                            Err(e) => return Err(e.into()),
                            _ => (),
                        }
                    }
                    self.strings = strings;
                },
                Err(ZipError::FileNotFound) => (),
                Err(e) => return Err(e.into()),
            }
        }

        Ok(())
    }

    /// Read workbook relationships
    fn read_relationships(&mut self) -> Result<()> {
        if self.relationships.is_empty() {
            let z = match self.zip {
                FileType::CFB(_) => return Err("read_shared_strings not implemented for CFB files".into()),
                FileType::Zip(ref mut z) => z
            };
            match z.by_name("xl/_rels/workbook.xml.rels") {
                Ok(f) => {
                    let mut xml = XmlReader::from_reader(BufReader::new(f))
                        .with_check(false)
                        .trim_text(false);

                    while let Some(res_event) = xml.next() {
                        match res_event {
                            Ok(Event::Start(ref e)) if e.name() == b"Relationship" => {
                                let mut id = Vec::new();
                                let mut target = String::new();
                                for a in e.attributes() {
                                    match try!(a) {
                                        (b"Id", v) => id.extend_from_slice(v),
                                        (b"Target", v) => target = try!(v.as_str()).to_string(),
                                        _ => (),
                                    }
                                }
                                self.relationships.insert(id, target);
                            }
                            Err(e) => return Err(e.into()),
                            _ => (),
                        }
                    }
                },
                Err(ZipError::FileNotFound) => (),
                Err(e) => return Err(e.into()),
            }
        }

        Ok(())
    }

}

impl Range {

    /// open a xml `ZipFile` reader and read content of *sheetData* and *dimension* node
    fn from_worksheet(xml: ZipFile, strings: &[String]) -> Result<Range> {
        let mut xml = XmlReader::from_reader(BufReader::new(xml))
            .with_check(false)
            .trim_text(false);
        let mut data = Range::default();
        while let Some(res_event) = xml.next() {
            match res_event {
                Err(e) => return Err(e.into()),
                Ok(Event::Start(ref e)) => {
                    match e.name() {
                        b"dimension" => match e.attributes().filter_map(|a| a.ok())
                                .find(|&(key, _)| key == b"ref") {
                            Some((_, dim)) => {
                                let (position, size) = try!(get_dimension(try!(dim.as_str())));
                                data.position = position;
                                data.size = (size.0 as usize, size.1 as usize);
                                data.inner.reserve_exact(data.size.0 * data.size.1);
                            },
                            None => unexp!("Expecting dimension, got {:?}", e),
                        },
                        b"sheetData" => {
                            let _ = try!(data.read_sheet_data(&mut xml, strings));
                        }
                        _ => (),
                    }
                },
                _ => (),
            }
        }
        data.inner.shrink_to_fit();
        Ok(data)
    }

    /// get worksheet position (row, column)
    pub fn get_position(&self) -> (u32, u32) {
        self.position
    }

    /// get size
    pub fn get_size(&self) -> (usize, usize) {
        self.size
    }

    /// get cell value
    pub fn get_value(&self, i: usize, j: usize) -> &DataType {
        let idx = i * self.size.0 + j;
        &self.inner[idx]
    }

    /// get an iterator over inner rows
    pub fn rows(&self) -> Rows {
        let width = self.size.0;
        Rows { inner: self.inner.chunks(width) }
    }

    /// read sheetData node
    fn read_sheet_data(&mut self, xml: &mut XmlReader<BufReader<ZipFile>>, strings: &[String])
        -> Result<()>
    {
        while let Some(res_event) = xml.next() {
            match res_event {
                Err(e) => return Err(e.into()),
                Ok(Event::Start(ref c_element)) => {
                    if c_element.name() == b"c" {
                        loop {
                            match xml.next() {
                                Some(Err(e)) => return Err(e.into()),
                                Some(Ok(Event::Start(ref e))) => {
                                    if e.name() == b"v" {
                                        let v = try!(xml.read_text(b"v"));
                                        let value = match c_element.attributes()
                                            .filter_map(|a| a.ok())
                                            .find(|&(k, _)| k == b"t") {
                                                Some((_, b"s")) => {
                                                    let idx: usize = try!(v.parse());
                                                    DataType::String(strings[idx].clone())
                                                },
                                                // TODO: check in styles to know which type is
                                                // supposed to be used
                                                _ => match v.parse() {
                                                    Ok(i) => DataType::Int(i),
                                                    Err(_) => try!(v.parse()
                                                                   .map(DataType::Float)),
                                                },
                                            };
                                        self.inner.push(value);
                                        break;
                                    } else {
                                        unexp!("not v node");
                                    }
                                },
                                Some(Ok(Event::End(ref e))) => {
                                    if e.name() == b"c" {
                                        self.inner.push(DataType::Empty);
                                        break;
                                    }
                                }
                                None => unexp!("End of xml"),
                                _ => (),
                            }
                        }
                    }
                },
                Ok(Event::End(ref e)) if e.name() == b"sheetData" => return Ok(()),
                _ => (),
            }
        }
        unexp!("Could not find </sheetData>")
    }

}

impl<'a> Iterator for Rows<'a> {
    type Item = &'a [DataType];
    fn next(&mut self) -> Option<&'a [DataType]> {
        self.inner.next()
    }
}

/// converts a text representation (e.g. "A6:G67") of a dimension into integers
/// - top left (row, column),
/// - size (width, height)
fn get_dimension(dimension: &str) -> Result<((u32, u32), (u32, u32))> {
    match dimension.chars().position(|c| c == ':') {
        None => {
            get_row_column(dimension).map(|position| (position, (1, 1)))
        },
        Some(p) => {
            let top_left = try!(get_row_column(&dimension[..p]));
            let bottom_right = try!(get_row_column(&dimension[p + 1..]));
            Ok((top_left, (bottom_right.0 - top_left.0 + 1, bottom_right.1 - top_left.1 + 1)))
        }
    }
}

/// converts a text range name into its position (row, column)
fn get_row_column(range: &str) -> Result<(u32, u32)> {
    let mut col = 0;
    let mut pow = 1;
    let mut rowpos = range.len();
    let mut readrow = true;
    for c in range.chars().rev() {
        match c {
            '0'...'9' => {
                if readrow {
                    rowpos -= 1;
                } else {
                    unexp!("Numeric character are only allowed at the end of the range: {}", c);
                }
            }
            c @ 'A'...'Z' => {
                readrow = false;
                col += ((c as u8 - b'A') as u32 + 1) * pow;
                pow *= 26;
            },
            c @ 'a'...'z' => {
                readrow = false;
                col += ((c as u8 - b'a') as u32 + 1) * pow;
                pow *= 26;
            },
            _ => unexp!("Expecting alphanumeric character, got {:?}", c),
        }
    }
    let row = try!(range[rowpos..].parse());
    Ok((row, col))
}

<commit_msg>swap row/column offsets<commit_after>#![feature(conservative_impl_trait)]

extern crate zip;
extern crate quick_xml;
extern crate encoding;
extern crate byteorder;
#[macro_use]
extern crate error_chain;

#[macro_use]
extern crate log;

mod errors;
pub mod vba;

use std::path::Path;
use std::fs::File;
use std::io::BufReader;
use std::collections::HashMap;
use std::slice::Chunks;

pub use errors::*;
use vba::VbaProject;

use zip::read::{ZipFile, ZipArchive};
use zip::result::ZipError;
use quick_xml::{XmlReader, Event, AsStr};

macro_rules! unexp {
    ($pat: expr) => {
        {
            return Err($pat.into());
        }
    };
    ($pat: expr, $($args: expr)* ) => {
        {
            return Err(format!($pat, $($args)*).into());
        }
    };
}

#[derive(Debug, Clone, PartialEq)]
pub enum DataType {
    Int(i64),
    Float(f64),
    String(String),
    Empty,
}

enum FileType {
    /// Compound File Binary Format [MS-CFB]
    CFB(File),
    Zip(ZipArchive<File>),
}

pub struct Excel {
    zip: FileType,
    strings: Vec<String>,
    relationships: HashMap<Vec<u8>, String>,
    /// Map of sheet names/sheet path within zip archive
    sheets: HashMap<String, String>,
}

#[derive(Debug, Default)]
pub struct Range {
    position: (u32, u32),
    size: (usize, usize),
    inner: Vec<DataType>,
}

/// An iterator to read `Range` struct row by row
pub struct Rows<'a> {
    inner: Chunks<'a, DataType>,
}

impl Excel {

    /// Opens a new workbook
    pub fn open<P: AsRef<Path>>(path: P) -> Result<Excel> {
        let f = try!(File::open(&path));
        let zip = match path.as_ref().extension().and_then(|s| s.to_str()) {
            Some("xls") | Some("xla") => FileType::CFB(f),
            Some("xlsx") | Some("xlsb") | Some("xlsm") |
                Some("xlam") => FileType::Zip(try!(ZipArchive::new(f))),
            Some(e) => return Err(format!("unrecognized extension: {:?}", e).into()),
            None => return Err("expecting a file with an extension".into()),
        };
        Ok(Excel {
            zip: zip,
            strings: vec![],
            relationships: HashMap::new(),
            sheets: HashMap::new(),
        })
    }

    /// Does the workbook contain a vba project
    pub fn has_vba(&mut self) -> bool {
        match self.zip {
            FileType::CFB(_) => true,
            FileType::Zip(ref mut z) => z.by_name("xl/vbaProject.bin").is_ok()
        }
    }

    /// Gets vba project
    pub fn vba_project(&mut self) -> Result<VbaProject> {
        match self.zip {
            FileType::CFB(ref mut f) => {
                let len = try!(f.metadata()).len() as usize;
                VbaProject::new(f, len)
            },
            FileType::Zip(ref mut z) => {
                let f = try!(z.by_name("xl/vbaProject.bin"));
                let len = f.size() as usize;
                VbaProject::new(f, len)
            }
        }
    }

    /// Get all data from `Worksheet`
    pub fn worksheet_range(&mut self, name: &str) -> Result<Range> {
        try!(self.read_shared_strings());
        try!(self.read_relationships());
        try!(self.read_sheets_names());
        let strings = &self.strings;
        let z = match self.zip {
            FileType::CFB(_) => return Err("worksheet_range not implemented for CFB files".into()),
            FileType::Zip(ref mut z) => z
        };
        let ws = match self.sheets.get(name) {
            Some(p) => try!(z.by_name(p)),
            None => unexp!("Sheet '{}' does not exist", name),
        };
        Range::from_worksheet(ws, strings)
    }

    /// Read sheets from workbook.xml and get their corresponding path from relationships
    fn read_sheets_names(&mut self) -> Result<()> {
        if self.sheets.is_empty() {
            let z = match self.zip {
                FileType::CFB(_) => return Err("read_sheet_names not implemented for CFB files".into()),
                FileType::Zip(ref mut z) => z
            };

            match z.by_name("xl/workbook.xml") {
                Ok(f) => {
                    let mut xml = XmlReader::from_reader(BufReader::new(f))
                        .with_check(false)
                        .trim_text(false);

                    while let Some(res_event) = xml.next() {
                        match res_event {
                            Ok(Event::Start(ref e)) if e.name() == b"sheet" => {
                                let mut name = String::new();
                                let mut path = String::new();
                                for a in e.attributes() {
                                    match try!(a) {
                                        (b"name", v) => name = try!(v.as_str()).to_string(),
                                        (b"r:id", v) => path = format!("xl/{}", self.relationships[v]),
                                        _ => (),
                                    }
                                }
                                self.sheets.insert(name, path);
                            }
                            Err(e) => return Err(e.into()),
                            _ => (),
                        }
                    }
                },
                Err(ZipError::FileNotFound) => (),
                Err(e) => return Err(e.into()),
            }
        }
        Ok(())
    }

    /// Read shared string list
    fn read_shared_strings(&mut self) -> Result<()> {
        if self.strings.is_empty() {
            let z = match self.zip {
                FileType::CFB(_) => return Err("read_shared_strings not implemented for CFB files".into()),
                FileType::Zip(ref mut z) => z
            };
            match z.by_name("xl/sharedStrings.xml") {
                Ok(f) => {
                    let mut xml = XmlReader::from_reader(BufReader::new(f))
                        .with_check(false)
                        .trim_text(false);

                    let mut strings = Vec::new();
                    while let Some(res_event) = xml.next() {
                        match res_event {
                            Ok(Event::Start(ref e)) if e.name() == b"t" => {
                                strings.push(try!(xml.read_text(b"t")));
                            }
                            Err(e) => return Err(e.into()),
                            _ => (),
                        }
                    }
                    self.strings = strings;
                },
                Err(ZipError::FileNotFound) => (),
                Err(e) => return Err(e.into()),
            }
        }

        Ok(())
    }

    /// Read workbook relationships
    fn read_relationships(&mut self) -> Result<()> {
        if self.relationships.is_empty() {
            let z = match self.zip {
                FileType::CFB(_) => return Err("read_shared_strings not implemented for CFB files".into()),
                FileType::Zip(ref mut z) => z
            };
            match z.by_name("xl/_rels/workbook.xml.rels") {
                Ok(f) => {
                    let mut xml = XmlReader::from_reader(BufReader::new(f))
                        .with_check(false)
                        .trim_text(false);

                    while let Some(res_event) = xml.next() {
                        match res_event {
                            Ok(Event::Start(ref e)) if e.name() == b"Relationship" => {
                                let mut id = Vec::new();
                                let mut target = String::new();
                                for a in e.attributes() {
                                    match try!(a) {
                                        (b"Id", v) => id.extend_from_slice(v),
                                        (b"Target", v) => target = try!(v.as_str()).to_string(),
                                        _ => (),
                                    }
                                }
                                self.relationships.insert(id, target);
                            }
                            Err(e) => return Err(e.into()),
                            _ => (),
                        }
                    }
                },
                Err(ZipError::FileNotFound) => (),
                Err(e) => return Err(e.into()),
            }
        }

        Ok(())
    }

}

impl Range {

    /// open a xml `ZipFile` reader and read content of *sheetData* and *dimension* node
    fn from_worksheet(xml: ZipFile, strings: &[String]) -> Result<Range> {
        let mut xml = XmlReader::from_reader(BufReader::new(xml))
            .with_check(false)
            .trim_text(false);
        let mut data = Range::default();
        while let Some(res_event) = xml.next() {
            match res_event {
                Err(e) => return Err(e.into()),
                Ok(Event::Start(ref e)) => {
                    match e.name() {
                        b"dimension" => match e.attributes().filter_map(|a| a.ok())
                                .find(|&(key, _)| key == b"ref") {
                            Some((_, dim)) => {
                                let (position, size) = try!(get_dimension(try!(dim.as_str())));
                                data.position = position;
                                data.size = (size.0 as usize, size.1 as usize);
                                data.inner.reserve_exact(data.size.0 * data.size.1);
                            },
                            None => unexp!("Expecting dimension, got {:?}", e),
                        },
                        b"sheetData" => {
                            let _ = try!(data.read_sheet_data(&mut xml, strings));
                        }
                        _ => (),
                    }
                },
                _ => (),
            }
        }
        data.inner.shrink_to_fit();
        Ok(data)
    }

    /// get worksheet position (row, column)
    pub fn get_position(&self) -> (u32, u32) {
        self.position
    }

    /// get size
    pub fn get_size(&self) -> (usize, usize) {
        self.size
    }

    /// get cell value
    pub fn get_value(&self, i: usize, j: usize) -> &DataType {
        assert!((i, j) < self.size);
        let idx = i * self.size.1 + j;
        &self.inner[idx]
    }

    /// get an iterator over inner rows
    pub fn rows(&self) -> Rows {
        let width = self.size.1;
        Rows { inner: self.inner.chunks(width) }
    }

    /// read sheetData node
    fn read_sheet_data(&mut self, xml: &mut XmlReader<BufReader<ZipFile>>, strings: &[String])
        -> Result<()>
    {
        while let Some(res_event) = xml.next() {
            match res_event {
                Err(e) => return Err(e.into()),
                Ok(Event::Start(ref c_element)) => {
                    if c_element.name() == b"c" {
                        loop {
                            match xml.next() {
                                Some(Err(e)) => return Err(e.into()),
                                Some(Ok(Event::Start(ref e))) => {
                                    if e.name() == b"v" {
                                        let v = try!(xml.read_text(b"v"));
                                        let value = match c_element.attributes()
                                            .filter_map(|a| a.ok())
                                            .find(|&(k, _)| k == b"t") {
                                                Some((_, b"s")) => {
                                                    let idx: usize = try!(v.parse());
                                                    DataType::String(strings[idx].clone())
                                                },
                                                // TODO: check in styles to know which type is
                                                // supposed to be used
                                                _ => match v.parse() {
                                                    Ok(i) => DataType::Int(i),
                                                    Err(_) => try!(v.parse()
                                                                   .map(DataType::Float)),
                                                },
                                            };
                                        self.inner.push(value);
                                        break;
                                    } else {
                                        unexp!("not v node");
                                    }
                                },
                                Some(Ok(Event::End(ref e))) => {
                                    if e.name() == b"c" {
                                        self.inner.push(DataType::Empty);
                                        break;
                                    }
                                }
                                None => unexp!("End of xml"),
                                _ => (),
                            }
                        }
                    }
                },
                Ok(Event::End(ref e)) if e.name() == b"sheetData" => return Ok(()),
                _ => (),
            }
        }
        unexp!("Could not find </sheetData>")
    }

}

impl<'a> Iterator for Rows<'a> {
    type Item = &'a [DataType];
    fn next(&mut self) -> Option<&'a [DataType]> {
        self.inner.next()
    }
}

/// converts a text representation (e.g. "A6:G67") of a dimension into integers
/// - top left (row, column),
/// - size (width, height)
fn get_dimension(dimension: &str) -> Result<((u32, u32), (u32, u32))> {
    match dimension.chars().position(|c| c == ':') {
        None => {
            get_row_column(dimension).map(|position| (position, (1, 1)))
        },
        Some(p) => {
            let top_left = try!(get_row_column(&dimension[..p]));
            let bottom_right = try!(get_row_column(&dimension[p + 1..]));
            Ok((top_left, (bottom_right.0 - top_left.0 + 1, bottom_right.1 - top_left.1 + 1)))
        }
    }
}

/// converts a text range name into its position (row, column)
fn get_row_column(range: &str) -> Result<(u32, u32)> {
    let mut col = 0;
    let mut pow = 1;
    let mut rowpos = range.len();
    let mut readrow = true;
    for c in range.chars().rev() {
        match c {
            '0'...'9' => {
                if readrow {
                    rowpos -= 1;
                } else {
                    unexp!("Numeric character are only allowed at the end of the range: {}", c);
                }
            }
            c @ 'A'...'Z' => {
                readrow = false;
                col += ((c as u8 - b'A') as u32 + 1) * pow;
                pow *= 26;
            },
            c @ 'a'...'z' => {
                readrow = false;
                col += ((c as u8 - b'a') as u32 + 1) * pow;
                pow *= 26;
            },
            _ => unexp!("Expecting alphanumeric character, got {:?}", c),
        }
    }
    let row = try!(range[rowpos..].parse());
    Ok((row, col))
}

<|endoftext|>


<commit_before>// Copyright 2016 Mozilla Foundation
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

#![deny(rust_2018_idioms)]
#![allow(unknown_lints)]
#![allow(clippy::type_complexity, clippy::new_without_default)]
#![recursion_limit = "256"]

#[macro_use]
extern crate clap;
#[macro_use]
extern crate counted_array;
#[macro_use]
extern crate futures;
#[cfg(feature = "jsonwebtoken")]
use jsonwebtoken as jwt;
#[macro_use]
extern crate lazy_static;
#[macro_use]
extern crate log;
#[cfg(feature = "rouille")]
#[macro_use(router)]
extern crate rouille;
#[macro_use]
extern crate serde_derive;

// To get macros in scope, this has to be first.
#[cfg(test)]
#[macro_use]
mod test;

#[macro_use]
pub mod errors;

#[cfg(feature = "azure")]
mod azure;
mod cache;
mod client;
mod cmdline;
mod commands;
mod compiler;
pub mod config;
pub mod dist;
mod jobserver;
pub mod lru_disk_cache;
mod mock_command;
mod protocol;
pub mod server;
#[cfg(feature = "simple-s3")]
mod simples3;
#[doc(hidden)]
pub mod util;

use std::env;

const LOGGING_ENV: &str = "SCCACHE_LOG";

pub fn main() {
    init_logging();
    std::process::exit(match cmdline::parse() {
        Ok(cmd) => match commands::run_command(cmd) {
            Ok(s) => s,
            Err(e) => {
                eprintln!("sccache: error: {}", e);
                for e in e.chain().skip(1) {
                    eprintln!("sccache: caused by: {}", e);
                }
                2
            }
        },
        Err(e) => {
            println!("sccache: {}", e);
            for e in e.chain().skip(1) {
                println!("sccache: caused by: {}", e);
            }
            cmdline::get_app().print_help().unwrap();
            println!();
            1
        }
    });
}

fn init_logging() {
    if env::var(LOGGING_ENV).is_ok() {
        match env_logger::Builder::from_env(LOGGING_ENV).try_init() {
            Ok(_) => (),
            Err(e) => panic!(format!("Failed to initalize logging: {:?}", e)),
        }
    }
}
<commit_msg>Don't allow unknown lints by default<commit_after>// Copyright 2016 Mozilla Foundation
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

#![deny(rust_2018_idioms)]
#![allow(clippy::type_complexity, clippy::new_without_default)]
#![recursion_limit = "256"]

#[macro_use]
extern crate clap;
#[macro_use]
extern crate counted_array;
#[macro_use]
extern crate futures;
#[cfg(feature = "jsonwebtoken")]
use jsonwebtoken as jwt;
#[macro_use]
extern crate lazy_static;
#[macro_use]
extern crate log;
#[cfg(feature = "rouille")]
#[macro_use(router)]
extern crate rouille;
#[macro_use]
extern crate serde_derive;

// To get macros in scope, this has to be first.
#[cfg(test)]
#[macro_use]
mod test;

#[macro_use]
pub mod errors;

#[cfg(feature = "azure")]
mod azure;
mod cache;
mod client;
mod cmdline;
mod commands;
mod compiler;
pub mod config;
pub mod dist;
mod jobserver;
pub mod lru_disk_cache;
mod mock_command;
mod protocol;
pub mod server;
#[cfg(feature = "simple-s3")]
mod simples3;
#[doc(hidden)]
pub mod util;

use std::env;

const LOGGING_ENV: &str = "SCCACHE_LOG";

pub fn main() {
    init_logging();
    std::process::exit(match cmdline::parse() {
        Ok(cmd) => match commands::run_command(cmd) {
            Ok(s) => s,
            Err(e) => {
                eprintln!("sccache: error: {}", e);
                for e in e.chain().skip(1) {
                    eprintln!("sccache: caused by: {}", e);
                }
                2
            }
        },
        Err(e) => {
            println!("sccache: {}", e);
            for e in e.chain().skip(1) {
                println!("sccache: caused by: {}", e);
            }
            cmdline::get_app().print_help().unwrap();
            println!();
            1
        }
    });
}

fn init_logging() {
    if env::var(LOGGING_ENV).is_ok() {
        match env_logger::Builder::from_env(LOGGING_ENV).try_init() {
            Ok(_) => (),
            Err(e) => panic!(format!("Failed to initalize logging: {:?}", e)),
        }
    }
}
<|endoftext|>


<commit_before>//! A machine learning library for supervised regression trainings
//!
//! This library wants to enable its user to write teachers
//! independent of the model trained or the cost function tried to
//! minimize.
//! Consequently its three most important traits are `Model`, `Cost`
//! and `Training`.
//!
//! # Examples
//!
//! Estimate mean
//!
//! ```
//! use vikos::{model, cost, teacher, teach_history, Model};
//! //We are only training a constant so our feature vector is empty
//! //Any other vector would be fine, too.
//! let features = ();
//! let history = [1f64, 3.0, 4.0, 7.0, 8.0, 11.0, 29.0]; //mean is 9
//! // Give each 'truth' its feature vector and stretch our history to 100 elements
//! let history = history.iter().cycle().map(|&y|((),y)).take(100);
//!
//! let cost = cost::LeastSquares{};
//! let mut model = model::Constant::new(0.0);
//!
//! let teacher = teacher::GradientDescentAl{ l0 : 0.3, t : 4.0 };
//! teach_history(&teacher, &cost, &mut model, history);
//! println!("{}", model.predict(&features));
//! ```
#![warn(missing_docs)]

extern crate num;

use std::iter::IntoIterator;
use num::Float;

/// A Model is defines how to predict a target from an input
///
/// A model usually depends on several coefficents whose values
/// are derived using a training algorithm
pub trait Model : Clone{
    /// Input features
    type Input;
    /// Target type
    type Target : Float;

    /// Predicts a target for the inputs based on the internal coefficents
    fn predict(&self, &Self::Input) -> Self::Target;

    /// The number of internal coefficents this model depends on
    fn num_coefficents(&self) -> usize;

    /// Value predict derived by the n-th `coefficent` at `input`
    fn gradient(&self, coefficent : usize, input : &Self::Input) -> Self::Target;

    /// Mutable reference to the n-th `coefficent`
    fn coefficent(& mut self, coefficent : usize) -> & mut Self::Target;
}

/// Cost functions those value is supposed be minimized by the training algorithm
pub trait Cost{

    /// Error type used by the cost function
    ///
    /// Usually `f64` or `f32`
    type Error : Float;

    /// Value of the cost function derived by the n-th coefficent at x expressed in Error(x) and dY(x)/dx
    ///
    /// This method is called by SGD based training algorithm in order to
    /// determine the delta of the coefficents
    fn gradient(&self, prediction : Self::Error, truth : Self::Error, gradient_error_by_coefficent : Self::Error) -> Self::Error;
}

/// Teaches event to a `Model`
pub trait Training{

    /// `Model` changed by this `Training`
    ///
    /// A `Training` is strictly associated with a `Model` type. One could
    /// even argue that an instance of `Training` strictly belongs to an
    /// instance of `Model`
    type Model : Model;

    /// Changes `model`s coefficents so they minimize the `cost` function (hopefully)
    fn teach_event<C>(
        &mut self, cost : &C, model : &mut Self::Model,
        features : &<Self::Model as Model>::Input,
        truth : <Self::Model as Model>::Target
    ) where
        C : Cost<Error=<Self::Model as Model>::Target>;
}

/// `Teachers` are used to train `Models`, based on events and a `Cost` function
pub trait Teacher<M : Model>{

    /// Contains state which changes during the training, but is not required by the expert
    ///
    /// Examples are the velocity of the coefficents (in SGD) or the number of events already learned.
    /// This may also be empty
    type Training : Training<Model=M>;

    /// Creates a new `Training` object to hold training state
    fn new_training(&self, model : &M) -> Self::Training;
}

/// Teaches `model` all events in `history`
pub fn teach_history<M, C, T, H>(teacher : &T, cost : &C, model : &mut M, history : H)
    where M : Model,
    C : Cost<Error=M::Target>,
    T : Teacher<M>,
    H : IntoIterator<Item=(M::Input, M::Target)>
{
    let mut training = teacher.new_training(&model);
    for (features, truth) in history{

        training.teach_event(cost, model, &features, truth);
    }
}

/// Implementations of `Model` trait
pub mod model;
/// Implementations of `Cost` trait
pub mod cost;
/// Implementations of `Training` trait
///
/// You can use `Teacher`s to create `Training` instances
pub mod training;
/// Implementatios of `Teacher` trait
pub mod teacher;
/// Defines linear algebra traits used for some model parameters
pub mod linear_algebra;

#[cfg(test)]
mod tests {

    #[test]
    fn estimate_median() {

        use model::Constant;
        use cost::LeastAbsoluteDeviation;
        use training::GradientDescentAl;
        use Training;

        let features = ();
        let history = [1.0, 3.0, 4.0, 7.0, 8.0, 11.0, 29.0]; //median is seven

        let cost = LeastAbsoluteDeviation{};
        let mut model = Constant::new(0.0);

        let mut training = GradientDescentAl{ l0 : 0.9, t : 9.0, learned_events : 0.0 };

        for &truth in history.iter().cycle().take(150){

            training.teach_event(&cost, &mut model, &features, truth);
            println!("model: {:?}, learning_rate: {:?}", model, training.learning_rate());
        }

        assert!(model.c < 7.1);
        assert!(model.c > 6.9);
    }

    #[test]
    fn estimate_mean() {

        use model::Constant;
        use cost::LeastSquares;
        use training::GradientDescentAl;
        use Training;

        let features = ();
        let history = [1f64, 3.0, 4.0, 7.0, 8.0, 11.0, 29.0]; //mean is 9

        let cost = LeastSquares{};
        let mut model = Constant::new(0.0);

        let mut training = GradientDescentAl{ l0 : 0.3, t : 4.0, learned_events : 0.0 };

        for &truth in history.iter().cycle().take(100){

            training.teach_event(&cost, &mut model, &features, truth);
            println!("model: {:?}, learning_rate: {:?}", model, training.learning_rate());
        }

        assert!(model.c < 9.1);
        assert!(model.c > 8.9);
    }

    #[test]
    fn linear_stochastic_gradient_descent() {

        use cost::LeastSquares;
        use model::Linear;
        use teacher::GradientDescent;
        use teach_history;

        let history = [(0f64, 3f64), (1.0, 4.0), (2.0, 5.0)];

        let mut model = Linear{m : 0.0, c : 0.0};

        let teacher = GradientDescent{ learning_rate : 0.2 };

        let cost = LeastSquares{};
        teach_history(&teacher, &cost, &mut model, history.iter().cycle().take(20).cloned());

        assert!(model.m < 1.1);
        assert!(model.m > 0.9);
        assert!(model.c < 3.1);
        assert!(model.c > 2.9);
    }

    #[test]
    fn linear_stochastic_gradient_descent_iter() {

        use model::Linear;
        use cost::LeastSquares;
        use teacher::GradientDescent;
        use Teacher;
        use Training;

        let history = [(0f64, 3f64), (1.0, 4.0), (2.0, 5.0)];

        let cost = LeastSquares{};
        let mut model = Linear{m : 0.0, c : 0.0};

        let teacher = GradientDescent{ learning_rate : 0.2 };
        let mut training = teacher.new_training(&model);

        for &(features, truth) in history.iter().cycle().take(20){

            training.teach_event(&cost, &mut model, &features, truth);
            println!("model: {:?}", model);
        }

        assert!(model.m < 1.1);
        assert!(model.m > 0.9);
        assert!(model.c < 3.1);
        assert!(model.c > 2.9);
    }

    #[test]
    fn linear_sgd_2d(){
        use cost::LeastSquares;
        use model::Linear;
        use teacher::Momentum;
        use teach_history;

        let history = [([0.0, 7.0], 17.0), ([1.0, 2.0], 8.0), ([2.0, -2.0], 1.0)];
        let mut model = Linear{m : [0.0, 0.0], c : 0.0};
        let cost = LeastSquares{};
        let teacher = Momentum{ l0 : 0.01, t : 10000000.0, inertia : 0.9 };

        teach_history(&teacher, &cost, &mut model, history.iter().cycle().take(15000).cloned());

        println!("{:?}", model);

        assert!(model.m[0] < 1.1);
        assert!(model.m[0] > 0.9);
        assert!(model.m[1] < 2.1);
        assert!(model.m[1] > 1.9);
        assert!(model.c < 3.1);
        assert!(model.c > 2.9);
    }

    #[test]
    fn logistic_sgd_2d_least_squares(){
        use cost::LeastSquares;
        use model::{Logistic, Linear};
        use teacher::GradientDescent;
        use teach_history;

        use Model;

        let history = [
            ([2.7, 2.5], 0.0),
            ([1.4, 2.3], 0.0),
            ([3.3, 4.4], 0.0),
            ([1.3, 1.8], 0.0),
            ([3.0, 3.0], 0.0),
            ([7.6, 2.7], 1.0),
            ([5.3, 2.0], 1.0),
            ([6.9, 1.7], 1.0),
            ([8.6, -0.2], 1.0),
            ([7.6, 3.5], 1.0)
        ];

        let mut model = Logistic{ linear: Linear{m : [0.0, 0.0], c : 0.0}};
        let teacher = GradientDescent{ learning_rate : 0.3 };
        let cost = LeastSquares{};

        teach_history(
            &teacher, &cost, &mut model,
            history.iter().cycle().take(40).cloned(),
        );

        println!("{:?}", model.linear);

        let classification_errors = history.iter()
            .map(|&(input, truth)| model.predict(&input).round() == truth)
            .fold(0, |errors, correct| if correct { errors } else { errors + 1 });

        assert_eq!(0, classification_errors);
    }

    #[test]
    fn logistic_sgd_2d_max_likelihood(){
        use cost::MaxLikelihood;
        use model::{Logistic, Linear};
        use teacher::GradientDescent;
        use teach_history;
        use Model;

        let history = [
            ([2.7, 2.5], 0.0),
            ([1.4, 2.3], 0.0),
            ([3.3, 4.4], 0.0),
            ([1.3, 1.8], 0.0),
            ([3.0, 3.0], 0.0),
            ([7.6, 2.7], 1.0),
            ([5.3, 2.0], 1.0),
            ([6.9, 1.7], 1.0),
            ([8.6, -0.2], 1.0),
            ([7.6, 3.5], 1.0)
        ];

        let mut model = Logistic{ linear: Linear{m : [0.0, 0.0], c : 0.0}};
        let teacher = GradientDescent{ learning_rate : 0.3 };
        let cost = MaxLikelihood{};

        teach_history(
            &teacher, &cost, &mut model,
            history.iter().cycle().take(20).cloned(),
        );

        println!("{:?}", model.linear);

        let classification_errors = history.iter()
            .map(|&(input, truth)| model.predict(&input).round() == truth)
            .fold(0, |errors, correct| if correct { errors } else { errors + 1 });

        assert_eq!(0, classification_errors);
    }
}
<commit_msg>introduced tutorial section<commit_after>//! A machine learning library for supervised regression trainings
//!
//! This library wants to enable its user to write teachers
//! independent of the model trained or the cost function tried to
//! minimize.
//! Consequently its three most important traits are `Model`, `Cost`
//! and `Training`.
//!
//! # Tutorial
//!
//! ## Estimating mean
//!
//! ```
//! use vikos::{model, cost, teacher, teach_history, Model};
//! // First element of the tuple is our feature vector
//! // the second element represents the truth.
//! let history = [
//!    (2.0, 1.0), (3.0, 3.0), (3.5, 4.0),
//!    (5.0, 7.0), (5.5, 8.0), (7.0, 11.0),
//!    (16.0, 29.0)
//! ]; //mean is 9, but we don't know that yet of course
//!
//! // The mean is just a simple number ...
//! let mut model = model::Constant::new(0.0);
//! // ... which minimizes the square error
//! let cost = cost::LeastSquares{};
//! // Use Stochasic Gradient Descent with an annealed learning rate
//! let teacher = teacher::GradientDescentAl{ l0 : 0.3, t : 4.0 };
//! // Train 100 (admitettly repetitive) events
//! teach_history(&teacher, &cost, &mut model, history.iter().cycle().take(100).cloned());
//! // We need an input vector for predictions, the 42 won't influence the mean
//! println!("{}", model.predict(&42.0));
//! // Since we know models type is `Constant` we could just access the members
//! println!("{}", model.c);
//! ```
//! As far as the mean is concerned the first element is just
//! ignored. The code would also compile if the first
//! element would be an empty tuple or any other type for
//! that matter.
//!
//! ## Estimating median
//!
//! If we want to estimate the median instead we only need to change
//! our cost function
//!
//! ```
//! use vikos::{model, cost, teacher, teach_history, Model};
//! // First element of the tuple is our feature vector
//! // the second element represents the truth.
//! let history = [
//!    (2.0, 1.0), (3.0, 3.0), (3.5, 4.0),
//!    (5.0, 7.0), (5.5, 8.0), (7.0, 11.0),
//!    (16.0, 29.0)
//! ]; //median is 7, but we don't know that yet of course
//!
//! // The median is just a simple number ...
//! let mut model = model::Constant::new(0.0);
//! // ... which minimizes the absolute error
//! let cost = cost::LeastAbsoluteDeviation{};
//! // Use Stochasic Gradient Descent with an annealed learning rate
//! let teacher = teacher::GradientDescentAl{ l0 : 1.0, t : 9.0 };
//! // Train 100 (admitettly repetitive) events
//! teach_history(&teacher, &cost, &mut model, history.iter().cycle().take(100).cloned());
//! // We need an input vector for predictions, the 42 won't influence the median
//! println!("{}", model.predict(&42.0));
//! ```
//! Most notably we changed the cost function to train for the median. We also increased
//! our learning rate , but the old one would have worked fine too, if we had increased
//! the number of events instead.
#![warn(missing_docs)]

extern crate num;

use std::iter::IntoIterator;
use num::Float;

/// A Model is defines how to predict a target from an input
///
/// A model usually depends on several coefficents whose values
/// are derived using a training algorithm
pub trait Model : Clone{
    /// Input features
    type Input;
    /// Target type
    type Target : Float;

    /// Predicts a target for the inputs based on the internal coefficents
    fn predict(&self, &Self::Input) -> Self::Target;

    /// The number of internal coefficents this model depends on
    fn num_coefficents(&self) -> usize;

    /// Value predict derived by the n-th `coefficent` at `input`
    fn gradient(&self, coefficent : usize, input : &Self::Input) -> Self::Target;

    /// Mutable reference to the n-th `coefficent`
    fn coefficent(& mut self, coefficent : usize) -> & mut Self::Target;
}

/// Cost functions those value is supposed be minimized by the training algorithm
pub trait Cost{

    /// Error type used by the cost function
    ///
    /// Usually `f64` or `f32`
    type Error : Float;

    /// Value of the cost function derived by the n-th coefficent at x expressed in Error(x) and dY(x)/dx
    ///
    /// This method is called by SGD based training algorithm in order to
    /// determine the delta of the coefficents
    fn gradient(&self, prediction : Self::Error, truth : Self::Error, gradient_error_by_coefficent : Self::Error) -> Self::Error;
}

/// Teaches event to a `Model`
pub trait Training{

    /// `Model` changed by this `Training`
    ///
    /// A `Training` is strictly associated with a `Model` type. One could
    /// even argue that an instance of `Training` strictly belongs to an
    /// instance of `Model`
    type Model : Model;

    /// Changes `model`s coefficents so they minimize the `cost` function (hopefully)
    fn teach_event<C>(
        &mut self, cost : &C, model : &mut Self::Model,
        features : &<Self::Model as Model>::Input,
        truth : <Self::Model as Model>::Target
    ) where
        C : Cost<Error=<Self::Model as Model>::Target>;
}

/// `Teachers` are used to train `Models`, based on events and a `Cost` function
pub trait Teacher<M : Model>{

    /// Contains state which changes during the training, but is not required by the expert
    ///
    /// Examples are the velocity of the coefficents (in SGD) or the number of events already learned.
    /// This may also be empty
    type Training : Training<Model=M>;

    /// Creates a new `Training` object to hold training state
    fn new_training(&self, model : &M) -> Self::Training;
}

/// Teaches `model` all events in `history`
pub fn teach_history<M, C, T, H>(teacher : &T, cost : &C, model : &mut M, history : H)
    where M : Model,
    C : Cost<Error=M::Target>,
    T : Teacher<M>,
    H : IntoIterator<Item=(M::Input, M::Target)>
{
    let mut training = teacher.new_training(&model);
    for (features, truth) in history{

        training.teach_event(cost, model, &features, truth);
    }
}

/// Implementations of `Model` trait
pub mod model;
/// Implementations of `Cost` trait
pub mod cost;
/// Implementations of `Training` trait
///
/// You can use `Teacher`s to create `Training` instances
pub mod training;
/// Implementatios of `Teacher` trait
pub mod teacher;
/// Defines linear algebra traits used for some model parameters
pub mod linear_algebra;

#[cfg(test)]
mod tests {

    #[test]
    fn estimate_median() {

        use model::Constant;
        use cost::LeastAbsoluteDeviation;
        use training::GradientDescentAl;
        use Training;

        let features = ();
        let history = [1.0, 3.0, 4.0, 7.0, 8.0, 11.0, 29.0]; //median is seven

        let cost = LeastAbsoluteDeviation{};
        let mut model = Constant::new(0.0);

        let mut training = GradientDescentAl{ l0 : 0.9, t : 9.0, learned_events : 0.0 };

        for &truth in history.iter().cycle().take(150){

            training.teach_event(&cost, &mut model, &features, truth);
            println!("model: {:?}, learning_rate: {:?}", model, training.learning_rate());
        }

        assert!(model.c < 7.1);
        assert!(model.c > 6.9);
    }

    #[test]
    fn estimate_mean() {

        use model::Constant;
        use cost::LeastSquares;
        use training::GradientDescentAl;
        use Training;

        let features = ();
        let history = [1f64, 3.0, 4.0, 7.0, 8.0, 11.0, 29.0]; //mean is 9

        let cost = LeastSquares{};
        let mut model = Constant::new(0.0);

        let mut training = GradientDescentAl{ l0 : 0.3, t : 4.0, learned_events : 0.0 };

        for &truth in history.iter().cycle().take(100){

            training.teach_event(&cost, &mut model, &features, truth);
            println!("model: {:?}, learning_rate: {:?}", model, training.learning_rate());
        }

        assert!(model.c < 9.1);
        assert!(model.c > 8.9);
    }

    #[test]
    fn linear_stochastic_gradient_descent() {

        use cost::LeastSquares;
        use model::Linear;
        use teacher::GradientDescent;
        use teach_history;

        let history = [(0f64, 3f64), (1.0, 4.0), (2.0, 5.0)];

        let mut model = Linear{m : 0.0, c : 0.0};

        let teacher = GradientDescent{ learning_rate : 0.2 };

        let cost = LeastSquares{};
        teach_history(&teacher, &cost, &mut model, history.iter().cycle().take(20).cloned());

        assert!(model.m < 1.1);
        assert!(model.m > 0.9);
        assert!(model.c < 3.1);
        assert!(model.c > 2.9);
    }

    #[test]
    fn linear_stochastic_gradient_descent_iter() {

        use model::Linear;
        use cost::LeastSquares;
        use teacher::GradientDescent;
        use Teacher;
        use Training;

        let history = [(0f64, 3f64), (1.0, 4.0), (2.0, 5.0)];

        let cost = LeastSquares{};
        let mut model = Linear{m : 0.0, c : 0.0};

        let teacher = GradientDescent{ learning_rate : 0.2 };
        let mut training = teacher.new_training(&model);

        for &(features, truth) in history.iter().cycle().take(20){

            training.teach_event(&cost, &mut model, &features, truth);
            println!("model: {:?}", model);
        }

        assert!(model.m < 1.1);
        assert!(model.m > 0.9);
        assert!(model.c < 3.1);
        assert!(model.c > 2.9);
    }

    #[test]
    fn linear_sgd_2d(){
        use cost::LeastSquares;
        use model::Linear;
        use teacher::Momentum;
        use teach_history;

        let history = [([0.0, 7.0], 17.0), ([1.0, 2.0], 8.0), ([2.0, -2.0], 1.0)];
        let mut model = Linear{m : [0.0, 0.0], c : 0.0};
        let cost = LeastSquares{};
        let teacher = Momentum{ l0 : 0.01, t : 10000000.0, inertia : 0.9 };

        teach_history(&teacher, &cost, &mut model, history.iter().cycle().take(15000).cloned());

        println!("{:?}", model);

        assert!(model.m[0] < 1.1);
        assert!(model.m[0] > 0.9);
        assert!(model.m[1] < 2.1);
        assert!(model.m[1] > 1.9);
        assert!(model.c < 3.1);
        assert!(model.c > 2.9);
    }

    #[test]
    fn logistic_sgd_2d_least_squares(){
        use cost::LeastSquares;
        use model::{Logistic, Linear};
        use teacher::GradientDescent;
        use teach_history;

        use Model;

        let history = [
            ([2.7, 2.5], 0.0),
            ([1.4, 2.3], 0.0),
            ([3.3, 4.4], 0.0),
            ([1.3, 1.8], 0.0),
            ([3.0, 3.0], 0.0),
            ([7.6, 2.7], 1.0),
            ([5.3, 2.0], 1.0),
            ([6.9, 1.7], 1.0),
            ([8.6, -0.2], 1.0),
            ([7.6, 3.5], 1.0)
        ];

        let mut model = Logistic{ linear: Linear{m : [0.0, 0.0], c : 0.0}};
        let teacher = GradientDescent{ learning_rate : 0.3 };
        let cost = LeastSquares{};

        teach_history(
            &teacher, &cost, &mut model,
            history.iter().cycle().take(40).cloned(),
        );

        println!("{:?}", model.linear);

        let classification_errors = history.iter()
            .map(|&(input, truth)| model.predict(&input).round() == truth)
            .fold(0, |errors, correct| if correct { errors } else { errors + 1 });

        assert_eq!(0, classification_errors);
    }

    #[test]
    fn logistic_sgd_2d_max_likelihood(){
        use cost::MaxLikelihood;
        use model::{Logistic, Linear};
        use teacher::GradientDescent;
        use teach_history;
        use Model;

        let history = [
            ([2.7, 2.5], 0.0),
            ([1.4, 2.3], 0.0),
            ([3.3, 4.4], 0.0),
            ([1.3, 1.8], 0.0),
            ([3.0, 3.0], 0.0),
            ([7.6, 2.7], 1.0),
            ([5.3, 2.0], 1.0),
            ([6.9, 1.7], 1.0),
            ([8.6, -0.2], 1.0),
            ([7.6, 3.5], 1.0)
        ];

        let mut model = Logistic{ linear: Linear{m : [0.0, 0.0], c : 0.0}};
        let teacher = GradientDescent{ learning_rate : 0.3 };
        let cost = MaxLikelihood{};

        teach_history(
            &teacher, &cost, &mut model,
            history.iter().cycle().take(20).cloned(),
        );

        println!("{:?}", model.linear);

        let classification_errors = history.iter()
            .map(|&(input, truth)| model.predict(&input).round() == truth)
            .fold(0, |errors, correct| if correct { errors } else { errors + 1 });

        assert_eq!(0, classification_errors);
    }
}
<|endoftext|>


<commit_before>#![deny(warnings)]
#![feature(if_let)]

mod just;
pub mod led;
<commit_msg>fix(lib): add #![feature(slicing_syntax)]<commit_after>#![deny(warnings)]
#![feature(if_let, slicing_syntax)]

mod just;
pub mod led;
<|endoftext|>


<commit_before>extern crate libc;

use std::c_str::CString;

mod ffi;

pub fn is_present() -> bool {
    unsafe {
        ffi::cpuid_present() == 1
    }
}

pub fn version() -> String {
    let version_string = unsafe {
        let ptr = ffi::cpuid_lib_version();
        CString::new(ptr, false)
    };
    version_string.as_str().unwrap().to_string()
}

pub fn error() -> String {
    let error_string = unsafe {
        let ptr = ffi::cpuid_error();
        CString::new(ptr, false)
    };
    error_string.as_str().unwrap().to_string()
}

pub fn identify() -> bool {
    let mut raw = ffi::cpu_raw_data_t {
        basic_cpuid: [[0, ..ffi::MAX_CPUID_LEVEL], ..4u],
        ext_cpuid: [[0, ..ffi::MAX_EXT_CPUID_LEVEL], ..4u],
        intel_fn4: [[0, ..ffi::MAX_INTELFN4_LEVEL], ..4u],
        intel_fn11: [[0, ..ffi::MAX_INTELFN11_LEVEL], ..4u],
    };
    let result = unsafe {
        ffi::cpuid_get_raw_data(&mut raw)
    };
    // TODO: use raw in the call to cpu_identify
    result == 0
}

#[test]
fn test_is_present() {
    assert!(is_present());
}
<commit_msg>Actual call to cpu_identify.<commit_after>extern crate libc;

use std::c_str::CString;

mod ffi;

pub fn is_present() -> bool {
    unsafe {
        ffi::cpuid_present() == 1
    }
}

pub fn version() -> String {
    let version_string = unsafe {
        let ptr = ffi::cpuid_lib_version();
        CString::new(ptr, false)
    };
    version_string.as_str().unwrap().to_string()
}

pub fn error() -> String {
    let error_string = unsafe {
        let ptr = ffi::cpuid_error();
        CString::new(ptr, false)
    };
    error_string.as_str().unwrap().to_string()
}

pub fn identify() -> bool {
    let mut raw = ffi::cpu_raw_data_t {
        basic_cpuid: [[0, ..ffi::MAX_CPUID_LEVEL], ..4u],
        ext_cpuid: [[0, ..ffi::MAX_EXT_CPUID_LEVEL], ..4u],
        intel_fn4: [[0, ..ffi::MAX_INTELFN4_LEVEL], ..4u],
        intel_fn11: [[0, ..ffi::MAX_INTELFN11_LEVEL], ..4u],
    };
    let raw_result = unsafe {
        ffi::cpuid_get_raw_data(&mut raw)
    };
    // TODO: use raw in the call to cpu_identify
    let mut data = ffi::cpu_id_t {
        vendor_str: [0, ..ffi::VENDOR_STR_MAX],
        brand_str: [0, ..ffi::BRAND_STR_MAX],
        vendor: 0,
        flags: [0u8, ..ffi::CPU_FLAGS_MAX],
        family: 0,
        model: 0,
        stepping: 0,
        ext_family: 0,
        ext_model: 0,
        num_cores: 0,
        num_logical_cpus: 0,
        total_logical_cpus: 0,
        l1_data_cache: 0,
        l1_instruction_cache: 0,
        l2_cache: 0,
        l3_cache: 0,
        l1_assoc: 0,
        l2_assoc: 0,
        l3_assoc: 0,
        l1_cacheline: 0,
        l2_cacheline: 0,
        l3_cacheline: 0,
        cpu_codename: [0, ..64],
        sse_size: 0,
        detection_hints: [0u8, ..ffi::CPU_HINTS_MAX],
    };
    let identify_result = unsafe {
        ffi::cpu_identify(&mut raw, &mut data)
    };
    // TODO: convert data to a more friendly representation and return
    identify_result == 0
}

#[test]
fn test_is_present() {
    assert!(is_present());
}
<|endoftext|>


<commit_before>//! `primal` puts raw power into prime numbers.
//!
//! This crates includes
//!
//! - optimised prime sieves
//! - checking for primality
//! - enumerating primes
//! - factorising numbers
//! - estimating upper and lower bounds for π(*n*) (the number of primes
//!   below *n*) and *pk* (the kth prime)
//!
//! This uses a state-of-the-art cache-friendly Sieve of Eratosthenes
//! to enumerate the primes up to some fixed bound (in a memory
//! efficient manner), and then allows this cached information to be
//! used for things like enumerating and counting primes.
//!
//! `primal` takes around 2.8 seconds and less than 3MB of RAM to
//! count the exact number of primes below 1010 (455052511)
//! on the author's laptop (i7-3517U).
//!
//! [*Source*](http://github.com/huonw/primal)
//!
//! # Using this library
//!
//! Just add the following to your [`Cargo.toml`](http://crates.io/):
//!
//! ```toml
//! [dependencies]
//! primal = "0.2"
//! ```
//!
//! # Example
//!
//! Let's find the 10001st prime. The easiest way is to enumerate the
//! primes, and find the 10001st:
//!
//! ```rust
//! // (.nth is zero indexed.)
//! let p = primal::Primes::all().nth(10001 - 1).unwrap();
//! println!("The 10001st prime is {}", p); // 104743
//! ```
//!
//! `Primes` is flexible at the cost of performance, so if we wish to
//! optimise we could instead explicitly sieve and then find the
//! appropriate prime.  Unfortunately, `Sieve` requires a limit to
//! know how far to sieve: we need some way to find an upper bound to
//! be guaranteed to be at least as large the 10001st prime. We could
//! guess that, say, 108 will be large enough and use that,
//! but that's probably a huge overestimate. We could also try
//! filtering with exponentially larger upper bounds until we find one
//! that works (e.g. doubling each time), or... we could just take a
//! shortcut and use deeper mathematics via
//! [`estimate_nth_prime`](fn.estimate_nth_prime.html).
//!
//! ```rust
//! // find our upper bound
//! let (_lo, hi) = primal::estimate_nth_prime(10001);
//!
//! // find the primes up to this upper bound
//! let sieve = primal::Sieve::new(hi as usize);
//!
//! let p = sieve.nth_prime(10001);
//! println!("The 10001st prime is {}", p); // 104743
//! ```
//!

#![cfg_attr(all(test, feature = "unstable"), feature(test, step_by))]

extern crate primal_estimate;
extern crate primal_check;
extern crate primal_sieve;

#[cfg(all(test, feature = "unstable"))] extern crate test;

pub use primal_estimate::prime_pi as estimate_prime_pi;
pub use primal_estimate::nth_prime as estimate_nth_prime;
pub use primal_check::miller_rabin as is_prime;
pub use primal_check::{as_perfect_power, as_prime_power};

pub use primal_sieve::{StreamingSieve, Sieve, SievePrimes, Primes};

#[cfg(all(test, feature = "unstable"))]
mod benches {
    extern crate test;

    use super::{Sieve, is_prime};
    use self::test::Bencher;

    const N: usize = 1_000_000;
    const STEP: usize = 101;
    #[bench]
    fn bench_miller_rabin_tests(b: &mut Bencher) {
        b.iter(|| {
            (1..N).step_by(STEP)
                .filter(|&n| is_prime(n as u64)).count()
        })
    }
    #[bench]
    fn bench_sieve_tests(b: &mut Bencher) {
        b.iter(|| {
            let sieve = Sieve::new(1_000_000);
            (1..N).step_by(STEP)
                .filter(|&n| sieve.is_prime(n)).count()
        })
    }
}
<commit_msg>primal: make the main example use `StreamingSieve` too.<commit_after>//! `primal` puts raw power into prime numbers.
//!
//! This crates includes
//!
//! - optimised prime sieves
//! - checking for primality
//! - enumerating primes
//! - factorising numbers
//! - estimating upper and lower bounds for π(*n*) (the number of primes
//!   below *n*) and *pk* (the kth prime)
//!
//! This uses a state-of-the-art cache-friendly Sieve of Eratosthenes
//! to enumerate the primes up to some fixed bound (in a memory
//! efficient manner), and then allows this cached information to be
//! used for things like enumerating and counting primes.
//!
//! `primal` takes around 2.8 seconds and less than 3MB of RAM to
//! count the exact number of primes below 1010 (455052511)
//! on my laptop (i7-3517U).
//!
//! [*Source*](http://github.com/huonw/primal)
//!
//! # Using this library
//!
//! Just add the following to your [`Cargo.toml`](http://crates.io/):
//!
//! ```toml
//! [dependencies]
//! primal = "0.2"
//! ```
//!
//! # Examples
//!
//! Let's find the 10001st prime. The easiest way is to enumerate the
//! primes, and find the 10001st:
//!
//! ```rust
//! // (.nth is zero indexed.)
//! let p = primal::Primes::all().nth(10001 - 1).unwrap();
//! println!("The 10001st prime is {}", p); // 104743
//! ```
//!
//! This takes around 400 microseconds on my computer, which seems
//! nice and quick, but, `Primes` is flexible at the cost of
//! performance: we can make it faster. The `StreamingSieve` type
//! offers a specialised `nth_prime` function:
//!
//! ```rust
//! let p = primal::StreamingSieve::nth_prime(10001);
//! println!("The 10001st prime is {}", p); // 104743
//! ```
//!
//! This runs in only 10 microseconds! `StreamingSieve` is extremely
//! efficient and uses very little memory. It is the best way to solve
//! this task with `primal`.
//!
//! Since that was so easy, let's now make the problem bigger and
//! harder: find the sum of the 100,000th, 200,000th, 300,000th, ...,
//! 10,000,000th primes (100 in total).
//!
//! We could call `StreamingSieve::nth_prime` repeatedly:
//!
//! ```rust,no_run
//! // the primes we want to find
//! let ns = (1..100 + 1).map(|x| x * 100_000).collect::<Vec<_>>();
//!
//! // search and sum them up
//! let sum = ns.iter()
//!             .map(|n| primal::StreamingSieve::nth_prime(*n))
//!             .fold(0, |a, b| a + b);
//! println!("the sum is {}", sum);
//! ```
//!
//! This takes around 1.6s seconds to print `the sum is 8795091674`;
//! not so speedy. Each call to `nth_prime` is individually fast (400
//! microseconds for 100,000 to 40 milliseconds for 10,000,000) but
//! they add up to something bad. Every one is starting from the start
//! and redoing work that previous calls have done... wouldn't it be
//! nice if we could just do the computation for 10,000,000 and reuse
//! that for the smaller ones?
//!
//! The `Sieve` type is a wrapper around `StreamingSieve` that
//! caches information, allowing repeated queries to be answered
//! efficiently.
//!
//! There's one hitch: `Sieve` requires a limit to know how far to
//! sieve: we need some way to find an upper bound to be guaranteed to
//! be at least as large as all our primes. We could guess that, say,
//! 1010 will be large enough and use that, but that's a
//! huge overestimate (spoilers: the 10,000,000th prime is around
//! 2×108). We could also try filtering with
//! exponentially larger upper bounds until we find one that works
//! (e.g. doubling each time), or, we could just take a shortcut and
//! use deeper mathematics via
//! [`estimate_nth_prime`](fn.estimate_nth_prime.html).
//!
//! ```rust
//! // the primes we want to find
//! let ns = (1..100 + 1).map(|x| x * 100_000).collect::<Vec<_>>();
//!
//! // find our upper bound
//! let (_lo, hi) = primal::estimate_nth_prime(10_000_000);
//!
//! // find the primes up to this upper bound
//! let sieve = primal::Sieve::new(hi as usize);
//!
//! // now we can efficiently sum them up
//! let sum = ns.iter()
//!             .map(|n| sieve.nth_prime(*n))
//!             .fold(0, |a, b| a + b);
//! println!("the sum is {}", sum);
//! ```
//!
//! This takes around 40 milliseconds, and gives the same output: much
//! better!
//!
//! (By the way, the version using 1010 as the bound
//! instead of the more accurate estimate still only takes ~3
//! seconds.)

#![cfg_attr(all(test, feature = "unstable"), feature(test, step_by))]

extern crate primal_estimate;
extern crate primal_check;
extern crate primal_sieve;

#[cfg(all(test, feature = "unstable"))] extern crate test;

pub use primal_estimate::prime_pi as estimate_prime_pi;
pub use primal_estimate::nth_prime as estimate_nth_prime;
pub use primal_check::miller_rabin as is_prime;
pub use primal_check::{as_perfect_power, as_prime_power};

pub use primal_sieve::{StreamingSieve, Sieve, SievePrimes, Primes};

#[cfg(all(test, feature = "unstable"))]
mod benches {
    extern crate test;

    use super::{Sieve, is_prime};
    use self::test::Bencher;

    const N: usize = 1_000_000;
    const STEP: usize = 101;
    #[bench]
    fn bench_miller_rabin_tests(b: &mut Bencher) {
        b.iter(|| {
            (1..N).step_by(STEP)
                .filter(|&n| is_prime(n as u64)).count()
        })
    }
    #[bench]
    fn bench_sieve_tests(b: &mut Bencher) {
        b.iter(|| {
            let sieve = Sieve::new(1_000_000);
            (1..N).step_by(STEP)
                .filter(|&n| sieve.is_prime(n)).count()
        })
    }
}
<|endoftext|>


<commit_before>// Copyright 2014 The Rust Project Developers. See the COPYRIGHT
// file at the top-level directory of this distribution and at
// http://rust-lang.org/COPYRIGHT.
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.

//! Simple numerics.
//!
//! This crate contains arbitrary-sized integer, rational, and complex types.
//!
//! ## Example
//!
//! This example uses the BigRational type and [Newton's method][newt] to
//! approximate a square root to arbitrary precision:
//!
//! ```
//! extern crate num;
//!
//! use num::bigint::BigInt;
//! use num::rational::{Ratio, BigRational};
//!
//! fn approx_sqrt(number: u64, iterations: uint) -> BigRational {
//!     let start: Ratio<BigInt> = Ratio::from_integer(FromPrimitive::from_u64(number).unwrap());
//!     let mut approx = start.clone();
//!
//!     for _ in range(0, iterations) {
//!         approx = (approx + (start / approx)) /
//!             Ratio::from_integer(FromPrimitive::from_u64(2).unwrap());
//!     }
//!
//!     approx
//! }
//!
//! fn main() {
//!     println!("{}", approx_sqrt(10, 4)); // prints 4057691201/1283082416
//! }
//! ```
//!
//! [newt]: https://en.wikipedia.org/wiki/Methods_of_computing_square_roots#Babylonian_method

#![feature(macro_rules)]
#![feature(default_type_params)]
#![feature(slicing_syntax)]

#![crate_name = "num"]
#![experimental]
#![crate_type = "rlib"]
#![crate_type = "dylib"]
#![license = "MIT/ASL2"]
#![doc(html_logo_url = "http://www.rust-lang.org/logos/rust-logo-128x128-blk-v2.png",
       html_favicon_url = "http://www.rust-lang.org/favicon.ico",
       html_root_url = "http://doc.rust-lang.org/master/",
       html_playground_url = "http://play.rust-lang.org/")]
#![allow(deprecated)] // from_str_radix

extern crate rand;
extern crate serialize;

pub use bigint::{BigInt, BigUint};
pub use rational::{Rational, BigRational};
pub use complex::Complex;
pub use integer::Integer;
pub use iter::{range, range_inclusive, range_step, range_step_inclusive};
pub use traits::{Num, Zero, One, Signed, Unsigned, Bounded,
                 Saturating, CheckedAdd, CheckedSub, CheckedMul, CheckedDiv};

pub mod bigint;
pub mod complex;
pub mod integer;
pub mod iter;
pub mod traits;
pub mod rational;

/// Returns the additive identity, `0`.
#[inline(always)] pub fn zero<T: Zero>() -> T { Zero::zero() }

/// Returns the multiplicative identity, `1`.
#[inline(always)] pub fn one<T: One>() -> T { One::one() }

/// Computes the absolute value.
///
/// For `f32` and `f64`, `NaN` will be returned if the number is `NaN`
///
/// For signed integers, `::MIN` will be returned if the number is `::MIN`.
#[inline(always)]
pub fn abs<T: Signed>(value: T) -> T {
    value.abs()
}

/// The positive difference of two numbers.
///
/// Returns zero if `x` is less than or equal to `y`, otherwise the difference
/// between `x` and `y` is returned.
#[inline(always)]
pub fn abs_sub<T: Signed>(x: T, y: T) -> T {
    x.abs_sub(&y)
}

/// Returns the sign of the number.
///
/// For `f32` and `f64`:
///
/// * `1.0` if the number is positive, `+0.0` or `INFINITY`
/// * `-1.0` if the number is negative, `-0.0` or `NEG_INFINITY`
/// * `NaN` if the number is `NaN`
///
/// For signed integers:
///
/// * `0` if the number is zero
/// * `1` if the number is positive
/// * `-1` if the number is negative
#[inline(always)] pub fn signum<T: Signed>(value: T) -> T { value.signum() }

/// Raises a value to the power of exp, using exponentiation by squaring.
///
/// # Example
///
/// ```rust
/// use num;
///
/// assert_eq!(num::pow(2i, 4), 16);
/// ```
#[inline]
pub fn pow<T: One + Mul<T, T>>(mut base: T, mut exp: uint) -> T {
    if exp == 1 { base }
    else {
        let mut acc = one::<T>();
        while exp > 0 {
            if (exp & 1) == 1 {
                acc = acc * base;
            }
            base = base * base;
            exp = exp >> 1;
        }
        acc
    }
}
<commit_msg>remove license attribute<commit_after>// Copyright 2014 The Rust Project Developers. See the COPYRIGHT
// file at the top-level directory of this distribution and at
// http://rust-lang.org/COPYRIGHT.
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.

//! Simple numerics.
//!
//! This crate contains arbitrary-sized integer, rational, and complex types.
//!
//! ## Example
//!
//! This example uses the BigRational type and [Newton's method][newt] to
//! approximate a square root to arbitrary precision:
//!
//! ```
//! extern crate num;
//!
//! use num::bigint::BigInt;
//! use num::rational::{Ratio, BigRational};
//!
//! fn approx_sqrt(number: u64, iterations: uint) -> BigRational {
//!     let start: Ratio<BigInt> = Ratio::from_integer(FromPrimitive::from_u64(number).unwrap());
//!     let mut approx = start.clone();
//!
//!     for _ in range(0, iterations) {
//!         approx = (approx + (start / approx)) /
//!             Ratio::from_integer(FromPrimitive::from_u64(2).unwrap());
//!     }
//!
//!     approx
//! }
//!
//! fn main() {
//!     println!("{}", approx_sqrt(10, 4)); // prints 4057691201/1283082416
//! }
//! ```
//!
//! [newt]: https://en.wikipedia.org/wiki/Methods_of_computing_square_roots#Babylonian_method

#![feature(macro_rules)]
#![feature(default_type_params)]
#![feature(slicing_syntax)]

#![crate_name = "num"]
#![experimental]
#![crate_type = "rlib"]
#![crate_type = "dylib"]
#![doc(html_logo_url = "http://www.rust-lang.org/logos/rust-logo-128x128-blk-v2.png",
       html_favicon_url = "http://www.rust-lang.org/favicon.ico",
       html_root_url = "http://doc.rust-lang.org/master/",
       html_playground_url = "http://play.rust-lang.org/")]
#![allow(deprecated)] // from_str_radix

extern crate rand;
extern crate serialize;

pub use bigint::{BigInt, BigUint};
pub use rational::{Rational, BigRational};
pub use complex::Complex;
pub use integer::Integer;
pub use iter::{range, range_inclusive, range_step, range_step_inclusive};
pub use traits::{Num, Zero, One, Signed, Unsigned, Bounded,
                 Saturating, CheckedAdd, CheckedSub, CheckedMul, CheckedDiv};

pub mod bigint;
pub mod complex;
pub mod integer;
pub mod iter;
pub mod traits;
pub mod rational;

/// Returns the additive identity, `0`.
#[inline(always)] pub fn zero<T: Zero>() -> T { Zero::zero() }

/// Returns the multiplicative identity, `1`.
#[inline(always)] pub fn one<T: One>() -> T { One::one() }

/// Computes the absolute value.
///
/// For `f32` and `f64`, `NaN` will be returned if the number is `NaN`
///
/// For signed integers, `::MIN` will be returned if the number is `::MIN`.
#[inline(always)]
pub fn abs<T: Signed>(value: T) -> T {
    value.abs()
}

/// The positive difference of two numbers.
///
/// Returns zero if `x` is less than or equal to `y`, otherwise the difference
/// between `x` and `y` is returned.
#[inline(always)]
pub fn abs_sub<T: Signed>(x: T, y: T) -> T {
    x.abs_sub(&y)
}

/// Returns the sign of the number.
///
/// For `f32` and `f64`:
///
/// * `1.0` if the number is positive, `+0.0` or `INFINITY`
/// * `-1.0` if the number is negative, `-0.0` or `NEG_INFINITY`
/// * `NaN` if the number is `NaN`
///
/// For signed integers:
///
/// * `0` if the number is zero
/// * `1` if the number is positive
/// * `-1` if the number is negative
#[inline(always)] pub fn signum<T: Signed>(value: T) -> T { value.signum() }

/// Raises a value to the power of exp, using exponentiation by squaring.
///
/// # Example
///
/// ```rust
/// use num;
///
/// assert_eq!(num::pow(2i, 4), 16);
/// ```
#[inline]
pub fn pow<T: One + Mul<T, T>>(mut base: T, mut exp: uint) -> T {
    if exp == 1 { base }
    else {
        let mut acc = one::<T>();
        while exp > 0 {
            if (exp & 1) == 1 {
                acc = acc * base;
            }
            base = base * base;
            exp = exp >> 1;
        }
        acc
    }
}
<|endoftext|>


<commit_before>//! Guessing of MIME types by file extension.
//!
//! Uses a static list of file-extension : MIME type mappings.
#![cfg_attr(feature = "bench", feature(test))]

extern crate mime;
extern crate phf;
extern crate unicase;

use mime::Mime;
use unicase::UniCase;

use std::ffi::OsStr;
use std::path::Path;

include!("mime_types_generated.rs");

struct TopLevelExts {
    start: usize,
    end: usize,
    subs: phf::Map<UniCase<&'static str>, (usize, usize)>,
}

macro_rules! try_opt (
    ($expr:expr) => (
        match $expr {
            Some(val) => val,
            None => return None,
        }
    )
);

#[cfg(test)]
#[path = "mime_types.rs"]
mod mime_types_src;

/// Guess the MIME type of `path` by its extension (as defined by `Path::extension()`).
///
/// If `path` has no extension, or its extension has no known MIME type mapping,
/// then the MIME type is assumed to be `application/octet-stream`.
///
/// ##Note
/// **Guess** is the operative word here, as there are no guarantees that the contents of the file
/// that `path` points to match the MIME type associated with the path's extension.
///
/// Take care when processing files with assumptions based on the return value of this function.
pub fn guess_mime_type<P: AsRef<Path>>(path: P) -> Mime {
    guess_mime_type_opt(path)
        .unwrap_or_else(octet_stream)
}

/// Guess the MIME type of `path` by its extension (as defined by `Path::extension()`).
///
/// If `path` has no extension, or its extension has no known MIME type mapping,
/// then `None` is returned.
///
/// ##Note
/// **Guess** is the operative word here, as there are no guarantees that the contents of the file
/// that `path` points to match the MIME type associated with the path's extension.
///
/// Take care when processing files with assumptions based on the return value of this function.
pub fn guess_mime_type_opt<P: AsRef<Path>>(path: P) -> Option<Mime> {
    let ext = path.as_ref().extension().and_then(OsStr::to_str).unwrap_or("");

    get_mime_type_opt(ext)
}

/// Get the MIME type associated with a file extension.
///
/// If there is no association for the extension, or `ext` is empty,
/// `application/octet-stream` is returned.
pub fn get_mime_type(search_ext: &str) -> Mime {
    get_mime_type_opt(search_ext)
        .unwrap_or_else(octet_stream)
}

/// Get the MIME type associated with a file extension.
///
/// If there is no association for the extension, or `ext` is empty,
/// `None` is returned.
pub fn get_mime_type_opt(search_ext: &str) -> Option<Mime> {
    get_mime_type_str(search_ext)
        .map(|mime| mime.parse::<Mime>().unwrap())
}

/// Get the MIME type string associated with a file extension. Case-insensitive.
///
/// If `search_ext` is not already lowercase,
/// it will be converted to lowercase to facilitate the search.
///
/// Returns `None` if `search_ext` is empty or an associated extension was not found.
pub fn get_mime_type_str(search_ext: &str) -> Option<&'static str> {
    if search_ext.is_empty() { return None; }

    map_lookup(&MIME_TYPES, search_ext).cloned()
}

/// Get a list of known extensions for a given `Mime`.
///
/// Ignores parameters (only searches with `<main type>/<subtype>`). Case-insensitive (for extension types).
///
/// Returns `None` if the MIME type is unknown.
///
/// ###Wildcards
/// If the top-level of the MIME type is a wildcard (`*`), returns all extensions.
///
/// If the sub-level of the MIME type is a wildcard, returns all extensions for the top-level.
pub fn get_mime_extensions(mime: &Mime) -> Option<&'static [&'static str]> {
    get_extensions(&mime.0, &mime.1)
}

/// Get a list of known extensions for a MIME type string.
///
/// Ignores parameters (only searches `<main type>/<subtype>`). Case-insensitive.
///
/// Returns `None` if the MIME type is unknown.
///
/// ###Wildcards
/// If the top-level of the MIME type is a wildcard (`*`), returns all extensions.
///
/// If the sub-level of the MIME type is a wildcard, returns all extensions for the top-level.
///
/// ###Panics
/// If `mime_str` is not a valid MIME type specifier (naive).
pub fn get_mime_extensions_str(mut mime_str: &str) -> Option<&'static [&'static str]> {
    mime_str = mime_str.trim();

    if let Some(sep_idx) = mime_str.find(';') {
        mime_str = &mime_str[..sep_idx];
    }

    let (top, sub) = {
        let split_idx = mime_str.find('/').unwrap();
        (&mime_str[..split_idx], &mime_str[split_idx + 1 ..])
    };

    get_extensions(top, sub)
}

/// Get the extensions for a given top-level and sub-level of a MIME type
/// (`{toplevel}/{sublevel}`).
///
/// Returns `None` if `toplevel` or `sublevel` are unknown.
///
/// ###Wildcards
/// If the top-level of the MIME type is a wildcard (`*`), returns all extensions.
///
/// If the sub-level of the MIME type is a wildcard, returns all extensions for the top-level.
pub fn get_extensions(toplevel: &str, sublevel: &str) -> Option<&'static [&'static str]> {
    if toplevel == "*" {
        return Some(EXTS);
    }

    let top = try_opt!(map_lookup(&REV_MAPPINGS, toplevel));

    if sublevel == "*" {
        return Some(&EXTS[top.start .. top.end]);
    }

    let sub = try_opt!(map_lookup(&top.subs, sublevel));
    Some(&EXTS[sub.0 .. sub.1])
}

/// Get the MIME type for `application/octet-stream` (generic binary stream)
pub fn octet_stream() -> Mime {
    "application/octet-stream".parse().unwrap()
}

fn map_lookup<'map, V>(map: &'map phf::Map<UniCase<&'static str>, V>, key: &str) -> Option<&'map V> {
    // This transmute should be safe as `get` will not store the reference with
    // the expanded lifetime. This is due to `Borrow` being overly strict and
    // can't have an impl for `&'static str` to `Borrow<&'a str>`.
    //
    // See https://github.com/rust-lang/rust/issues/28853#issuecomment-158735548
    let key = unsafe { ::std::mem::transmute::<_, &'static str>(key) };
    map.get(&UniCase(key))
}

#[cfg(test)]
mod tests {
    use mime::Mime;
    use std::ascii::AsciiExt;
    use std::path::Path;
    use super::{get_mime_type, guess_mime_type, MIME_TYPES};
    use super::{get_mime_type_opt, guess_mime_type_opt};

    #[test]
    fn test_mime_type_guessing() {
        assert_eq!(get_mime_type("gif").to_string(), "image/gif".to_string());
        assert_eq!(get_mime_type("TXT").to_string(), "text/plain".to_string());
        assert_eq!(get_mime_type("blahblah").to_string(), "application/octet-stream".to_string());

        assert_eq!(guess_mime_type(Path::new("/path/to/file.gif")).to_string(), "image/gif".to_string());
        assert_eq!(guess_mime_type("/path/to/file.gif").to_string(), "image/gif".to_string());
    }

    #[test]
    fn test_mime_type_guessing_opt() {
        assert_eq!(get_mime_type_opt("gif").unwrap().to_string(), "image/gif".to_string());
        assert_eq!(get_mime_type_opt("TXT").unwrap().to_string(), "text/plain".to_string());
        assert_eq!(get_mime_type_opt("blahblah"), None);

        assert_eq!(guess_mime_type_opt("/path/to/file.gif").unwrap().to_string(), "image/gif".to_string());
        assert_eq!(guess_mime_type_opt("/path/to/file"), None);
    }

    #[test]
    fn test_are_mime_types_parseable() {
        for (_, mime) in &MIME_TYPES {
            mime.parse::<Mime>().unwrap();
        }
    }

    // RFC: Is this test necessary anymore? --@cybergeek94, 2/1/2016
    #[test]
    fn test_are_extensions_ascii() {
        for (ext, _) in &MIME_TYPES {
            assert!(ext.is_ascii(), "Extension not ASCII: {:?}", ext);
        }
    }

    #[test]
    fn test_are_extensions_sorted() {
        use mime_types_src::MIME_TYPES;

        for (&(ext, _), &(n_ext, _)) in MIME_TYPES.iter().zip(MIME_TYPES.iter().skip(1)) {
            assert!(
                ext <= n_ext,
                "Extensions in src/mime_types should be sorted alphabetically
                in ascending order. Failed assert: {:?} <= {:?}",
                ext, n_ext
            );
        }
    }

}

#[cfg(feature = "bench")]
mod bench {
    extern crate test;

    use self::test::Bencher;

    use super::{get_mime_type_str, MIME_TYPES};

    /// WARNING: this may take a while!
    #[bench]
    fn bench_mime_str(b: &mut Bencher) {
        for (mime_ext, _) in &MIME_TYPES {
            b.iter(|| {
                get_mime_type_str(mime_ext).expect(mime_ext);
            });
        }
    }

    #[bench]
    fn bench_mime_str_uppercase(b: &mut Bencher) {
        let uppercased : Vec<_> = MIME_TYPES.into_iter().map(|(s, _)| s.to_uppercase()).collect();

        for mime_ext in &uppercased {
            b.iter(|| {
                get_mime_type_str(mime_ext).expect(mime_ext);
            });
        }
    }
}
<commit_msg>Republish Mime trait<commit_after>//! Guessing of MIME types by file extension.
//!
//! Uses a static list of file-extension : MIME type mappings.
#![cfg_attr(feature = "bench", feature(test))]

extern crate mime;
extern crate phf;
extern crate unicase;

pub use mime::Mime;
use unicase::UniCase;

use std::ffi::OsStr;
use std::path::Path;

include!("mime_types_generated.rs");

struct TopLevelExts {
    start: usize,
    end: usize,
    subs: phf::Map<UniCase<&'static str>, (usize, usize)>,
}

macro_rules! try_opt (
    ($expr:expr) => (
        match $expr {
            Some(val) => val,
            None => return None,
        }
    )
);

#[cfg(test)]
#[path = "mime_types.rs"]
mod mime_types_src;

/// Guess the MIME type of `path` by its extension (as defined by `Path::extension()`).
///
/// If `path` has no extension, or its extension has no known MIME type mapping,
/// then the MIME type is assumed to be `application/octet-stream`.
///
/// ##Note
/// **Guess** is the operative word here, as there are no guarantees that the contents of the file
/// that `path` points to match the MIME type associated with the path's extension.
///
/// Take care when processing files with assumptions based on the return value of this function.
pub fn guess_mime_type<P: AsRef<Path>>(path: P) -> Mime {
    guess_mime_type_opt(path)
        .unwrap_or_else(octet_stream)
}

/// Guess the MIME type of `path` by its extension (as defined by `Path::extension()`).
///
/// If `path` has no extension, or its extension has no known MIME type mapping,
/// then `None` is returned.
///
/// ##Note
/// **Guess** is the operative word here, as there are no guarantees that the contents of the file
/// that `path` points to match the MIME type associated with the path's extension.
///
/// Take care when processing files with assumptions based on the return value of this function.
pub fn guess_mime_type_opt<P: AsRef<Path>>(path: P) -> Option<Mime> {
    let ext = path.as_ref().extension().and_then(OsStr::to_str).unwrap_or("");

    get_mime_type_opt(ext)
}

/// Get the MIME type associated with a file extension.
///
/// If there is no association for the extension, or `ext` is empty,
/// `application/octet-stream` is returned.
pub fn get_mime_type(search_ext: &str) -> Mime {
    get_mime_type_opt(search_ext)
        .unwrap_or_else(octet_stream)
}

/// Get the MIME type associated with a file extension.
///
/// If there is no association for the extension, or `ext` is empty,
/// `None` is returned.
pub fn get_mime_type_opt(search_ext: &str) -> Option<Mime> {
    get_mime_type_str(search_ext)
        .map(|mime| mime.parse::<Mime>().unwrap())
}

/// Get the MIME type string associated with a file extension. Case-insensitive.
///
/// If `search_ext` is not already lowercase,
/// it will be converted to lowercase to facilitate the search.
///
/// Returns `None` if `search_ext` is empty or an associated extension was not found.
pub fn get_mime_type_str(search_ext: &str) -> Option<&'static str> {
    if search_ext.is_empty() { return None; }

    map_lookup(&MIME_TYPES, search_ext).cloned()
}

/// Get a list of known extensions for a given `Mime`.
///
/// Ignores parameters (only searches with `<main type>/<subtype>`). Case-insensitive (for extension types).
///
/// Returns `None` if the MIME type is unknown.
///
/// ###Wildcards
/// If the top-level of the MIME type is a wildcard (`*`), returns all extensions.
///
/// If the sub-level of the MIME type is a wildcard, returns all extensions for the top-level.
pub fn get_mime_extensions(mime: &Mime) -> Option<&'static [&'static str]> {
    get_extensions(&mime.0, &mime.1)
}

/// Get a list of known extensions for a MIME type string.
///
/// Ignores parameters (only searches `<main type>/<subtype>`). Case-insensitive.
///
/// Returns `None` if the MIME type is unknown.
///
/// ###Wildcards
/// If the top-level of the MIME type is a wildcard (`*`), returns all extensions.
///
/// If the sub-level of the MIME type is a wildcard, returns all extensions for the top-level.
///
/// ###Panics
/// If `mime_str` is not a valid MIME type specifier (naive).
pub fn get_mime_extensions_str(mut mime_str: &str) -> Option<&'static [&'static str]> {
    mime_str = mime_str.trim();

    if let Some(sep_idx) = mime_str.find(';') {
        mime_str = &mime_str[..sep_idx];
    }

    let (top, sub) = {
        let split_idx = mime_str.find('/').unwrap();
        (&mime_str[..split_idx], &mime_str[split_idx + 1 ..])
    };

    get_extensions(top, sub)
}

/// Get the extensions for a given top-level and sub-level of a MIME type
/// (`{toplevel}/{sublevel}`).
///
/// Returns `None` if `toplevel` or `sublevel` are unknown.
///
/// ###Wildcards
/// If the top-level of the MIME type is a wildcard (`*`), returns all extensions.
///
/// If the sub-level of the MIME type is a wildcard, returns all extensions for the top-level.
pub fn get_extensions(toplevel: &str, sublevel: &str) -> Option<&'static [&'static str]> {
    if toplevel == "*" {
        return Some(EXTS);
    }

    let top = try_opt!(map_lookup(&REV_MAPPINGS, toplevel));

    if sublevel == "*" {
        return Some(&EXTS[top.start .. top.end]);
    }

    let sub = try_opt!(map_lookup(&top.subs, sublevel));
    Some(&EXTS[sub.0 .. sub.1])
}

/// Get the MIME type for `application/octet-stream` (generic binary stream)
pub fn octet_stream() -> Mime {
    "application/octet-stream".parse().unwrap()
}

fn map_lookup<'map, V>(map: &'map phf::Map<UniCase<&'static str>, V>, key: &str) -> Option<&'map V> {
    // This transmute should be safe as `get` will not store the reference with
    // the expanded lifetime. This is due to `Borrow` being overly strict and
    // can't have an impl for `&'static str` to `Borrow<&'a str>`.
    //
    // See https://github.com/rust-lang/rust/issues/28853#issuecomment-158735548
    let key = unsafe { ::std::mem::transmute::<_, &'static str>(key) };
    map.get(&UniCase(key))
}

#[cfg(test)]
mod tests {
    use mime::Mime;
    use std::ascii::AsciiExt;
    use std::path::Path;
    use super::{get_mime_type, guess_mime_type, MIME_TYPES};
    use super::{get_mime_type_opt, guess_mime_type_opt};

    #[test]
    fn test_mime_type_guessing() {
        assert_eq!(get_mime_type("gif").to_string(), "image/gif".to_string());
        assert_eq!(get_mime_type("TXT").to_string(), "text/plain".to_string());
        assert_eq!(get_mime_type("blahblah").to_string(), "application/octet-stream".to_string());

        assert_eq!(guess_mime_type(Path::new("/path/to/file.gif")).to_string(), "image/gif".to_string());
        assert_eq!(guess_mime_type("/path/to/file.gif").to_string(), "image/gif".to_string());
    }

    #[test]
    fn test_mime_type_guessing_opt() {
        assert_eq!(get_mime_type_opt("gif").unwrap().to_string(), "image/gif".to_string());
        assert_eq!(get_mime_type_opt("TXT").unwrap().to_string(), "text/plain".to_string());
        assert_eq!(get_mime_type_opt("blahblah"), None);

        assert_eq!(guess_mime_type_opt("/path/to/file.gif").unwrap().to_string(), "image/gif".to_string());
        assert_eq!(guess_mime_type_opt("/path/to/file"), None);
    }

    #[test]
    fn test_are_mime_types_parseable() {
        for (_, mime) in &MIME_TYPES {
            mime.parse::<Mime>().unwrap();
        }
    }

    // RFC: Is this test necessary anymore? --@cybergeek94, 2/1/2016
    #[test]
    fn test_are_extensions_ascii() {
        for (ext, _) in &MIME_TYPES {
            assert!(ext.is_ascii(), "Extension not ASCII: {:?}", ext);
        }
    }

    #[test]
    fn test_are_extensions_sorted() {
        use mime_types_src::MIME_TYPES;

        for (&(ext, _), &(n_ext, _)) in MIME_TYPES.iter().zip(MIME_TYPES.iter().skip(1)) {
            assert!(
                ext <= n_ext,
                "Extensions in src/mime_types should be sorted alphabetically
                in ascending order. Failed assert: {:?} <= {:?}",
                ext, n_ext
            );
        }
    }

}

#[cfg(feature = "bench")]
mod bench {
    extern crate test;

    use self::test::Bencher;

    use super::{get_mime_type_str, MIME_TYPES};

    /// WARNING: this may take a while!
    #[bench]
    fn bench_mime_str(b: &mut Bencher) {
        for (mime_ext, _) in &MIME_TYPES {
            b.iter(|| {
                get_mime_type_str(mime_ext).expect(mime_ext);
            });
        }
    }

    #[bench]
    fn bench_mime_str_uppercase(b: &mut Bencher) {
        let uppercased : Vec<_> = MIME_TYPES.into_iter().map(|(s, _)| s.to_uppercase()).collect();

        for mime_ext in &uppercased {
            b.iter(|| {
                get_mime_type_str(mime_ext).expect(mime_ext);
            });
        }
    }
}
<|endoftext|>