<commit_before>// Iterates over every crate versions ever uploaded and (re-)renders their
// readme using the Markdown renderer from the cargo_registry crate.
//
// Warning: this can take a lot of time.
//
// Usage:
//     cargo run --bin render-readmes [page-size: optional = 25]
// The page-size argument dictate how much versions should be queried and processed at once.

#![deny(warnings)]

#[macro_use]
extern crate serde_derive;

extern crate cargo_registry;
extern crate curl;
extern crate diesel;
extern crate flate2;
extern crate s3;
extern crate tar;
extern crate time;
extern crate toml;
extern crate url;

use curl::easy::{Easy, List};
use diesel::prelude::*;
use flate2::read::GzDecoder;
use std::env;
use std::io::{Cursor, Read};
use std::path::Path;
use std::thread;
use tar::Archive;
use url::Url;

use cargo_registry::{Config, Version};
use cargo_registry::version::EncodableVersion;
use cargo_registry::schema::*;
use cargo_registry::render::markdown_to_html;

const DEFAULT_PAGE_SIZE: i64 = 25;

fn main() {
    let config: Config = Default::default();
    let conn = cargo_registry::db::connect_now().unwrap();
    let versions_count = versions::table
        .select(versions::all_columns)
        .count()
        .get_result::<i64>(&conn)
        .expect("error counting versions");
    let page_size = match env::args().nth(1) {
        None => DEFAULT_PAGE_SIZE,
        Some(s) => s.parse::<i64>().unwrap_or(DEFAULT_PAGE_SIZE),
    };
    let pages = if versions_count % page_size == 0 {
        versions_count / page_size
    } else {
        versions_count / page_size + 1
    };
    for current_page in 0..pages {
        let versions: Vec<EncodableVersion> = versions::table
            .inner_join(crates::table)
            .select((versions::all_columns, crates::name))
            .limit(page_size)
            .offset(current_page * page_size)
            .load::<(Version, String)>(&conn)
            .expect("error loading versions")
            .into_iter()
            .map(|(version, crate_name)| version.encodable(&crate_name))
            .collect();
        let mut tasks = Vec::with_capacity(page_size as usize);
        for version in versions {
            let config = config.clone();
            let handle = thread::spawn(move || {
                println!("[{}-{}] Rendering README...", version.krate, version.num);
                let readme = get_readme(&config, &version);
                if readme.is_none() {
                    return;
                }
                let readme = readme.unwrap();
                let readme_path = format!(
                    "readmes/{}/{}-{}.html",
                    version.krate,
                    version.krate,
                    version.num
                );
                let readme_len = readme.len();
                let mut body = Cursor::new(readme.into_bytes());
                config
                    .uploader
                    .upload(
                        Easy::new(),
                        &readme_path,
                        &mut body,
                        "text/html",
                        readme_len as u64,
                    )
                    .expect(&format!(
                        "[{}-{}] Couldn't upload file to S3",
                        version.krate,
                        version.num
                    ));
            });
            tasks.push(handle);
        }
        for handle in tasks {
            if let Err(err) = handle.join() {
                println!("Thead panicked: {:?}", err);
            }
        }
    }
}

/// Renders the readme of an uploaded crate version.
fn get_readme(config: &Config, version: &EncodableVersion) -> Option<String> {
    let mut handle = Easy::new();
    let location = match config.uploader.crate_location(&version.krate, &version.num) {
        Some(l) => l,
        None => return None,
    };
    let date = time::now().rfc822z().to_string();
    let url = Url::parse(&location).expect(&format!(
        "[{}-{}] Couldn't parse crate URL",
        version.krate,
        version.num
    ));

    let mut headers = List::new();
    headers
        .append(&format!("Host: {}", url.host().unwrap()))
        .unwrap();
    headers.append(&format!("Date: {}", date)).unwrap();

    handle.url(url.as_str()).unwrap();
    handle.get(true).unwrap();
    handle.http_headers(headers).unwrap();

    let mut response = Vec::new();
    {
        let mut req = handle.transfer();
        req.write_function(|data| {
            response.extend(data);
            Ok(data.len())
        }).unwrap();
        if let Err(err) = req.perform() {
            println!(
                "[{}-{}] Unable to fetch crate: {}",
                version.krate,
                version.num,
                err
            );
            return None;
        }
    }
    if handle.response_code().unwrap() != 200 {
        let response = String::from_utf8_lossy(&response);
        println!(
            "[{}-{}] Failed to get a 200 response: {}",
            version.krate,
            version.num,
            response
        );
        return None;
    }
    let reader = Cursor::new(response);
    let reader = GzDecoder::new(reader).expect(&format!(
        "[{}-{}] Invalid gzip header",
        version.krate,
        version.num
    ));
    let mut archive = Archive::new(reader);
    let mut entries = archive.entries().expect(&format!(
        "[{}-{}] Invalid tar archive entries",
        version.krate,
        version.num
    ));
    let manifest: Manifest = {
        let path = format!("{}-{}/Cargo.toml", version.krate, version.num);
        let contents = find_file_by_path(&mut entries, Path::new(&path), &version);
        toml::from_str(&contents).expect(&format!(
            "[{}-{}] Syntax error in manifest file",
            version.krate,
            version.num
        ))
    };
    if manifest.package.readme.is_none() {
        return None;
    }
    let rendered = {
        let path = format!(
            "{}-{}/{}",
            version.krate,
            version.num,
            manifest.package.readme.unwrap()
        );
        let contents = find_file_by_path(&mut entries, Path::new(&path), &version);
        markdown_to_html(&contents).expect(&format!(
            "[{}-{}] Couldn't render README",
            version.krate,
            version.num
        ))
    };
    return Some(rendered);
    #[derive(Deserialize)]
    struct Package {
        readme: Option<String>,
    }
    #[derive(Deserialize)]
    struct Manifest {
        package: Package,
    }
}

/// Search an entry by its path in a Tar archive.
fn find_file_by_path<R: Read>(
    mut entries: &mut tar::Entries<R>,
    path: &Path,
    version: &EncodableVersion,
) -> String {
    let mut file = entries
        .find(|entry| match *entry {
            Err(_) => false,
            Ok(ref file) => {
                let filepath = match file.path() {
                    Ok(p) => p,
                    Err(_) => return false,
                };
                return filepath == path;
            }
        })
        .expect(&format!(
            "[{}-{}] couldn't open file: {}",
            version.krate,
            version.num,
            path.display()
        ))
        .expect(&format!(
            "[{}-{}] file is not present: {}",
            version.krate,
            version.num,
            path.display()
        ));
    let mut contents = String::new();
    file.read_to_string(&mut contents).expect(&format!(
        "[{}-{}] Couldn't read file contents",
        version.krate,
        version.num
    ));
    contents
}
<commit_msg>Make `entries` binding immutable for &mut type<commit_after>// Iterates over every crate versions ever uploaded and (re-)renders their
// readme using the Markdown renderer from the cargo_registry crate.
//
// Warning: this can take a lot of time.
//
// Usage:
//     cargo run --bin render-readmes [page-size: optional = 25]
// The page-size argument dictate how much versions should be queried and processed at once.

#![deny(warnings)]

#[macro_use]
extern crate serde_derive;

extern crate cargo_registry;
extern crate curl;
extern crate diesel;
extern crate flate2;
extern crate s3;
extern crate tar;
extern crate time;
extern crate toml;
extern crate url;

use curl::easy::{Easy, List};
use diesel::prelude::*;
use flate2::read::GzDecoder;
use std::env;
use std::io::{Cursor, Read};
use std::path::Path;
use std::thread;
use tar::Archive;
use url::Url;

use cargo_registry::{Config, Version};
use cargo_registry::version::EncodableVersion;
use cargo_registry::schema::*;
use cargo_registry::render::markdown_to_html;

const DEFAULT_PAGE_SIZE: i64 = 25;

fn main() {
    let config: Config = Default::default();
    let conn = cargo_registry::db::connect_now().unwrap();
    let versions_count = versions::table
        .select(versions::all_columns)
        .count()
        .get_result::<i64>(&conn)
        .expect("error counting versions");
    let page_size = match env::args().nth(1) {
        None => DEFAULT_PAGE_SIZE,
        Some(s) => s.parse::<i64>().unwrap_or(DEFAULT_PAGE_SIZE),
    };
    let pages = if versions_count % page_size == 0 {
        versions_count / page_size
    } else {
        versions_count / page_size + 1
    };
    for current_page in 0..pages {
        let versions: Vec<EncodableVersion> = versions::table
            .inner_join(crates::table)
            .select((versions::all_columns, crates::name))
            .limit(page_size)
            .offset(current_page * page_size)
            .load::<(Version, String)>(&conn)
            .expect("error loading versions")
            .into_iter()
            .map(|(version, crate_name)| version.encodable(&crate_name))
            .collect();
        let mut tasks = Vec::with_capacity(page_size as usize);
        for version in versions {
            let config = config.clone();
            let handle = thread::spawn(move || {
                println!("[{}-{}] Rendering README...", version.krate, version.num);
                let readme = get_readme(&config, &version);
                if readme.is_none() {
                    return;
                }
                let readme = readme.unwrap();
                let readme_path = format!(
                    "readmes/{}/{}-{}.html",
                    version.krate,
                    version.krate,
                    version.num
                );
                let readme_len = readme.len();
                let mut body = Cursor::new(readme.into_bytes());
                config
                    .uploader
                    .upload(
                        Easy::new(),
                        &readme_path,
                        &mut body,
                        "text/html",
                        readme_len as u64,
                    )
                    .expect(&format!(
                        "[{}-{}] Couldn't upload file to S3",
                        version.krate,
                        version.num
                    ));
            });
            tasks.push(handle);
        }
        for handle in tasks {
            if let Err(err) = handle.join() {
                println!("Thead panicked: {:?}", err);
            }
        }
    }
}

/// Renders the readme of an uploaded crate version.
fn get_readme(config: &Config, version: &EncodableVersion) -> Option<String> {
    let mut handle = Easy::new();
    let location = match config.uploader.crate_location(&version.krate, &version.num) {
        Some(l) => l,
        None => return None,
    };
    let date = time::now().rfc822z().to_string();
    let url = Url::parse(&location).expect(&format!(
        "[{}-{}] Couldn't parse crate URL",
        version.krate,
        version.num
    ));

    let mut headers = List::new();
    headers
        .append(&format!("Host: {}", url.host().unwrap()))
        .unwrap();
    headers.append(&format!("Date: {}", date)).unwrap();

    handle.url(url.as_str()).unwrap();
    handle.get(true).unwrap();
    handle.http_headers(headers).unwrap();

    let mut response = Vec::new();
    {
        let mut req = handle.transfer();
        req.write_function(|data| {
            response.extend(data);
            Ok(data.len())
        }).unwrap();
        if let Err(err) = req.perform() {
            println!(
                "[{}-{}] Unable to fetch crate: {}",
                version.krate,
                version.num,
                err
            );
            return None;
        }
    }
    if handle.response_code().unwrap() != 200 {
        let response = String::from_utf8_lossy(&response);
        println!(
            "[{}-{}] Failed to get a 200 response: {}",
            version.krate,
            version.num,
            response
        );
        return None;
    }
    let reader = Cursor::new(response);
    let reader = GzDecoder::new(reader).expect(&format!(
        "[{}-{}] Invalid gzip header",
        version.krate,
        version.num
    ));
    let mut archive = Archive::new(reader);
    let mut entries = archive.entries().expect(&format!(
        "[{}-{}] Invalid tar archive entries",
        version.krate,
        version.num
    ));
    let manifest: Manifest = {
        let path = format!("{}-{}/Cargo.toml", version.krate, version.num);
        let contents = find_file_by_path(&mut entries, Path::new(&path), &version);
        toml::from_str(&contents).expect(&format!(
            "[{}-{}] Syntax error in manifest file",
            version.krate,
            version.num
        ))
    };
    if manifest.package.readme.is_none() {
        return None;
    }
    let rendered = {
        let path = format!(
            "{}-{}/{}",
            version.krate,
            version.num,
            manifest.package.readme.unwrap()
        );
        let contents = find_file_by_path(&mut entries, Path::new(&path), &version);
        markdown_to_html(&contents).expect(&format!(
            "[{}-{}] Couldn't render README",
            version.krate,
            version.num
        ))
    };
    return Some(rendered);
    #[derive(Deserialize)]
    struct Package {
        readme: Option<String>,
    }
    #[derive(Deserialize)]
    struct Manifest {
        package: Package,
    }
}

/// Search an entry by its path in a Tar archive.
fn find_file_by_path<R: Read>(
    entries: &mut tar::Entries<R>,
    path: &Path,
    version: &EncodableVersion,
) -> String {
    let mut file = entries
        .find(|entry| match *entry {
            Err(_) => false,
            Ok(ref file) => {
                let filepath = match file.path() {
                    Ok(p) => p,
                    Err(_) => return false,
                };
                return filepath == path;
            }
        })
        .expect(&format!(
            "[{}-{}] couldn't open file: {}",
            version.krate,
            version.num,
            path.display()
        ))
        .expect(&format!(
            "[{}-{}] file is not present: {}",
            version.krate,
            version.num,
            path.display()
        ));
    let mut contents = String::new();
    file.read_to_string(&mut contents).expect(&format!(
        "[{}-{}] Couldn't read file contents",
        version.krate,
        version.num
    ));
    contents
}
<|endoftext|>


<commit_before>use crate::Msg;
use chrono::{NaiveDate};
use seed::prelude::*;

struct NewsItem {
    pub content: Vec<Node<Msg>>,
    pub date: NaiveDate,
}

const NUM_NEWS_ITEMS :usize = 2;
fn get_news_items() -> [NewsItem; NUM_NEWS_ITEMS] {
    let news_items: [NewsItem; NUM_NEWS_ITEMS] = [
        NewsItem{
            content: vec![
                a!["Ethereum bounty contracts", attrs!{At::Class => "link", At::Href => "https://github.com/craigmayhew/ethereum-contracts"}],
                span!["have now been added for anyone that would like to try to push the boundaries of human knowledge.
                The first contract is a competition for the correct solution to a³+b³+c³ = 33.
                Tweet me if you need assistance using these contracts!"]
            ],
            date: NaiveDate::from_ymd(2019, 1, 22)
        },
        NewsItem{
            content: vec![
                div!["huh"]
            ],
            date: NaiveDate::from_ymd(2019, 1, 22)
        }
    ];
    news_items
}

pub fn render() -> seed::dom_types::Node<Msg> {
    let news_items = get_news_items();
    let mut table_rows = vec![div![]; NUM_NEWS_ITEMS];
    for i in 0..news_items.len() {
        let news_item = &news_items[i];
        table_rows[i] = tr![
            td![news_item.date.to_string()],
            td![news_item.content.to_vec()]
        ];
    };

    div![
        h1!["News"],
        br![],
        br![],
        br![],
        table![table_rows]
    ]
}<commit_msg>Removed superfluous variable assignment<commit_after>use crate::Msg;
use chrono::{NaiveDate};
use seed::prelude::*;

struct NewsItem {
    pub content: Vec<Node<Msg>>,
    pub date: NaiveDate,
}

const NUM_NEWS_ITEMS :usize = 2;
fn get_news_items() -> [NewsItem; NUM_NEWS_ITEMS] {
    [
        NewsItem{
            content: vec![
                a!["Ethereum bounty contracts", attrs!{At::Class => "link", At::Href => "https://github.com/craigmayhew/ethereum-contracts"}],
                span!["have now been added for anyone that would like to try to push the boundaries of human knowledge.
                The first contract is a competition for the correct solution to a³+b³+c³ = 33.
                Tweet me if you need assistance using these contracts!"]
            ],
            date: NaiveDate::from_ymd(2019, 1, 22)
        },
        NewsItem{
            content: vec![
                div!["huh"]
            ],
            date: NaiveDate::from_ymd(2019, 1, 22)
        }
    ]
}

pub fn render() -> seed::dom_types::Node<Msg> {
    let news_items = get_news_items();
    let mut table_rows = vec![div![]; NUM_NEWS_ITEMS];
    for i in 0..news_items.len() {
        let news_item = &news_items[i];
        table_rows[i] = tr![
            td![news_item.date.to_string()],
            td![news_item.content.to_vec()]
        ];
    };

    div![
        h1!["News"],
        br![],
        br![],
        br![],
        table![table_rows]
    ]
}<|endoftext|>


<commit_before>use domination::Domination;
use multi_objective::MultiObjective;
use crowding_distance::SolutionRankDist;
use selection::select_solutions;
use std::iter::FromIterator;
use rayon::par_iter::*;
use selection::tournament_selection_fast;
use rand::Rng;

/// An unrated Population of individuals.
pub struct UnratedPopulation<I>
    where I: Clone
{
    individuals: Vec<I>,
}

/// A rated Population of individuals, i.e.
/// each individual has a fitness assigned.
pub struct RatedPopulation<I, F>
    where I: Clone,
          F: MultiObjective + Clone
{
    individuals: Vec<I>,
    fitness: Vec<F>,
}

/// A selected (ranked) Population.
pub struct SelectedPopulation<I, F>
    where I: Clone,
          F: MultiObjective + Clone
{
    individuals: Vec<I>,
    fitness: Vec<F>,
    rank_dist: Vec<SolutionRankDist>,
}

impl<I> From<Vec<I>> for UnratedPopulation<I> where I: Clone
{
    fn from(v: Vec<I>) -> UnratedPopulation<I> {
        UnratedPopulation { individuals: v }
    }
}

impl<I> FromIterator<I> for UnratedPopulation<I> where I: Clone
{
    fn from_iter<T>(iterator: T) -> UnratedPopulation<I>
        where T: IntoIterator<Item = I>
    {
        UnratedPopulation { individuals: iterator.into_iter().collect() }
    }
}

impl<I> AsRef<[I]> for UnratedPopulation<I> where I: Clone
{
    fn as_ref(&self) -> &[I] {
        &self.individuals
    }
}

impl<I> UnratedPopulation<I> where I: Clone + Sync
{
    pub fn individuals(&self) -> &[I] {
        &self.individuals
    }

    pub fn new() -> UnratedPopulation<I> {
        UnratedPopulation { individuals: Vec::new() }
    }

    pub fn push(&mut self, ind: I) {
        self.individuals.push(ind);
    }

    pub fn len(&self) -> usize {
        self.individuals.len()
    }

    /// Rate the individuals in parallel.

    pub fn rate_in_parallel<F, E>(self, eval: &E, weight: f64) -> RatedPopulation<I, F>
        where F: MultiObjective + Clone + Send,
              E: Sync + Fn(&I) -> F
    {
        let mut fitness = Vec::new();
        self.individuals.into_par_iter().map(eval).weight(weight).collect_into(&mut fitness);
        assert!(fitness.len() == self.individuals.len());

        RatedPopulation {
            individuals: self.individuals,
            fitness: fitness,
        }
    }
}

impl<I, F> RatedPopulation<I, F>
    where I: Clone,
          F: MultiObjective + Clone
{
    pub fn select<D>(self,
                     population_size: usize,
                     num_objectives: usize,
                     domination: &mut D)
                     -> SelectedPopulation<I, F>
        where D: Domination<F>
    {
        // evaluate rank and crowding distance
        let rank_dist = select_solutions(&self.fitness,
                                         population_size,
                                         num_objectives,
                                         domination);
        assert!(rank_dist.len() <= population_size);
        SelectedPopulation {
            individuals: self.individuals,
            fitness: self.fitness,
            rank_dist: rank_dist,
        }
    }

    pub fn new() -> RatedPopulation<I, F> {
        RatedPopulation {
            individuals: Vec::new(),
            fitness: Vec::new(),
        }
    }

    pub fn len(&self) -> usize {
        self.individuals.len()
    }

    pub fn fitness(&self) -> &[F] {
        &self.fitness
    }
}

impl<I, F> SelectedPopulation<I, F>
    where I: Clone,
          F: MultiObjective + Clone
{
    /// Generate an offspring population.
    /// XXX: Factor out selection into a separate Trait  SelectionMethod
    pub fn reproduce<R, M>(&self,
                           rng: &mut R,
                           offspring_size: usize,
                           tournament_k: usize,
                           mate: &M)
                           -> UnratedPopulation<I>
        where R: Rng,
              M: Fn(&mut R, &I, &I) -> I
    {
        assert!(self.individuals.len() == self.fitness.len());
        assert!(self.individuals.len() >= self.rank_dist.len());
        assert!(tournament_k > 0);

        let rank_dist = &self.rank_dist[..];

        // create `offspring_size` new offspring using k-tournament (
        // select the best individual out of k randomly choosen individuals)
        let offspring: Vec<I> =
            (0..offspring_size)
                .map(|_| {

                    // first parent. k candidates
                    let p1 = tournament_selection_fast(rng,
                                                       |i1, i2| rank_dist[i1] < rank_dist[i2],
                                                       rank_dist.len(),
                                                       tournament_k);

                    // second parent. k candidates
                    let p2 = tournament_selection_fast(rng,
                                                       |i1, i2| rank_dist[i1] < rank_dist[i2],
                                                       rank_dist.len(),
                                                       tournament_k);

                    // cross-over the two parents and produce one child (throw away
                    // second child XXX)

                    // The potentially dominating individual is gives as first
                    // parameter.
                    let (p1, p2) = if rank_dist[p1] < rank_dist[p2] {
                        (p1, p2)
                    } else if rank_dist[p2] < rank_dist[p1] {
                        (p2, p1)
                    } else {
                        (p1, p2)
                    };

                    mate(rng, &self.individuals[p1], &self.individuals[p2])
                })
                .collect();

        assert!(offspring.len() == offspring_size);

        UnratedPopulation { individuals: offspring }
    }

    pub fn new() -> SelectedPopulation<I, F> {
        SelectedPopulation {
            individuals: Vec::new(),
            fitness: Vec::new(),
            rank_dist: Vec::new(),
        }
    }

    pub fn len(&self) -> usize {
        // NOTE: the population and fitness arrays might still be larger.
        self.rank_dist.len()
    }

    /// Merging a selected population result in a rated population as the selection
    /// criteria is no longer met.
    pub fn merge(self, offspring: RatedPopulation<I, F>) -> RatedPopulation<I, F> {
        let mut new_ind = Vec::with_capacity(self.len() + offspring.len());
        let mut new_fit = Vec::with_capacity(self.len() + offspring.len());
        self.all(&mut |ind, fit| {
            // XXX: Optimize. Avoid clone
            new_ind.push(ind.clone());
            new_fit.push(fit.clone());
        });

        new_ind.extend(offspring.individuals);
        new_fit.extend(offspring.fitness);

        assert!(new_ind.len() == new_fit.len());

        RatedPopulation {
            individuals: new_ind,
            fitness: new_fit,
        }
    }

    pub fn all<C>(&self, f: &mut C)
        where C: FnMut(&I, &F)
    {
        for rd in self.rank_dist.iter() {
            f(&self.individuals[rd.idx], &self.fitness[rd.idx]);
        }
    }

    pub fn all_with_rank_dist<C>(&self, f: &mut C)
        where C: FnMut(&I, &F, usize, f32)
    {
        for rd in self.rank_dist.iter() {
            f(&self.individuals[rd.idx],
              &self.fitness[rd.idx],
              rd.rank as usize,
              rd.dist);
        }
    }

    // XXX: fitness_iter()
    pub fn fitness_to_vec(&self) -> Vec<F> {
        let mut v = Vec::new();
        self.all(&mut |_, f| v.push(f.clone()));
        v
    }

    pub fn max_rank(&self) -> Option<usize> {
        self.rank_dist.iter().map(|rd| rd.rank).max().map(|r| r as usize)
    }

    pub fn all_of_rank<C>(&self, rank: usize, f: &mut C)
        where C: FnMut(&I, &F)
    {
        for rd in self.rank_dist.iter() {
            if rd.rank as usize == rank {
                f(&self.individuals[rd.idx], &self.fitness[rd.idx]);
            }
        }
    }
}
<commit_msg>Add fitness_mut()<commit_after>use domination::Domination;
use multi_objective::MultiObjective;
use crowding_distance::SolutionRankDist;
use selection::select_solutions;
use std::iter::FromIterator;
use rayon::par_iter::*;
use selection::tournament_selection_fast;
use rand::Rng;

/// An unrated Population of individuals.
pub struct UnratedPopulation<I>
    where I: Clone
{
    individuals: Vec<I>,
}

/// A rated Population of individuals, i.e.
/// each individual has a fitness assigned.
pub struct RatedPopulation<I, F>
    where I: Clone,
          F: MultiObjective + Clone
{
    individuals: Vec<I>,
    fitness: Vec<F>,
}

/// A selected (ranked) Population.
pub struct SelectedPopulation<I, F>
    where I: Clone,
          F: MultiObjective + Clone
{
    individuals: Vec<I>,
    fitness: Vec<F>,
    rank_dist: Vec<SolutionRankDist>,
}

impl<I> From<Vec<I>> for UnratedPopulation<I> where I: Clone
{
    fn from(v: Vec<I>) -> UnratedPopulation<I> {
        UnratedPopulation { individuals: v }
    }
}

impl<I> FromIterator<I> for UnratedPopulation<I> where I: Clone
{
    fn from_iter<T>(iterator: T) -> UnratedPopulation<I>
        where T: IntoIterator<Item = I>
    {
        UnratedPopulation { individuals: iterator.into_iter().collect() }
    }
}

impl<I> AsRef<[I]> for UnratedPopulation<I> where I: Clone
{
    fn as_ref(&self) -> &[I] {
        &self.individuals
    }
}

impl<I> UnratedPopulation<I> where I: Clone + Sync
{
    pub fn individuals(&self) -> &[I] {
        &self.individuals
    }

    pub fn new() -> UnratedPopulation<I> {
        UnratedPopulation { individuals: Vec::new() }
    }

    pub fn push(&mut self, ind: I) {
        self.individuals.push(ind);
    }

    pub fn len(&self) -> usize {
        self.individuals.len()
    }

    /// Rate the individuals in parallel.

    pub fn rate_in_parallel<F, E>(self, eval: &E, weight: f64) -> RatedPopulation<I, F>
        where F: MultiObjective + Clone + Send,
              E: Sync + Fn(&I) -> F
    {
        let mut fitness = Vec::new();
        self.individuals.into_par_iter().map(eval).weight(weight).collect_into(&mut fitness);
        assert!(fitness.len() == self.individuals.len());

        RatedPopulation {
            individuals: self.individuals,
            fitness: fitness,
        }
    }
}

impl<I, F> RatedPopulation<I, F>
    where I: Clone,
          F: MultiObjective + Clone
{
    pub fn select<D>(self,
                     population_size: usize,
                     num_objectives: usize,
                     domination: &mut D)
                     -> SelectedPopulation<I, F>
        where D: Domination<F>
    {
        // evaluate rank and crowding distance
        let rank_dist = select_solutions(&self.fitness,
                                         population_size,
                                         num_objectives,
                                         domination);
        assert!(rank_dist.len() <= population_size);
        SelectedPopulation {
            individuals: self.individuals,
            fitness: self.fitness,
            rank_dist: rank_dist,
        }
    }

    pub fn new() -> RatedPopulation<I, F> {
        RatedPopulation {
            individuals: Vec::new(),
            fitness: Vec::new(),
        }
    }

    pub fn len(&self) -> usize {
        self.individuals.len()
    }

    pub fn fitness(&self) -> &[F] {
        &self.fitness
    }
    pub fn fitness_mut(&mut self) -> &mut [F] {
        &mut self.fitness
    }

}

impl<I, F> SelectedPopulation<I, F>
    where I: Clone,
          F: MultiObjective + Clone
{
    /// Generate an offspring population.
    /// XXX: Factor out selection into a separate Trait  SelectionMethod
    pub fn reproduce<R, M>(&self,
                           rng: &mut R,
                           offspring_size: usize,
                           tournament_k: usize,
                           mate: &M)
                           -> UnratedPopulation<I>
        where R: Rng,
              M: Fn(&mut R, &I, &I) -> I
    {
        assert!(self.individuals.len() == self.fitness.len());
        assert!(self.individuals.len() >= self.rank_dist.len());
        assert!(tournament_k > 0);

        let rank_dist = &self.rank_dist[..];

        // create `offspring_size` new offspring using k-tournament (
        // select the best individual out of k randomly choosen individuals)
        let offspring: Vec<I> =
            (0..offspring_size)
                .map(|_| {

                    // first parent. k candidates
                    let p1 = tournament_selection_fast(rng,
                                                       |i1, i2| rank_dist[i1] < rank_dist[i2],
                                                       rank_dist.len(),
                                                       tournament_k);

                    // second parent. k candidates
                    let p2 = tournament_selection_fast(rng,
                                                       |i1, i2| rank_dist[i1] < rank_dist[i2],
                                                       rank_dist.len(),
                                                       tournament_k);

                    // cross-over the two parents and produce one child (throw away
                    // second child XXX)

                    // The potentially dominating individual is gives as first
                    // parameter.
                    let (p1, p2) = if rank_dist[p1] < rank_dist[p2] {
                        (p1, p2)
                    } else if rank_dist[p2] < rank_dist[p1] {
                        (p2, p1)
                    } else {
                        (p1, p2)
                    };

                    mate(rng, &self.individuals[p1], &self.individuals[p2])
                })
                .collect();

        assert!(offspring.len() == offspring_size);

        UnratedPopulation { individuals: offspring }
    }

    pub fn new() -> SelectedPopulation<I, F> {
        SelectedPopulation {
            individuals: Vec::new(),
            fitness: Vec::new(),
            rank_dist: Vec::new(),
        }
    }

    pub fn len(&self) -> usize {
        // NOTE: the population and fitness arrays might still be larger.
        self.rank_dist.len()
    }

    /// Merging a selected population result in a rated population as the selection
    /// criteria is no longer met.
    pub fn merge(self, offspring: RatedPopulation<I, F>) -> RatedPopulation<I, F> {
        let mut new_ind = Vec::with_capacity(self.len() + offspring.len());
        let mut new_fit = Vec::with_capacity(self.len() + offspring.len());
        self.all(&mut |ind, fit| {
            // XXX: Optimize. Avoid clone
            new_ind.push(ind.clone());
            new_fit.push(fit.clone());
        });

        new_ind.extend(offspring.individuals);
        new_fit.extend(offspring.fitness);

        assert!(new_ind.len() == new_fit.len());

        RatedPopulation {
            individuals: new_ind,
            fitness: new_fit,
        }
    }

    pub fn all<C>(&self, f: &mut C)
        where C: FnMut(&I, &F)
    {
        for rd in self.rank_dist.iter() {
            f(&self.individuals[rd.idx], &self.fitness[rd.idx]);
        }
    }

    pub fn all_with_rank_dist<C>(&self, f: &mut C)
        where C: FnMut(&I, &F, usize, f32)
    {
        for rd in self.rank_dist.iter() {
            f(&self.individuals[rd.idx],
              &self.fitness[rd.idx],
              rd.rank as usize,
              rd.dist);
        }
    }

    // XXX: fitness_iter()
    pub fn fitness_to_vec(&self) -> Vec<F> {
        let mut v = Vec::new();
        self.all(&mut |_, f| v.push(f.clone()));
        v
    }

    pub fn max_rank(&self) -> Option<usize> {
        self.rank_dist.iter().map(|rd| rd.rank).max().map(|r| r as usize)
    }

    pub fn all_of_rank<C>(&self, rank: usize, f: &mut C)
        where C: FnMut(&I, &F)
    {
        for rd in self.rank_dist.iter() {
            if rd.rank as usize == rank {
                f(&self.individuals[rd.idx], &self.fitness[rd.idx]);
            }
        }
    }
}
<|endoftext|>


<commit_before>extern crate time;

use std::fs::{File, read_dir};
use std::io::{BufReader, BufRead, Read, Write};
use std::io;
use std::path::Path;
use std::collections::HashMap;
use std::error::Error;
use std::fmt;
use models::LanguageModelBuilder;

#[derive(Debug)]
pub struct FormatError;
impl fmt::Display for FormatError {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        write!(f, "Invalid format for vocabulary file")
    }
}

impl Error for FormatError {
    fn description(&self) -> &str {
        "The format of the vocabulary file was Invalid"
    }
    fn cause(&self) -> Option<&Error> {
        None
    }
}

pub fn find_most_common_words(corpus: &Path, num: usize) -> Vec<(String, u32)> {
    let words = files(corpus).flat_map(|file| read_words(file));

    let mut word_counts = HashMap::new();
    for word in words {
        *word_counts.entry(word).or_insert(0) += 1;
    }

    let mut counts: Vec<_> = word_counts.into_iter().collect();
    counts.sort_by(|a, b| b.1.cmp(&a.1));
    counts.truncate(num);

    counts
}

pub fn save_words(path: &Path, words: &Vec<(String, u32)>) -> io::Result<()> {
    let mut out = try!(File::create(path));
    for &(ref word, count) in words {
        try!(out.write_all(format!("{}, {}\n", &word, count).as_bytes()));
    }

    Ok(())
}

pub fn load_most_common_words(filename: &str,
                              num: usize)
                              -> Result<Vec<(String, u32)>, Box<Error>> {
    let file = try!(File::open(&Path::new(filename)));
    let reader = BufReader::new(file);
    let mut words = vec![];
    for line in reader.lines().take(num) {
        if let Ok(line) = line {
            let mut columns = line.split(',');
            let word = columns.next().map(|w| w.trim().to_string());
            let num = columns.next().map(|n| n.trim().parse());

            match (word, num) {
                (Some(word), Some(num)) => words.push((word, try!(num))),
                _ => return Err(Box::new(FormatError)),
            }
        } else {
            return Err(Box::new(FormatError));
        }
    }

    Ok(words)
}

fn sentences<T: Read + 'static>(reader: BufReader<T>) -> Box<Iterator<Item = String>> {
    Box::new(reader.split('.' as u8)
                   .filter_map(|v| {
                       let lowercase = v.unwrap();
                       String::from_utf8(lowercase).ok().map(|s| s.to_lowercase())
                   }))
}

fn read_words<R: BufRead + 'static>(reader: R) -> Box<Iterator<Item = String>> {
    Box::new(reader.lines()
                   .filter_map(|line| line.ok())
                   .flat_map(|line| {
                       line.split(|c| {
                               match c {
                                   'a'...'z' | 'A'...'Z' => false,
                                   _ => true,
                               }
                           })
                           .filter(|word| !word.is_empty())
                           .map(|word| word.to_lowercase())
                           .collect::<Vec<_>>()
                           .into_iter()
                   }))
}

fn files(path: &Path) -> Box<Iterator<Item = BufReader<File>>> {
    Box::new(read_dir(path)
                 .unwrap()
                 .map(|path| path.unwrap().path())
                 .filter(|path| path.to_str().unwrap().ends_with(".txt"))
                 .map(|path| {
                     let file = File::open(path).unwrap();
                     BufReader::new(file)
                 }))
}

pub fn create_model(corpus: &Path, words: Vec<String>) -> LanguageModelBuilder {
    let mut builder = LanguageModelBuilder::new(10, words);

    for sentence in files(corpus).take(10).flat_map(sentences) {
        let mut acc = builder.new_sentence();

        let words = sentence.split(|c| {
                                match c {
                                    'a'...'z' => false,
                                    _ => true,
                                }
                            })
                            .filter(|w| !w.is_empty());
        for word in words {
            acc.add_word(word);
        }
    }

    builder
}
<commit_msg>Ooops didn't mean to leave that there<commit_after>extern crate time;

use std::fs::{File, read_dir};
use std::io::{BufReader, BufRead, Read, Write};
use std::io;
use std::path::Path;
use std::collections::HashMap;
use std::error::Error;
use std::fmt;
use models::LanguageModelBuilder;

#[derive(Debug)]
pub struct FormatError;
impl fmt::Display for FormatError {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        write!(f, "Invalid format for vocabulary file")
    }
}

impl Error for FormatError {
    fn description(&self) -> &str {
        "The format of the vocabulary file was Invalid"
    }
    fn cause(&self) -> Option<&Error> {
        None
    }
}

pub fn find_most_common_words(corpus: &Path, num: usize) -> Vec<(String, u32)> {
    let words = files(corpus).flat_map(|file| read_words(file));

    let mut word_counts = HashMap::new();
    for word in words {
        *word_counts.entry(word).or_insert(0) += 1;
    }

    let mut counts: Vec<_> = word_counts.into_iter().collect();
    counts.sort_by(|a, b| b.1.cmp(&a.1));
    counts.truncate(num);

    counts
}

pub fn save_words(path: &Path, words: &Vec<(String, u32)>) -> io::Result<()> {
    let mut out = try!(File::create(path));
    for &(ref word, count) in words {
        try!(out.write_all(format!("{}, {}\n", &word, count).as_bytes()));
    }

    Ok(())
}

pub fn load_most_common_words(filename: &str,
                              num: usize)
                              -> Result<Vec<(String, u32)>, Box<Error>> {
    let file = try!(File::open(&Path::new(filename)));
    let reader = BufReader::new(file);
    let mut words = vec![];
    for line in reader.lines().take(num) {
        if let Ok(line) = line {
            let mut columns = line.split(',');
            let word = columns.next().map(|w| w.trim().to_string());
            let num = columns.next().map(|n| n.trim().parse());

            match (word, num) {
                (Some(word), Some(num)) => words.push((word, try!(num))),
                _ => return Err(Box::new(FormatError)),
            }
        } else {
            return Err(Box::new(FormatError));
        }
    }

    Ok(words)
}

fn sentences<T: Read + 'static>(reader: BufReader<T>) -> Box<Iterator<Item = String>> {
    Box::new(reader.split('.' as u8)
                   .filter_map(|v| {
                       let lowercase = v.unwrap();
                       String::from_utf8(lowercase).ok().map(|s| s.to_lowercase())
                   }))
}

fn read_words<R: BufRead + 'static>(reader: R) -> Box<Iterator<Item = String>> {
    Box::new(reader.lines()
                   .filter_map(|line| line.ok())
                   .flat_map(|line| {
                       line.split(|c| {
                               match c {
                                   'a'...'z' | 'A'...'Z' => false,
                                   _ => true,
                               }
                           })
                           .filter(|word| !word.is_empty())
                           .map(|word| word.to_lowercase())
                           .collect::<Vec<_>>()
                           .into_iter()
                   }))
}

fn files(path: &Path) -> Box<Iterator<Item = BufReader<File>>> {
    Box::new(read_dir(path)
                 .unwrap()
                 .map(|path| path.unwrap().path())
                 .filter(|path| path.to_str().unwrap().ends_with(".txt"))
                 .map(|path| {
                     let file = File::open(path).unwrap();
                     BufReader::new(file)
                 }))
}

pub fn create_model(corpus: &Path, words: Vec<String>) -> LanguageModelBuilder {
    let mut builder = LanguageModelBuilder::new(10, words);

    for sentence in files(corpus).flat_map(sentences) {
        let mut acc = builder.new_sentence();

        let words = sentence.split(|c| {
                                match c {
                                    'a'...'z' => false,
                                    _ => true,
                                }
                            })
                            .filter(|w| !w.is_empty());
        for word in words {
            acc.add_word(word);
        }
    }

    builder
}
<|endoftext|>


<commit_before>//  Copyright (C) 2020 Éloïs SANCHEZ.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as
// published by the Free Software Foundation, either version 3 of the
// License, or (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program.  If not, see <https://www.gnu.org/licenses/>.

use crate::*;
use dubp::{
    block::parser::parse_json_block_from_serde_value, block::parser::ParseJsonBlockError,
    block::prelude::DubpBlockTrait, block::DubpBlock, common::prelude::BlockNumber,
};
use duniter_dbs::BcV1DbReadable;
use fast_threadpool::{ThreadPool, ThreadPoolConfig};
use std::path::PathBuf;

const CHUNK_SIZE: usize = 250;

pub(crate) fn migrate(profile_path: PathBuf) -> anyhow::Result<()> {
    let start_time = Instant::now();
    let dbs = duniter_dbs::open_dbs(Some(profile_path.as_path()));

    let data_path = profile_path.join(crate::DATA_DIR);
    let duniter_js_db = BcV1Db::<LevelDb>::open(LevelDbConf {
        db_path: data_path.as_path().join("leveldb"),
        ..Default::default()
    })?;

    let dbs_pool = ThreadPool::start(ThreadPoolConfig::default(), dbs).into_sync_handler();

    if let Some(target) = get_target_block_number(&duniter_js_db)? {
        println!("target block: #{}", target.0);
        /*let chunk_count = if (target.0 as usize + 1) % CHUNK_SIZE == 0 {
            (target.0 as usize + 1) / CHUNK_SIZE
        } else {
            (target.0 as usize / CHUNK_SIZE) + 1
        };*/

        let (s, r) = std::sync::mpsc::channel();
        let reader_handle = std::thread::spawn(move || {
            duniter_js_db.main_blocks().iter(.., |it| {
                it.values()
                    .map(|block_res| s.send(block_res).map_err(|_| anyhow!("fail to send")))
                    .collect::<anyhow::Result<()>>()
            })
        });
        let (s2, r2) = std::sync::mpsc::channel();
        let parser_handle = std::thread::spawn(move || {
            let target_u64 = target.0 as u64;
            let mut db_blocks = Vec::with_capacity(CHUNK_SIZE);
            while let Ok(db_block_res) = r.recv() {
                let db_block = db_block_res?;
                let db_block_number = db_block.number;
                db_blocks.push(db_block);
                if db_blocks.len() == CHUNK_SIZE || db_block_number == target_u64 {
                    let blocks = std::mem::take(&mut db_blocks)
                        .into_par_iter()
                        .map(|db_block| match serde_json::to_value(&db_block) {
                            Ok(json_block) => {
                                match parse_json_block_from_serde_value(&json_block) {
                                    Ok(block) => match block {
                                        DubpBlock::V10(block_v10) => Ok(block_v10),
                                    },
                                    Err(e) => Err(anyhow::Error::new::<ParseJsonBlockError>(e)),
                                }
                            }
                            Err(e) => Err(anyhow::Error::new::<serde_json::Error>(e)),
                        })
                        .collect::<anyhow::Result<Vec<_>>>()?;
                    s2.send(blocks).map_err(|_| anyhow!("fail to send"))?;
                    db_blocks.reserve_exact(CHUNK_SIZE);
                }
            }
            Ok::<(), anyhow::Error>(())
        });

        let mut current = None;
        while let Ok(chunk) = r2.recv() {
            if !chunk.is_empty() {
                println!(
                    "Apply chunk #{}-#{} ..",
                    chunk[0].number(),
                    chunk[chunk.len() - 1].number()
                );
                current = Some(duniter_dbs_write_ops::apply_block::apply_chunk(
                    current, &dbs_pool, chunk, true,
                )?);
            }
        }

        reader_handle.join().expect("reader thread panic")?;
        parser_handle.join().expect("parser thread panic")?;

        dbs_pool.execute(|dbs| {
            dbs.bc_db.save()?;
            dbs.gva_db.save()?;
            Ok::<(), KvError>(())
        })??;

        if let Some(current) = current {
            if current.number != target.0 {
                Err(anyhow::anyhow!("Migration fail: current != target"))
            } else {
                let duration = start_time.elapsed();
                println!(
                    "Migration successfully completed on {} seconds.",
                    duration.as_secs()
                );
                Ok(())
            }
        } else {
            Err(anyhow::anyhow!("Migration fail: rust dbs are empty"))
        }
    } else {
        Err(anyhow::anyhow!("Empty blockchain"))
    }
}

fn get_target_block_number(duniter_js_db: &BcV1Db<LevelDb>) -> KvResult<Option<BlockNumber>> {
    duniter_js_db.main_blocks().iter(.., |it| {
        it.reverse()
            .keys()
            .map(|k_res| k_res.map(|bn| bn.0))
            .next_res()
    })
}

/*fn get_chunk(duniter_js_db: &BcV1Db<LevelDb>, i: u32) -> anyhow::Result<Vec<DubpBlockV10>> {
    let start = BlockNumberKeyV1(BlockNumber(i * CHUNK_SIZE));
    let end = BlockNumberKeyV1(BlockNumber(((i + 1) * CHUNK_SIZE) - 1));
    println!("get_chunk({}): range {}..{}", i, start.0, end.0);
    let db_blocks = duniter_js_db
        .main_blocks()
        .iter(start..=end, |it| it.values().collect::<KvResult<Vec<_>>>())?;

    db_blocks
        .into_par_iter()
        .map(|db_block| match serde_json::to_value(&db_block) {
            Ok(json_block) => match parse_json_block_from_serde_value(&json_block) {
                Ok(block) => match block {
                    DubpBlock::V10(block_v10) => Ok(block_v10),
                },
                Err(e) => Err(anyhow::Error::new::<ParseJsonBlockError>(e)),
            },
            Err(e) => Err(anyhow::Error::new::<serde_json::Error>(e)),
        })
        .collect()
}
*/
<commit_msg>[feat] dex:migrate: add message on flush step<commit_after>//  Copyright (C) 2020 Éloïs SANCHEZ.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as
// published by the Free Software Foundation, either version 3 of the
// License, or (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program.  If not, see <https://www.gnu.org/licenses/>.

use crate::*;
use dubp::{
    block::parser::parse_json_block_from_serde_value, block::parser::ParseJsonBlockError,
    block::prelude::DubpBlockTrait, block::DubpBlock, common::prelude::BlockNumber,
};
use duniter_dbs::BcV1DbReadable;
use fast_threadpool::{ThreadPool, ThreadPoolConfig};
use std::path::PathBuf;

const CHUNK_SIZE: usize = 250;

pub(crate) fn migrate(profile_path: PathBuf) -> anyhow::Result<()> {
    let start_time = Instant::now();
    let dbs = duniter_dbs::open_dbs(Some(profile_path.as_path()));

    let data_path = profile_path.join(crate::DATA_DIR);
    let duniter_js_db = BcV1Db::<LevelDb>::open(LevelDbConf {
        db_path: data_path.as_path().join("leveldb"),
        ..Default::default()
    })?;

    let dbs_pool = ThreadPool::start(ThreadPoolConfig::default(), dbs).into_sync_handler();

    if let Some(target) = get_target_block_number(&duniter_js_db)? {
        println!("target block: #{}", target.0);
        /*let chunk_count = if (target.0 as usize + 1) % CHUNK_SIZE == 0 {
            (target.0 as usize + 1) / CHUNK_SIZE
        } else {
            (target.0 as usize / CHUNK_SIZE) + 1
        };*/

        let (s, r) = std::sync::mpsc::channel();
        let reader_handle = std::thread::spawn(move || {
            duniter_js_db.main_blocks().iter(.., |it| {
                it.values()
                    .map(|block_res| s.send(block_res).map_err(|_| anyhow!("fail to send")))
                    .collect::<anyhow::Result<()>>()
            })
        });
        let (s2, r2) = std::sync::mpsc::channel();
        let parser_handle = std::thread::spawn(move || {
            let target_u64 = target.0 as u64;
            let mut db_blocks = Vec::with_capacity(CHUNK_SIZE);
            while let Ok(db_block_res) = r.recv() {
                let db_block = db_block_res?;
                let db_block_number = db_block.number;
                db_blocks.push(db_block);
                if db_blocks.len() == CHUNK_SIZE || db_block_number == target_u64 {
                    let blocks = std::mem::take(&mut db_blocks)
                        .into_par_iter()
                        .map(|db_block| match serde_json::to_value(&db_block) {
                            Ok(json_block) => {
                                match parse_json_block_from_serde_value(&json_block) {
                                    Ok(block) => match block {
                                        DubpBlock::V10(block_v10) => Ok(block_v10),
                                    },
                                    Err(e) => Err(anyhow::Error::new::<ParseJsonBlockError>(e)),
                                }
                            }
                            Err(e) => Err(anyhow::Error::new::<serde_json::Error>(e)),
                        })
                        .collect::<anyhow::Result<Vec<_>>>()?;
                    s2.send(blocks).map_err(|_| anyhow!("fail to send"))?;
                    db_blocks.reserve_exact(CHUNK_SIZE);
                }
            }
            Ok::<(), anyhow::Error>(())
        });

        let mut current = None;
        while let Ok(chunk) = r2.recv() {
            if !chunk.is_empty() {
                println!(
                    "Apply chunk #{}-#{} ..",
                    chunk[0].number(),
                    chunk[chunk.len() - 1].number()
                );
                current = Some(duniter_dbs_write_ops::apply_block::apply_chunk(
                    current, &dbs_pool, chunk, true,
                )?);
            }
        }

        reader_handle.join().expect("reader thread panic")?;
        parser_handle.join().expect("parser thread panic")?;

        println!("Flush DBs caches on disk...");
        dbs_pool.execute(|dbs| {
            dbs.bc_db.save()?;
            dbs.gva_db.save()?;
            Ok::<(), KvError>(())
        })??;

        if let Some(current) = current {
            if current.number != target.0 {
                Err(anyhow::anyhow!("Migration fail: current != target"))
            } else {
                let duration = start_time.elapsed();
                println!(
                    "Migration successfully completed on {} seconds.",
                    duration.as_secs()
                );
                Ok(())
            }
        } else {
            Err(anyhow::anyhow!("Migration fail: rust dbs are empty"))
        }
    } else {
        Err(anyhow::anyhow!("Empty blockchain"))
    }
}

fn get_target_block_number(duniter_js_db: &BcV1Db<LevelDb>) -> KvResult<Option<BlockNumber>> {
    duniter_js_db.main_blocks().iter(.., |it| {
        it.reverse()
            .keys()
            .map(|k_res| k_res.map(|bn| bn.0))
            .next_res()
    })
}

/*fn get_chunk(duniter_js_db: &BcV1Db<LevelDb>, i: u32) -> anyhow::Result<Vec<DubpBlockV10>> {
    let start = BlockNumberKeyV1(BlockNumber(i * CHUNK_SIZE));
    let end = BlockNumberKeyV1(BlockNumber(((i + 1) * CHUNK_SIZE) - 1));
    println!("get_chunk({}): range {}..{}", i, start.0, end.0);
    let db_blocks = duniter_js_db
        .main_blocks()
        .iter(start..=end, |it| it.values().collect::<KvResult<Vec<_>>>())?;

    db_blocks
        .into_par_iter()
        .map(|db_block| match serde_json::to_value(&db_block) {
            Ok(json_block) => match parse_json_block_from_serde_value(&json_block) {
                Ok(block) => match block {
                    DubpBlock::V10(block_v10) => Ok(block_v10),
                },
                Err(e) => Err(anyhow::Error::new::<ParseJsonBlockError>(e)),
            },
            Err(e) => Err(anyhow::Error::new::<serde_json::Error>(e)),
        })
        .collect()
}
*/
<|endoftext|>


<commit_before>// Copyright © 2016, Canal TP and/or its affiliates. All rights reserved.
//
// This file is part of Navitia,
//     the software to build cool stuff with public transport.
//
// Hope you'll enjoy and contribute to this project,
//     powered by Canal TP (www.canaltp.fr).
// Help us simplify mobility and open public transport:
//     a non ending quest to the responsive locomotion way of traveling!
//
// LICENCE: This program is free software; you can redistribute it
// and/or modify it under the terms of the GNU Affero General Public
// License as published by the Free Software Foundation, either
// version 3 of the License, or (at your option) any later version.
//
// This program is distributed in the hope that it will be useful, but
// WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
// Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public
// License along with this program. If not, see
// <http://www.gnu.org/licenses/>.
//
// Stay tuned using
// twitter @navitia
// IRC #navitia on freenode
// https://groups.google.com/d/forum/navitia
// www.navitia.io

extern crate docopt;
#[macro_use]
extern crate log;
extern crate mimir;
extern crate mimirsbrunn;
extern crate rustc_serialize;

use mimir::rubber::Rubber;
use mimirsbrunn::osm_reader::admin::administrative_regions;
use mimirsbrunn::osm_reader::poi::{PoiTypes, pois, default_amenity_types,
                                              default_leisure_types};
use mimirsbrunn::osm_reader::street::streets;
use mimirsbrunn::osm_reader::parse_osm_pbf;
use mimirsbrunn::admin_geofinder::AdminGeoFinder;

#[derive(RustcDecodable, Debug)]
struct Args {
    flag_input: String,
    flag_level: Vec<u32>,
    flag_city_level: u32,
    flag_connection_string: String,
    flag_import_way: bool,
    flag_import_admin: bool,
    flag_import_poi: bool,
    flag_dataset: String,
}

static USAGE: &'static str =
    "
Usage:
    osm2mimir --help
    osm2mimir --input=<file> \
     [--connection-string=<connection-string>] [--import-way] [--import-admin] [--import-poi] \
     [--dataset=<dataset>] [--city-level=<level>] --level=<level> ...

Options:
    -h, --help              \
     Show this message.
    -i, --input=<file>      OSM PBF file.
    -l, --level=<level>     \
     Admin levels to keep.
    -C, --city-level=<level>
                            City level to \
     calculate weight, [default: 8]
    -w, --import-way        Import ways
    -a, \
     --import-admin      Import admins
    -p, --import-poi        Import POIs
    -c, \
     --connection-string=<connection-string>
                            Elasticsearch \
     parameters, [default: http://localhost:9200/munin]
    -d, --dataset=<dataset>
                            \
     Name of the dataset, [default: fr]
";

fn main() {
    mimir::logger_init().unwrap();
    let args: Args = docopt::Docopt::new(USAGE)
        .and_then(|d| d.decode())
        .unwrap_or_else(|e| e.exit());

    let levels = args.flag_level.iter().cloned().collect();
    let city_level = args.flag_city_level;
    let mut parsed_pbf = parse_osm_pbf(&args.flag_input);
    debug!("creation of indexes");
    let mut rubber = Rubber::new(&args.flag_connection_string);

    info!("creating adminstrative regions");
    let admins = administrative_regions(&mut parsed_pbf, levels);
	let admins_geofinder = admins.iter().cloned().collect::<AdminGeoFinder>();
	{
        info!("computing city weight");
        let mut streets = streets(&mut parsed_pbf, &admins_geofinder, city_level);

        for st in &mut streets {
            for admin in &mut st.administrative_regions {
                admin.weight.set(admin.weight.get() + 1);
                if admin.level == city_level {
                    st.weight = admin.weight.get();
                }
            }
        }

        if args.flag_import_way {
            info!("importing streets into Mimir");
            let nb_streets = rubber.index("way", &args.flag_dataset, streets.into_iter())
                .unwrap();
            info!("Nb of indexed street: {}", nb_streets);
        }
	}
    let nb_admins = rubber.index("admin", &args.flag_dataset, admins.iter())
        .unwrap();
    info!("Nb of indexed admin: {}", nb_admins);

    if args.flag_import_poi {
        let mut poi_types = PoiTypes::new();
        poi_types.insert("amenity".to_string(), default_amenity_types());
        poi_types.insert("leisure".to_string(), default_leisure_types());

        info!("Extracting pois from osm");
        let pois = pois(&mut parsed_pbf, poi_types, &admins_geofinder, city_level);

        info!("Importing pois into Mimir");
        let nb_pois = rubber.index("poi", &args.flag_dataset, pois.iter())
            .unwrap();

        info!("Nb of indexed pois: {}", nb_pois);
    }
}
<commit_msg>comment added<commit_after>// Copyright © 2016, Canal TP and/or its affiliates. All rights reserved.
//
// This file is part of Navitia,
//     the software to build cool stuff with public transport.
//
// Hope you'll enjoy and contribute to this project,
//     powered by Canal TP (www.canaltp.fr).
// Help us simplify mobility and open public transport:
//     a non ending quest to the responsive locomotion way of traveling!
//
// LICENCE: This program is free software; you can redistribute it
// and/or modify it under the terms of the GNU Affero General Public
// License as published by the Free Software Foundation, either
// version 3 of the License, or (at your option) any later version.
//
// This program is distributed in the hope that it will be useful, but
// WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
// Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public
// License along with this program. If not, see
// <http://www.gnu.org/licenses/>.
//
// Stay tuned using
// twitter @navitia
// IRC #navitia on freenode
// https://groups.google.com/d/forum/navitia
// www.navitia.io

extern crate docopt;
#[macro_use]
extern crate log;
extern crate mimir;
extern crate mimirsbrunn;
extern crate rustc_serialize;

use mimir::rubber::Rubber;
use mimirsbrunn::osm_reader::admin::administrative_regions;
use mimirsbrunn::osm_reader::poi::{PoiTypes, pois, default_amenity_types,
                                              default_leisure_types};
use mimirsbrunn::osm_reader::street::streets;
use mimirsbrunn::osm_reader::parse_osm_pbf;
use mimirsbrunn::admin_geofinder::AdminGeoFinder;

#[derive(RustcDecodable, Debug)]
struct Args {
    flag_input: String,
    flag_level: Vec<u32>,
    flag_city_level: u32,
    flag_connection_string: String,
    flag_import_way: bool,
    flag_import_admin: bool,
    flag_import_poi: bool,
    flag_dataset: String,
}

static USAGE: &'static str =
    "
Usage:
    osm2mimir --help
    osm2mimir --input=<file> \
     [--connection-string=<connection-string>] [--import-way] [--import-admin] [--import-poi] \
     [--dataset=<dataset>] [--city-level=<level>] --level=<level> ...

Options:
    -h, --help              \
     Show this message.
    -i, --input=<file>      OSM PBF file.
    -l, --level=<level>     \
     Admin levels to keep.
    -C, --city-level=<level>
                            City level to \
     calculate weight, [default: 8]
    -w, --import-way        Import ways
    -a, \
     --import-admin      Import admins
    -p, --import-poi        Import POIs
    -c, \
     --connection-string=<connection-string>
                            Elasticsearch \
     parameters, [default: http://localhost:9200/munin]
    -d, --dataset=<dataset>
                            \
     Name of the dataset, [default: fr]
";

fn main() {
    mimir::logger_init().unwrap();
    let args: Args = docopt::Docopt::new(USAGE)
        .and_then(|d| d.decode())
        .unwrap_or_else(|e| e.exit());

    let levels = args.flag_level.iter().cloned().collect();
    let city_level = args.flag_city_level;
    let mut parsed_pbf = parse_osm_pbf(&args.flag_input);
    debug!("creation of indexes");
    let mut rubber = Rubber::new(&args.flag_connection_string);

    info!("creating adminstrative regions");
    let admins = administrative_regions(&mut parsed_pbf, levels);
	let admins_geofinder = admins.iter().cloned().collect::<AdminGeoFinder>();
	{
        info!("Extracting streets from osm");
        let mut streets = streets(&mut parsed_pbf, &admins_geofinder, city_level);

    	info!("computing city, street weight");
        for st in &mut streets {
            for admin in &mut st.administrative_regions {
                admin.weight.set(admin.weight.get() + 1);
                if admin.level == city_level {
                    st.weight = admin.weight.get();
                }
            }
        }

        if args.flag_import_way {
            info!("importing streets into Mimir");
            let nb_streets = rubber.index("way", &args.flag_dataset, streets.into_iter())
                .unwrap();
            info!("Nb of indexed street: {}", nb_streets);
        }
	}
    let nb_admins = rubber.index("admin", &args.flag_dataset, admins.iter())
        .unwrap();
    info!("Nb of indexed admin: {}", nb_admins);

    if args.flag_import_poi {
        let mut poi_types = PoiTypes::new();
        poi_types.insert("amenity".to_string(), default_amenity_types());
        poi_types.insert("leisure".to_string(), default_leisure_types());

        info!("Extracting pois from osm");
        let pois = pois(&mut parsed_pbf, poi_types, &admins_geofinder, city_level);

        info!("Importing pois into Mimir");
        let nb_pois = rubber.index("poi", &args.flag_dataset, pois.iter())
            .unwrap();

        info!("Nb of indexed pois: {}", nb_pois);
    }
}
<|endoftext|>


<commit_before>use std::path::Path;

use builder::context::Context;
use builder::distrib::{Unknown,Distribution};
use builder::error::{Error};
use builder::commands::{composer, gem, npm, pip};
use builder::packages;
use build_step::BuildStep;
use container::util::clean_dir;
use container::mount::{unmount, mount_system_dirs};
use file_util::{create_dir, copy};

pub struct Guard<'a> {
    pub ctx: Context<'a>,
    pub distro: Box<Distribution>,
}

impl<'a> Guard<'a> {
    pub fn build(ctx: Context) -> Result<(), Error> {
        Guard {
            ctx: ctx,
            distro: Box::new(Unknown),
        }.run_process()
    }

    fn run_process(&mut self) -> Result<(), Error> {
        try!(self.start());

        for b in self.ctx.container_config.setup.iter() {
            debug!("Building step: {:?}", b);
            try!(b.build(self, true)
                .map_err(|e| Error::Step(b.0.clone(), e)));
        }

        try!(self.finish());
        Ok(())
    }

    pub fn start(&mut self) -> Result<(), String> {
        try!(mount_system_dirs());
        try_msg!(create_dir("/vagga/root/etc", false),
             "Error creating /etc dir: {err}");
        try!(copy("/etc/resolv.conf", "/vagga/root/etc/resolv.conf")
            .map_err(|e| format!("Error copying /etc/resolv.conf: {}", e)));
        try!(self.ctx.timelog.mark(format_args!("Prepare"))
            .map_err(|e| format!("Can't write timelog: {}", e)));
        Ok(())
    }

    pub fn finish(&mut self) -> Result<(), String> {
        // Pip
        if self.ctx.featured_packages.contains(&packages::PipPy2) ||
           self.ctx.featured_packages.contains(&packages::PipPy3)
        {
            try!(pip::freeze(&mut self.ctx));
        }
        // Npm
        if self.ctx.featured_packages.contains(&packages::Npm) {
            try!(npm::list(&mut self.ctx));
        }
        // Composer
        if self.ctx.featured_packages.contains(&packages::Composer) {
            try!(composer::finish(&mut self.ctx));
        }
        // Gem
        if self.ctx.featured_packages.contains(&packages::Bundler) {
            try!(gem::list(&mut self.ctx));
        }

        try!(self.distro.finish(&mut self.ctx));

        let base = Path::new("/vagga/root");

        for path in self.ctx.mounted.iter().rev() {
            try!(unmount(path));
        }

        for dir in self.ctx.remove_dirs.iter() {
            try!(clean_dir(&base.join(dir), true)
                .map_err(|e| format!("Error removing dir: {}", e)));
        }

        for dir in self.ctx.empty_dirs.iter() {
            try!(clean_dir(&base.join(dir), false));
        }

        for dir in self.ctx.ensure_dirs.iter() {
            let fulldir = base.join(dir);
            try_msg!(create_dir(&fulldir, true),
                "Error creating dir: {err}");
        }

        try!(self.ctx.timelog.mark(format_args!("Finish"))
            .map_err(|e| format!("Can't write timelog: {}", e)));

        return Ok(());
    }
}
<commit_msg>Fix timelog<commit_after>use std::path::Path;

use builder::context::Context;
use builder::distrib::{Unknown,Distribution};
use builder::error::{Error};
use builder::commands::{composer, gem, npm, pip};
use builder::packages;
use build_step::BuildStep;
use container::util::clean_dir;
use container::mount::{unmount, mount_system_dirs};
use file_util::{create_dir, copy};

pub struct Guard<'a> {
    pub ctx: Context<'a>,
    pub distro: Box<Distribution>,
}

impl<'a> Guard<'a> {
    pub fn build(ctx: Context) -> Result<(), Error> {
        Guard {
            ctx: ctx,
            distro: Box::new(Unknown),
        }.run_process()
    }

    fn run_process(&mut self) -> Result<(), Error> {
        try!(self.start());

        for b in self.ctx.container_config.setup.iter() {
            debug!("Building step: {:?}", b);
            try!(self.ctx.timelog.mark(format_args!("Step: {:?}", b))
                .map_err(|e| format!("Can't write timelog: {}", e)));
            try!(b.build(self, true)
                .map_err(|e| Error::Step(b.0.clone(), e)));
        }

        try!(self.finish());
        Ok(())
    }

    pub fn start(&mut self) -> Result<(), String> {
        try!(mount_system_dirs());
        try_msg!(create_dir("/vagga/root/etc", false),
             "Error creating /etc dir: {err}");
        try!(copy("/etc/resolv.conf", "/vagga/root/etc/resolv.conf")
            .map_err(|e| format!("Error copying /etc/resolv.conf: {}", e)));
        try!(self.ctx.timelog.mark(format_args!("Prepare"))
            .map_err(|e| format!("Can't write timelog: {}", e)));
        Ok(())
    }

    pub fn finish(&mut self) -> Result<(), String> {
        // Pip
        if self.ctx.featured_packages.contains(&packages::PipPy2) ||
           self.ctx.featured_packages.contains(&packages::PipPy3)
        {
            try!(pip::freeze(&mut self.ctx));
        }
        // Npm
        if self.ctx.featured_packages.contains(&packages::Npm) {
            try!(npm::list(&mut self.ctx));
        }
        // Composer
        if self.ctx.featured_packages.contains(&packages::Composer) {
            try!(composer::finish(&mut self.ctx));
        }
        // Gem
        if self.ctx.featured_packages.contains(&packages::Bundler) {
            try!(gem::list(&mut self.ctx));
        }

        try!(self.distro.finish(&mut self.ctx));

        let base = Path::new("/vagga/root");

        for path in self.ctx.mounted.iter().rev() {
            try!(unmount(path));
        }

        for dir in self.ctx.remove_dirs.iter() {
            try!(clean_dir(&base.join(dir), true)
                .map_err(|e| format!("Error removing dir: {}", e)));
        }

        for dir in self.ctx.empty_dirs.iter() {
            try!(clean_dir(&base.join(dir), false));
        }

        for dir in self.ctx.ensure_dirs.iter() {
            let fulldir = base.join(dir);
            try_msg!(create_dir(&fulldir, true),
                "Error creating dir: {err}");
        }

        try!(self.ctx.timelog.mark(format_args!("Finish"))
            .map_err(|e| format!("Can't write timelog: {}", e)));

        return Ok(());
    }
}
<|endoftext|>


<commit_before>use std::marker::PhantomData;
use translate::*;

/// Wrapper implementations for refcounted types. See `glib_wrapper!`.
#[macro_export]
macro_rules! glib_refcounted_wrapper {
    ([$($attr:meta)*] $name:ident, $ffi_name:path, @ref $ref_arg:ident $ref_expr:expr,
     @unref $unref_arg:ident $unref_expr:expr) => {
        $(#[$attr])*
        pub struct $name($crate::refcounted::Refcounted<$ffi_name, MemoryManager>);

        #[doc(hidden)]
        pub struct MemoryManager;

        impl $crate::refcounted::RefcountedMemoryManager<$ffi_name> for MemoryManager {
            #[inline]
            unsafe fn ref_($ref_arg: *mut $ffi_name) {
                $ref_expr;
            }

            #[inline]
            unsafe fn unref($unref_arg: *mut $ffi_name) {
                $unref_expr
            }
        }

        impl<'a> $crate::translate::ToGlibPtr<'a, *mut $ffi_name> for $name {
            type Storage = &'a $crate::refcounted::Refcounted<$ffi_name, MemoryManager>;

            #[inline]
            fn to_glib_none(&'a self) -> $crate::translate::Stash<'a, *mut $ffi_name, Self> {
                let stash = self.0.to_glib_none();
                $crate::translate::Stash(stash.0, stash.1)
            }

            #[inline]
            fn to_glib_full(&self) -> *mut $ffi_name {
                (&self.0).to_glib_full()
            }
        }

        impl $crate::translate::FromGlibPtr<*mut $ffi_name> for $name {
            #[inline]
            unsafe fn from_glib_none(ptr: *mut $ffi_name) -> Self {
                $name($crate::translate::from_glib_none(ptr))
            }

            #[inline]
            unsafe fn from_glib_full(ptr: *mut $ffi_name) -> Self {
                $name($crate::translate::from_glib_full(ptr))
            }

            #[inline]
            unsafe fn from_glib_borrow(ptr: *mut $ffi_name) -> Self {
                $name($crate::translate::from_glib_borrow(ptr))
            }
        }

        impl Clone for $name {
            fn clone(&self) -> Self {
                $name(self.0.clone())
            }
        }
    }
}

pub trait RefcountedMemoryManager<T> {
    unsafe fn ref_(ptr: *mut T);
    unsafe fn unref(ptr: *mut T);
}

/// Encapsulates memory management logic for refcounted types.
#[allow(raw_pointer_derive)]
#[derive(Debug)]
pub struct Refcounted<T, MM: RefcountedMemoryManager<T>> {
    inner: *mut T,
    borrowed: bool,
    mm: PhantomData<*const MM>,
}

impl<T, MM: RefcountedMemoryManager<T>> Drop for Refcounted<T, MM> {
    fn drop(&mut self) {
        if !self.borrowed {
            unsafe { MM::unref(self.inner); }
        }
    }
}

impl<T, MM: RefcountedMemoryManager<T>> Clone for Refcounted<T, MM> {
    fn clone(&self) -> Self {
        unsafe { MM::ref_(self.inner); }
        Refcounted {
            inner: self.inner,
            borrowed: false,
            mm: PhantomData,
        }
    }
}

impl<'a, T: 'static, MM> ToGlibPtr<'a, *mut T> for Refcounted<T, MM>
where MM: RefcountedMemoryManager<T> + 'static {
    type Storage = &'a Self;

    #[inline]
    fn to_glib_none(&'a self) -> Stash<'a, *mut T, Self> {
        Stash(self.inner, self)
    }

    #[inline]
    fn to_glib_full(&self) -> *mut T {
        unsafe { MM::ref_(self.inner); }
        self.inner
    }
}

impl<T: 'static, MM: RefcountedMemoryManager<T>> FromGlibPtr<*mut T> for Refcounted<T, MM> {
    #[inline]
    unsafe fn from_glib_none(ptr: *mut T) -> Self {
        assert!(!ptr.is_null());
        MM::ref_(ptr);
        Refcounted {
            inner: ptr,
            borrowed: false,
            mm: PhantomData,
        }
    }

    #[inline]
    unsafe fn from_glib_full(ptr: *mut T) -> Self {
        assert!(!ptr.is_null());
        Refcounted {
            inner: ptr,
            borrowed: false,
            mm: PhantomData,
        }
    }

    #[inline]
    unsafe fn from_glib_borrow(ptr: *mut T) -> Self {
        assert!(!ptr.is_null());
        Refcounted {
            inner: ptr,
            borrowed: true,
            mm: PhantomData,
        }
    }
}
<commit_msg>Sunset the raw_pointer_derive lint<commit_after>use std::marker::PhantomData;
use translate::*;

/// Wrapper implementations for refcounted types. See `glib_wrapper!`.
#[macro_export]
macro_rules! glib_refcounted_wrapper {
    ([$($attr:meta)*] $name:ident, $ffi_name:path, @ref $ref_arg:ident $ref_expr:expr,
     @unref $unref_arg:ident $unref_expr:expr) => {
        $(#[$attr])*
        pub struct $name($crate::refcounted::Refcounted<$ffi_name, MemoryManager>);

        #[doc(hidden)]
        pub struct MemoryManager;

        impl $crate::refcounted::RefcountedMemoryManager<$ffi_name> for MemoryManager {
            #[inline]
            unsafe fn ref_($ref_arg: *mut $ffi_name) {
                $ref_expr;
            }

            #[inline]
            unsafe fn unref($unref_arg: *mut $ffi_name) {
                $unref_expr
            }
        }

        impl<'a> $crate::translate::ToGlibPtr<'a, *mut $ffi_name> for $name {
            type Storage = &'a $crate::refcounted::Refcounted<$ffi_name, MemoryManager>;

            #[inline]
            fn to_glib_none(&'a self) -> $crate::translate::Stash<'a, *mut $ffi_name, Self> {
                let stash = self.0.to_glib_none();
                $crate::translate::Stash(stash.0, stash.1)
            }

            #[inline]
            fn to_glib_full(&self) -> *mut $ffi_name {
                (&self.0).to_glib_full()
            }
        }

        impl $crate::translate::FromGlibPtr<*mut $ffi_name> for $name {
            #[inline]
            unsafe fn from_glib_none(ptr: *mut $ffi_name) -> Self {
                $name($crate::translate::from_glib_none(ptr))
            }

            #[inline]
            unsafe fn from_glib_full(ptr: *mut $ffi_name) -> Self {
                $name($crate::translate::from_glib_full(ptr))
            }

            #[inline]
            unsafe fn from_glib_borrow(ptr: *mut $ffi_name) -> Self {
                $name($crate::translate::from_glib_borrow(ptr))
            }
        }

        impl Clone for $name {
            fn clone(&self) -> Self {
                $name(self.0.clone())
            }
        }
    }
}

pub trait RefcountedMemoryManager<T> {
    unsafe fn ref_(ptr: *mut T);
    unsafe fn unref(ptr: *mut T);
}

/// Encapsulates memory management logic for refcounted types.
#[derive(Debug)]
pub struct Refcounted<T, MM: RefcountedMemoryManager<T>> {
    inner: *mut T,
    borrowed: bool,
    mm: PhantomData<*const MM>,
}

impl<T, MM: RefcountedMemoryManager<T>> Drop for Refcounted<T, MM> {
    fn drop(&mut self) {
        if !self.borrowed {
            unsafe { MM::unref(self.inner); }
        }
    }
}

impl<T, MM: RefcountedMemoryManager<T>> Clone for Refcounted<T, MM> {
    fn clone(&self) -> Self {
        unsafe { MM::ref_(self.inner); }
        Refcounted {
            inner: self.inner,
            borrowed: false,
            mm: PhantomData,
        }
    }
}

impl<'a, T: 'static, MM> ToGlibPtr<'a, *mut T> for Refcounted<T, MM>
where MM: RefcountedMemoryManager<T> + 'static {
    type Storage = &'a Self;

    #[inline]
    fn to_glib_none(&'a self) -> Stash<'a, *mut T, Self> {
        Stash(self.inner, self)
    }

    #[inline]
    fn to_glib_full(&self) -> *mut T {
        unsafe { MM::ref_(self.inner); }
        self.inner
    }
}

impl<T: 'static, MM: RefcountedMemoryManager<T>> FromGlibPtr<*mut T> for Refcounted<T, MM> {
    #[inline]
    unsafe fn from_glib_none(ptr: *mut T) -> Self {
        assert!(!ptr.is_null());
        MM::ref_(ptr);
        Refcounted {
            inner: ptr,
            borrowed: false,
            mm: PhantomData,
        }
    }

    #[inline]
    unsafe fn from_glib_full(ptr: *mut T) -> Self {
        assert!(!ptr.is_null());
        Refcounted {
            inner: ptr,
            borrowed: false,
            mm: PhantomData,
        }
    }

    #[inline]
    unsafe fn from_glib_borrow(ptr: *mut T) -> Self {
        assert!(!ptr.is_null());
        Refcounted {
            inner: ptr,
            borrowed: true,
            mm: PhantomData,
        }
    }
}
<|endoftext|>


<commit_before>pub trait Channel {
    static pure fn channel_max() -> self;

    static pure fn from_channel<T:Channel>(val: T) -> self;
    pure fn convert_channel<T:Channel>(&self) -> T;

    pure fn to_channel_u8(&self)    -> u8;
    pure fn to_channel_u16(&self)   -> u16;
    pure fn to_channel_u32(&self)   -> u32;
    pure fn to_channel_u64(&self)   -> u64;
    pure fn to_channel_f32(&self)   -> f32;
    pure fn to_channel_f64(&self)   -> f64;
    pure fn to_channel_float(&self) -> float;

    pure fn inverse(&self) -> self;
}

pub pure fn convert_channel<T:Channel, U:Channel>(val: T) -> U { val.convert_channel() }

pub impl u8: Channel {
    #[inline(always)] static pure fn channel_max() -> u8 { 0xFF }  // 2^8

    #[inline(always)] static pure fn from_channel<T:Channel>(val: T) -> u8 { val.to_channel_u8() }
    #[inline(always)] pure fn convert_channel<T:Channel>(&self) -> T { from_channel(*self) }

    #[inline(always)] pure fn to_channel_u8(&self)    -> u8    { (*self) }
    #[inline(always)] pure fn to_channel_u16(&self)   -> u16   { (*self as u16 << 8) | (*self) as u16 }
    #[inline(always)] pure fn to_channel_u32(&self)   -> u32   { (self.to_channel_u16() as u32 << 16) | self.to_channel_u16() as u32 }
    #[inline(always)] pure fn to_channel_u64(&self)   -> u64   { (self.to_channel_u32() as u64 << 32) | self.to_channel_u32() as u64 }
    #[inline(always)] pure fn to_channel_f32(&self)   -> f32   { (*self as f32)   / (0xFF as f32)   }
    #[inline(always)] pure fn to_channel_f64(&self)   -> f64   { (*self as f64)   / (0xFF as f64)   }
    #[inline(always)] pure fn to_channel_float(&self) -> float { (*self as float) / (0xFF as float) }

    #[inline(always)] pure fn inverse(&self) -> u8 { !(*self) }
}

pub impl u16: Channel {
    #[inline(always)] static pure fn channel_max() -> u16 { 0xFFFF }   // 2^16

    #[inline(always)] static pure fn from_channel<T:Channel>(val: T) -> u16 { val.to_channel_u16() }
    #[inline(always)] pure fn convert_channel<T:Channel>(&self) -> T { from_channel(*self) }

    #[inline(always)] pure fn to_channel_u8(&self)    -> u8    { (*self >> 8) as u8 }    // this is the equivalent of `self/256`. Some folks prefer to do `self/257`
    #[inline(always)] pure fn to_channel_u16(&self)   -> u16   { (*self) }
    #[inline(always)] pure fn to_channel_u32(&self)   -> u32   { (*self as u32 << 16) | (*self) as u32 }
    #[inline(always)] pure fn to_channel_u64(&self)   -> u64   { (self.to_channel_u32() as u64 << 32) | self.to_channel_u32() as u64 }
    #[inline(always)] pure fn to_channel_f32(&self)   -> f32   { (*self) / 0xFFFF as f32   }
    #[inline(always)] pure fn to_channel_f64(&self)   -> f64   { (*self) / 0xFFFF as f64   }
    #[inline(always)] pure fn to_channel_float(&self) -> float { (*self) / 0xFFFF as float }

    #[inline(always)] pure fn inverse(&self) -> u16 { !(*self) }
}

pub impl u32: Channel {
    #[inline(always)] static pure fn channel_max() -> u32 { 0xFFFF_FFFF }   // 2^32

    #[inline(always)] static pure fn from_channel<T:Channel>(val: T) -> u32 { val.to_channel_u32() }
    #[inline(always)] pure fn convert_channel<T:Channel>(&self) -> T { from_channel(*self) }

    #[inline(always)] pure fn to_channel_u8(&self)    -> u8    { (*self >> 24) as u8 }
    #[inline(always)] pure fn to_channel_u16(&self)   -> u16   { (*self >> 16) as u16 }
    #[inline(always)] pure fn to_channel_u32(&self)   -> u32   { (*self) }
    #[inline(always)] pure fn to_channel_u64(&self)   -> u64   { (*self as u64 << 32) | (*self) as u64 }
    #[inline(always)] pure fn to_channel_f32(&self)   -> f32   { (*self) / 0xFFFF_FFFF as f32   }
    #[inline(always)] pure fn to_channel_f64(&self)   -> f64   { (*self) / 0xFFFF_FFFF as f64   }
    #[inline(always)] pure fn to_channel_float(&self) -> float { (*self) / 0xFFFF_FFFF as float }

    #[inline(always)] pure fn inverse(&self) -> u32 { !(*self) }
}

pub impl u64: Channel {
    #[inline(always)] static pure fn channel_max() -> u64 { 0xFFFF_FFFF_FFFF_FFFF_u64 }   // 2^64

    #[inline(always)] static pure fn from_channel<T:Channel>(val: T) -> u64 { val.to_channel_u64() }
    #[inline(always)] pure fn convert_channel<T:Channel>(&self) -> T { from_channel(*self) }

    #[inline(always)] pure fn to_channel_u8(&self)    -> u8    { (*self >> 56) as u8 }
    #[inline(always)] pure fn to_channel_u16(&self)   -> u16   { (*self >> 48) as u16 }
    #[inline(always)] pure fn to_channel_u32(&self)   -> u32   { (*self >> 32) as u32 }
    #[inline(always)] pure fn to_channel_u64(&self)   -> u64   { (*self) }
    #[inline(always)] pure fn to_channel_f32(&self)   -> f32   { (*self) / 0xFFFF_FFFF_FFFF_FFFF_u64 as f32   }
    #[inline(always)] pure fn to_channel_f64(&self)   -> f64   { (*self) / 0xFFFF_FFFF_FFFF_FFFF_u64 as f64   }
    #[inline(always)] pure fn to_channel_float(&self) -> float { (*self) / 0xFFFF_FFFF_FFFF_FFFF_u64 as float }

    #[inline(always)] pure fn inverse(&self) -> u64 { !(*self) }
}

pub impl f32: Channel {
    #[inline(always)] static pure fn channel_max() -> f32 { 1f32 }

    #[inline(always)] static pure fn from_channel<T:Channel>(val: T) -> f32 { val.to_channel_f32() }
    #[inline(always)] pure fn convert_channel<T:Channel>(&self) -> T { from_channel(*self) }

    #[inline(always)] pure fn to_channel_u8(&self)    -> u8    { (*self) * (0xFF_u8         as f32) as u8  }
    #[inline(always)] pure fn to_channel_u16(&self)   -> u16   { (*self) * (0xFFFF_u16      as f32) as u16 }
    #[inline(always)] pure fn to_channel_u32(&self)   -> u32   { fail(~"to_channel_u32 not yet implemented for f32") }
    #[inline(always)] pure fn to_channel_u64(&self)   -> u64   { fail(~"to_channel_u64 not yet implemented for f32") }
    #[inline(always)] pure fn to_channel_f32(&self)   -> f32   { (*self) }
    #[inline(always)] pure fn to_channel_f64(&self)   -> f64   { (*self) as f64 }
    #[inline(always)] pure fn to_channel_float(&self) -> float { (*self) as float }

    #[inline(always)] pure fn inverse(&self) -> f32 { 1f32 - (*self) }
}

pub impl f64: Channel {
    #[inline(always)] static pure fn channel_max() -> f64 { 1f64 }

    #[inline(always)] static pure fn from_channel<T:Channel>(val: T) -> f64 { val.to_channel_f64() }
    #[inline(always)] pure fn convert_channel<T:Channel>(&self) -> T { from_channel(*self) }

    #[inline(always)] pure fn to_channel_u8(&self)    -> u8    { (*self) * (0xFFu8    as f64) as u8  }
    #[inline(always)] pure fn to_channel_u16(&self)   -> u16   { (*self) * (0xFFFFu16 as f64) as u16 }
    #[inline(always)] pure fn to_channel_u32(&self)   -> u32   { fail(~"to_channel_u32 not yet implemented for f64") }
    #[inline(always)] pure fn to_channel_u64(&self)   -> u64   { fail(~"to_channel_u64 not yet implemented for f64") }
    #[inline(always)] pure fn to_channel_f32(&self)   -> f32   { (*self) as f32 }
    #[inline(always)] pure fn to_channel_f64(&self)   -> f64   { (*self) }
    #[inline(always)] pure fn to_channel_float(&self) -> float { (*self) as float }

    #[inline(always)] pure fn inverse(&self) -> f64 { 1f64 - (*self) }
}

pub impl float: Channel {
    #[inline(always)] static pure fn channel_max() -> float { 1f }

    #[inline(always)] static pure fn from_channel<T:Channel>(val: T) -> float { val.to_channel_float() }
    #[inline(always)] pure fn convert_channel<T:Channel>(&self) -> T { from_channel(*self) }

    #[inline(always)] pure fn to_channel_u8(&self)    -> u8    { (*self) * (0xFFu8    as float) as u8  }
    #[inline(always)] pure fn to_channel_u16(&self)   -> u16   { (*self) * (0xFFFFu16 as float) as u16 }
    #[inline(always)] pure fn to_channel_u32(&self)   -> u32   { fail(~"to_channel_u32 not yet implemented for float") }
    #[inline(always)] pure fn to_channel_u64(&self)   -> u64   { fail(~"to_channel_u64 not yet implemented for float") }
    #[inline(always)] pure fn to_channel_f32(&self)   -> f32   { (*self) as f32 }
    #[inline(always)] pure fn to_channel_f64(&self)   -> f64   { (*self) as f64 }
    #[inline(always)] pure fn to_channel_float(&self) -> float { (*self) }

    #[inline(always)] pure fn inverse(&self) -> float { 1f - (*self) }
}<commit_msg>Rename functions and inherit Number trait<commit_after>use num::kinds::Number;

pub trait Channel: Number {
    static pure fn max() -> self;

    static pure fn from<T:Channel>(val: T) -> self;
    pure fn convert_channel<T:Channel>(&self) -> T;

    pure fn to_channel_u8(&self)    -> u8;
    pure fn to_channel_u16(&self)   -> u16;
    pure fn to_channel_u32(&self)   -> u32;
    pure fn to_channel_u64(&self)   -> u64;
    pure fn to_channel_f32(&self)   -> f32;
    pure fn to_channel_f64(&self)   -> f64;
    pure fn to_channel_float(&self) -> float;

    pure fn inverse(&self) -> self;
}

pub impl u8: Channel {
    #[inline(always)] static pure fn max() -> u8 { 0xFF }  // 2^8

    #[inline(always)] static pure fn from<T:Channel>(val: T) -> u8 { val.to_channel_u8() }
    #[inline(always)] pure fn convert_channel<T:Channel>(&self) -> T { Channel::from(*self) }

    #[inline(always)] pure fn to_channel_u8(&self)    -> u8    { (*self) }
    #[inline(always)] pure fn to_channel_u16(&self)   -> u16   { (*self as u16 << 8) | (*self) as u16 }
    #[inline(always)] pure fn to_channel_u32(&self)   -> u32   { (self.to_channel_u16() as u32 << 16) | self.to_channel_u16() as u32 }
    #[inline(always)] pure fn to_channel_u64(&self)   -> u64   { (self.to_channel_u32() as u64 << 32) | self.to_channel_u32() as u64 }
    #[inline(always)] pure fn to_channel_f32(&self)   -> f32   { (*self as f32)   / (0xFF as f32)   }
    #[inline(always)] pure fn to_channel_f64(&self)   -> f64   { (*self as f64)   / (0xFF as f64)   }
    #[inline(always)] pure fn to_channel_float(&self) -> float { (*self as float) / (0xFF as float) }

    #[inline(always)] pure fn inverse(&self) -> u8 { !(*self) }
}

pub impl u16: Channel {
    #[inline(always)] static pure fn max() -> u16 { 0xFFFF }   // 2^16

    #[inline(always)] static pure fn from<T:Channel>(val: T) -> u16 { val.to_channel_u16() }
    #[inline(always)] pure fn convert_channel<T:Channel>(&self) -> T { Channel::from(*self) }

    #[inline(always)] pure fn to_channel_u8(&self)    -> u8    { (*self >> 8) as u8 }    // this is the equivalent of `self/256`. Some folks prefer to do `self/257`
    #[inline(always)] pure fn to_channel_u16(&self)   -> u16   { (*self) }
    #[inline(always)] pure fn to_channel_u32(&self)   -> u32   { (*self as u32 << 16) | (*self) as u32 }
    #[inline(always)] pure fn to_channel_u64(&self)   -> u64   { (self.to_channel_u32() as u64 << 32) | self.to_channel_u32() as u64 }
    #[inline(always)] pure fn to_channel_f32(&self)   -> f32   { (*self) / 0xFFFF as f32   }
    #[inline(always)] pure fn to_channel_f64(&self)   -> f64   { (*self) / 0xFFFF as f64   }
    #[inline(always)] pure fn to_channel_float(&self) -> float { (*self) / 0xFFFF as float }

    #[inline(always)] pure fn inverse(&self) -> u16 { !(*self) }
}

pub impl u32: Channel {
    #[inline(always)] static pure fn max() -> u32 { 0xFFFF_FFFF }   // 2^32

    #[inline(always)] static pure fn from<T:Channel>(val: T) -> u32 { val.to_channel_u32() }
    #[inline(always)] pure fn convert_channel<T:Channel>(&self) -> T { Channel::from(*self) }

    #[inline(always)] pure fn to_channel_u8(&self)    -> u8    { (*self >> 24) as u8 }
    #[inline(always)] pure fn to_channel_u16(&self)   -> u16   { (*self >> 16) as u16 }
    #[inline(always)] pure fn to_channel_u32(&self)   -> u32   { (*self) }
    #[inline(always)] pure fn to_channel_u64(&self)   -> u64   { (*self as u64 << 32) | (*self) as u64 }
    #[inline(always)] pure fn to_channel_f32(&self)   -> f32   { (*self) / 0xFFFF_FFFF as f32   }
    #[inline(always)] pure fn to_channel_f64(&self)   -> f64   { (*self) / 0xFFFF_FFFF as f64   }
    #[inline(always)] pure fn to_channel_float(&self) -> float { (*self) / 0xFFFF_FFFF as float }

    #[inline(always)] pure fn inverse(&self) -> u32 { !(*self) }
}

pub impl u64: Channel {
    #[inline(always)] static pure fn max() -> u64 { 0xFFFF_FFFF_FFFF_FFFF_u64 }   // 2^64

    #[inline(always)] static pure fn from<T:Channel>(val: T) -> u64 { val.to_channel_u64() }
    #[inline(always)] pure fn convert_channel<T:Channel>(&self) -> T { Channel::from(*self) }

    #[inline(always)] pure fn to_channel_u8(&self)    -> u8    { (*self >> 56) as u8 }
    #[inline(always)] pure fn to_channel_u16(&self)   -> u16   { (*self >> 48) as u16 }
    #[inline(always)] pure fn to_channel_u32(&self)   -> u32   { (*self >> 32) as u32 }
    #[inline(always)] pure fn to_channel_u64(&self)   -> u64   { (*self) }
    #[inline(always)] pure fn to_channel_f32(&self)   -> f32   { (*self) / 0xFFFF_FFFF_FFFF_FFFF_u64 as f32   }
    #[inline(always)] pure fn to_channel_f64(&self)   -> f64   { (*self) / 0xFFFF_FFFF_FFFF_FFFF_u64 as f64   }
    #[inline(always)] pure fn to_channel_float(&self) -> float { (*self) / 0xFFFF_FFFF_FFFF_FFFF_u64 as float }

    #[inline(always)] pure fn inverse(&self) -> u64 { !(*self) }
}

pub impl f32: Channel {
    #[inline(always)] static pure fn max() -> f32 { 1f32 }

    #[inline(always)] static pure fn from<T:Channel>(val: T) -> f32 { val.to_channel_f32() }
    #[inline(always)] pure fn convert_channel<T:Channel>(&self) -> T { Channel::from(*self) }

    #[inline(always)] pure fn to_channel_u8(&self)    -> u8    { (*self) * (0xFF_u8    as f32) as u8  }
    #[inline(always)] pure fn to_channel_u16(&self)   -> u16   { (*self) * (0xFFFF_u16 as f32) as u16 }
    #[inline(always)] pure fn to_channel_u32(&self)   -> u32   { fail(~"to_channel_u32 not yet implemented for f32") }
    #[inline(always)] pure fn to_channel_u64(&self)   -> u64   { fail(~"to_channel_u64 not yet implemented for f32") }
    #[inline(always)] pure fn to_channel_f32(&self)   -> f32   { (*self) }
    #[inline(always)] pure fn to_channel_f64(&self)   -> f64   { (*self) as f64 }
    #[inline(always)] pure fn to_channel_float(&self) -> float { (*self) as float }

    #[inline(always)] pure fn inverse(&self) -> f32 { 1f32 - (*self) }
}

pub impl f64: Channel {
    #[inline(always)] static pure fn max() -> f64 { 1f64 }

    #[inline(always)] static pure fn from<T:Channel>(val: T) -> f64 { val.to_channel_f64() }
    #[inline(always)] pure fn convert_channel<T:Channel>(&self) -> T { Channel::from(*self) }

    #[inline(always)] pure fn to_channel_u8(&self)    -> u8    { (*self) * (0xFF_u8    as f64) as u8  }
    #[inline(always)] pure fn to_channel_u16(&self)   -> u16   { (*self) * (0xFFFF_u16 as f64) as u16 }
    #[inline(always)] pure fn to_channel_u32(&self)   -> u32   { fail(~"to_channel_u32 not yet implemented for f64") }
    #[inline(always)] pure fn to_channel_u64(&self)   -> u64   { fail(~"to_channel_u64 not yet implemented for f64") }
    #[inline(always)] pure fn to_channel_f32(&self)   -> f32   { (*self) as f32 }
    #[inline(always)] pure fn to_channel_f64(&self)   -> f64   { (*self) }
    #[inline(always)] pure fn to_channel_float(&self) -> float { (*self) as float }

    #[inline(always)] pure fn inverse(&self) -> f64 { 1f64 - (*self) }
}

pub impl float: Channel {
    #[inline(always)] static pure fn max() -> float { 1f }

    #[inline(always)] static pure fn from<T:Channel>(val: T) -> float { val.to_channel_float() }
    #[inline(always)] pure fn convert_channel<T:Channel>(&self) -> T { Channel::from(*self) }

    #[inline(always)] pure fn to_channel_u8(&self)    -> u8    { (*self) * (0xFF_u8    as float) as u8  }
    #[inline(always)] pure fn to_channel_u16(&self)   -> u16   { (*self) * (0xFFFF_u16 as float) as u16 }
    #[inline(always)] pure fn to_channel_u32(&self)   -> u32   { fail(~"to_channel_u32 not yet implemented for float") }
    #[inline(always)] pure fn to_channel_u64(&self)   -> u64   { fail(~"to_channel_u64 not yet implemented for float") }
    #[inline(always)] pure fn to_channel_f32(&self)   -> f32   { (*self) as f32 }
    #[inline(always)] pure fn to_channel_f64(&self)   -> f64   { (*self) as f64 }
    #[inline(always)] pure fn to_channel_float(&self) -> float { (*self) }

    #[inline(always)] pure fn inverse(&self) -> float { 1f - (*self) }
}<|endoftext|>