<commit_before>// Copyright 2012 The Rust Project Developers. See the COPYRIGHT
// file at the top-level directory of this distribution and at
// http://rust-lang.org/COPYRIGHT.
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.

// The crate store - a central repo for information collected about external
// crates and libraries

use metadata::decoder;
use metadata::loader;

use std::cell::RefCell;
use std::hashmap::HashMap;
use syntax::ast;
use syntax::parse::token::ident_interner;

// A map from external crate numbers (as decoded from some crate file) to
// local crate numbers (as generated during this session). Each external
// crate may refer to types in other external crates, and each has their
// own crate numbers.
pub type cnum_map = @mut HashMap<ast::CrateNum, ast::CrateNum>;

pub enum MetadataBlob {
    MetadataVec(~[u8]),
    MetadataArchive(loader::ArchiveMetadata),
}

pub struct crate_metadata {
    name: @str,
    data: MetadataBlob,
    cnum_map: cnum_map,
    cnum: ast::CrateNum
}

#[deriving(Eq)]
pub enum LinkagePreference {
    RequireDynamic,
    RequireStatic,
}

#[deriving(Eq, FromPrimitive)]
pub enum NativeLibaryKind {
    NativeStatic,    // native static library (.a archive)
    NativeFramework, // OSX-specific
    NativeUnknown,   // default way to specify a dynamic library
}

// Where a crate came from on the local filesystem. One of these two options
// must be non-None.
#[deriving(Eq)]
pub struct CrateSource {
    dylib: Option<Path>,
    rlib: Option<Path>,
    cnum: ast::CrateNum,
}

pub struct CStore {
    priv metas: RefCell<HashMap<ast::CrateNum, @crate_metadata>>,
    priv extern_mod_crate_map: extern_mod_crate_map,
    priv used_crate_sources: ~[CrateSource],
    priv used_libraries: ~[(~str, NativeLibaryKind)],
    priv used_link_args: ~[~str],
    intr: @ident_interner
}

// Map from NodeId's of local extern mod statements to crate numbers
type extern_mod_crate_map = HashMap<ast::NodeId, ast::CrateNum>;

impl CStore {
    pub fn new(intr: @ident_interner) -> CStore {
        CStore {
            metas: RefCell::new(HashMap::new()),
            extern_mod_crate_map: HashMap::new(),
            used_crate_sources: ~[],
            used_libraries: ~[],
            used_link_args: ~[],
            intr: intr
        }
    }

    pub fn get_crate_data(&self, cnum: ast::CrateNum) -> @crate_metadata {
        let metas = self.metas.borrow();
        *metas.get().get(&cnum)
    }

    pub fn get_crate_hash(&self, cnum: ast::CrateNum) -> @str {
        let cdata = self.get_crate_data(cnum);
        decoder::get_crate_hash(cdata.data())
    }

    pub fn get_crate_vers(&self, cnum: ast::CrateNum) -> @str {
        let cdata = self.get_crate_data(cnum);
        decoder::get_crate_vers(cdata.data())
    }

    pub fn set_crate_data(&mut self, cnum: ast::CrateNum, data: @crate_metadata) {
        let mut metas = self.metas.borrow_mut();
        metas.get().insert(cnum, data);
    }

    pub fn have_crate_data(&self, cnum: ast::CrateNum) -> bool {
        let metas = self.metas.borrow();
        metas.get().contains_key(&cnum)
    }

    pub fn iter_crate_data(&self, i: |ast::CrateNum, @crate_metadata|) {
        let metas = self.metas.borrow();
        for (&k, &v) in metas.get().iter() {
            i(k, v);
        }
    }

    pub fn add_used_crate_source(&mut self, src: CrateSource) {
        if !self.used_crate_sources.contains(&src) {
            self.used_crate_sources.push(src);
        }
    }

    pub fn get_used_crate_sources<'a>(&'a self) -> &'a [CrateSource] {
        self.used_crate_sources.as_slice()
    }

    pub fn get_used_crates(&self, prefer: LinkagePreference)
                           -> ~[(ast::CrateNum, Option<Path>)] {
        self.used_crate_sources.iter()
            .map(|src| (src.cnum, match prefer {
                RequireDynamic => src.dylib.clone(),
                RequireStatic => src.rlib.clone(),
            }))
            .collect()
    }

    pub fn add_used_library(&mut self, lib: ~str, kind: NativeLibaryKind) -> bool {
        assert!(!lib.is_empty());
        if self.used_libraries.iter().any(|&(ref x, _)| x == &lib) {
            return false;
        }
        self.used_libraries.push((lib, kind));
        true
    }

    pub fn get_used_libraries<'a>(&'a self) -> &'a [(~str, NativeLibaryKind)] {
        self.used_libraries.as_slice()
    }

    pub fn add_used_link_args(&mut self, args: &str) {
        for s in args.split(' ') {
            self.used_link_args.push(s.to_owned());
        }
    }

    pub fn get_used_link_args<'a>(&'a self) -> &'a [~str] {
        self.used_link_args.as_slice()
    }

    pub fn add_extern_mod_stmt_cnum(&mut self,
                                    emod_id: ast::NodeId,
                                    cnum: ast::CrateNum) {
        self.extern_mod_crate_map.insert(emod_id, cnum);
    }

    pub fn find_extern_mod_stmt_cnum(&self, emod_id: ast::NodeId)
                                     -> Option<ast::CrateNum> {
        self.extern_mod_crate_map.find(&emod_id).map(|x| *x)
    }

    // returns hashes of crates directly used by this crate. Hashes are sorted by
    // (crate name, crate version, crate hash) in lexicographic order (not semver)
    pub fn get_dep_hashes(&self) -> ~[@str] {
        let mut result = ~[];

        for (_, &cnum) in self.extern_mod_crate_map.iter() {
            let cdata = self.get_crate_data(cnum);
            let hash = decoder::get_crate_hash(cdata.data());
            let vers = decoder::get_crate_vers(cdata.data());
            debug!("Add hash[{}]: {} {}", cdata.name, vers, hash);
            result.push(crate_hash {
                name: cdata.name,
                vers: vers,
                hash: hash
            });
        }

        result.sort();

        debug!("sorted:");
        for x in result.iter() {
            debug!("  hash[{}]: {}", x.name, x.hash);
        }

        result.map(|ch| ch.hash)
    }
}

#[deriving(Clone, TotalEq, TotalOrd)]
struct crate_hash {
    name: @str,
    vers: @str,
    hash: @str,
}

impl crate_metadata {
    pub fn data<'a>(&'a self) -> &'a [u8] { self.data.as_slice() }
}

impl MetadataBlob {
    pub fn as_slice<'a>(&'a self) -> &'a [u8] {
        match *self {
            MetadataVec(ref vec) => vec.as_slice(),
            MetadataArchive(ref ar) => ar.as_slice(),
        }
    }
}
<commit_msg>librustc: De-`@mut` `CStore::extern_mod_crate_map`<commit_after>// Copyright 2012 The Rust Project Developers. See the COPYRIGHT
// file at the top-level directory of this distribution and at
// http://rust-lang.org/COPYRIGHT.
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.

// The crate store - a central repo for information collected about external
// crates and libraries

use metadata::decoder;
use metadata::loader;

use std::cell::RefCell;
use std::hashmap::HashMap;
use syntax::ast;
use syntax::parse::token::ident_interner;

// A map from external crate numbers (as decoded from some crate file) to
// local crate numbers (as generated during this session). Each external
// crate may refer to types in other external crates, and each has their
// own crate numbers.
pub type cnum_map = @mut HashMap<ast::CrateNum, ast::CrateNum>;

pub enum MetadataBlob {
    MetadataVec(~[u8]),
    MetadataArchive(loader::ArchiveMetadata),
}

pub struct crate_metadata {
    name: @str,
    data: MetadataBlob,
    cnum_map: cnum_map,
    cnum: ast::CrateNum
}

#[deriving(Eq)]
pub enum LinkagePreference {
    RequireDynamic,
    RequireStatic,
}

#[deriving(Eq, FromPrimitive)]
pub enum NativeLibaryKind {
    NativeStatic,    // native static library (.a archive)
    NativeFramework, // OSX-specific
    NativeUnknown,   // default way to specify a dynamic library
}

// Where a crate came from on the local filesystem. One of these two options
// must be non-None.
#[deriving(Eq)]
pub struct CrateSource {
    dylib: Option<Path>,
    rlib: Option<Path>,
    cnum: ast::CrateNum,
}

pub struct CStore {
    priv metas: RefCell<HashMap<ast::CrateNum, @crate_metadata>>,
    priv extern_mod_crate_map: RefCell<extern_mod_crate_map>,
    priv used_crate_sources: ~[CrateSource],
    priv used_libraries: ~[(~str, NativeLibaryKind)],
    priv used_link_args: ~[~str],
    intr: @ident_interner
}

// Map from NodeId's of local extern mod statements to crate numbers
type extern_mod_crate_map = HashMap<ast::NodeId, ast::CrateNum>;

impl CStore {
    pub fn new(intr: @ident_interner) -> CStore {
        CStore {
            metas: RefCell::new(HashMap::new()),
            extern_mod_crate_map: RefCell::new(HashMap::new()),
            used_crate_sources: ~[],
            used_libraries: ~[],
            used_link_args: ~[],
            intr: intr
        }
    }

    pub fn get_crate_data(&self, cnum: ast::CrateNum) -> @crate_metadata {
        let metas = self.metas.borrow();
        *metas.get().get(&cnum)
    }

    pub fn get_crate_hash(&self, cnum: ast::CrateNum) -> @str {
        let cdata = self.get_crate_data(cnum);
        decoder::get_crate_hash(cdata.data())
    }

    pub fn get_crate_vers(&self, cnum: ast::CrateNum) -> @str {
        let cdata = self.get_crate_data(cnum);
        decoder::get_crate_vers(cdata.data())
    }

    pub fn set_crate_data(&mut self, cnum: ast::CrateNum, data: @crate_metadata) {
        let mut metas = self.metas.borrow_mut();
        metas.get().insert(cnum, data);
    }

    pub fn have_crate_data(&self, cnum: ast::CrateNum) -> bool {
        let metas = self.metas.borrow();
        metas.get().contains_key(&cnum)
    }

    pub fn iter_crate_data(&self, i: |ast::CrateNum, @crate_metadata|) {
        let metas = self.metas.borrow();
        for (&k, &v) in metas.get().iter() {
            i(k, v);
        }
    }

    pub fn add_used_crate_source(&mut self, src: CrateSource) {
        if !self.used_crate_sources.contains(&src) {
            self.used_crate_sources.push(src);
        }
    }

    pub fn get_used_crate_sources<'a>(&'a self) -> &'a [CrateSource] {
        self.used_crate_sources.as_slice()
    }

    pub fn get_used_crates(&self, prefer: LinkagePreference)
                           -> ~[(ast::CrateNum, Option<Path>)] {
        self.used_crate_sources.iter()
            .map(|src| (src.cnum, match prefer {
                RequireDynamic => src.dylib.clone(),
                RequireStatic => src.rlib.clone(),
            }))
            .collect()
    }

    pub fn add_used_library(&mut self, lib: ~str, kind: NativeLibaryKind) -> bool {
        assert!(!lib.is_empty());
        if self.used_libraries.iter().any(|&(ref x, _)| x == &lib) {
            return false;
        }
        self.used_libraries.push((lib, kind));
        true
    }

    pub fn get_used_libraries<'a>(&'a self) -> &'a [(~str, NativeLibaryKind)] {
        self.used_libraries.as_slice()
    }

    pub fn add_used_link_args(&mut self, args: &str) {
        for s in args.split(' ') {
            self.used_link_args.push(s.to_owned());
        }
    }

    pub fn get_used_link_args<'a>(&'a self) -> &'a [~str] {
        self.used_link_args.as_slice()
    }

    pub fn add_extern_mod_stmt_cnum(&mut self,
                                    emod_id: ast::NodeId,
                                    cnum: ast::CrateNum) {
        let mut extern_mod_crate_map = self.extern_mod_crate_map.borrow_mut();
        extern_mod_crate_map.get().insert(emod_id, cnum);
    }

    pub fn find_extern_mod_stmt_cnum(&self, emod_id: ast::NodeId)
                                     -> Option<ast::CrateNum> {
        let extern_mod_crate_map = self.extern_mod_crate_map.borrow();
        extern_mod_crate_map.get().find(&emod_id).map(|x| *x)
    }

    // returns hashes of crates directly used by this crate. Hashes are sorted by
    // (crate name, crate version, crate hash) in lexicographic order (not semver)
    pub fn get_dep_hashes(&self) -> ~[@str] {
        let mut result = ~[];

        let extern_mod_crate_map = self.extern_mod_crate_map.borrow();
        for (_, &cnum) in extern_mod_crate_map.get().iter() {
            let cdata = self.get_crate_data(cnum);
            let hash = decoder::get_crate_hash(cdata.data());
            let vers = decoder::get_crate_vers(cdata.data());
            debug!("Add hash[{}]: {} {}", cdata.name, vers, hash);
            result.push(crate_hash {
                name: cdata.name,
                vers: vers,
                hash: hash
            });
        }

        result.sort();

        debug!("sorted:");
        for x in result.iter() {
            debug!("  hash[{}]: {}", x.name, x.hash);
        }

        result.map(|ch| ch.hash)
    }
}

#[deriving(Clone, TotalEq, TotalOrd)]
struct crate_hash {
    name: @str,
    vers: @str,
    hash: @str,
}

impl crate_metadata {
    pub fn data<'a>(&'a self) -> &'a [u8] { self.data.as_slice() }
}

impl MetadataBlob {
    pub fn as_slice<'a>(&'a self) -> &'a [u8] {
        match *self {
            MetadataVec(ref vec) => vec.as_slice(),
            MetadataArchive(ref ar) => ar.as_slice(),
        }
    }
}
<|endoftext|>


<commit_before>use cucumber::{t, Steps};
use mimir2::{
    adapters::primary::bragi::autocomplete::{build_query, Filters},
    adapters::secondary::elasticsearch,
    domain::ports::remote::Remote,
    domain::ports::search::SearchParameters,
    domain::usecases::search_documents::{SearchDocuments, SearchDocumentsParameters},
    domain::usecases::UseCase,
};
use mimir2::{
    adapters::secondary::elasticsearch::{
        internal::{IndexConfiguration, IndexMappings, IndexParameters, IndexSettings},
        remote::{connection_test_pool, Error as PoolError},
    },
    domain::ports::remote::Error as ConnectionError,
};
use places::{addr::Addr, admin::Admin, poi::Poi, stop::Stop, street::Street, MimirObject};
use snafu::{ResultExt, Snafu};

use std::path::PathBuf;
use tokio::io::AsyncWriteExt;
use url::Url;

pub fn steps() -> Steps<crate::MyWorld> {
    let mut steps: Steps<crate::MyWorld> = Steps::new();

    steps.given_regex_async(
        "osm file has been downloaded for (.*)",
        t!(|mut world, ctx| {
            let region = ctx.matches[1].clone();
            world
                .processing_step
                .insert(download_osm(&region).await.unwrap());
            world
        }),
    );

    steps.given_regex_async(
        "osm file has been processed by cosmogony for (.*)",
        t!(|mut world, ctx| {
            let region = ctx.matches[1].clone();
            let previous = world.processing_step.clone().unwrap(); // we must have done something before, file either skipped or downloaded.
            world
                .processing_step
                .insert(generate_cosmogony(&region, previous).await.unwrap());
            world
        }),
    );

    steps.given_regex_async(
        "cosmogony file has been indexed for (.*)",
        t!(|mut world, ctx| {
            let region = ctx.matches[1].clone();
            let previous = world.processing_step.clone().unwrap(); // we must have done something before, file either skipped or downloaded.
            world
                .processing_step
                .insert(index_cosmogony(&region, previous).await.unwrap());
            world
        }),
    );

    steps.when_regex_async(
        "the user searches for \"(.*)\"",
        t!(|mut world, ctx| {
            let pool = elasticsearch::remote::connection_test_pool().await.unwrap();

            let client = pool.conn().await.unwrap();

            let search_documents = SearchDocuments::new(Box::new(client));

            let filters = Filters::default();

            let query = build_query(&ctx.matches[1], filters, &["fr"], &world.query_settings);

            let parameters = SearchDocumentsParameters {
                parameters: SearchParameters {
                    dsl: query,
                    doc_types: vec![String::from(Admin::doc_type())],
                },
            };
            world.search_result = search_documents.execute(parameters).await.unwrap();
            world
        }),
    );

    steps.then_regex("he finds \"(.*)\" as the first result", |world, ctx| {
        let rank = rank(&ctx.matches[1], &world.search_result).unwrap();
        assert_eq!(rank, 0);
        world
    });

    steps
}

#[derive(PartialEq, Clone)]
pub enum ProcessingStep {
    Downloaded,
    Generated,
    Indexed,
    Skipped,
}

#[derive(Debug, Snafu)]
pub enum Error {
    #[snafu(display("Invalid Download URL: {} ({})", source, details))]
    InvalidUrl {
        details: String,
        source: url::ParseError,
    },
    #[snafu(display("Invalid IO: {} ({})", source, details))]
    InvalidIO {
        details: String,
        source: std::io::Error,
    },
    #[snafu(display("Download Error: {} ({})", source, details))]
    DownloadError {
        details: String,
        source: reqwest::Error,
    },
    #[snafu(display("Elasticsearch Pool Error: {} ({})", source, details))]
    ElasticsearchPoolError { details: String, source: PoolError },
    #[snafu(display("Elasticsearch Connection Error: {} ({})", source, details))]
    ElasticsearchConnectionError {
        details: String,
        source: ConnectionError,
    },
    #[snafu(display("Indexing Error: {}", details))]
    IndexingError { details: String },

    #[snafu(display("JSON Error: {} ({})", details, source))]
    JsonError {
        details: String,
        source: serde_json::Error,
    },
}

// Given the name of a french region, it will download the matching OSM file
// If that file is already in the local filesystem, then we skip the download.
// This function makes several assumptions:
// 1. The name of the region is one found in http://download.geofabrik.de/europe/france.html
// 2. The file will be downloaded to `tests/data/osm` under the project's root (identified
//    by the CARGO_MANIFEST_DIR environment variable
// The function returns either `ProcessingStep::Skipped` or `ProcessingStep::Downloaded`,
// or an error.
async fn download_osm(region: &str) -> Result<ProcessingStep, Error> {
    // Here we make sure that there is a folder tests/data/osm at the root of the project.
    let path: &'static str = env!("CARGO_MANIFEST_DIR");
    let mut path = PathBuf::from(path);
    path.push("tests");
    path.push("data");
    if tokio::fs::metadata(&path).await.is_err() {
        tokio::fs::create_dir(&path).await.context(InvalidIO {
            details: format!("could no create directory {}", path.display()),
        })?;
    }
    path.push("osm");
    if tokio::fs::metadata(&path).await.is_err() {
        tokio::fs::create_dir(&path).await.context(InvalidIO {
            details: format!("could no create directory {}", path.display()),
        })?;
    }
    // Then we try to see if there is already a file with the expected name in that
    // folder, in which case we skip the actual download, to save time.
    let filename = format!("{}-latest.osm.pbf", region);
    path.push(&filename);
    if tokio::fs::metadata(&path).await.is_ok() {
        return Ok(ProcessingStep::Skipped);
    }

    // Ok, directory structure is fine, no file in 'cache', so we download
    let url = format!(
        "https://download.geofabrik.de/europe/france/{}-latest.osm.pbf",
        region
    );
    let url = Url::parse(&url).context(InvalidUrl {
        details: String::from(url),
    })?;

    let resp = reqwest::get(url.clone()).await.context(DownloadError {
        details: format!("could not download url {}", url),
    })?;

    // If we got a response which is an error (eg 404), then turn it
    // into an Error.
    let mut resp = resp.error_for_status().context(DownloadError {
        details: format!("download response error {}", url),
    })?;

    let file = tokio::fs::File::create(&path).await.context(InvalidIO {
        details: format!("could no create file {}", path.display()),
    })?;

    // Do an asynchronous, buffered copy of the download to the output file
    let mut file = tokio::io::BufWriter::new(file);

    while let Some(chunk) = resp.chunk().await.context(DownloadError {
        details: String::from("read chunk"),
    })? {
        file.write(&chunk).await.context(InvalidIO {
            details: String::from("write chunk"),
        })?;
    }

    file.flush().await.context(InvalidIO {
        details: String::from("flush"),
    })?;

    Ok(ProcessingStep::Downloaded)
}

// Generate a cosmogony file given the region
// The function makes several assumptions:
// 1. The OSM file has previously been downloaded into the expected folder (tests/data/osm)
// 2. The output is a jsonl.gz
// 3. The output will be stored in tests/data/cosmogony
//
// The OSM file will be processed if:
// 1. The output file is not found
// 2. If the output file is found and the previous step is 'downloaded' (that is it's probably a
//    new OSM file and we need to generate a new cosmogony file.
async fn generate_cosmogony(
    region: &str,
    previous: ProcessingStep,
) -> Result<ProcessingStep, Error> {
    let path: &'static str = env!("CARGO_MANIFEST_DIR");
    let mut input_path: PathBuf = [path, "tests", "data", "osm"].iter().collect();
    let filename = format!("{}-latest.osm.pbf", region);
    input_path.push(&filename);
    let mut output_path: PathBuf = [path, "tests", "data", "cosmogony"].iter().collect();
    if tokio::fs::metadata(&output_path).await.is_err() {
        tokio::fs::create_dir(&output_path)
            .await
            .context(InvalidIO {
                details: format!("could no create directory {}", output_path.display()),
            })?;
    }
    let filename = format!("{}.jsonl.gz", region);
    output_path.push(&filename);
    // if the output already exists, and the input is not a new file, then skip the generation
    if (tokio::fs::metadata(&output_path).await.is_ok()) && (previous != ProcessingStep::Downloaded)
    {
        return Ok(ProcessingStep::Skipped);
    }
    let mut child = tokio::process::Command::new(
        "/home/matt/lab/rust/kisio/cosmogony/target/release/cosmogony",
    )
    .arg("--country-code")
    .arg("FR")
    .arg("--input")
    .arg(&input_path)
    .arg("--output")
    .arg(&output_path)
    .spawn()
    .expect("failed to spawn cosmogony");

    let status = child.wait().await.context(InvalidIO {
        details: format!(
            "failed to generate cosmogony with input {} and output {}",
            input_path.display(),
            output_path.display()
        ),
    })?;

    // TODO Do something with the status?

    Ok(ProcessingStep::Generated)
}

async fn index_cosmogony(region: &str, previous: ProcessingStep) -> Result<ProcessingStep, Error> {
    // if the previous step is 'generated', then we need to index the cosmogony file.
    // Otherwise, we skip.
    if previous != ProcessingStep::Generated {
        return Ok(ProcessingStep::Skipped);
    }
    let pool = connection_test_pool()
        .await
        .context(ElasticsearchPoolError {
            details: String::from("Could not retrieve Elasticsearch test pool"),
        })?;

    let client = pool.conn().await.context(ElasticsearchConnectionError {
        details: String::from("Could not establish connection to Elasticsearch"),
    })?;

    let path: &'static str = env!("CARGO_MANIFEST_DIR");
    let config_path: PathBuf = [path, "config", "admin"].iter().collect();
    let mut settings_path = config_path.clone();
    settings_path.push("settings.json");
    let settings = tokio::fs::read_to_string(&settings_path)
        .await
        .context(InvalidIO {
            details: format!(
                "could not read settings file from '{}'",
                settings_path.display()
            ),
        })?;

    let settings = serde_json::from_str(&settings).context(JsonError {
        details: String::from("Could not deserialize settings"),
    })?;

    let mut mappings_path = config_path.clone();
    mappings_path.push("mappings.json");
    let mappings = tokio::fs::read_to_string(&mappings_path)
        .await
        .context(InvalidIO {
            details: format!(
                "could not read mappings file from '{}'",
                mappings_path.display()
            ),
        })?;

    let mappings = serde_json::from_str(&mappings).context(JsonError {
        details: String::from("Could not deserialize settings"),
    })?;

    let config = IndexConfiguration {
        name: String::from("test"),
        parameters: IndexParameters {
            timeout: String::from("10s"),
            wait_for_active_shards: String::from("1"), // only the primary shard
        },
        settings: IndexSettings { value: settings },
        mappings: IndexMappings { value: mappings },
    };

    let mut input_path: PathBuf = [path, "tests", "data", "cosmogony"].iter().collect();
    let filename = format!("{}.jsonl.gz", region);
    input_path.push(&filename);
    mimirsbrunn::admin::index_cosmogony(
        input_path.into_os_string().into_string().unwrap(),
        vec![String::from("fr")],
        config,
        client,
    )
    .await
    .map_err(|err| Error::IndexingError {
        details: format!("could not index cosmogony: {}", err.to_string(),),
    })?;
    Ok(ProcessingStep::Indexed)
}

pub fn rank(id: &str, list: &[serde_json::Value]) -> Option<usize> {
    list.iter()
        .enumerate()
        // .find(|(_i, v)| v.as_object().unwrap().get("id").unwrap().as_str().unwrap() == id)
        .find(|(_i, v)| {
            let idr = v.as_object().unwrap().get("id").unwrap().as_str().unwrap();
            idr == id
        })
        .map(|(i, _r)| i)
}
<commit_msg>Fix Query paramater in admin integration tests<commit_after>use cucumber::{t, Steps};
use mimir2::{
    adapters::primary::bragi::autocomplete::{build_query, Filters},
    adapters::secondary::elasticsearch,
    domain::model::query::Query,
    domain::ports::remote::Remote,
    domain::ports::search::SearchParameters,
    domain::usecases::search_documents::{SearchDocuments, SearchDocumentsParameters},
    domain::usecases::UseCase,
};
use mimir2::{
    adapters::secondary::elasticsearch::{
        internal::{IndexConfiguration, IndexMappings, IndexParameters, IndexSettings},
        remote::{connection_test_pool, Error as PoolError},
    },
    domain::ports::remote::Error as ConnectionError,
};
use places::{addr::Addr, admin::Admin, poi::Poi, stop::Stop, street::Street, MimirObject};
use snafu::{ResultExt, Snafu};

use std::path::PathBuf;
use tokio::io::AsyncWriteExt;
use url::Url;

pub fn steps() -> Steps<crate::MyWorld> {
    let mut steps: Steps<crate::MyWorld> = Steps::new();

    steps.given_regex_async(
        "osm file has been downloaded for (.*)",
        t!(|mut world, ctx| {
            let region = ctx.matches[1].clone();
            world
                .processing_step
                .insert(download_osm(&region).await.unwrap());
            world
        }),
    );

    steps.given_regex_async(
        "osm file has been processed by cosmogony for (.*)",
        t!(|mut world, ctx| {
            let region = ctx.matches[1].clone();
            let previous = world.processing_step.clone().unwrap(); // we must have done something before, file either skipped or downloaded.
            world
                .processing_step
                .insert(generate_cosmogony(&region, previous).await.unwrap());
            world
        }),
    );

    steps.given_regex_async(
        "cosmogony file has been indexed for (.*)",
        t!(|mut world, ctx| {
            let region = ctx.matches[1].clone();
            let previous = world.processing_step.clone().unwrap(); // we must have done something before, file either skipped or downloaded.
            world
                .processing_step
                .insert(index_cosmogony(&region, previous).await.unwrap());
            world
        }),
    );

    steps.when_regex_async(
        "the user searches for \"(.*)\"",
        t!(|mut world, ctx| {
            let pool = elasticsearch::remote::connection_test_pool().await.unwrap();

            let client = pool.conn().await.unwrap();

            let search_documents = SearchDocuments::new(Box::new(client));

            let filters = Filters::default();

            let dsl = build_query(&ctx.matches[1], filters, &["fr"], &world.query_settings);

            let parameters = SearchDocumentsParameters {
                parameters: SearchParameters {
                    query: Query::QueryDSL(dsl),
                    doc_types: vec![String::from(Admin::doc_type())],
                },
            };
            world.search_result = search_documents.execute(parameters).await.unwrap();
            world
        }),
    );

    steps.then_regex("he finds \"(.*)\" as the first result", |world, ctx| {
        let rank = rank(&ctx.matches[1], &world.search_result).unwrap();
        assert_eq!(rank, 0);
        world
    });

    steps
}

#[derive(PartialEq, Clone)]
pub enum ProcessingStep {
    Downloaded,
    Generated,
    Indexed,
    Skipped,
}

#[derive(Debug, Snafu)]
pub enum Error {
    #[snafu(display("Invalid Download URL: {} ({})", source, details))]
    InvalidUrl {
        details: String,
        source: url::ParseError,
    },
    #[snafu(display("Invalid IO: {} ({})", source, details))]
    InvalidIO {
        details: String,
        source: std::io::Error,
    },
    #[snafu(display("Download Error: {} ({})", source, details))]
    DownloadError {
        details: String,
        source: reqwest::Error,
    },
    #[snafu(display("Elasticsearch Pool Error: {} ({})", source, details))]
    ElasticsearchPoolError { details: String, source: PoolError },
    #[snafu(display("Elasticsearch Connection Error: {} ({})", source, details))]
    ElasticsearchConnectionError {
        details: String,
        source: ConnectionError,
    },
    #[snafu(display("Indexing Error: {}", details))]
    IndexingError { details: String },

    #[snafu(display("JSON Error: {} ({})", details, source))]
    JsonError {
        details: String,
        source: serde_json::Error,
    },
}

// Given the name of a french region, it will download the matching OSM file
// If that file is already in the local filesystem, then we skip the download.
// This function makes several assumptions:
// 1. The name of the region is one found in http://download.geofabrik.de/europe/france.html
// 2. The file will be downloaded to `tests/data/osm` under the project's root (identified
//    by the CARGO_MANIFEST_DIR environment variable
// The function returns either `ProcessingStep::Skipped` or `ProcessingStep::Downloaded`,
// or an error.
async fn download_osm(region: &str) -> Result<ProcessingStep, Error> {
    // Here we make sure that there is a folder tests/data/osm at the root of the project.
    let path: &'static str = env!("CARGO_MANIFEST_DIR");
    let mut path = PathBuf::from(path);
    path.push("tests");
    path.push("data");
    if tokio::fs::metadata(&path).await.is_err() {
        tokio::fs::create_dir(&path).await.context(InvalidIO {
            details: format!("could no create directory {}", path.display()),
        })?;
    }
    path.push("osm");
    if tokio::fs::metadata(&path).await.is_err() {
        tokio::fs::create_dir(&path).await.context(InvalidIO {
            details: format!("could no create directory {}", path.display()),
        })?;
    }
    // Then we try to see if there is already a file with the expected name in that
    // folder, in which case we skip the actual download, to save time.
    let filename = format!("{}-latest.osm.pbf", region);
    path.push(&filename);
    if tokio::fs::metadata(&path).await.is_ok() {
        return Ok(ProcessingStep::Skipped);
    }

    // Ok, directory structure is fine, no file in 'cache', so we download
    let url = format!(
        "https://download.geofabrik.de/europe/france/{}-latest.osm.pbf",
        region
    );
    let url = Url::parse(&url).context(InvalidUrl {
        details: String::from(url),
    })?;

    let resp = reqwest::get(url.clone()).await.context(DownloadError {
        details: format!("could not download url {}", url),
    })?;

    // If we got a response which is an error (eg 404), then turn it
    // into an Error.
    let mut resp = resp.error_for_status().context(DownloadError {
        details: format!("download response error {}", url),
    })?;

    let file = tokio::fs::File::create(&path).await.context(InvalidIO {
        details: format!("could no create file {}", path.display()),
    })?;

    // Do an asynchronous, buffered copy of the download to the output file
    let mut file = tokio::io::BufWriter::new(file);

    while let Some(chunk) = resp.chunk().await.context(DownloadError {
        details: String::from("read chunk"),
    })? {
        file.write(&chunk).await.context(InvalidIO {
            details: String::from("write chunk"),
        })?;
    }

    file.flush().await.context(InvalidIO {
        details: String::from("flush"),
    })?;

    Ok(ProcessingStep::Downloaded)
}

// Generate a cosmogony file given the region
// The function makes several assumptions:
// 1. The OSM file has previously been downloaded into the expected folder (tests/data/osm)
// 2. The output is a jsonl.gz
// 3. The output will be stored in tests/data/cosmogony
//
// The OSM file will be processed if:
// 1. The output file is not found
// 2. If the output file is found and the previous step is 'downloaded' (that is it's probably a
//    new OSM file and we need to generate a new cosmogony file.
async fn generate_cosmogony(
    region: &str,
    previous: ProcessingStep,
) -> Result<ProcessingStep, Error> {
    let path: &'static str = env!("CARGO_MANIFEST_DIR");
    let mut input_path: PathBuf = [path, "tests", "data", "osm"].iter().collect();
    let filename = format!("{}-latest.osm.pbf", region);
    input_path.push(&filename);
    let mut output_path: PathBuf = [path, "tests", "data", "cosmogony"].iter().collect();
    if tokio::fs::metadata(&output_path).await.is_err() {
        tokio::fs::create_dir(&output_path)
            .await
            .context(InvalidIO {
                details: format!("could no create directory {}", output_path.display()),
            })?;
    }
    let filename = format!("{}.jsonl.gz", region);
    output_path.push(&filename);
    // if the output already exists, and the input is not a new file, then skip the generation
    if (tokio::fs::metadata(&output_path).await.is_ok()) && (previous != ProcessingStep::Downloaded)
    {
        return Ok(ProcessingStep::Skipped);
    }
    let mut child = tokio::process::Command::new(
        "/home/matt/lab/rust/kisio/cosmogony/target/release/cosmogony",
    )
    .arg("--country-code")
    .arg("FR")
    .arg("--input")
    .arg(&input_path)
    .arg("--output")
    .arg(&output_path)
    .spawn()
    .expect("failed to spawn cosmogony");

    let status = child.wait().await.context(InvalidIO {
        details: format!(
            "failed to generate cosmogony with input {} and output {}",
            input_path.display(),
            output_path.display()
        ),
    })?;

    // TODO Do something with the status?

    Ok(ProcessingStep::Generated)
}

async fn index_cosmogony(region: &str, previous: ProcessingStep) -> Result<ProcessingStep, Error> {
    // if the previous step is 'generated', then we need to index the cosmogony file.
    // Otherwise, we skip.
    if previous != ProcessingStep::Generated {
        return Ok(ProcessingStep::Skipped);
    }
    let pool = connection_test_pool()
        .await
        .context(ElasticsearchPoolError {
            details: String::from("Could not retrieve Elasticsearch test pool"),
        })?;

    let client = pool.conn().await.context(ElasticsearchConnectionError {
        details: String::from("Could not establish connection to Elasticsearch"),
    })?;

    let path: &'static str = env!("CARGO_MANIFEST_DIR");
    let config_path: PathBuf = [path, "config", "admin"].iter().collect();
    let mut settings_path = config_path.clone();
    settings_path.push("settings.json");
    let settings = tokio::fs::read_to_string(&settings_path)
        .await
        .context(InvalidIO {
            details: format!(
                "could not read settings file from '{}'",
                settings_path.display()
            ),
        })?;

    let settings = serde_json::from_str(&settings).context(JsonError {
        details: String::from("Could not deserialize settings"),
    })?;

    let mut mappings_path = config_path.clone();
    mappings_path.push("mappings.json");
    let mappings = tokio::fs::read_to_string(&mappings_path)
        .await
        .context(InvalidIO {
            details: format!(
                "could not read mappings file from '{}'",
                mappings_path.display()
            ),
        })?;

    let mappings = serde_json::from_str(&mappings).context(JsonError {
        details: String::from("Could not deserialize settings"),
    })?;

    let config = IndexConfiguration {
        name: String::from("test"),
        parameters: IndexParameters {
            timeout: String::from("10s"),
            wait_for_active_shards: String::from("1"), // only the primary shard
        },
        settings: IndexSettings { value: settings },
        mappings: IndexMappings { value: mappings },
    };

    let mut input_path: PathBuf = [path, "tests", "data", "cosmogony"].iter().collect();
    let filename = format!("{}.jsonl.gz", region);
    input_path.push(&filename);
    mimirsbrunn::admin::index_cosmogony(
        input_path.into_os_string().into_string().unwrap(),
        vec![String::from("fr")],
        config,
        client,
    )
    .await
    .map_err(|err| Error::IndexingError {
        details: format!("could not index cosmogony: {}", err.to_string(),),
    })?;
    Ok(ProcessingStep::Indexed)
}

pub fn rank(id: &str, list: &[serde_json::Value]) -> Option<usize> {
    list.iter()
        .enumerate()
        // .find(|(_i, v)| v.as_object().unwrap().get("id").unwrap().as_str().unwrap() == id)
        .find(|(_i, v)| {
            let idr = v.as_object().unwrap().get("id").unwrap().as_str().unwrap();
            idr == id
        })
        .map(|(i, _r)| i)
}
<|endoftext|>


<commit_before>/// Easy to use Let's Encrypt client to issue and renew TLS certs

extern crate acme_client;
extern crate clap;
extern crate env_logger;

use std::io::{self, Write};
use acme_client::Directory;
use acme_client::error::Result;
use clap::{Arg, App, SubCommand, ArgMatches};

fn main() {
    let matches = App::new(env!("CARGO_PKG_NAME"))
        .version(env!("CARGO_PKG_VERSION"))
        .about(env!("CARGO_PKG_DESCRIPTION"))
        .usage("letsencrypt-rs sign -D example.org -P /var/www -k domain.key -o domain.crt\
                \n    letsencrypt-rs revoke -K user_or_domain.key -C signed.crt")
        .subcommand(SubCommand::with_name("sign")
            .about("Signs a certificate")
            .display_order(1)
            .arg(Arg::with_name("USER_KEY_PATH")
                .help("User private key path to use it in account registration.")
                .long("user-key")
                .short("U")
                .takes_value(true))
            .arg(Arg::with_name("DOMAIN_KEY_PATH")
                .help("Domain private key path to use it in CSR generation.")
                .short("K")
                .long("domain-key")
                .takes_value(true))
            .arg(Arg::with_name("DOMAIN")
                .help("Domain name to obtain certificate. You can use more than one domain name.")
                .short("D")
                .long("domain")
                .multiple(true)
                .required(true)
                .takes_value(true))
            .arg(Arg::with_name("PUBLIC_DIR")
                .help("Directory to save ACME simple http challenge. This option is required.")
                .short("P")
                .long("public-dir")
                .takes_value(true))
            .arg(Arg::with_name("EMAIL")
                .help("Contact email address (optional).")
                .short("E")
                .long("email")
                .takes_value(true))
            .arg(Arg::with_name("DOMAIN_CSR")
                .help("Path to domain certificate signing request.")
                .short("C")
                .long("domain-csr")
                .takes_value(true))
            .arg(Arg::with_name("SAVE_USER_KEY")
                .help("Path to save private user key.")
                .long("save-user-key")
                .short("u")
                .takes_value(true))
            .arg(Arg::with_name("SAVE_DOMAIN_KEY")
                .help("Path to save domain private key.")
                .short("k")
                .long("save-domain-key")
                .takes_value(true))
            .arg(Arg::with_name("SAVE_DOMAIN_CSR")
                .help("Path to save domain certificate signing request.")
                .long("save-csr")
                .short("S")
                .takes_value(true))
            .arg(Arg::with_name("SAVE_SIGNED_CERTIFICATE")
                .help("Path to save signed certificate. Default is STDOUT.")
                .short("o")
                .long("save-crt")
                .takes_value(true))
            .arg(Arg::with_name("CHAIN")
                .help("Chains the signed certificate with Let's Encrypt Authority X3 \
                       (IdenTrust cross-signed) intermediate certificate.")
                .short("c")
                .long("chain")
                .takes_value(false))
            .arg(Arg::with_name("DNS_CHALLENGE")
                 .help("Use DNS challenge instead of HTTP. This option requires user \
                        to generate a TXT record for domain")
                 .short("d")
                 .long("dns")
                 .takes_value(false)))
        .subcommand(SubCommand::with_name("revoke")
            .about("Revokes a signed certificate")
            .display_order(2)
            .arg(Arg::with_name("USER_KEY")
                .help("User or domain private key path.")
                .long("user-key")
                .short("K")
                .required(true)
                .takes_value(true))
            .arg(Arg::with_name("SIGNED_CRT")
                .help("Path to signed domain certificate to revoke.")
                .long("signed-crt")
                .short("C")
                .required(true)
                .takes_value(true)))
        .arg(Arg::with_name("verbose")
             .help("Show verbose output")
             .short("v")
             .multiple(true))
        .get_matches();

    init_logger(matches.occurrences_of("verbose"));

    let res = if let Some(matches) = matches.subcommand_matches("sign") {
        sign_certificate(matches)
    } else if let Some(matches) = matches.subcommand_matches("revoke") {
        revoke_certificate(matches)
    } else {
        println!("{}", matches.usage());
        Ok(())
    };

    if let Err(e) = res {
        writeln!(io::stderr(), "{}", e.description()).expect("Failed to write stderr");
        ::std::process::exit(1);
    }
}

fn sign_certificate(matches: &ArgMatches) -> Result<()> {
    let directory = Directory::lets_encrypt()?;

    let mut account_registration = directory.account_registration();

    if let Some(email) = matches.value_of("EMAIL") {
        account_registration = account_registration.email(email);
    }

    if let Some(user_key_path) = matches.value_of("USER_KEY_PATH") {
        account_registration = account_registration.pkey_from_file(user_key_path)?;
    }

    let account = account_registration.register()?;
    let domains: Vec<_> = matches.values_of("DOMAIN")
        .ok_or("You need to provide at least one domain name")?
        .collect();
    for domain in &domains {
        let authorization = account.authorization(domain)?;
        if !matches.is_present("DNS_CHALLENGE") {
            let challenge = authorization.get_http_challenge().ok_or("HTTP challenge not found")?;
            challenge.save_key_authorization(matches.value_of("PUBLIC_DIR")
                                                 .ok_or("--public-dir not defined. \
                                                            You need to define a public \
                                                            directory to use http challenge \
                                                            verification")?)?;
            challenge.validate()?;
        } else {
            let challenge = authorization.get_dns_challenge().ok_or("DNS challenge not found")?;
            println!("Please create a TXT record for _acme-challenge.{}: {}\n\
                      Press enter to continue",
                     domain,
                     challenge.signature()?);
            io::stdin().read_line(&mut String::new()).unwrap();
            challenge.validate()?;
        }
    }

    let mut certificate_signer = account.certificate_signer(&domains);

    if let Some(domain_key_path) = matches.value_of("DOMAIN_KEY_PATH") {
        if let Some(csr_path) = matches.value_of("DOMAIN_CSR") {
            certificate_signer = certificate_signer.csr_from_file(domain_key_path, csr_path)?;
        } else {
            certificate_signer = certificate_signer.pkey_from_file(domain_key_path)?;
        }
    }

    let certificate = certificate_signer.sign_certificate()?;
    let signed_certificate_path = matches.value_of("SAVE_SIGNED_CERTIFICATE")
        .ok_or("You need to save signed certificate")?;
    if matches.is_present("CHAIN") {
        certificate.save_signed_certificate_and_chain(None, signed_certificate_path)?;
    } else {
        certificate.save_signed_certificate(signed_certificate_path)?;
    }

    if let Some(path) = matches.value_of("SAVE_DOMAIN_KEY") {
        certificate.save_private_key(path)?;
    }
    if let Some(path) = matches.value_of("SAVE_DOMAIN_CSR") {
        certificate.save_csr(path)?;
    }
    if let Some(path) = matches.value_of("SAVE_USER_KEY") {
        account.save_private_key(path)?;
    }

    Ok(())
}

fn revoke_certificate(matches: &ArgMatches) -> Result<()> {
    let directory = Directory::lets_encrypt()?;
    let account = directory.account_registration()
        .pkey_from_file(matches.value_of("USER_KEY_PATH")
                            .ok_or("You need to provide user \
                                   or domain private key used \
                                   to sign certificate.")?)?
        .register()?;
    account.revoke_certificate_from_file(matches.value_of("SIGNED_CRT")
                                             .ok_or("You need to provide \
                                                    a signed certificate to \
                                                    revoke.")?)?;
    Ok(())
}

fn init_logger(level: u64) {
    let level = match level {
        0 => "",
        1 => "acme_client=info",
        _ => "acme_client=debug",
    };
    let mut builder = env_logger::LogBuilder::new();
    builder.parse(&::std::env::var("RUST_LOG").unwrap_or(level.to_owned()));
    let _ = builder.init();
}
<commit_msg>same parameter name for USER_KEY<commit_after>/// Easy to use Let's Encrypt client to issue and renew TLS certs

extern crate acme_client;
extern crate clap;
extern crate env_logger;

use std::io::{self, Write};
use acme_client::Directory;
use acme_client::error::Result;
use clap::{Arg, App, SubCommand, ArgMatches};

fn main() {
    let matches = App::new(env!("CARGO_PKG_NAME"))
        .version(env!("CARGO_PKG_VERSION"))
        .about(env!("CARGO_PKG_DESCRIPTION"))
        .usage("letsencrypt-rs sign -D example.org -P /var/www -k domain.key -o domain.crt\
                \n    letsencrypt-rs revoke -K user_or_domain.key -C signed.crt")
        .subcommand(SubCommand::with_name("sign")
            .about("Signs a certificate")
            .display_order(1)
            .arg(Arg::with_name("USER_KEY_PATH")
                .help("User private key path to use it in account registration.")
                .long("user-key")
                .short("U")
                .takes_value(true))
            .arg(Arg::with_name("DOMAIN_KEY_PATH")
                .help("Domain private key path to use it in CSR generation.")
                .short("K")
                .long("domain-key")
                .takes_value(true))
            .arg(Arg::with_name("DOMAIN")
                .help("Domain name to obtain certificate. You can use more than one domain name.")
                .short("D")
                .long("domain")
                .multiple(true)
                .required(true)
                .takes_value(true))
            .arg(Arg::with_name("PUBLIC_DIR")
                .help("Directory to save ACME simple http challenge. This option is required.")
                .short("P")
                .long("public-dir")
                .takes_value(true))
            .arg(Arg::with_name("EMAIL")
                .help("Contact email address (optional).")
                .short("E")
                .long("email")
                .takes_value(true))
            .arg(Arg::with_name("DOMAIN_CSR")
                .help("Path to domain certificate signing request.")
                .short("C")
                .long("domain-csr")
                .takes_value(true))
            .arg(Arg::with_name("SAVE_USER_KEY")
                .help("Path to save private user key.")
                .long("save-user-key")
                .short("u")
                .takes_value(true))
            .arg(Arg::with_name("SAVE_DOMAIN_KEY")
                .help("Path to save domain private key.")
                .short("k")
                .long("save-domain-key")
                .takes_value(true))
            .arg(Arg::with_name("SAVE_DOMAIN_CSR")
                .help("Path to save domain certificate signing request.")
                .long("save-csr")
                .short("S")
                .takes_value(true))
            .arg(Arg::with_name("SAVE_SIGNED_CERTIFICATE")
                .help("Path to save signed certificate. Default is STDOUT.")
                .short("o")
                .long("save-crt")
                .takes_value(true))
            .arg(Arg::with_name("CHAIN")
                .help("Chains the signed certificate with Let's Encrypt Authority X3 \
                       (IdenTrust cross-signed) intermediate certificate.")
                .short("c")
                .long("chain")
                .takes_value(false))
            .arg(Arg::with_name("DNS_CHALLENGE")
                 .help("Use DNS challenge instead of HTTP. This option requires user \
                        to generate a TXT record for domain")
                 .short("d")
                 .long("dns")
                 .takes_value(false)))
        .subcommand(SubCommand::with_name("revoke")
            .about("Revokes a signed certificate")
            .display_order(2)
            .arg(Arg::with_name("USER_KEY")
                .help("User or domain private key path.")
                .long("user-key")
                .short("K")
                .required(true)
                .takes_value(true))
            .arg(Arg::with_name("SIGNED_CRT")
                .help("Path to signed domain certificate to revoke.")
                .long("signed-crt")
                .short("C")
                .required(true)
                .takes_value(true)))
        .arg(Arg::with_name("verbose")
             .help("Show verbose output")
             .short("v")
             .multiple(true))
        .get_matches();

    init_logger(matches.occurrences_of("verbose"));

    let res = if let Some(matches) = matches.subcommand_matches("sign") {
        sign_certificate(matches)
    } else if let Some(matches) = matches.subcommand_matches("revoke") {
        revoke_certificate(matches)
    } else {
        println!("{}", matches.usage());
        Ok(())
    };

    if let Err(e) = res {
        writeln!(io::stderr(), "{}", e.description()).expect("Failed to write stderr");
        ::std::process::exit(1);
    }
}

fn sign_certificate(matches: &ArgMatches) -> Result<()> {
    let directory = Directory::lets_encrypt()?;

    let mut account_registration = directory.account_registration();

    if let Some(email) = matches.value_of("EMAIL") {
        account_registration = account_registration.email(email);
    }

    if let Some(user_key_path) = matches.value_of("USER_KEY_PATH") {
        account_registration = account_registration.pkey_from_file(user_key_path)?;
    }

    let account = account_registration.register()?;
    let domains: Vec<_> = matches.values_of("DOMAIN")
        .ok_or("You need to provide at least one domain name")?
        .collect();
    for domain in &domains {
        let authorization = account.authorization(domain)?;
        if !matches.is_present("DNS_CHALLENGE") {
            let challenge = authorization.get_http_challenge().ok_or("HTTP challenge not found")?;
            challenge.save_key_authorization(matches.value_of("PUBLIC_DIR")
                                                 .ok_or("--public-dir not defined. \
                                                            You need to define a public \
                                                            directory to use http challenge \
                                                            verification")?)?;
            challenge.validate()?;
        } else {
            let challenge = authorization.get_dns_challenge().ok_or("DNS challenge not found")?;
            println!("Please create a TXT record for _acme-challenge.{}: {}\n\
                      Press enter to continue",
                     domain,
                     challenge.signature()?);
            io::stdin().read_line(&mut String::new()).unwrap();
            challenge.validate()?;
        }
    }

    let mut certificate_signer = account.certificate_signer(&domains);

    if let Some(domain_key_path) = matches.value_of("DOMAIN_KEY_PATH") {
        if let Some(csr_path) = matches.value_of("DOMAIN_CSR") {
            certificate_signer = certificate_signer.csr_from_file(domain_key_path, csr_path)?;
        } else {
            certificate_signer = certificate_signer.pkey_from_file(domain_key_path)?;
        }
    }

    let certificate = certificate_signer.sign_certificate()?;
    let signed_certificate_path = matches.value_of("SAVE_SIGNED_CERTIFICATE")
        .ok_or("You need to save signed certificate")?;
    if matches.is_present("CHAIN") {
        certificate.save_signed_certificate_and_chain(None, signed_certificate_path)?;
    } else {
        certificate.save_signed_certificate(signed_certificate_path)?;
    }

    if let Some(path) = matches.value_of("SAVE_DOMAIN_KEY") {
        certificate.save_private_key(path)?;
    }
    if let Some(path) = matches.value_of("SAVE_DOMAIN_CSR") {
        certificate.save_csr(path)?;
    }
    if let Some(path) = matches.value_of("SAVE_USER_KEY") {
        account.save_private_key(path)?;
    }

    Ok(())
}

fn revoke_certificate(matches: &ArgMatches) -> Result<()> {
    let directory = Directory::lets_encrypt()?;
    let account = directory.account_registration()
        .pkey_from_file(matches.value_of("USER_KEY")
                            .ok_or("You need to provide user \
                                   or domain private key used \
                                   to sign certificate.")?)?
        .register()?;
    account.revoke_certificate_from_file(matches.value_of("SIGNED_CRT")
                                             .ok_or("You need to provide \
                                                    a signed certificate to \
                                                    revoke.")?)?;
    Ok(())
}

fn init_logger(level: u64) {
    let level = match level {
        0 => "",
        1 => "acme_client=info",
        _ => "acme_client=debug",
    };
    let mut builder = env_logger::LogBuilder::new();
    builder.parse(&::std::env::var("RUST_LOG").unwrap_or(level.to_owned()));
    let _ = builder.init();
}
<|endoftext|>


<commit_before>//! A trait to represent a stream

use native_tls::{Protocol, TlsConnector, TlsStream};
use smtp::client::mock::MockStream;
use std::io::{self, ErrorKind, Read, Write};
use std::net::{Ipv4Addr, Shutdown, SocketAddr, SocketAddrV4, TcpStream};
use std::time::Duration;

/// Parameters to use for secure clients
#[derive(Clone)]
#[allow(missing_debug_implementations)]
pub struct ClientTlsParameters {
    /// A connector from `native-tls`
    pub connector: TlsConnector,
    /// The domain to send during the TLS handshake
    pub domain: String,
}

impl ClientTlsParameters {
    /// Creates a `ClientTlsParameters`
    pub fn new(domain: String, connector: TlsConnector) -> ClientTlsParameters {
        ClientTlsParameters { connector, domain }
    }
}

/// Accepted protocols by default.
/// This removes TLS 1.0 compared to tls-native defaults.
pub const DEFAULT_TLS_PROTOCOLS: &[Protocol] = &[Protocol::Tlsv11, Protocol::Tlsv12];

#[derive(Debug)]
/// Represents the different types of underlying network streams
pub enum NetworkStream {
    /// Plain TCP stream
    Tcp(TcpStream),
    /// Encrypted TCP stream
    Tls(TlsStream<TcpStream>),
    /// Mock stream
    Mock(MockStream),
}

impl NetworkStream {
    /// Returns peer's address
    pub fn peer_addr(&self) -> io::Result<SocketAddr> {
        match *self {
            NetworkStream::Tcp(ref s) => s.peer_addr(),
            NetworkStream::Tls(ref s) => s.get_ref().peer_addr(),
            NetworkStream::Mock(_) => Ok(SocketAddr::V4(SocketAddrV4::new(
                Ipv4Addr::new(127, 0, 0, 1),
                80,
            ))),
        }
    }

    /// Shutdowns the connection
    pub fn shutdown(&self, how: Shutdown) -> io::Result<()> {
        match *self {
            NetworkStream::Tcp(ref s) => s.shutdown(how),
            NetworkStream::Tls(ref s) => s.get_ref().shutdown(how),
            NetworkStream::Mock(_) => Ok(()),
        }
    }
}

impl Read for NetworkStream {
    fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> {
        match *self {
            NetworkStream::Tcp(ref mut s) => s.read(buf),
            NetworkStream::Tls(ref mut s) => s.read(buf),
            NetworkStream::Mock(ref mut s) => s.read(buf),
        }
    }
}

impl Write for NetworkStream {
    fn write(&mut self, buf: &[u8]) -> io::Result<usize> {
        match *self {
            NetworkStream::Tcp(ref mut s) => s.write(buf),
            NetworkStream::Tls(ref mut s) => s.write(buf),
            NetworkStream::Mock(ref mut s) => s.write(buf),
        }
    }

    fn flush(&mut self) -> io::Result<()> {
        match *self {
            NetworkStream::Tcp(ref mut s) => s.flush(),
            NetworkStream::Tls(ref mut s) => s.flush(),
            NetworkStream::Mock(ref mut s) => s.flush(),
        }
    }
}

/// A trait for the concept of opening a stream
pub trait Connector: Sized {
    /// Opens a connection to the given IP socket
    fn connect(addr: &SocketAddr, tls_parameters: Option<&ClientTlsParameters>)
        -> io::Result<Self>;
    /// Upgrades to TLS connection
    fn upgrade_tls(&mut self, tls_parameters: &ClientTlsParameters) -> io::Result<()>;
    /// Is the NetworkStream encrypted
    fn is_encrypted(&self) -> bool;
}

impl Connector for NetworkStream {
    fn connect(
        addr: &SocketAddr,
        tls_parameters: Option<&ClientTlsParameters>,
    ) -> io::Result<NetworkStream> {
        let tcp_stream = TcpStream::connect(addr)?;

        match tls_parameters {
            Some(context) => context
                .connector
                .connect(context.domain.as_ref(), tcp_stream)
                .map(NetworkStream::Tls)
                .map_err(|e| io::Error::new(ErrorKind::Other, e)),
            None => Ok(NetworkStream::Tcp(tcp_stream)),
        }
    }

    #[cfg_attr(feature = "cargo-clippy", allow(match_same_arms))]
    fn upgrade_tls(&mut self, tls_parameters: &ClientTlsParameters) -> io::Result<()> {
        *self = match *self {
            NetworkStream::Tcp(ref mut stream) => match tls_parameters
                .connector
                .connect(tls_parameters.domain.as_ref(), stream.try_clone().unwrap())
            {
                Ok(tls_stream) => NetworkStream::Tls(tls_stream),
                Err(err) => return Err(io::Error::new(ErrorKind::Other, err)),
            },
            NetworkStream::Tls(_) => return Ok(()),
            NetworkStream::Mock(_) => return Ok(()),
        };

        Ok(())
    }

    #[cfg_attr(feature = "cargo-clippy", allow(match_same_arms))]
    fn is_encrypted(&self) -> bool {
        match *self {
            NetworkStream::Tcp(_) => false,
            NetworkStream::Tls(_) => true,
            NetworkStream::Mock(_) => false,
        }
    }
}

/// A trait for read and write timeout support
pub trait Timeout: Sized {
    /// Set read timeout for IO calls
    fn set_read_timeout(&mut self, duration: Option<Duration>) -> io::Result<()>;
    /// Set write timeout for IO calls
    fn set_write_timeout(&mut self, duration: Option<Duration>) -> io::Result<()>;
}

impl Timeout for NetworkStream {
    fn set_read_timeout(&mut self, duration: Option<Duration>) -> io::Result<()> {
        match *self {
            NetworkStream::Tcp(ref mut stream) => stream.set_read_timeout(duration),
            NetworkStream::Tls(ref mut stream) => stream.get_ref().set_read_timeout(duration),
            NetworkStream::Mock(_) => Ok(()),
        }
    }

    /// Set write timeout for IO calls
    fn set_write_timeout(&mut self, duration: Option<Duration>) -> io::Result<()> {
        match *self {
            NetworkStream::Tcp(ref mut stream) => stream.set_write_timeout(duration),
            NetworkStream::Tls(ref mut stream) => stream.get_ref().set_write_timeout(duration),
            NetworkStream::Mock(_) => Ok(()),
        }
    }
}
<commit_msg>feat(transport): Remove TLS 1.1 in accepted protocols by default (only allow TLS 1.2)<commit_after>//! A trait to represent a stream

use native_tls::{Protocol, TlsConnector, TlsStream};
use smtp::client::mock::MockStream;
use std::io::{self, ErrorKind, Read, Write};
use std::net::{Ipv4Addr, Shutdown, SocketAddr, SocketAddrV4, TcpStream};
use std::time::Duration;

/// Parameters to use for secure clients
#[derive(Clone)]
#[allow(missing_debug_implementations)]
pub struct ClientTlsParameters {
    /// A connector from `native-tls`
    pub connector: TlsConnector,
    /// The domain to send during the TLS handshake
    pub domain: String,
}

impl ClientTlsParameters {
    /// Creates a `ClientTlsParameters`
    pub fn new(domain: String, connector: TlsConnector) -> ClientTlsParameters {
        ClientTlsParameters { connector, domain }
    }
}

/// Accepted protocols by default.
/// This removes TLS 1.0 and 1.1 compared to tls-native defaults.
pub const DEFAULT_TLS_PROTOCOLS: &[Protocol] = &[Protocol::Tlsv12];

#[derive(Debug)]
/// Represents the different types of underlying network streams
pub enum NetworkStream {
    /// Plain TCP stream
    Tcp(TcpStream),
    /// Encrypted TCP stream
    Tls(TlsStream<TcpStream>),
    /// Mock stream
    Mock(MockStream),
}

impl NetworkStream {
    /// Returns peer's address
    pub fn peer_addr(&self) -> io::Result<SocketAddr> {
        match *self {
            NetworkStream::Tcp(ref s) => s.peer_addr(),
            NetworkStream::Tls(ref s) => s.get_ref().peer_addr(),
            NetworkStream::Mock(_) => Ok(SocketAddr::V4(SocketAddrV4::new(
                Ipv4Addr::new(127, 0, 0, 1),
                80,
            ))),
        }
    }

    /// Shutdowns the connection
    pub fn shutdown(&self, how: Shutdown) -> io::Result<()> {
        match *self {
            NetworkStream::Tcp(ref s) => s.shutdown(how),
            NetworkStream::Tls(ref s) => s.get_ref().shutdown(how),
            NetworkStream::Mock(_) => Ok(()),
        }
    }
}

impl Read for NetworkStream {
    fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> {
        match *self {
            NetworkStream::Tcp(ref mut s) => s.read(buf),
            NetworkStream::Tls(ref mut s) => s.read(buf),
            NetworkStream::Mock(ref mut s) => s.read(buf),
        }
    }
}

impl Write for NetworkStream {
    fn write(&mut self, buf: &[u8]) -> io::Result<usize> {
        match *self {
            NetworkStream::Tcp(ref mut s) => s.write(buf),
            NetworkStream::Tls(ref mut s) => s.write(buf),
            NetworkStream::Mock(ref mut s) => s.write(buf),
        }
    }

    fn flush(&mut self) -> io::Result<()> {
        match *self {
            NetworkStream::Tcp(ref mut s) => s.flush(),
            NetworkStream::Tls(ref mut s) => s.flush(),
            NetworkStream::Mock(ref mut s) => s.flush(),
        }
    }
}

/// A trait for the concept of opening a stream
pub trait Connector: Sized {
    /// Opens a connection to the given IP socket
    fn connect(addr: &SocketAddr, tls_parameters: Option<&ClientTlsParameters>)
        -> io::Result<Self>;
    /// Upgrades to TLS connection
    fn upgrade_tls(&mut self, tls_parameters: &ClientTlsParameters) -> io::Result<()>;
    /// Is the NetworkStream encrypted
    fn is_encrypted(&self) -> bool;
}

impl Connector for NetworkStream {
    fn connect(
        addr: &SocketAddr,
        tls_parameters: Option<&ClientTlsParameters>,
    ) -> io::Result<NetworkStream> {
        let tcp_stream = TcpStream::connect(addr)?;

        match tls_parameters {
            Some(context) => context
                .connector
                .connect(context.domain.as_ref(), tcp_stream)
                .map(NetworkStream::Tls)
                .map_err(|e| io::Error::new(ErrorKind::Other, e)),
            None => Ok(NetworkStream::Tcp(tcp_stream)),
        }
    }

    #[cfg_attr(feature = "cargo-clippy", allow(match_same_arms))]
    fn upgrade_tls(&mut self, tls_parameters: &ClientTlsParameters) -> io::Result<()> {
        *self = match *self {
            NetworkStream::Tcp(ref mut stream) => match tls_parameters
                .connector
                .connect(tls_parameters.domain.as_ref(), stream.try_clone().unwrap())
            {
                Ok(tls_stream) => NetworkStream::Tls(tls_stream),
                Err(err) => return Err(io::Error::new(ErrorKind::Other, err)),
            },
            NetworkStream::Tls(_) => return Ok(()),
            NetworkStream::Mock(_) => return Ok(()),
        };

        Ok(())
    }

    #[cfg_attr(feature = "cargo-clippy", allow(match_same_arms))]
    fn is_encrypted(&self) -> bool {
        match *self {
            NetworkStream::Tcp(_) => false,
            NetworkStream::Tls(_) => true,
            NetworkStream::Mock(_) => false,
        }
    }
}

/// A trait for read and write timeout support
pub trait Timeout: Sized {
    /// Set read timeout for IO calls
    fn set_read_timeout(&mut self, duration: Option<Duration>) -> io::Result<()>;
    /// Set write timeout for IO calls
    fn set_write_timeout(&mut self, duration: Option<Duration>) -> io::Result<()>;
}

impl Timeout for NetworkStream {
    fn set_read_timeout(&mut self, duration: Option<Duration>) -> io::Result<()> {
        match *self {
            NetworkStream::Tcp(ref mut stream) => stream.set_read_timeout(duration),
            NetworkStream::Tls(ref mut stream) => stream.get_ref().set_read_timeout(duration),
            NetworkStream::Mock(_) => Ok(()),
        }
    }

    /// Set write timeout for IO calls
    fn set_write_timeout(&mut self, duration: Option<Duration>) -> io::Result<()> {
        match *self {
            NetworkStream::Tcp(ref mut stream) => stream.set_write_timeout(duration),
            NetworkStream::Tls(ref mut stream) => stream.get_ref().set_write_timeout(duration),
            NetworkStream::Mock(_) => Ok(()),
        }
    }
}
<|endoftext|>


<commit_before>extern crate csv;
extern crate rustc_serialize;
extern crate stopwatch;

#[macro_use]
extern crate text_io;

use std::collections::HashMap;
use std::collections::hash_map::Entry::{Occupied, Vacant};
use stopwatch::Stopwatch;

struct IndexedDB(HashMap<String, HashMap<String, f32>>, HashMap<String, HashMap<String, f32>>);

fn pearson_coef(x: &Vec<f32>, y: &Vec<f32>) -> f32 {
    if x.len() != y.len() {
        panic!("Should compare vectors of same size");
    }

    let n_dims = x.len();

    let mut prod_xy = 0.0;
    let mut sum_x = 0.0;
    let mut sum_y = 0.0;

    for i in 0..n_dims {
        sum_x += x[i];
        sum_y += y[i];
        prod_xy += x[i] * y[i];
    }

    let avg_sqr_x = sum_x.powf(2.0) / n_dims as f32;
    let avg_sqr_y = sum_y.powf(2.0) / n_dims as f32;
    let mut sqr_diff_x = 0.0;
    let mut sqr_diff_y = 0.0;

    for i in 0..x.len() {
        sqr_diff_x += x[i].powf(2.0);
        sqr_diff_y += y[i].powf(2.0);
    }

    (prod_xy - (sum_x * sum_y) / n_dims as f32) /
    ((sqr_diff_x - avg_sqr_x).sqrt() * (sqr_diff_y - avg_sqr_y).sqrt())
}

fn load_db(path: &str) -> IndexedDB {
    // let mut rdr = csv::Reader::from_file(path).unwrap().has_headers(false);
    let mut rdr = csv::Reader::from_file(path).unwrap().has_headers(true);
    let mut ratings: HashMap<String, HashMap<String, f32>> = HashMap::new();
    let mut features: HashMap<String, HashMap<String, f32>> = HashMap::new();

    for record in rdr.decode() {
        let (user_id, feat_id, rating): (String, String, f32) = record.unwrap();
        let (user_id2, feat_id2, rating2) = (user_id.clone(), feat_id.clone(), rating);

        match ratings.entry(user_id) {
            Vacant(entry) => {
                let mut user_ratings = HashMap::new();
                user_ratings.insert(feat_id, rating);
                entry.insert(user_ratings);
            }
            Occupied(entry) => {
                entry.into_mut().insert(feat_id, rating);
            }
        }

        match features.entry(feat_id2) {
            Vacant(entry) => {
                let mut movie_ratings = HashMap::new();
                movie_ratings.insert(user_id2, rating2);
                entry.insert(movie_ratings);
            }
            Occupied(entry) => {
                entry.into_mut().insert(user_id2, rating2);
            }
        }
    }

    IndexedDB(ratings, features)
}

fn deviation(db: &IndexedDB, feat_1: &str, feat_2: &str) -> (f32, usize) {
    let mut vec_a: Vec<f32> = Vec::new();
    let mut vec_b: Vec<f32> = Vec::new();

    if let Some(ref ratings_f1) = db.1.get(&String::from(feat_1)) {
        for (user_id, &rating_f1) in ratings_f1.iter() {
            if let Some(ratings_f2) = db.1.get(&String::from(feat_2)) {
                if let Some(&rating_f2) = ratings_f2.get(user_id) {
                    vec_a.push(rating_f1);
                    vec_b.push(rating_f2);
                }
            } else {
                panic!("feat_2 is not found!");
            }
        }
    } else {
        panic!("feat_1 is not found!");
    }

    let length = vec_a.len();
    let mut deviation = 0.0;
    for i in 0..length {
        deviation += (vec_a[i] - vec_b[i]) / length as f32;
    }

    (deviation, length)
}

fn knn_pearson_coef(db: &IndexedDB, user: &str, k: u32) -> IndexedDB {
    let ref ratings_u = db.0.get(&String::from(user)).unwrap();
    let mut pearson_coefs: Vec<(f32, String)> = Vec::new();

    for (user_2, ref ratings_2) in db.0.iter() {
        let mut vec_a: Vec<f32> = Vec::new();
        let mut vec_b: Vec<f32> = Vec::new();

        if user_2 != user {
            for (feat_u, &rating_u) in ratings_u.iter() {
                if let Some(&rating) = ratings_2.get(feat_u) {
                    vec_a.push(rating_u);
                    vec_b.push(rating);
                } else {
                    vec_a.push(rating_u);
                    vec_b.push(0.0);
                }
            }

            for (feat_2, &rating_2) in ratings_2.iter() {
                if let Some(_) = ratings_u.get(feat_2) {
                } else {
                    vec_a.push(0.0);
                    vec_b.push(rating_2);
                }
            }
            pearson_coefs.push((pearson_coef(&vec_a, &vec_b), user_2.clone()));
        }
    }

    pearson_coefs.sort_by(|a, b| a.0.partial_cmp(&b.0).unwrap());

    let mut db_ratings: HashMap<String, HashMap<String, f32>> = HashMap::new();
    let mut db_features: HashMap<String, HashMap<String, f32>> = HashMap::new();

    for i in 0..k {
        let (_, user_id) = pearson_coefs.pop().unwrap();

        if let Some(ratings) = db.0.get(&user_id) {
            for (ref feat_id, &rating) in ratings.iter() {
                let (user_id2, feat_id2, rating2) = (user_id.clone(), feat_id.clone(), rating);

                match db_ratings.entry(user_id) {
                    Vacant(entry) => {
                        let mut usr_ratings = HashMap::new();
                        usr_ratings.insert(feat_id, rating);
                        entry.insert(usr_ratings);
                    }
                    Occupied(entry) => {
                        entry.into_mut().insert(feat_id, rating);
                    }
                }

                match db_features.entry(*feat_id2) {
                    Vacant(entry) => {
                        let mut movie_ratings = HashMap::new();
                        movie_ratings.insert(user_id2, rating2);
                        entry.insert(movie_ratings);
                    }
                    Occupied(entry) => {
                        entry.into_mut().insert(user_id2, rating2);
                    }
                }
            }
        } else {
            println!("Couldn't find ratings for user");
        }
    }

    IndexedDB(db_ratings, db_features)
}

fn improved_slope_one(db: &IndexedDB, user: &str, feat: &str) -> f32 {
    let mut num = 0.0;
    let mut den = 0.0;

    let s_k = knn_pearson_coef(db, user, 10);

    if let Some(ref ratings) = s_k.0.get(&String::from(user)) {
        for (feat_id, &rating) in ratings.iter() {
            let (dev, card) = deviation(&s_k, feat, feat_id);
            num += (dev + rating) * card as f32;
            den += card as f32;
        }
    } else {
        panic!("user is not found!");
    }

    num / den
}

fn prediction_input(db: &mut IndexedDB, sim_fun: fn(&IndexedDB, &str, &str) -> f32) -> f32 {
    let user_id: String;
    let feat_id: String;
    println!("Enter {{user}} y {{feature}}:");
    scan!("{} {}", user_id, feat_id);

    if let Some(_) = db.0.get(&user_id) {
    } else {
        println!("\nUser not found, you'll be requested to enter some ratings.\
                  \nEnter number of ratings...");
        let n_ratings: u32;
        scan!("{}", n_ratings);

        let mut ratings: HashMap<String, f32> = HashMap::new();
        let mut feat: String;
        let mut rating: f32;

        for i in 0..n_ratings {
            println!("{}. Enter feature and rating:", i);
            scan!("{} {}", feat, rating);

            let ins_feat = feat.clone();

            match db.1.entry(feat) {
                Vacant(entry) => {
                    let mut feat_ratings = HashMap::new();
                    feat_ratings.insert(user_id.clone(), rating);
                    entry.insert(feat_ratings);
                }
                Occupied(entry) => {
                    entry.into_mut().insert(user_id.clone(), rating);
                }
            }

            ratings.insert(ins_feat, rating);
        }

        db.0.insert(user_id.clone(), ratings);
    }

    let sw = Stopwatch::start_new();
    let similarity = sim_fun(&db, user_id.as_str(), feat_id.as_str());
    println!("Execution time: {}ms", sw.elapsed_ms());

    similarity
}

fn main() {
    println!("Loading database, please wait...");
    // let mut db = load_db("../report01/data/BX-Dump/BX-Book-Ratings.csv");
    // let mut db = load_db("../../../Downloads/ml-20m/ratings.csv");
    let mut db = load_db("../../../Downloads/ml-latest-small/ratings.csv"); // 311 1479
    println!("Database ready!\n---------------------------------------------");

    let mut ender: u32;
    loop {
        println!("Press 1 to make a query with Slope One.\n\
                  Press 0 to exit: ");
        scan!("{}", ender);
        match ender {
            1 => {
                println!("S1p: {}", prediction_input(&mut db, improved_slope_one));
            }
            0 => {
                break;
            }
            _ => {
                println!("Try again");
            }
        }
        println!("\nNum. of Records: {} items, {} records",
                 db.0.len(),
                 db.1.len());
        println!("\n-------------------------------------------------------");
    }
    println!("\nBye bytes...");
}
<commit_msg>snall changes<commit_after>extern crate csv;
extern crate rustc_serialize;
extern crate stopwatch;

#[macro_use]
extern crate text_io;

use std::collections::HashMap;
use std::collections::hash_map::Entry::{Occupied, Vacant};
use stopwatch::Stopwatch;

struct IndexedDB(HashMap<String, HashMap<String, f32>>, HashMap<String, HashMap<String, f32>>);

fn pearson_coef(x: &Vec<f32>, y: &Vec<f32>) -> f32 {
    if x.len() != y.len() {
        panic!("Should compare vectors of same size");
    }

    let n_dims = x.len();

    let mut prod_xy = 0.0;
    let mut sum_x = 0.0;
    let mut sum_y = 0.0;

    for i in 0..n_dims {
        sum_x += x[i];
        sum_y += y[i];
        prod_xy += x[i] * y[i];
    }

    let avg_sqr_x = sum_x.powf(2.0) / n_dims as f32;
    let avg_sqr_y = sum_y.powf(2.0) / n_dims as f32;
    let mut sqr_diff_x = 0.0;
    let mut sqr_diff_y = 0.0;

    for i in 0..x.len() {
        sqr_diff_x += x[i].powf(2.0);
        sqr_diff_y += y[i].powf(2.0);
    }

    (prod_xy - (sum_x * sum_y) / n_dims as f32) /
    ((sqr_diff_x - avg_sqr_x).sqrt() * (sqr_diff_y - avg_sqr_y).sqrt())
}

fn load_db(path: &str) -> IndexedDB {
    // let mut rdr = csv::Reader::from_file(path).unwrap().has_headers(false);
    let mut rdr = csv::Reader::from_file(path).unwrap().has_headers(true);
    let mut ratings: HashMap<String, HashMap<String, f32>> = HashMap::new();
    let mut features: HashMap<String, HashMap<String, f32>> = HashMap::new();

    for record in rdr.decode() {
        let (user_id, feat_id, rating): (String, String, f32) = record.unwrap();
        let (user_id2, feat_id2, rating2) = (user_id.clone(), feat_id.clone(), rating);

        match ratings.entry(user_id) {
            Vacant(entry) => {
                let mut user_ratings = HashMap::new();
                user_ratings.insert(feat_id, rating);
                entry.insert(user_ratings);
            }
            Occupied(entry) => {
                entry.into_mut().insert(feat_id, rating);
            }
        }

        match features.entry(feat_id2) {
            Vacant(entry) => {
                let mut movie_ratings = HashMap::new();
                movie_ratings.insert(user_id2, rating2);
                entry.insert(movie_ratings);
            }
            Occupied(entry) => {
                entry.into_mut().insert(user_id2, rating2);
            }
        }
    }

    IndexedDB(ratings, features)
}

fn deviation(db: &IndexedDB, feat_1: &str, feat_2: &str) -> (f32, usize) {
    let mut vec_a: Vec<f32> = Vec::new();
    let mut vec_b: Vec<f32> = Vec::new();

    if let Some(ref ratings_f1) = db.1.get(&String::from(feat_1)) {
        for (user_id, &rating_f1) in ratings_f1.iter() {
            if let Some(ratings_f2) = db.1.get(&String::from(feat_2)) {
                if let Some(&rating_f2) = ratings_f2.get(user_id) {
                    vec_a.push(rating_f1);
                    vec_b.push(rating_f2);
                }
            } else {
                panic!("feat_2 is not found!");
            }
        }
    } else {
        panic!("feat_1 is not found!");
    }

    let length = vec_a.len();
    let mut deviation = 0.0;
    for i in 0..length {
        deviation += (vec_a[i] - vec_b[i]) / length as f32;
    }

    (deviation, length)
}

fn knn_pearson_coef(db: &IndexedDB, user: &str, k: u32) -> IndexedDB {
    let ref ratings_u = db.0.get(&String::from(user)).unwrap();
    let mut pearson_coefs: Vec<(f32, String)> = Vec::new();

    for (user_2, ref ratings_2) in db.0.iter() {
        let mut vec_a: Vec<f32> = Vec::new();
        let mut vec_b: Vec<f32> = Vec::new();

        if user_2 != user {
            for (feat_u, &rating_u) in ratings_u.iter() {
                if let Some(&rating) = ratings_2.get(feat_u) {
                    vec_a.push(rating_u);
                    vec_b.push(rating);
                } else {
                    vec_a.push(rating_u);
                    vec_b.push(0.0);
                }
            }

            for (feat_2, &rating_2) in ratings_2.iter() {
                if let Some(_) = ratings_u.get(feat_2) {
                } else {
                    vec_a.push(0.0);
                    vec_b.push(rating_2);
                }
            }
            pearson_coefs.push((pearson_coef(&vec_a, &vec_b), user_2.clone()));
        }
    }

    pearson_coefs.sort_by(|a, b| a.0.partial_cmp(&b.0).unwrap());

    let mut db_ratings: HashMap<String, HashMap<String, f32>> = HashMap::new();
    let mut db_features: HashMap<String, HashMap<String, f32>> = HashMap::new();

    for i in 0..k {
        let (_, user_id) = pearson_coefs.pop().unwrap();

        if let Some(ratings) = db.0.get(&user_id) {
            for (ref feat_id, &rating) in ratings.iter() {
                let (user_id2, feat_id2, rating2) = (user_id.clone(), feat_id.clone(), rating);

                match db_ratings.entry(user_id) {
                    Vacant(entry) => {
                        let mut usr_ratings = HashMap::new();
                        usr_ratings.insert(feat_id, rating);
                        entry.insert(usr_ratings);
                    }
                    Occupied(entry) => {
                        entry.into_mut().insert(feat_id, rating);
                    }
                }

                match db_features.entry(*feat_id2) {
                    Vacant(entry) => {
                        let mut movie_ratings = HashMap::new();
                        movie_ratings.insert(user_id2, rating2);
                        entry.insert(movie_ratings);
                    }
                    Occupied(entry) => {
                        entry.into_mut().insert(user_id2, rating2);
                    }
                }
            }
        } else {
            println!("Couldn't find ratings for user");
        }
    }

    IndexedDB(db_ratings, db_features)
}

fn improved_slope_one(db: &IndexedDB, user: &str, feat: &str) -> f32 {
    let mut num = 0.0;
    let mut den = 0.0;

    // let s_k = knn_pearson_coef(db, user, 10);
    let s_k = knn_pearson_coef(db, user, 2);

    if let Some(ref ratings) = s_k.0.get(&String::from(user)) {
        for (feat_id, &rating) in ratings.iter() {
            let (dev, card) = deviation(&s_k, feat, feat_id);
            num += (dev + rating) * card as f32;
            den += card as f32;
        }
    } else {
        panic!("user is not found!");
    }

    num / den
}

fn prediction_input(db: &mut IndexedDB, sim_fun: fn(&IndexedDB, &str, &str) -> f32) -> f32 {
    let user_id: String;
    let feat_id: String;
    println!("Enter {{user}} y {{feature}}:");
    scan!("{} {}", user_id, feat_id);

    if let Some(_) = db.0.get(&user_id) {
    } else {
        println!("\nUser not found, you'll be requested to enter some ratings.\
                  \nEnter number of ratings...");
        let n_ratings: u32;
        scan!("{}", n_ratings);

        let mut ratings: HashMap<String, f32> = HashMap::new();
        let mut feat: String;
        let mut rating: f32;

        for i in 0..n_ratings {
            println!("{}. Enter feature and rating:", i);
            scan!("{} {}", feat, rating);

            let ins_feat = feat.clone();

            match db.1.entry(feat) {
                Vacant(entry) => {
                    let mut feat_ratings = HashMap::new();
                    feat_ratings.insert(user_id.clone(), rating);
                    entry.insert(feat_ratings);
                }
                Occupied(entry) => {
                    entry.into_mut().insert(user_id.clone(), rating);
                }
            }

            ratings.insert(ins_feat, rating);
        }

        db.0.insert(user_id.clone(), ratings);
    }

    let sw = Stopwatch::start_new();
    let similarity = sim_fun(&db, user_id.as_str(), feat_id.as_str());
    println!("Execution time: {}ms", sw.elapsed_ms());

    similarity
}

fn main() {
    println!("Loading database, please wait...");
    // let mut db = load_db("../report01/data/BX-Dump/BX-Book-Ratings.csv");
    // let mut db = load_db("../../../Downloads/ml-20m/ratings.csv");
    // let mut db = load_db("../../../Downloads/ml-latest-small/ratings.csv"); // 311 1479
    let mut db = load_db("../slope-one/data/db2.csv");
    println!("Database ready!\n---------------------------------------------");

    let mut ender: u32;
    loop {
        println!("Press 1 to make a query with Slope One.\n\
                  Press 0 to exit: ");
        scan!("{}", ender);
        match ender {
            1 => {
                println!("S1p: {}", prediction_input(&mut db, improved_slope_one));
            }
            0 => {
                break;
            }
            _ => {
                println!("Try again");
            }
        }
        println!("\nNum. of Records: {} items, {} records",
                 db.0.len(),
                 db.1.len());
        println!("\n-------------------------------------------------------");
    }
    println!("\nBye bytes...");
}
<|endoftext|>


<commit_before>use lazy_static::lazy_static;
use markup5ever::local_name;
use regex::{Captures, Regex};
use std::borrow::Cow;
use std::cell::RefCell;

use crate::drawing_ctx::DrawingCtx;
use crate::error::RenderingError;
use crate::node::*;
use crate::property_bag::PropertyBag;

#[derive(Default)]
pub struct NodeLink {
    link: RefCell<Option<String>>,
}

impl NodeTrait for NodeLink {
    fn set_atts(&mut self, _: Option<&RsvgNode>, pbag: &PropertyBag<'_>) -> NodeResult {
        for (attr, value) in pbag.iter() {
            match attr {
                local_name!("xlink:href") => *self.link.borrow_mut() = Some(value.to_owned()),

                _ => (),
            }
        }

        Ok(())
    }

    fn draw(
        &self,
        node: &RsvgNode,
        cascaded: &CascadedValues<'_>,
        draw_ctx: &mut DrawingCtx,
        clipping: bool,
    ) -> Result<(), RenderingError> {
        let link = self.link.borrow();

        let cascaded = CascadedValues::new(cascaded, node);
        let values = cascaded.get();

        draw_ctx.with_discrete_layer(node, values, clipping, &mut |dc| match link.as_ref() {
            Some(l) if !l.is_empty() => {
                const CAIRO_TAG_LINK: &str = "Link";

                let attributes = format!("uri='{}'", escape_value(l));

                let cr = dc.get_cairo_context();

                cr.tag_begin(CAIRO_TAG_LINK, &attributes);

                let res = node.draw_children(&cascaded, dc, clipping);

                cr.tag_end(CAIRO_TAG_LINK);

                res
            }
            _ => node.draw_children(&cascaded, dc, clipping),
        })
    }
}

/// escape quotes and backslashes with backslash
fn escape_value(value: &str) -> Cow<'_, str> {
    lazy_static! {
        static ref REGEX: Regex = Regex::new(r"['\\]").unwrap();
    }

    REGEX.replace_all(value, |caps: &Captures<'_>| {
        match caps.get(0).unwrap().as_str() {
            "'" => "\\'".to_owned(),
            "\\" => "\\\\".to_owned(),
            _ => unreachable!(),
        }
    })
}
<commit_msg>link: remove interior mutability<commit_after>use lazy_static::lazy_static;
use markup5ever::local_name;
use regex::{Captures, Regex};
use std::borrow::Cow;

use crate::drawing_ctx::DrawingCtx;
use crate::error::RenderingError;
use crate::node::*;
use crate::property_bag::PropertyBag;

#[derive(Default)]
pub struct NodeLink {
    link: Option<String>,
}

impl NodeTrait for NodeLink {
    fn set_atts(&mut self, _: Option<&RsvgNode>, pbag: &PropertyBag<'_>) -> NodeResult {
        for (attr, value) in pbag.iter() {
            match attr {
                local_name!("xlink:href") => self.link = Some(value.to_owned()),
                _ => (),
            }
        }

        Ok(())
    }

    fn draw(
        &self,
        node: &RsvgNode,
        cascaded: &CascadedValues<'_>,
        draw_ctx: &mut DrawingCtx,
        clipping: bool,
    ) -> Result<(), RenderingError> {
        let cascaded = CascadedValues::new(cascaded, node);
        let values = cascaded.get();

        draw_ctx.with_discrete_layer(node, values, clipping, &mut |dc| match self.link.as_ref() {
            Some(l) if !l.is_empty() => {
                const CAIRO_TAG_LINK: &str = "Link";

                let attributes = format!("uri='{}'", escape_value(l));

                let cr = dc.get_cairo_context();

                cr.tag_begin(CAIRO_TAG_LINK, &attributes);

                let res = node.draw_children(&cascaded, dc, clipping);

                cr.tag_end(CAIRO_TAG_LINK);

                res
            }
            _ => node.draw_children(&cascaded, dc, clipping),
        })
    }
}

/// escape quotes and backslashes with backslash
fn escape_value(value: &str) -> Cow<'_, str> {
    lazy_static! {
        static ref REGEX: Regex = Regex::new(r"['\\]").unwrap();
    }

    REGEX.replace_all(value, |caps: &Captures<'_>| {
        match caps.get(0).unwrap().as_str() {
            "'" => "\\'".to_owned(),
            "\\" => "\\\\".to_owned(),
            _ => unreachable!(),
        }
    })
}
<|endoftext|>


<commit_before>extern crate hoedown;
use hoedown::renderer::Render;

use std::env::args as cmd_args;
use std::io::{Write, BufReader};

const USAGE: &'static str  = "Usage: md_render <files>...\n
";

const EXT_TABLES: u32            = 1 << 0;
const EXT_FENCED_CODE: u32       = 1 << 1;
const EXT_FOOTNOTES: u32         = 1 << 2;
const EXT_AUTOLINK: u32          = 1 << 3;
const EXT_STRIKETHROUGH: u32     = 1 << 4;
const EXT_UNDERLINE: u32         = 1 << 5;
const EXT_HIGHLIGHT: u32         = 1 << 6;
const EXT_QUOTE: u32             = 1 << 7;
const EXT_NO_INTRA_EMPHASIS: u32 = 1 << 11;
const EXT_SPACE_HEADERS: u32     = 1 << 12;

macro_rules! is_file {
    ($path:expr) => { std::fs::metadata($path).unwrap().is_file() }
}

#[inline]
fn usage() {
    println!("{}", USAGE);
}

fn main() {
    if cmd_args().len() < 2 { return usage(); }

    let md_ext = hoedown::Extension::from_bits_truncate(EXT_TABLES |
                                                        EXT_FENCED_CODE |
                                                        EXT_FOOTNOTES |
                                                        EXT_AUTOLINK |
                                                        EXT_STRIKETHROUGH |
                                                        EXT_UNDERLINE |
                                                        EXT_HIGHLIGHT |
                                                        EXT_QUOTE |
                                                        EXT_NO_INTRA_EMPHASIS |
                                                        EXT_SPACE_HEADERS);
    let flags = hoedown::renderer::html::Flags::empty();
    let mut html_render = hoedown::renderer::html::Html::new(flags, 0);
    for arg in cmd_args().skip(1) {
        if is_file!(&arg) {
            if !arg.ends_with(".md") { continue; }

            println!("Render - {}", &arg);
            let html_path = std::path::Path::new(&arg).with_extension("html");

            if let Ok(file) = std::fs::File::open(&arg) {
                let mut html_file = std::fs::File::create(&html_path).unwrap();

                let file = BufReader::new(file);
                let file = hoedown::Markdown::read_from(file).extensions(md_ext);
                let render_result = html_render.render(&file);

                html_file.write_all(&render_result).unwrap();
                println!(">>>Result: {}", &html_path.to_str().unwrap_or(""));
            }
            else {
                println!(">>>{}: failed to open", &arg);
            }
            println!("");
        }
        else if let Ok(dir_content) = std::fs::read_dir(&arg) {
            println!("Render in directory - {}\n", &arg);
            for entry in dir_content.map(|elem| elem.unwrap())
                                    .filter(|elem| elem.file_type().map(|elem| elem.is_file()).unwrap_or(false))
                                    .map(|elem| elem.path())
                                    .filter(|path| path.extension().map(|ext| ext == "md").unwrap_or(false)) {

                println!("File: {}", entry.to_str().unwrap_or(""));
                let html_path = std::path::Path::new(&entry).with_extension("html");

                if let Ok(file) = std::fs::File::open(&entry) {
                    let mut html_file = std::fs::File::create(&html_path).unwrap();

                    let file = BufReader::new(file);
                    let file = hoedown::Markdown::read_from(file).extensions(md_ext);
                    let render_result = html_render.render(&file);

                    html_file.write_all(&render_result).unwrap();
                    println!(">>>Result: {}", &html_path.to_str().unwrap_or(""));
                }
                else {
                    println!(">>>{}: failed to open", entry.to_str().unwrap_or(""));
                }

                println!("");
            }
        }
        else {
            println!(">>>{}: cannot access\n", &arg);
            continue;
        }

    }
}
<commit_msg>Rust: md_render add options to specify css<commit_after>extern crate hoedown;
use hoedown::renderer::Render;

use std::env::args as cmd_args;
use std::io::{Read, Write, BufReader};

const USAGE: &'static str  = "Usage: md_render <files>... [options]

Options:
--css=<file_name> - specify name of css file to link with.
--inline-css=<file_name> - specify the name of css file which style will be inlined.
";

const EXT_TABLES: u32            = 1 << 0;
const EXT_FENCED_CODE: u32       = 1 << 1;
const EXT_FOOTNOTES: u32         = 1 << 2;
const EXT_AUTOLINK: u32          = 1 << 3;
const EXT_STRIKETHROUGH: u32     = 1 << 4;
const EXT_UNDERLINE: u32         = 1 << 5;
const EXT_HIGHLIGHT: u32         = 1 << 6;
const EXT_QUOTE: u32             = 1 << 7;
const EXT_NO_INTRA_EMPHASIS: u32 = 1 << 11;
const EXT_SPACE_HEADERS: u32     = 1 << 12;

macro_rules! is_file {
    ($path:expr) => { std::fs::metadata($path).unwrap().is_file() }
}

#[inline]
fn usage() {
    println!("{}", USAGE);
}

fn main() {
    if cmd_args().len() < 2 { return usage(); }

    let mut css = String::new();
    let mut md_files: Vec<String> = vec![];
    for arg in cmd_args().skip(1) {
        if arg.starts_with("--") {
            let arg = &arg[2..];
            let eq_pos = arg.find('=');

            if arg.starts_with("css") {
                if let Some(eq_pos) = eq_pos {
                    std::fmt::Write::write_fmt(&mut css,
                                               format_args!("<link rel=\"stylesheet\" href=\"{}\">\n", &arg[eq_pos+1..])).unwrap();
                }
                else {
                    println!(">>>Invalid use of options. CSS File is not specified!");
                }

            }
            else if arg.starts_with("inline-css") {
                if let Some(eq_pos) = eq_pos {
                    if let Ok(mut css_file) = std::fs::File::open(&arg[eq_pos+1..]) {
                        css.push_str("<style>\n");
                        css_file.read_to_string(&mut css).unwrap();
                        css.push_str("</style>\n");
                    }
                    else {
                        println!(">>>{}: no such CSS", &arg[eq_pos+1..]);
                    }
                }
                else {
                    println!(">>>Invalid use of options. CSS File is not specified!");
                }
            }
        }
        else if is_file!(&arg) {
            md_files.push(arg);
        }
        else {
            println!(">>>{}: No such file!\n", arg);
        }
    };

    if md_files.len() == 0 {
        println!(">>>No Markdown files are  specified!\n");
        usage();
        return;
    }

    let md_ext = hoedown::Extension::from_bits_truncate(EXT_TABLES |
                                                        EXT_FENCED_CODE |
                                                        EXT_FOOTNOTES |
                                                        EXT_AUTOLINK |
                                                        EXT_STRIKETHROUGH |
                                                        EXT_UNDERLINE |
                                                        EXT_HIGHLIGHT |
                                                        EXT_QUOTE |
                                                        EXT_NO_INTRA_EMPHASIS |
                                                        EXT_SPACE_HEADERS);
    let flags = hoedown::renderer::html::Flags::empty();
    let mut html_render = hoedown::renderer::html::Html::new(flags, 0);
    for arg in md_files {
        println!("Render - {}", &arg);
        let html_path = std::path::Path::new(&arg).with_extension("html");

        if let Ok(file) = std::fs::File::open(&arg) {
            let mut html_file = std::fs::File::create(&html_path).unwrap();

            let file = BufReader::new(file);
            let file = hoedown::Markdown::read_from(file).extensions(md_ext);
            let render_result = html_render.render(&file);

            html_file.write_all(css.as_bytes()).unwrap();
            html_file.write_all(&render_result).unwrap();
            println!(">>>Result: {}", &html_path.to_str().unwrap_or(""));
        }
        else {
            println!(">>>{}: failed to open", &arg);
        }
        println!("");
    }
}
<|endoftext|>