<commit_before>
#[macro_use(s)]
extern crate ndarray;
extern crate ndarray_rand;
extern crate rand;

use ndarray_rand::{RandomExt, F32};
use rand::Rng;

use ndarray::prelude::*;
use ndarray::{
    Data,
    DataMut,
    RemoveAxis,
    LinalgScalar,
};

use rand::distributions::Normal;

// simple, slow, correct (hopefully) mat mul
fn reference_mat_mul<A, S, S2>(lhs: &ArrayBase<S, Ix2>, rhs: &ArrayBase<S2, Ix2>)
    -> Array<A, Ix2>
    where A: LinalgScalar,
          S: Data<Elem=A>,
          S2: Data<Elem=A>,
{
    let ((m, k), (_, n)) = (lhs.dim(), rhs.dim());
    let mut res_elems = Vec::<A>::with_capacity(m * n);
    unsafe {
        res_elems.set_len(m * n);
    }

    let mut i = 0;
    let mut j = 0;
    for rr in &mut res_elems {
        unsafe {
            *rr = (0..k).fold(A::zero(),
                move |s, x| s + *lhs.uget((i, x)) * *rhs.uget((x, j)));
        }
        j += 1;
        if j == n {
            j = 0;
            i += 1;
        }
    }
    unsafe {
        ArrayBase::from_shape_vec_unchecked((m, n), res_elems)
    }
}

fn gen<D>(d: D) -> Array<f32, D>
    where D: Dimension,
{
    Array::random(d, F32(Normal::new(0., 1.)))
}
fn gen_f64<D>(d: D) -> Array<f64, D>
    where D: Dimension,
{
    Array::random(d, Normal::new(0., 1.))
}

#[test]
fn accurate_eye_f32() {
    for i in 0..20 {
        let eye = Array::eye(i);
        for j in 0..20 {
            let a = gen(Ix2(i, j));
            let a2 = eye.dot(&a);
            if !a.all_close(&a2, 1e-6) {
                panic!("Arrays are not equal:\n{:?}\n{:?}\n{:?}", a, a2, &a2 - &a);
            }
            let a3 = a.t().dot(&eye);
            if !a.t().all_close(&a3, 1e-6) {
                panic!("Arrays are not equal:\n{:?}\n{:?}\n{:?}", a.t(), a3, &a3 - &a.t());
            }
        }
    }
    // pick a few random sizes
    let mut rng = rand::weak_rng();
    for _ in 0..10 {
        let i = rng.gen_range(15, 512);
        let j = rng.gen_range(15, 512);
        println!("Testing size {} by {}", i, j);
        let a = gen(Ix2(i, j));
        let eye = Array::eye(i);
        let a2 = eye.dot(&a);
        if !a.all_close(&a2, 1e-6) {
            panic!("Arrays are not equal:\n{:?}\n{:?}\n{:?}", a, a2, &a2 - &a);
        }
        let a3 = a.t().dot(&eye);
        if !a.t().all_close(&a3, 1e-6) {
            panic!("Arrays are not equal:\n{:?}\n{:?}\n{:?}", a.t(), a3, &a3 - &a.t());
        }
    }
}

#[test]
fn accurate_eye_f64() {
    let abs_tol = 1e-15;
    for i in 0..20 {
        let eye = Array::eye(i);
        for j in 0..20 {
            let a = gen_f64(Ix2(i, j));
            let a2 = eye.dot(&a);
            if !a.all_close(&a2, abs_tol) {
                panic!("Arrays are not equal:\n{:?}\n{:?}\n{:?}", a, a2, &a2 - &a);
            }
            let a3 = a.t().dot(&eye);
            if !a.t().all_close(&a3, abs_tol) {
                panic!("Arrays are not equal:\n{:?}\n{:?}\n{:?}", a.t(), a3, &a3 - &a.t());
            }
        }
    }
    // pick a few random sizes
    let mut rng = rand::weak_rng();
    for _ in 0..10 {
        let i = rng.gen_range(15, 512);
        let j = rng.gen_range(15, 512);
        println!("Testing size {} by {}", i, j);
        let a = gen_f64(Ix2(i, j));
        let eye = Array::eye(i);
        let a2 = eye.dot(&a);
        if !a.all_close(&a2, 1e-6) {
            panic!("Arrays are not equal:\n{:?}\n{:?}\n{:?}", a, a2, &a2 - &a);
        }
        let a3 = a.t().dot(&eye);
        if !a.t().all_close(&a3, 1e-6) {
            panic!("Arrays are not equal:\n{:?}\n{:?}\n{:?}", a.t(), a3, &a3 - &a.t());
        }
    }
}

#[test]
fn accurate_mul_f32() {
    // pick a few random sizes
    let mut rng = rand::weak_rng();
    for i in 0..20 {
        let m = rng.gen_range(15, 512);
        let k = rng.gen_range(15, 512);
        let n = rng.gen_range(15, 1560);
        let a = gen(Ix2(m, k));
        let b = gen(Ix2(n, k));
        let b = b.t();
        let (a, b) = if i > 10 {
            (a.slice(s![..;2, ..;2]),
             b.slice(s![..;2, ..;2]))
        } else { (a.view(), b) };

        println!("Testing size {} by {} by {}", a.shape()[0], a.shape()[1], b.shape()[1]);
        let c = a.dot(&b);
        let reference = reference_mat_mul(&a, &b);
        let diff = (&c - &reference).mapv_into(f32::abs);

        let rtol = 1e-3;
        let atol = 1e-4;
        let crtol = c.mapv(|x| x.abs() * rtol);
        let tol = crtol + atol;
        let tol_m_diff = &diff - &tol;
        let maxdiff = *tol_m_diff.max();
        println!("diff offset from tolerance level= {:.2e}", maxdiff);
        if maxdiff > 0. {
            panic!("results differ");
        }
    }
}

#[test]
fn accurate_mul_f64() {
    // pick a few random sizes
    let mut rng = rand::weak_rng();
    for i in 0..20 {
        let m = rng.gen_range(15, 512);
        let k = rng.gen_range(15, 512);
        let n = rng.gen_range(15, 1560);
        let a = gen_f64(Ix2(m, k));
        let b = gen_f64(Ix2(n, k));
        let b = b.t();
        let (a, b) = if i > 10 {
            (a.slice(s![..;2, ..;2]),
             b.slice(s![..;2, ..;2]))
        } else { (a.view(), b) };

        println!("Testing size {} by {} by {}", a.shape()[0], a.shape()[1], b.shape()[1]);
        let c = a.dot(&b);
        let reference = reference_mat_mul(&a, &b);
        let diff = (&c - &reference).mapv_into(f64::abs);

        let rtol = 1e-7;
        let atol = 1e-12;
        let crtol = c.mapv(|x| x.abs() * rtol);
        let tol = crtol + atol;
        let tol_m_diff = &diff - &tol;
        let maxdiff = *tol_m_diff.max();
        println!("diff offset from tolerance level= {:.2e}", maxdiff);
        if maxdiff > 0. {
            panic!("results differ");
        }
    }
}

trait Utils {
    type Elem;
    type Dim;
    type Data;
    fn max(&self) -> &Self::Elem
        where Self::Elem: PartialOrd;
    fn lift<F>(&mut self, F)
        where F: FnMut(Self::Elem) -> Self::Elem, Self::Elem: Copy,
              Self::Data: DataMut;
    fn fold_axis<F>(&self, axis: Axis, f: F) -> Array<Self::Elem, <Self::Dim as RemoveAxis>::Smaller>
        where Self::Dim: RemoveAxis,
              Self::Elem: Clone,
              F: FnMut(&Self::Elem, &Self::Elem) -> Self::Elem;
}

impl<A, S, D> Utils for ArrayBase<S, D>
    where S: Data<Elem=A>,
          D: Dimension,
{
    type Elem = A;
    type Dim = D;
    type Data = S;

    fn max(&self) -> &A
        where A: PartialOrd
    {
        let mut iter = self.iter();
        if let Some(mut max) = iter.next() {
            for elt in iter {
                if elt > max {
                    max = elt;
                }
            }
            max
        } else {
            panic!("empty");
        }
    }
    fn lift<F>(&mut self, mut f: F)
        where F: FnMut(A) -> A, A: Copy,
              S: DataMut<Elem=A>

    {
        for elt in self {
            *elt = f(*elt);
        }
    }

    fn fold_axis<F>(&self, axis: Axis, mut f: F) -> Array<A, D::Smaller>
        where D: RemoveAxis,
              F: FnMut(&A, &A) -> A,
              A: Clone,
    {
        let len = self.shape()[axis.axis()];
        let mut row = self.subview(axis, 0).to_owned();
        for i in 1..len {
            let r2 = self.subview(axis, i);
            for (a, b) in row.iter_mut().zip(r2) {
                *a = f(a, b);
            }
        }
        row
    }
}
<commit_msg>Remove unused code in numeric-tests<commit_after>
#[macro_use(s)]
extern crate ndarray;
extern crate ndarray_rand;
extern crate rand;

use ndarray_rand::{RandomExt, F32};
use rand::Rng;

use ndarray::prelude::*;
use ndarray::{
    Data,
    LinalgScalar,
};

use rand::distributions::Normal;

// simple, slow, correct (hopefully) mat mul
fn reference_mat_mul<A, S, S2>(lhs: &ArrayBase<S, Ix2>, rhs: &ArrayBase<S2, Ix2>)
    -> Array<A, Ix2>
    where A: LinalgScalar,
          S: Data<Elem=A>,
          S2: Data<Elem=A>,
{
    let ((m, k), (_, n)) = (lhs.dim(), rhs.dim());
    let mut res_elems = Vec::<A>::with_capacity(m * n);
    unsafe {
        res_elems.set_len(m * n);
    }

    let mut i = 0;
    let mut j = 0;
    for rr in &mut res_elems {
        unsafe {
            *rr = (0..k).fold(A::zero(),
                move |s, x| s + *lhs.uget((i, x)) * *rhs.uget((x, j)));
        }
        j += 1;
        if j == n {
            j = 0;
            i += 1;
        }
    }
    unsafe {
        ArrayBase::from_shape_vec_unchecked((m, n), res_elems)
    }
}

fn gen<D>(d: D) -> Array<f32, D>
    where D: Dimension,
{
    Array::random(d, F32(Normal::new(0., 1.)))
}
fn gen_f64<D>(d: D) -> Array<f64, D>
    where D: Dimension,
{
    Array::random(d, Normal::new(0., 1.))
}

#[test]
fn accurate_eye_f32() {
    for i in 0..20 {
        let eye = Array::eye(i);
        for j in 0..20 {
            let a = gen(Ix2(i, j));
            let a2 = eye.dot(&a);
            if !a.all_close(&a2, 1e-6) {
                panic!("Arrays are not equal:\n{:?}\n{:?}\n{:?}", a, a2, &a2 - &a);
            }
            let a3 = a.t().dot(&eye);
            if !a.t().all_close(&a3, 1e-6) {
                panic!("Arrays are not equal:\n{:?}\n{:?}\n{:?}", a.t(), a3, &a3 - &a.t());
            }
        }
    }
    // pick a few random sizes
    let mut rng = rand::weak_rng();
    for _ in 0..10 {
        let i = rng.gen_range(15, 512);
        let j = rng.gen_range(15, 512);
        println!("Testing size {} by {}", i, j);
        let a = gen(Ix2(i, j));
        let eye = Array::eye(i);
        let a2 = eye.dot(&a);
        if !a.all_close(&a2, 1e-6) {
            panic!("Arrays are not equal:\n{:?}\n{:?}\n{:?}", a, a2, &a2 - &a);
        }
        let a3 = a.t().dot(&eye);
        if !a.t().all_close(&a3, 1e-6) {
            panic!("Arrays are not equal:\n{:?}\n{:?}\n{:?}", a.t(), a3, &a3 - &a.t());
        }
    }
}

#[test]
fn accurate_eye_f64() {
    let abs_tol = 1e-15;
    for i in 0..20 {
        let eye = Array::eye(i);
        for j in 0..20 {
            let a = gen_f64(Ix2(i, j));
            let a2 = eye.dot(&a);
            if !a.all_close(&a2, abs_tol) {
                panic!("Arrays are not equal:\n{:?}\n{:?}\n{:?}", a, a2, &a2 - &a);
            }
            let a3 = a.t().dot(&eye);
            if !a.t().all_close(&a3, abs_tol) {
                panic!("Arrays are not equal:\n{:?}\n{:?}\n{:?}", a.t(), a3, &a3 - &a.t());
            }
        }
    }
    // pick a few random sizes
    let mut rng = rand::weak_rng();
    for _ in 0..10 {
        let i = rng.gen_range(15, 512);
        let j = rng.gen_range(15, 512);
        println!("Testing size {} by {}", i, j);
        let a = gen_f64(Ix2(i, j));
        let eye = Array::eye(i);
        let a2 = eye.dot(&a);
        if !a.all_close(&a2, 1e-6) {
            panic!("Arrays are not equal:\n{:?}\n{:?}\n{:?}", a, a2, &a2 - &a);
        }
        let a3 = a.t().dot(&eye);
        if !a.t().all_close(&a3, 1e-6) {
            panic!("Arrays are not equal:\n{:?}\n{:?}\n{:?}", a.t(), a3, &a3 - &a.t());
        }
    }
}

#[test]
fn accurate_mul_f32() {
    // pick a few random sizes
    let mut rng = rand::weak_rng();
    for i in 0..20 {
        let m = rng.gen_range(15, 512);
        let k = rng.gen_range(15, 512);
        let n = rng.gen_range(15, 1560);
        let a = gen(Ix2(m, k));
        let b = gen(Ix2(n, k));
        let b = b.t();
        let (a, b) = if i > 10 {
            (a.slice(s![..;2, ..;2]),
             b.slice(s![..;2, ..;2]))
        } else { (a.view(), b) };

        println!("Testing size {} by {} by {}", a.shape()[0], a.shape()[1], b.shape()[1]);
        let c = a.dot(&b);
        let reference = reference_mat_mul(&a, &b);
        let diff = (&c - &reference).mapv_into(f32::abs);

        let rtol = 1e-3;
        let atol = 1e-4;
        let crtol = c.mapv(|x| x.abs() * rtol);
        let tol = crtol + atol;
        let tol_m_diff = &diff - &tol;
        let maxdiff = *tol_m_diff.max();
        println!("diff offset from tolerance level= {:.2e}", maxdiff);
        if maxdiff > 0. {
            panic!("results differ");
        }
    }
}

#[test]
fn accurate_mul_f64() {
    // pick a few random sizes
    let mut rng = rand::weak_rng();
    for i in 0..20 {
        let m = rng.gen_range(15, 512);
        let k = rng.gen_range(15, 512);
        let n = rng.gen_range(15, 1560);
        let a = gen_f64(Ix2(m, k));
        let b = gen_f64(Ix2(n, k));
        let b = b.t();
        let (a, b) = if i > 10 {
            (a.slice(s![..;2, ..;2]),
             b.slice(s![..;2, ..;2]))
        } else { (a.view(), b) };

        println!("Testing size {} by {} by {}", a.shape()[0], a.shape()[1], b.shape()[1]);
        let c = a.dot(&b);
        let reference = reference_mat_mul(&a, &b);
        let diff = (&c - &reference).mapv_into(f64::abs);

        let rtol = 1e-7;
        let atol = 1e-12;
        let crtol = c.mapv(|x| x.abs() * rtol);
        let tol = crtol + atol;
        let tol_m_diff = &diff - &tol;
        let maxdiff = *tol_m_diff.max();
        println!("diff offset from tolerance level= {:.2e}", maxdiff);
        if maxdiff > 0. {
            panic!("results differ");
        }
    }
}

trait Utils {
    type Elem;
    type Dim;
    type Data;
    fn max(&self) -> &Self::Elem
        where Self::Elem: PartialOrd;
}

impl<A, S, D> Utils for ArrayBase<S, D>
    where S: Data<Elem=A>,
          D: Dimension,
{
    type Elem = A;
    type Dim = D;
    type Data = S;

    fn max(&self) -> &A
        where A: PartialOrd
    {
        let mut iter = self.iter();
        if let Some(mut max) = iter.next() {
            for elt in iter {
                if elt > max {
                    max = elt;
                }
            }
            max
        } else {
            panic!("empty");
        }
    }
}
<|endoftext|>


<commit_before>// Copyright (c) The Libra Core Contributors
// SPDX-License-Identifier: Apache-2.0

use serde::{Deserialize, Serialize};

/// Dictates a set of permissions
#[derive(Debug, Default, Deserialize, PartialEq, Serialize)]
pub struct Policy {
    pub permissions: Vec<Permission>,
}

impl Policy {
    pub fn new(permissions: Vec<Permission>) -> Self {
        Self { permissions }
    }

    pub fn is_default(&self) -> bool {
        Self::default() == *self
    }

    pub fn public() -> Self {
        Self::new(vec![Permission::new(
            Identity::Anyone,
            vec![Capability::Read, Capability::Write],
        )])
    }
}

/// Maps an identity to a set of capabilities
#[derive(Debug, Deserialize, PartialEq, Serialize)]
pub struct Permission {
    pub id: Identity,
    pub capabilities: Vec<Capability>,
}

impl Permission {
    pub fn new(id: Identity, capabilities: Vec<Capability>) -> Self {
        Self { id, capabilities }
    }
}

/// Id represents a Libra internal identifier for a given process. For example, safety_rules or
/// key_manager. It is up to the Storage and its deployment to translate these identifiers into
/// verifiable material. For example, the process running safety_rules may have a token that is
/// intended for only safety_rules to own. The specifics are left to the implementation of the
/// storage backend interface layer.
#[derive(Debug, Deserialize, PartialEq, Serialize)]
pub enum Identity {
    User(String),
    Anyone,
    NoOne,
}

/// Represents actions
#[derive(Debug, Deserialize, PartialEq, Serialize)]
pub enum Capability {
    Export,
    Read,
    Rotate,
    Sign,
    Write,
}
<commit_msg>Removing is_default from secure/storage/src/policy.rs<commit_after>// Copyright (c) The Libra Core Contributors
// SPDX-License-Identifier: Apache-2.0

use serde::{Deserialize, Serialize};

/// Dictates a set of permissions
#[derive(Debug, Default, Deserialize, PartialEq, Serialize)]
pub struct Policy {
    pub permissions: Vec<Permission>,
}

impl Policy {
    pub fn new(permissions: Vec<Permission>) -> Self {
        Self { permissions }
    }

    pub fn public() -> Self {
        Self::new(vec![Permission::new(
            Identity::Anyone,
            vec![Capability::Read, Capability::Write],
        )])
    }
}

/// Maps an identity to a set of capabilities
#[derive(Debug, Deserialize, PartialEq, Serialize)]
pub struct Permission {
    pub id: Identity,
    pub capabilities: Vec<Capability>,
}

impl Permission {
    pub fn new(id: Identity, capabilities: Vec<Capability>) -> Self {
        Self { id, capabilities }
    }
}

/// Id represents a Libra internal identifier for a given process. For example, safety_rules or
/// key_manager. It is up to the Storage and its deployment to translate these identifiers into
/// verifiable material. For example, the process running safety_rules may have a token that is
/// intended for only safety_rules to own. The specifics are left to the implementation of the
/// storage backend interface layer.
#[derive(Debug, Deserialize, PartialEq, Serialize)]
pub enum Identity {
    User(String),
    Anyone,
    NoOne,
}

/// Represents actions
#[derive(Debug, Deserialize, PartialEq, Serialize)]
pub enum Capability {
    Export,
    Read,
    Rotate,
    Sign,
    Write,
}
<|endoftext|>


<commit_before>#![allow(non_snake_case, non_upper_case_globals)]
// #![cfg_attr(test, feature(test))]
extern crate time;
extern crate num_cpus;
extern crate scoped_threadpool;

use time::precise_time_s;
use scoped_threadpool::Pool;
use std::mem;

///////////////////////////////////////////////////////////////////////////////
// 1. Calibration
///////////////////////////////////////////////////////////////////////////////
const aalpha:f64 = 0.33333333333;  // Elasticity of output w.r.t. capital
const bbeta:f64 = 0.95;           // Discount factor

// Productivity values
const vProductivity:[f64; 5] = [0.9792, 0.9896, 1.0000, 1.0106, 1.0212];

// Transition matrix
static mTransition:[[f64; 5]; 5] = [[0.9727, 0.0273, 0.0000, 0.0000, 0.0000],
                                    [0.0041, 0.9806, 0.0153, 0.0000, 0.0000],
                                    [0.0000, 0.0082, 0.9837, 0.0082, 0.0000],
                                    [0.0000, 0.0000, 0.0153, 0.9806, 0.0041],
                                    [0.0000, 0.0000, 0.0000, 0.0273, 0.9727]];

// Dimensions to generate the grid of capital
const nGridCapital: usize = 17820;
const nGridProductivity: usize = 5;

fn solve(print: bool) -> f64 {
  /////////////////////////////////////////////////////////////////////////////
  // 2. Steady State
  /////////////////////////////////////////////////////////////////////////////

    let thread_count = num_cpus::get();
    let mut pool = Pool::new(thread_count as u32);

    let capitalSteadyState: f64 = (aalpha * bbeta).powf(1_f64 / (1_f64 - aalpha));
    let outputSteadyState: f64 = capitalSteadyState.powf(aalpha);
    let consumptionSteadyState: f64 = outputSteadyState - capitalSteadyState;

    if print {
        println!("Output = {}, Capital = {}, Consumption = {}",
                 outputSteadyState,
                 capitalSteadyState,
                 consumptionSteadyState);
    }

    let mut vGridCapital = [0f64; nGridCapital];

    for (i, val) in vGridCapital.iter_mut().enumerate() {
        *val = 0.5 * capitalSteadyState + 0.00001 * (i as f64)
    }

    // 3. Required matrices and vectors
    let mut mValueFunction: [Vec<f64>; nGridProductivity] = [
        vec![0f64; nGridCapital],
        vec![0f64; nGridCapital],
        vec![0f64; nGridCapital],
        vec![0f64; nGridCapital],
        vec![0f64; nGridCapital],
    ];

    let mut mPolicyFunction: [Vec<f64>; nGridProductivity] = [
        vec![0f64; nGridCapital],
        vec![0f64; nGridCapital],
        vec![0f64; nGridCapital],
        vec![0f64; nGridCapital],
        vec![0f64; nGridCapital],
    ];

    let mut expectedValueFunction: [Vec<f64>; nGridProductivity] = [
        vec![0f64; nGridCapital],
        vec![0f64; nGridCapital],
        vec![0f64; nGridCapital],
        vec![0f64; nGridCapital],
        vec![0f64; nGridCapital],
    ];

    // 4. We pre-build output for each point in the grid
    let mOutput : Vec<[f64; nGridProductivity]> = (0..nGridCapital).map(|nCapital| {
        let mut arr = [0.0; nGridProductivity];

        for (nProductivity, slot) in arr.iter_mut().enumerate() {
            *slot = vProductivity[nProductivity] * vGridCapital[nCapital].powf(aalpha)
        }

        arr
    }).collect();

  // 5. Main iteration
  // TODO: one could implement a macro for the multiple declarations
    let mut maxDifference = 10_f64;
    const tolerance: f64 = 0.0000001; // compiler warn: variable does not need to be mutable

    let mut iteration = 0;

    // small array to split where the writes are going, so we don't need a mutex
    let mut differences = [-100000.0; nGridProductivity];

    while maxDifference > tolerance {
        pool.scoped(|scoped| {
            for (expected_values, transitions) in expectedValueFunction.iter_mut()
                                                                       .zip(mTransition.iter()) {
                // Only capture refs
                let mValueFunction = &mValueFunction;

                scoped.execute(move || {
                    for (idx, expected_value) in expected_values.iter_mut().enumerate() {
                        *expected_value = transitions.iter()
                                                     .zip(mValueFunction.iter())
                                                     .fold(0.0f64, |acc, (transition, value_fns)| {
                            acc + (transition * value_fns[idx])
                        });
                    }
                });
            }
        });

        pool.scoped(|scoped| {
            for (nProductivity, (((policies, value_fns), maxDifference), expected_values)) in mPolicyFunction.iter_mut()
                                                                         .zip(mValueFunction.iter_mut())
                                                                         .zip(differences.iter_mut())
                                                                         .zip(expectedValueFunction.iter())
                                                                         .enumerate() {
                *maxDifference = -100000.0;

                // Only capture refs
                let mOutput = &mOutput;

                scoped.execute(move || {
                    // We start from previous choice (monotonicity of policy function)
                    let mut gridCapitalNextPeriod = 0;

                    for ((policy, output), value_fn) in policies.iter_mut()
                                                                .zip(mOutput.iter())
                                                                .zip(value_fns.iter_mut()) {
                        let mut valueHighSoFar = -100000.0;
                        let mut capitalChoice = vGridCapital[0];
                        let mOutput_cache = &output[nProductivity];

                        for nCapitalNextPeriod in gridCapitalNextPeriod..nGridCapital {
                            let consumption = mOutput_cache - &vGridCapital[nCapitalNextPeriod];
                            let valueProvisional = (1_f64 - bbeta) * (consumption.ln()) +
                                                    bbeta *
                                                    expected_values[nCapitalNextPeriod];

                            if valueProvisional > valueHighSoFar {
                                valueHighSoFar = valueProvisional;
                                capitalChoice = vGridCapital[nCapitalNextPeriod];
                                gridCapitalNextPeriod = nCapitalNextPeriod;
                            } else {
                                break; // We break when we have achieved the max
                            }
                        }

                        let old = mem::replace(value_fn, valueHighSoFar);
                        let diff = (old - valueHighSoFar).abs();
                        if diff > *maxDifference {
                            *maxDifference = diff
                        }

                        *policy = capitalChoice;
                    }
                })
            }
        });

        maxDifference = -100000.0;
        for &diff in &differences {
            if diff > maxDifference {
                maxDifference = diff
            }
        }

        iteration += 1;
        if print && (iteration % 10 == 0 || iteration == 1) {
            println!("Iteration = {}, Sup Diff = {}", iteration, maxDifference);
        }
    }

    if print {
        println!("Iteration = {}, Sup Diff = {}", iteration, maxDifference);
    }

    mPolicyFunction[2][999]
}

fn main() {
    let sample = false;

    if sample {
        let mut samples = (0..5).map(|i| {
            let cpu0 = precise_time_s();
            let result = solve(false);
            let cpu1 = precise_time_s();

            assert_eq!(result, 0.1465491436956954);

            let diff = cpu1 - cpu0;
            println!("Sample #{}, Time: {}s", i + 1, diff);
            diff
        }).collect::<Vec<f64>>();

        samples.sort_by(|a, b| a.partial_cmp(b).unwrap());
        println!("\nMedian time is = {}", samples[2]);
    } else {
        let cpu0 = precise_time_s();
        let result = solve(true);
        let cpu1 = precise_time_s();

        println!("My check = {}\n", result);
        assert_eq!(result, 0.1465491436956954);
        println!("Elapsed time is = {}", cpu1 - cpu0);
    }
}

// FIXME: Can't use bench as the loop is too slow (takes 8mins to get samples)
// #[cfg(test)]
// mod test {
//     extern crate test;
//     use self::test::Bencher;
//     use super::solve;

//     #[bench]
//     fn bench(b: &mut Bencher) {
//         b.iter(|| {
//             // use `test::black_box` to prevent compiler optimizations from disregarding
//             // unused values
//             test::black_box(solve(false));
//         });
//     }
// }
<commit_msg>Use --sample as an arg to run 5 iterations<commit_after>#![allow(non_snake_case, non_upper_case_globals)]
// #![cfg_attr(test, feature(test))]
extern crate time;
extern crate num_cpus;
extern crate scoped_threadpool;

use time::precise_time_s;
use scoped_threadpool::Pool;
use std::mem;

///////////////////////////////////////////////////////////////////////////////
// 1. Calibration
///////////////////////////////////////////////////////////////////////////////
const aalpha:f64 = 0.33333333333;  // Elasticity of output w.r.t. capital
const bbeta:f64 = 0.95;           // Discount factor

// Productivity values
const vProductivity:[f64; 5] = [0.9792, 0.9896, 1.0000, 1.0106, 1.0212];

// Transition matrix
static mTransition:[[f64; 5]; 5] = [[0.9727, 0.0273, 0.0000, 0.0000, 0.0000],
                                    [0.0041, 0.9806, 0.0153, 0.0000, 0.0000],
                                    [0.0000, 0.0082, 0.9837, 0.0082, 0.0000],
                                    [0.0000, 0.0000, 0.0153, 0.9806, 0.0041],
                                    [0.0000, 0.0000, 0.0000, 0.0273, 0.9727]];

// Dimensions to generate the grid of capital
const nGridCapital: usize = 17820;
const nGridProductivity: usize = 5;

fn solve(print: bool) -> f64 {
  /////////////////////////////////////////////////////////////////////////////
  // 2. Steady State
  /////////////////////////////////////////////////////////////////////////////

    let thread_count = num_cpus::get();
    let mut pool = Pool::new(thread_count as u32);

    let capitalSteadyState: f64 = (aalpha * bbeta).powf(1_f64 / (1_f64 - aalpha));
    let outputSteadyState: f64 = capitalSteadyState.powf(aalpha);
    let consumptionSteadyState: f64 = outputSteadyState - capitalSteadyState;

    if print {
        println!("Output = {}, Capital = {}, Consumption = {}",
                 outputSteadyState,
                 capitalSteadyState,
                 consumptionSteadyState);
    }

    let mut vGridCapital = [0f64; nGridCapital];

    for (i, val) in vGridCapital.iter_mut().enumerate() {
        *val = 0.5 * capitalSteadyState + 0.00001 * (i as f64)
    }

    // 3. Required matrices and vectors
    let mut mValueFunction: [Vec<f64>; nGridProductivity] = [
        vec![0f64; nGridCapital],
        vec![0f64; nGridCapital],
        vec![0f64; nGridCapital],
        vec![0f64; nGridCapital],
        vec![0f64; nGridCapital],
    ];

    let mut mPolicyFunction: [Vec<f64>; nGridProductivity] = [
        vec![0f64; nGridCapital],
        vec![0f64; nGridCapital],
        vec![0f64; nGridCapital],
        vec![0f64; nGridCapital],
        vec![0f64; nGridCapital],
    ];

    let mut expectedValueFunction: [Vec<f64>; nGridProductivity] = [
        vec![0f64; nGridCapital],
        vec![0f64; nGridCapital],
        vec![0f64; nGridCapital],
        vec![0f64; nGridCapital],
        vec![0f64; nGridCapital],
    ];

    // 4. We pre-build output for each point in the grid
    let mOutput : Vec<[f64; nGridProductivity]> = (0..nGridCapital).map(|nCapital| {
        let mut arr = [0.0; nGridProductivity];

        for (nProductivity, slot) in arr.iter_mut().enumerate() {
            *slot = vProductivity[nProductivity] * vGridCapital[nCapital].powf(aalpha)
        }

        arr
    }).collect();

  // 5. Main iteration
  // TODO: one could implement a macro for the multiple declarations
    let mut maxDifference = 10_f64;
    const tolerance: f64 = 0.0000001; // compiler warn: variable does not need to be mutable

    let mut iteration = 0;

    // small array to split where the writes are going, so we don't need a mutex
    let mut differences = [-100000.0; nGridProductivity];

    while maxDifference > tolerance {
        pool.scoped(|scoped| {
            for (expected_values, transitions) in expectedValueFunction.iter_mut()
                                                                       .zip(mTransition.iter()) {
                // Only capture refs
                let mValueFunction = &mValueFunction;

                scoped.execute(move || {
                    for (idx, expected_value) in expected_values.iter_mut().enumerate() {
                        *expected_value = transitions.iter()
                                                     .zip(mValueFunction.iter())
                                                     .fold(0.0f64, |acc, (transition, value_fns)| {
                            acc + (transition * value_fns[idx])
                        });
                    }
                });
            }
        });

        pool.scoped(|scoped| {
            for (nProductivity, (((policies, value_fns), maxDifference), expected_values)) in mPolicyFunction.iter_mut()
                                                                         .zip(mValueFunction.iter_mut())
                                                                         .zip(differences.iter_mut())
                                                                         .zip(expectedValueFunction.iter())
                                                                         .enumerate() {
                *maxDifference = -100000.0;

                // Only capture refs
                let mOutput = &mOutput;

                scoped.execute(move || {
                    // We start from previous choice (monotonicity of policy function)
                    let mut gridCapitalNextPeriod = 0;

                    for ((policy, output), value_fn) in policies.iter_mut()
                                                                .zip(mOutput.iter())
                                                                .zip(value_fns.iter_mut()) {
                        let mut valueHighSoFar = -100000.0;
                        let mut capitalChoice = vGridCapital[0];
                        let mOutput_cache = &output[nProductivity];

                        for nCapitalNextPeriod in gridCapitalNextPeriod..nGridCapital {
                            let consumption = mOutput_cache - &vGridCapital[nCapitalNextPeriod];
                            let valueProvisional = (1_f64 - bbeta) * (consumption.ln()) +
                                                    bbeta *
                                                    expected_values[nCapitalNextPeriod];

                            if valueProvisional > valueHighSoFar {
                                valueHighSoFar = valueProvisional;
                                capitalChoice = vGridCapital[nCapitalNextPeriod];
                                gridCapitalNextPeriod = nCapitalNextPeriod;
                            } else {
                                break; // We break when we have achieved the max
                            }
                        }

                        let old = mem::replace(value_fn, valueHighSoFar);
                        let diff = (old - valueHighSoFar).abs();
                        if diff > *maxDifference {
                            *maxDifference = diff
                        }

                        *policy = capitalChoice;
                    }
                })
            }
        });

        maxDifference = -100000.0;
        for &diff in &differences {
            if diff > maxDifference {
                maxDifference = diff
            }
        }

        iteration += 1;
        if print && (iteration % 10 == 0 || iteration == 1) {
            println!("Iteration = {}, Sup Diff = {}", iteration, maxDifference);
        }
    }

    if print {
        println!("Iteration = {}, Sup Diff = {}", iteration, maxDifference);
    }

    mPolicyFunction[2][999]
}

fn main() {
    use std::env::args;

    let sample = args().any(|arg| arg == "--sample");

    if sample {
        let mut samples = (0..5).map(|i| {
            let cpu0 = precise_time_s();
            let result = solve(false);
            let cpu1 = precise_time_s();

            assert_eq!(result, 0.1465491436956954);

            let diff = cpu1 - cpu0;
            println!("Sample #{}, Time: {}s", i + 1, diff);
            diff
        }).collect::<Vec<f64>>();

        samples.sort_by(|a, b| a.partial_cmp(b).unwrap());
        println!("\nMedian time is = {}", samples[2]);
    } else {
        let cpu0 = precise_time_s();
        let result = solve(true);
        let cpu1 = precise_time_s();

        println!("My check = {}\n", result);
        assert_eq!(result, 0.1465491436956954);
        println!("Elapsed time is = {}", cpu1 - cpu0);
    }
}

// FIXME: Can't use bench as the loop is too slow (takes 8mins to get samples)
// #[cfg(test)]
// mod test {
//     extern crate test;
//     use self::test::Bencher;
//     use super::solve;

//     #[bench]
//     fn bench(b: &mut Bencher) {
//         b.iter(|| {
//             // use `test::black_box` to prevent compiler optimizations from disregarding
//             // unused values
//             test::black_box(solve(false));
//         });
//     }
// }
<|endoftext|>


<commit_before>use std::{mem, raw};
use {Vector, Bitcast};

pub unsafe trait Convert<Out> {
    fn convert(self) -> Out;
}

unsafe impl<'a, In, Out> Convert<&'a [Out]> for &'a [In]
    where In: Vector, Out: Vector,
          <In as Vector>::Item: Bitcast<<Out as Vector>::Item> {
        #[inline]
        fn convert(self) -> &'a [Out] {
            // we need to ensure that the pointer has valid alignment
            // and that the slice has correct length to fit `Out`s
            // exactly; the first checks in each `assert` are
            // statically checkable, and so hopefully results in the
            // asserts being removed for "downcasts" (e.g. [u32x4] to
            // [u32]).
            assert!(mem::align_of::<In>() >= mem::align_of::<Out>() ||
                    self.as_ptr() as usize % mem::align_of::<Out>() == 0);
            assert!(mem::size_of::<In>() % mem::size_of::<Out>() == 0 ||
                    (self.len() * Vector::count(None::<In>)) % Vector::count(None::<Out>) == 0);

            unsafe {
                mem::transmute(raw::Slice {
                    data: self.as_ptr() as *mut Out,
                    len: self.len() * Vector::count(None::<In>) / Vector::count(None::<Out>)
                })
            }
        }
}
unsafe impl<'a, In, Out> Convert<&'a mut [Out]> for &'a mut [In]
    where In: Vector, Out: Vector,
          <In as Vector>::Item: Bitcast<<Out as Vector>::Item> {
        #[inline]
        fn convert(self) -> &'a mut [Out] {
            // we need to ensure that the pointer has valid alignment
            // and that the slice has correct length to fit `Out`s
            // exactly; the first checks in each `assert` are
            // statically checkable, and so hopefully results in the
            // asserts being removed for "downcasts" (e.g. [u32x4] to
            // [u32]).
            assert!(mem::align_of::<In>() >= mem::align_of::<Out>() ||
                    self.as_ptr() as usize % mem::align_of::<Out>() == 0);
            assert!(mem::size_of::<In>() % mem::size_of::<Out>() == 0 ||
                    (self.len() * Vector::count(None::<In>)) % Vector::count(None::<Out>) == 0);

            unsafe {
                mem::transmute(raw::Slice {
                    data: self.as_ptr() as *mut Out,
                    len: self.len() * Vector::count(None::<In>) / Vector::count(None::<Out>)
                })
            }
        }
}
<commit_msg>Add the `Tails` and `TailsMut` structs.<commit_after>use std::{cmp, mem, raw};
use {Vector, Bitcast};

pub unsafe trait Convert<Out> {
    fn convert(self) -> Out;
}

unsafe impl<'a, In, Out> Convert<&'a [Out]> for &'a [In]
    where In: Vector, Out: Vector,
          <In as Vector>::Item: Bitcast<<Out as Vector>::Item> {
        #[inline]
        fn convert(self) -> &'a [Out] {
            // we need to ensure that the pointer has valid alignment
            // and that the slice has correct length to fit `Out`s
            // exactly; the first checks in each `assert` are
            // statically checkable, and so hopefully results in the
            // asserts being removed for "downcasts" (e.g. [u32x4] to
            // [u32]).
            assert!(mem::align_of::<In>() >= mem::align_of::<Out>() ||
                    self.as_ptr() as usize % mem::align_of::<Out>() == 0);
            assert!(mem::size_of::<In>() % mem::size_of::<Out>() == 0 ||
                    (self.len() * Vector::count(None::<In>)) % Vector::count(None::<Out>) == 0);

            unsafe {
                mem::transmute(raw::Slice {
                    data: self.as_ptr() as *mut Out,
                    len: self.len() * Vector::count(None::<In>) / Vector::count(None::<Out>)
                })
            }
        }
}
unsafe impl<'a, In, Out> Convert<&'a mut [Out]> for &'a mut [In]
    where In: Vector, Out: Vector,
          <In as Vector>::Item: Bitcast<<Out as Vector>::Item> {
        #[inline]
        fn convert(self) -> &'a mut [Out] {
            // we need to ensure that the pointer has valid alignment
            // and that the slice has correct length to fit `Out`s
            // exactly; the first checks in each `assert` are
            // statically checkable, and so hopefully results in the
            // asserts being removed for "downcasts" (e.g. [u32x4] to
            // [u32]).
            assert!(mem::align_of::<In>() >= mem::align_of::<Out>() ||
                    self.as_ptr() as usize % mem::align_of::<Out>() == 0);
            assert!(mem::size_of::<In>() % mem::size_of::<Out>() == 0 ||
                    (self.len() * Vector::count(None::<In>)) % Vector::count(None::<Out>) == 0);

            unsafe {
                mem::transmute(raw::Slice {
                    data: self.as_ptr() as *mut Out,
                    len: self.len() * Vector::count(None::<In>) / Vector::count(None::<Out>)
                })
            }
        }
}

/// A type that repesents a `&[U]` converted from a `&[T]`, where the
/// conversion may be forced to leave leading/trailing elements (due
/// to alignment/size mismatches).
pub struct Tails<'a,T:'a,U:'a> {
    pub start: &'a [T],
    pub middle: &'a [U],
    pub end: &'a [T],
}
/// A type that repesents a `&mut [U]` converted from a `&mut [T]`,
/// where the conversion may be forced to leave leading/trailing
/// elements (due to alignment/size mismatches).
pub struct TailsMut<'a,T:'a,U:'a> {
    pub start: &'a mut [T],
    pub middle: &'a mut [U],
    pub end: &'a mut [T],
}

unsafe impl<'a, In, Out> Convert<Tails<'a, In, Out>> for &'a [In]
    where In: Vector, Out: Vector,
          <In as Vector>::Item: Bitcast<<Out as Vector>::Item>
{
    #[inline]
    fn convert(self) -> Tails<'a,In,Out> {
        let isize = mem::size_of::<In>();
        let osize = mem::size_of::<In>();
        let oalign = mem::align_of::<Out>();

        let start = self.as_ptr() as usize;
        let len = self.len();

        let offset = start % oalign;
        let slen = if offset == 0 {
            0
        } else {
            cmp::min(len, (oalign - offset) / isize)
        };
        let remaining = len - slen;

        let mlen_in_outs = remaining * isize / osize;
        let mlen_in_ins = mlen_in_outs * osize / isize;

        let (initial, tail) = self.split_at(slen);
        Tails {
            start: initial,
            middle: unsafe {mem::transmute(raw::Slice {
                data: tail.as_ptr() as *mut Out,
                len: mlen_in_outs,
            })},
            end: &tail[mlen_in_ins..]
        }
    }
}
unsafe impl<'a, In, Out> Convert<TailsMut<'a, In, Out>> for &'a mut [In]
    where In: Vector, Out: Vector,
          <In as Vector>::Item: Bitcast<<Out as Vector>::Item>
{
    #[inline]
    fn convert(self) -> TailsMut<'a,In,Out> {
        let isize = mem::size_of::<In>();
        let osize = mem::size_of::<In>();
        let oalign = mem::align_of::<Out>();

        let start = self.as_ptr() as usize;
        let len = self.len();

        let offset = start % oalign;
        let slen = if offset == 0 {
            0
        } else {
            cmp::min(len, (oalign - offset) / isize)
        };
        let remaining = len - slen;

        let mlen_in_outs = remaining * isize / osize;
        let mlen_in_ins = mlen_in_outs * osize / isize;

        let (start, tail) = self.split_at_mut(slen);
        TailsMut {
            start: start,
            middle: unsafe {mem::transmute(raw::Slice {
                data: tail.as_ptr() as *mut Out,
                len: mlen_in_outs,
            })},
            end: &mut tail[mlen_in_ins..]
        }
    }
}
<|endoftext|>


<commit_before>// Copyright 2014 Rich Lane.

// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

extern mod extra;
extern mod glfw;
extern mod gl;
extern mod cgmath;
extern mod noise;

use std::cast;

use gl::types::*;

use cgmath::vector::Vec3;

use noise::Perlin;

use CHUNK_SIZE;

pub fn make_noise_texture() -> GLuint {
    let mut pixels : ~[Vec3<u8>] = ~[];
    static length : i32 = CHUNK_SIZE as i32 * 16;
    let perlin = Perlin::from_seed([43 as uint]);

    for x in range(0, length) {
        for y in range(0, length) {
            let noise = perlin.gen([
                x as f64 * 0.6,
                y as f64 * 0.6
            ]);
            let g = ((noise * 0.3 + 0.4) * 255.0) as u8;
            pixels.push(Vec3 { x: 0, y: g, z: 0 });
        }
    }

    let mut tex : GLuint = 0;

    unsafe {
        gl::GenTextures(1, &mut tex);
    }

    gl::BindTexture(gl::TEXTURE_2D, tex);
    gl::TexParameteri(gl::TEXTURE_2D, gl::TEXTURE_MIN_FILTER, gl::NEAREST_MIPMAP_LINEAR as GLint);
    gl::TexParameteri(gl::TEXTURE_2D, gl::TEXTURE_MAG_FILTER, gl::NEAREST as GLint);

    unsafe {
        gl::TexImage2D(
            gl::TEXTURE_2D, 0,
            gl::RGB as GLint,
            length, length, 0,
            gl::RGB, gl::UNSIGNED_BYTE,
            cast::transmute(&pixels[0]));
    }

    gl::GenerateMipmap(gl::TEXTURE_2D);

    gl::BindTexture(gl::TEXTURE_2D, 0);

    tex
}
<commit_msg>use gl::LINEAR_MIPMAP_LINEAR<commit_after>// Copyright 2014 Rich Lane.

// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

extern mod extra;
extern mod glfw;
extern mod gl;
extern mod cgmath;
extern mod noise;

use std::cast;

use gl::types::*;

use cgmath::vector::Vec3;

use noise::Perlin;

use CHUNK_SIZE;

pub fn make_noise_texture() -> GLuint {
    let mut pixels : ~[Vec3<u8>] = ~[];
    static length : i32 = CHUNK_SIZE as i32 * 16;
    let perlin = Perlin::from_seed([43 as uint]);

    for x in range(0, length) {
        for y in range(0, length) {
            let noise = perlin.gen([
                x as f64 * 0.6,
                y as f64 * 0.6
            ]);
            let g = ((noise * 0.3 + 0.4) * 255.0) as u8;
            pixels.push(Vec3 { x: 0, y: g, z: 0 });
        }
    }

    let mut tex : GLuint = 0;

    unsafe {
        gl::GenTextures(1, &mut tex);
    }

    gl::BindTexture(gl::TEXTURE_2D, tex);
    gl::TexParameteri(gl::TEXTURE_2D, gl::TEXTURE_MIN_FILTER, gl::LINEAR_MIPMAP_LINEAR as GLint);
    gl::TexParameteri(gl::TEXTURE_2D, gl::TEXTURE_MAG_FILTER, gl::NEAREST as GLint);

    unsafe {
        gl::TexImage2D(
            gl::TEXTURE_2D, 0,
            gl::RGB as GLint,
            length, length, 0,
            gl::RGB, gl::UNSIGNED_BYTE,
            cast::transmute(&pixels[0]));
    }

    gl::GenerateMipmap(gl::TEXTURE_2D);

    gl::BindTexture(gl::TEXTURE_2D, 0);

    tex
}
<|endoftext|>


<commit_before>mod serialize;

use self::serialize::{Media, MediaDate, MediaListEntry, MediaStatus, ScoreFormat, User};
use super::{AnimeEntry, AnimeInfo, ScoreParser, Status, SyncBackend};
use config::Config;
use error::BackendError;
use input;
use process;
use reqwest::{Client, Response};
use serde_json as json;
use std::borrow::Cow;

const LOGIN_URL: &str =
    "https://anilist.co/api/v2/oauth/authorize?client_id=427&response_type=token";

const API_URL: &str = "https://graphql.anilist.co";

macro_rules! send_query {
    ($backend:expr, $query_str:expr, {$($vars:tt)*}, $($response_root:expr)=>*) => {{
        let vars = json!({
            $($vars)*
        });

        let query = minimize_graphql_query($query_str);

        $backend.send_json_request(&query, &vars).map(|json| json$([$response_root])*.clone())
    }};
}

fn minimize_graphql_query(query: &str) -> String {
    let mut minimized = String::from(query);
    minimized.retain(|c| c != ' ');

    minimized
}

pub struct AniList {
    client: Client,
    user: User,
    access_token: String,
}

impl AniList {
    fn send_request(
        &self,
        query_str: &str,
        variables: &json::Value,
    ) -> Result<Response, BackendError> {
        let body = json!({
            "query": query_str,
            "variables": variables,
        }).to_string();

        let response = self
            .client
            .post(API_URL)
            .header("Content-Type", "application/json")
            .header("Accept", "application/json")
            .header("Authorization", format!("Bearer {}", self.access_token))
            .body(body)
            .send()?;

        Ok(response)
    }

    fn send_json_request(
        &self,
        query_str: &str,
        variables: &json::Value,
    ) -> Result<json::Value, BackendError> {
        let text = self.send_request(query_str, variables)?.text()?;
        let json: json::Value = json::from_str(&text)?;

        if json["errors"] != json::Value::Null {
            // TODO: add error chaining
            let err = &json["errors"][0];

            let msg = err["message"].as_str().unwrap_or("unknown error");
            let status_code = err["status"].as_u64().unwrap_or(0) as u32;

            return Err(BackendError::BadResponse(status_code, msg.into()));
        }

        Ok(json)
    }

    fn request_user_info(&self) -> Result<User, BackendError> {
        let resp = send_query!(self,
            r#"
                query {
                    Viewer {
                        id
                        mediaListOptions {
                            scoreFormat
                        }
                    }
                }
            "#,
            {},
            "data" => "Viewer"
        )?;

        let user = json::from_value(resp)?;
        Ok(user)
    }

    fn prompt_for_access_token(open_url: bool) -> Result<String, BackendError> {
        if open_url {
            try_open_url(LOGIN_URL);
        }

        println!(
            "please authorize your account in the opened browser tab and paste the code below:"
        );

        let token = input::read_line()?;
        Ok(token)
    }

    fn login(&mut self, is_first_launch: bool, config: &mut Config) -> Result<(), BackendError> {
        let mut times_token_incorrect = 0;

        loop {
            match self.request_user_info() {
                Ok(user) => {
                    self.user = user;
                    break;
                }
                // As bad as checking for a specific error via its message is, the API does not provide
                // anything else to narrow it down to an invalid token error
                Err(BackendError::BadResponse(400, ref msg))
                    if msg.to_lowercase() == "invalid token" =>
                {
                    times_token_incorrect += 1;
                    println!("\ninvalid access token");

                    let should_open_url = !is_first_launch && times_token_incorrect <= 1;
                    let token = AniList::prompt_for_access_token(should_open_url)?;

                    self.access_token = token;
                }
                Err(err) => return Err(err),
            }
        }

        if times_token_incorrect > 0 {
            config.anilist.token.encode(&self.access_token);
        }

        Ok(())
    }
}

impl SyncBackend for AniList {
    fn name() -> &'static str {
        "AniList"
    }

    fn init(offline_mode: bool, config: &mut Config) -> Result<AniList, BackendError> {
        if offline_mode {
            let anilist = AniList {
                client: Client::new(),
                user: User::default(),
                access_token: String::new(),
            };

            return Ok(anilist);
        }

        let is_first_launch = !config.anilist.token.is_set();

        let access_token = if is_first_launch {
            let token = AniList::prompt_for_access_token(true)?;
            config.anilist.token.encode(&token);
            token
        } else {
            config.anilist.token.decode()?
        };

        let mut anilist = AniList {
            client: Client::new(),
            user: User::default(),
            access_token,
        };

        anilist.login(is_first_launch, config)?;

        Ok(anilist)
    }

    fn search_by_name(&self, name: &str) -> Result<Vec<AnimeInfo>, BackendError> {
        let resp = send_query!(self,
            r#"
                query ($name: String) {
                    Page (page: 1, perPage: 30) {
                        media (search: $name, type: ANIME) {
                            id
                            title {
                                romaji
                            }
                            episodes
                        }
                    }
                }
            "#,
            { "name": name },
            "data" => "Page" => "media"
        )?;

        use self::json::Value;
        let mut series = Vec::new();

        match resp {
            Value::Array(ref entries) => {
                for entry in entries {
                    let series_info: Media = json::from_value(entry.clone())?;
                    series.push(series_info.into());
                }
            }
            _ => return Err(BackendError::InvalidJsonResponse),
        }

        Ok(series)
    }

    fn get_series_info_by_id(&self, id: u32) -> Result<AnimeInfo, BackendError> {
        let resp = send_query!(self,
            r#"
                query ($id: Int) {
                    Media (id: $id) {
                        id
                        title {
                            romaji
                        }
                        episodes
                    }
                }
            "#,
            { "id": id },
            "data" => "Media"
        )?;

        let info: Media = json::from_value(resp)?;
        Ok(info.into())
    }

    fn get_list_entry(&self, info: AnimeInfo) -> Result<Option<AnimeEntry>, BackendError> {
        let resp = send_query!(self,
            r#"
                query ($id: Int, $userID: Int) {
                    MediaList(mediaId: $id, userId: $userID) {
                        progress
                        status
                        score
                        startedAt {
                            year
                            month
                            day
                        }
                        completedAt {
                            year
                            month
                            day
                        }
                    }
                }
            "#,
            { "id": info.id, "userID": self.user.id },
            "data" => "MediaList"
        );

        match resp {
            Ok(entry_json) => {
                let media_entry: MediaListEntry = json::from_value(entry_json)?;
                Ok(Some(media_entry.into_generic_entry(info)))
            }
            Err(BackendError::BadResponse(404, _)) => Ok(None),
            Err(err) => Err(err),
        }
    }

    fn update_list_entry(&self, entry: &AnimeEntry) -> Result<(), BackendError> {
        send_query!(self,
            r#"
                mutation (
                    $mediaId: Int,
                    $watched_eps: Int,
                    $score: Float,
                    $status: MediaListStatus,
                    $start_date: FuzzyDateInput,
                    $finish_date: FuzzyDateInput) {

                    SaveMediaListEntry (
                        mediaId: $mediaId,
                        progress: $watched_eps,
                        score: $score,
                        status: $status,
                        startedAt: $start_date,
                        completedAt: $finish_date) {

                        mediaId
                    }
                }
            "#,
            {
                "mediaId": entry.info.id,
                "watched_eps": entry.watched_episodes,
                "score": entry.score.unwrap_or(0.0),
                "status": MediaStatus::from(entry.status),
                "start_date": MediaDate::from_date(entry.start_date),
                "finish_date": MediaDate::from_date(entry.finish_date),
            },
        )?;

        Ok(())
    }
}

impl ScoreParser for AniList {
    fn formatted_score_range(&self) -> (Cow<str>, Cow<str>) {
        match self.user.list_options.score_format {
            ScoreFormat::Point3 => (":(".into(), ":)".into()),
            ScoreFormat::Point10Decimal => ("1.0".into(), "10.0".into()),
            format => ("1".into(), format.max_score().to_string().into()),
        }
    }

    fn parse_score(&self, input: &str) -> Result<f32, BackendError> {
        match self.user.list_options.score_format {
            ScoreFormat::Point3 => match input {
                ":(" => Ok(1.0),
                ":|" => Ok(2.0),
                ":)" => Ok(3.0),
                _ => Err(BackendError::UnknownScoreValue(input.into())),
            },
            format => {
                let value = input.parse::<f32>()?;
                let max_score = f32::from(format.max_score());

                if value < 1.0 || value > max_score {
                    return Err(BackendError::OutOfRangeScore);
                }

                Ok(value)
            }
        }
    }

    fn format_score(&self, raw_score: f32) -> Result<String, BackendError> {
        match self.user.list_options.score_format {
            ScoreFormat::Point3 => match raw_score.round() as u32 {
                1 => Ok(":(".into()),
                2 => Ok(":|".into()),
                3 => Ok(":)".into()),
                _ => Err(BackendError::OutOfRangeScore),
            },
            ScoreFormat::Point5 | ScoreFormat::Point10 | ScoreFormat::Point100 => {
                let value = raw_score.round() as u32;
                Ok(value.to_string())
            }
            ScoreFormat::Point10Decimal => Ok(raw_score.to_string()),
        }
    }
}

fn try_open_url(url: &str) {
    match process::open_with_default(url) {
        Ok(status) if status.success() => (),
        result => {
            eprintln!(
                "failed to open URL in default browser. please open it manually: {}",
                url
            );

            if let Err(err) = result {
                eprintln!("error message: {}", err);
            }
        }
    }
}
<commit_msg>[AniList] Further reduce the size of queries<commit_after>mod serialize;

use self::serialize::{Media, MediaDate, MediaListEntry, MediaStatus, ScoreFormat, User};
use super::{AnimeEntry, AnimeInfo, ScoreParser, Status, SyncBackend};
use config::Config;
use error::BackendError;
use input;
use process;
use reqwest::{Client, Response};
use serde_json as json;
use std::borrow::Cow;

const LOGIN_URL: &str =
    "https://anilist.co/api/v2/oauth/authorize?client_id=427&response_type=token";

const API_URL: &str = "https://graphql.anilist.co";

macro_rules! send_query {
    ($backend:expr, $query_str:expr, {$($vars:tt)*}, $($response_root:expr)=>*) => {{
        let vars = json!({
            $($vars)*
        });

        let query = minimize_graphql_query($query_str);

        $backend.send_json_request(&query, &vars).map(|json| json$([$response_root])*.clone())
    }};
}

fn minimize_graphql_query(query: &str) -> String {
    let mut minimized = String::from(query);
    minimized.retain(|c| c != ' ' && c != '\n');

    minimized
}

pub struct AniList {
    client: Client,
    user: User,
    access_token: String,
}

impl AniList {
    fn send_request(
        &self,
        query_str: &str,
        variables: &json::Value,
    ) -> Result<Response, BackendError> {
        let body = json!({
            "query": query_str,
            "variables": variables,
        })
        .to_string();

        let response = self
            .client
            .post(API_URL)
            .header("Content-Type", "application/json")
            .header("Accept", "application/json")
            .header("Authorization", format!("Bearer {}", self.access_token))
            .body(body)
            .send()?;

        Ok(response)
    }

    fn send_json_request(
        &self,
        query_str: &str,
        variables: &json::Value,
    ) -> Result<json::Value, BackendError> {
        let text = self.send_request(query_str, variables)?.text()?;
        let json: json::Value = json::from_str(&text)?;

        if json["errors"] != json::Value::Null {
            // TODO: add error chaining
            let err = &json["errors"][0];

            let msg = err["message"].as_str().unwrap_or("unknown error");
            let status_code = err["status"].as_u64().unwrap_or(0) as u32;

            return Err(BackendError::BadResponse(status_code, msg.into()));
        }

        Ok(json)
    }

    fn request_user_info(&self) -> Result<User, BackendError> {
        let resp = send_query!(self,
            r#"
                query {
                    Viewer {
                        id,
                        mediaListOptions {
                            scoreFormat
                        }
                    }
                }
            "#,
            {},
            "data" => "Viewer"
        )?;

        let user = json::from_value(resp)?;
        Ok(user)
    }

    fn prompt_for_access_token(open_url: bool) -> Result<String, BackendError> {
        if open_url {
            try_open_url(LOGIN_URL);
        }

        println!(
            "please authorize your account in the opened browser tab and paste the code below:"
        );

        let token = input::read_line()?;
        Ok(token)
    }

    fn login(&mut self, is_first_launch: bool, config: &mut Config) -> Result<(), BackendError> {
        let mut times_token_incorrect = 0;

        loop {
            match self.request_user_info() {
                Ok(user) => {
                    self.user = user;
                    break;
                }
                // As bad as checking for a specific error via its message is, the API does not provide
                // anything else to narrow it down to an invalid token error
                Err(BackendError::BadResponse(400, ref msg))
                    if msg.to_lowercase() == "invalid token" =>
                {
                    times_token_incorrect += 1;
                    println!("\ninvalid access token");

                    let should_open_url = !is_first_launch && times_token_incorrect <= 1;
                    let token = AniList::prompt_for_access_token(should_open_url)?;

                    self.access_token = token;
                }
                Err(err) => return Err(err),
            }
        }

        if times_token_incorrect > 0 {
            config.anilist.token.encode(&self.access_token);
        }

        Ok(())
    }
}

impl SyncBackend for AniList {
    fn name() -> &'static str {
        "AniList"
    }

    fn init(offline_mode: bool, config: &mut Config) -> Result<AniList, BackendError> {
        if offline_mode {
            let anilist = AniList {
                client: Client::new(),
                user: User::default(),
                access_token: String::new(),
            };

            return Ok(anilist);
        }

        let is_first_launch = !config.anilist.token.is_set();

        let access_token = if is_first_launch {
            let token = AniList::prompt_for_access_token(true)?;
            config.anilist.token.encode(&token);
            token
        } else {
            config.anilist.token.decode()?
        };

        let mut anilist = AniList {
            client: Client::new(),
            user: User::default(),
            access_token,
        };

        anilist.login(is_first_launch, config)?;

        Ok(anilist)
    }

    fn search_by_name(&self, name: &str) -> Result<Vec<AnimeInfo>, BackendError> {
        let resp = send_query!(self,
            r#"
                query ($name: String) {
                    Page (page: 1, perPage: 30) {
                        media (search: $name, type: ANIME) {
                            id,
                            title {
                                romaji
                            },
                            episodes
                        }
                    }
                }
            "#,
            { "name": name },
            "data" => "Page" => "media"
        )?;

        use self::json::Value;
        let mut series = Vec::new();

        match resp {
            Value::Array(ref entries) => {
                for entry in entries {
                    let series_info: Media = json::from_value(entry.clone())?;
                    series.push(series_info.into());
                }
            }
            _ => return Err(BackendError::InvalidJsonResponse),
        }

        Ok(series)
    }

    fn get_series_info_by_id(&self, id: u32) -> Result<AnimeInfo, BackendError> {
        let resp = send_query!(self,
            r#"
                query ($id: Int) {
                    Media (id: $id) {
                        id,
                        title {
                            romaji
                        },
                        episodes
                    }
                }
            "#,
            { "id": id },
            "data" => "Media"
        )?;

        let info: Media = json::from_value(resp)?;
        Ok(info.into())
    }

    fn get_list_entry(&self, info: AnimeInfo) -> Result<Option<AnimeEntry>, BackendError> {
        let resp = send_query!(self,
            r#"
                query ($id: Int, $userID: Int) {
                    MediaList(mediaId: $id, userId: $userID) {
                        progress,
                        status,
                        score,
                        startedAt {
                            year,
                            month,
                            day
                        },
                        completedAt {
                            year,
                            month,
                            day
                        }
                    }
                }
            "#,
            { "id": info.id, "userID": self.user.id },
            "data" => "MediaList"
        );

        match resp {
            Ok(entry_json) => {
                let media_entry: MediaListEntry = json::from_value(entry_json)?;
                Ok(Some(media_entry.into_generic_entry(info)))
            }
            Err(BackendError::BadResponse(404, _)) => Ok(None),
            Err(err) => Err(err),
        }
    }

    fn update_list_entry(&self, entry: &AnimeEntry) -> Result<(), BackendError> {
        send_query!(self,
            r#"
                mutation (
                    $mediaId: Int,
                    $watched_eps: Int,
                    $score: Float,
                    $status: MediaListStatus,
                    $start_date: FuzzyDateInput,
                    $finish_date: FuzzyDateInput) {

                    SaveMediaListEntry (
                        mediaId: $mediaId,
                        progress: $watched_eps,
                        score: $score,
                        status: $status,
                        startedAt: $start_date,
                        completedAt: $finish_date) {

                        mediaId
                    }
                }
            "#,
            {
                "mediaId": entry.info.id,
                "watched_eps": entry.watched_episodes,
                "score": entry.score.unwrap_or(0.0),
                "status": MediaStatus::from(entry.status),
                "start_date": MediaDate::from_date(entry.start_date),
                "finish_date": MediaDate::from_date(entry.finish_date),
            },
        )?;

        Ok(())
    }
}

impl ScoreParser for AniList {
    fn formatted_score_range(&self) -> (Cow<str>, Cow<str>) {
        match self.user.list_options.score_format {
            ScoreFormat::Point3 => (":(".into(), ":)".into()),
            ScoreFormat::Point10Decimal => ("1.0".into(), "10.0".into()),
            format => ("1".into(), format.max_score().to_string().into()),
        }
    }

    fn parse_score(&self, input: &str) -> Result<f32, BackendError> {
        match self.user.list_options.score_format {
            ScoreFormat::Point3 => match input {
                ":(" => Ok(1.0),
                ":|" => Ok(2.0),
                ":)" => Ok(3.0),
                _ => Err(BackendError::UnknownScoreValue(input.into())),
            },
            format => {
                let value = input.parse::<f32>()?;
                let max_score = f32::from(format.max_score());

                if value < 1.0 || value > max_score {
                    return Err(BackendError::OutOfRangeScore);
                }

                Ok(value)
            }
        }
    }

    fn format_score(&self, raw_score: f32) -> Result<String, BackendError> {
        match self.user.list_options.score_format {
            ScoreFormat::Point3 => match raw_score.round() as u32 {
                1 => Ok(":(".into()),
                2 => Ok(":|".into()),
                3 => Ok(":)".into()),
                _ => Err(BackendError::OutOfRangeScore),
            },
            ScoreFormat::Point5 | ScoreFormat::Point10 | ScoreFormat::Point100 => {
                let value = raw_score.round() as u32;
                Ok(value.to_string())
            }
            ScoreFormat::Point10Decimal => Ok(raw_score.to_string()),
        }
    }
}

fn try_open_url(url: &str) {
    match process::open_with_default(url) {
        Ok(status) if status.success() => (),
        result => {
            eprintln!(
                "failed to open URL in default browser. please open it manually: {}",
                url
            );

            if let Err(err) = result {
                eprintln!("error message: {}", err);
            }
        }
    }
}
<|endoftext|>


<commit_before>
/// Indexing macro for Dim<[usize; N]> this
/// gets the index at `$i` in the underlying array
macro_rules! get {
    ($dim:expr, $i:expr) => { $dim.ix()[$i] }
}
macro_rules! getm {
    ($dim:expr, $i:expr) => { $dim.ixm()[$i] }
}
<commit_msg>dimension: adjust get/getm macros<commit_after>
/// Indexing macro for Dim<[usize; N]> this
/// gets the index at `$i` in the underlying array
macro_rules! get {
    ($dim:expr, $i:expr) => { (*$dim.ix())[$i] }
}
macro_rules! getm {
    ($dim:expr, $i:expr) => { (*$dim.ixm())[$i] }
}
<|endoftext|>


<commit_before>// Copyright © 2018, Canal TP and/or its affiliates. All rights reserved.
//
// This file is part of Navitia,
//     the software to build cool stuff with public transport.
//
// Hope you'll enjoy and contribute to this project,
//     powered by Canal TP (www.canaltp.fr).
// Help us simplify mobility and open public transport:
//     a non ending quest to the responsive locomotion way of traveling!
//
// LICENCE: This program is free software; you can redistribute it
// and/or modify it under the terms of the GNU Affero General Public
// License as published by the Free Software Foundation, either
// version 3 of the License, or (at your option) any later version.
//
// This program is distributed in the hope that it will be useful, but
// WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
// Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public
// License along with this program. If not, see
// <http://www.gnu.org/licenses/>.
//
// Stay tuned using
// twitter @navitia
// IRC #navitia on freenode
// https://groups.google.com/d/forum/navitia
// www.navitia.io

#[macro_use]
extern crate slog;
#[macro_use]
extern crate slog_scope;

use cosmogony::{Zone, ZoneIndex};
use failure::Error;
use mimir::objects::Admin;
use mimir::rubber::{IndexSettings, Rubber};
use mimirsbrunn::osm_reader::admin;
use mimirsbrunn::osm_reader::osm_utils;
use mimirsbrunn::utils::normalize_weight;
use std::collections::BTreeMap;
use structopt::StructOpt;

trait IntoAdmin {
    fn into_admin(
        self,
        _: &BTreeMap<ZoneIndex, String>,
        langs: &[String],
        retrocompat_on_french_id: bool,
        max_weight: f64,
    ) -> Admin;
}

fn get_weight(tags: &osmpbfreader::Tags, center_tags: &osmpbfreader::Tags) -> f64 {
    // to have an admin weight we use the osm 'population' tag to priorize
    // the big zones over the small one.
    // Note: this tags is not often filled , so only some zones
    // will have a weight (but the main cities have it).
    tags.get("population")
        .and_then(|p| p.parse().ok())
        .or_else(|| center_tags.get("population")?.parse().ok())
        .unwrap_or(0.)
}

impl IntoAdmin for Zone {
    fn into_admin(
        self,
        zones_osm_id: &BTreeMap<ZoneIndex, String>,
        langs: &[String],
        french_id_retrocompatibility: bool,
        max_weight: f64,
    ) -> Admin {
        let insee = admin::read_insee(&self.tags).unwrap_or("");
        let zip_codes = admin::read_zip_codes(&self.tags);
        let label = self.label;
        let weight = get_weight(&self.tags, &self.center_tags);
        let center = self.center.map_or(mimir::Coord::default(), |c| {
            mimir::Coord::new(c.lng(), c.lat())
        });
        let format_id = |id| {
            if french_id_retrocompatibility && insee != "" {
                // for retrocompatibity reasons, Navitia needs the
                // french admins to have an id with the insee
                format!("admin:fr:{}", insee)
            } else {
                format!("admin:osm:{}", id)
            }
        };
        let parent_osm_id = self
            .parent
            .and_then(|id| zones_osm_id.get(&id))
            .map(format_id);
        Admin {
            id: format_id(&self.osm_id),
            insee: insee.into(),
            level: self.admin_level.unwrap_or(0),
            label: label,
            name: self.name,
            zip_codes: zip_codes,
            weight: normalize_weight(weight, max_weight),
            bbox: self.bbox,
            boundary: self.boundary,
            coord: center,
            zone_type: self.zone_type,
            parent_id: parent_osm_id,
            codes: osm_utils::get_osm_codes_from_tags(&self.tags),
            names: osm_utils::get_names_from_tags(&self.tags, &langs),
            labels: self
                .international_labels
                .into_iter()
                .filter(|(k, _)| langs.contains(&k))
                .collect(),
            distance: None,
        }
    }
}

fn send_to_es(
    admins: impl Iterator<Item = Admin>,
    cnx_string: &str,
    dataset: &str,
    index_settings: IndexSettings,
) -> Result<(), Error> {
    let mut rubber = Rubber::new(cnx_string);
    rubber.initialize_templates()?;
    let nb_admins = rubber.index(dataset, &index_settings, admins)?;
    info!("{} admins added.", nb_admins);
    Ok(())
}

fn read_zones(input: &str) -> Result<impl Iterator<Item = Zone>, Error> {
    Ok(cosmogony::read_zones_from_file(input)?.filter_map(|r| {
        r.map_err(|e| log::warn!("impossible to read zone: {}", e))
            .ok()
    }))
}

fn index_cosmogony(args: Args) -> Result<(), Error> {
    info!("building maps");

    let mut max_weight = 0.0;
    let mut cosmogony_id_to_osm_id = BTreeMap::new();
    for z in read_zones(&args.input)? {
        max_weight = f64::max(max_weight, get_weight(&z.tags, &z.center_tags));
        cosmogony_id_to_osm_id.insert(z.id.clone(), z.osm_id.clone());
    }
    let max_weight = max_weight;
    let cosmogony_id_to_osm_id = cosmogony_id_to_osm_id;

    info!("importing cosmogony into Mimir");
    let admins = read_zones(&args.input)?.map(|z| {
        z.into_admin(
            &cosmogony_id_to_osm_id,
            &args.langs,
            args.french_id_retrocompatibility,
            max_weight,
        )
    });

    let index_settings = IndexSettings {
        nb_shards: args.nb_shards,
        nb_replicas: args.nb_replicas,
    };
    send_to_es(
        admins,
        &args.connection_string,
        &args.dataset,
        index_settings,
    )?;

    Ok(())
}

#[derive(StructOpt, Debug)]
struct Args {
    /// cosmogony file
    #[structopt(short = "i", long = "input")]
    input: String,
    /// Elasticsearch parameters.
    #[structopt(
        short = "c",
        long = "connection-string",
        default_value = "http://localhost:9200/munin"
    )]
    connection_string: String,
    /// Name of the dataset.
    #[structopt(short = "d", long = "dataset", default_value = "fr")]
    dataset: String,
    /// Number of shards for the es index
    #[structopt(short = "s", long = "nb-shards", default_value = "1")]
    nb_shards: usize,
    /// Number of replicas for the es index
    #[structopt(short = "r", long = "nb-replicas", default_value = "1")]
    nb_replicas: usize,
    /// Languages codes, used to build i18n names and labels
    #[structopt(name = "lang", short, long)]
    langs: Vec<String>,
    /// Retrocompatibiilty on french admin id
    /// if activated, the french administrative regions will have an id like 'admin:fr:{insee}'
    /// instead of 'admin:osm:{osm_id}'
    #[structopt(long = "french-id-retrocompatibility")]
    french_id_retrocompatibility: bool,
}

fn main() {
    mimirsbrunn::utils::launch_run(index_cosmogony);
}
<commit_msg>set default value of max_weigth to 1<commit_after>// Copyright © 2018, Canal TP and/or its affiliates. All rights reserved.
//
// This file is part of Navitia,
//     the software to build cool stuff with public transport.
//
// Hope you'll enjoy and contribute to this project,
//     powered by Canal TP (www.canaltp.fr).
// Help us simplify mobility and open public transport:
//     a non ending quest to the responsive locomotion way of traveling!
//
// LICENCE: This program is free software; you can redistribute it
// and/or modify it under the terms of the GNU Affero General Public
// License as published by the Free Software Foundation, either
// version 3 of the License, or (at your option) any later version.
//
// This program is distributed in the hope that it will be useful, but
// WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
// Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public
// License along with this program. If not, see
// <http://www.gnu.org/licenses/>.
//
// Stay tuned using
// twitter @navitia
// IRC #navitia on freenode
// https://groups.google.com/d/forum/navitia
// www.navitia.io

#[macro_use]
extern crate slog;
#[macro_use]
extern crate slog_scope;

use cosmogony::{Zone, ZoneIndex};
use failure::Error;
use mimir::objects::Admin;
use mimir::rubber::{IndexSettings, Rubber};
use mimirsbrunn::osm_reader::admin;
use mimirsbrunn::osm_reader::osm_utils;
use mimirsbrunn::utils::normalize_weight;
use std::collections::BTreeMap;
use structopt::StructOpt;

trait IntoAdmin {
    fn into_admin(
        self,
        _: &BTreeMap<ZoneIndex, String>,
        langs: &[String],
        retrocompat_on_french_id: bool,
        max_weight: f64,
    ) -> Admin;
}

fn get_weight(tags: &osmpbfreader::Tags, center_tags: &osmpbfreader::Tags) -> f64 {
    // to have an admin weight we use the osm 'population' tag to priorize
    // the big zones over the small one.
    // Note: this tags is not often filled , so only some zones
    // will have a weight (but the main cities have it).
    tags.get("population")
        .and_then(|p| p.parse().ok())
        .or_else(|| center_tags.get("population")?.parse().ok())
        .unwrap_or(0.)
}

impl IntoAdmin for Zone {
    fn into_admin(
        self,
        zones_osm_id: &BTreeMap<ZoneIndex, String>,
        langs: &[String],
        french_id_retrocompatibility: bool,
        max_weight: f64,
    ) -> Admin {
        let insee = admin::read_insee(&self.tags).unwrap_or("");
        let zip_codes = admin::read_zip_codes(&self.tags);
        let label = self.label;
        let weight = get_weight(&self.tags, &self.center_tags);
        let center = self.center.map_or(mimir::Coord::default(), |c| {
            mimir::Coord::new(c.lng(), c.lat())
        });
        let format_id = |id| {
            if french_id_retrocompatibility && insee != "" {
                // for retrocompatibity reasons, Navitia needs the
                // french admins to have an id with the insee
                format!("admin:fr:{}", insee)
            } else {
                format!("admin:osm:{}", id)
            }
        };
        let parent_osm_id = self
            .parent
            .and_then(|id| zones_osm_id.get(&id))
            .map(format_id);
        Admin {
            id: format_id(&self.osm_id),
            insee: insee.into(),
            level: self.admin_level.unwrap_or(0),
            label: label,
            name: self.name,
            zip_codes: zip_codes,
            weight: normalize_weight(weight, max_weight),
            bbox: self.bbox,
            boundary: self.boundary,
            coord: center,
            zone_type: self.zone_type,
            parent_id: parent_osm_id,
            codes: osm_utils::get_osm_codes_from_tags(&self.tags),
            names: osm_utils::get_names_from_tags(&self.tags, &langs),
            labels: self
                .international_labels
                .into_iter()
                .filter(|(k, _)| langs.contains(&k))
                .collect(),
            distance: None,
        }
    }
}

fn send_to_es(
    admins: impl Iterator<Item = Admin>,
    cnx_string: &str,
    dataset: &str,
    index_settings: IndexSettings,
) -> Result<(), Error> {
    let mut rubber = Rubber::new(cnx_string);
    rubber.initialize_templates()?;
    let nb_admins = rubber.index(dataset, &index_settings, admins)?;
    info!("{} admins added.", nb_admins);
    Ok(())
}

fn read_zones(input: &str) -> Result<impl Iterator<Item = Zone>, Error> {
    Ok(cosmogony::read_zones_from_file(input)?.filter_map(|r| {
        r.map_err(|e| log::warn!("impossible to read zone: {}", e))
            .ok()
    }))
}

fn index_cosmogony(args: Args) -> Result<(), Error> {
    info!("building maps");

    let mut max_weight = 1.0;
    let mut cosmogony_id_to_osm_id = BTreeMap::new();
    for z in read_zones(&args.input)? {
        max_weight = f64::max(max_weight, get_weight(&z.tags, &z.center_tags));
        cosmogony_id_to_osm_id.insert(z.id.clone(), z.osm_id.clone());
    }
    let max_weight = max_weight;
    let cosmogony_id_to_osm_id = cosmogony_id_to_osm_id;

    info!("importing cosmogony into Mimir");
    let admins = read_zones(&args.input)?.map(|z| {
        z.into_admin(
            &cosmogony_id_to_osm_id,
            &args.langs,
            args.french_id_retrocompatibility,
            max_weight,
        )
    });

    let index_settings = IndexSettings {
        nb_shards: args.nb_shards,
        nb_replicas: args.nb_replicas,
    };
    send_to_es(
        admins,
        &args.connection_string,
        &args.dataset,
        index_settings,
    )?;

    Ok(())
}

#[derive(StructOpt, Debug)]
struct Args {
    /// cosmogony file
    #[structopt(short = "i", long = "input")]
    input: String,
    /// Elasticsearch parameters.
    #[structopt(
        short = "c",
        long = "connection-string",
        default_value = "http://localhost:9200/munin"
    )]
    connection_string: String,
    /// Name of the dataset.
    #[structopt(short = "d", long = "dataset", default_value = "fr")]
    dataset: String,
    /// Number of shards for the es index
    #[structopt(short = "s", long = "nb-shards", default_value = "1")]
    nb_shards: usize,
    /// Number of replicas for the es index
    #[structopt(short = "r", long = "nb-replicas", default_value = "1")]
    nb_replicas: usize,
    /// Languages codes, used to build i18n names and labels
    #[structopt(name = "lang", short, long)]
    langs: Vec<String>,
    /// Retrocompatibiilty on french admin id
    /// if activated, the french administrative regions will have an id like 'admin:fr:{insee}'
    /// instead of 'admin:osm:{osm_id}'
    #[structopt(long = "french-id-retrocompatibility")]
    french_id_retrocompatibility: bool,
}

fn main() {
    mimirsbrunn::utils::launch_run(index_cosmogony);
}
<|endoftext|>


<commit_before>//! Bounding volumes.

pub use self::circular_cone::CircularCone;
pub use self::spatialized_normal_cone::SpatializedNormalCone;
#[doc(inline)]
pub use crate::bounding_volume::aabb::{aabb, local_aabb, AABB};
pub use crate::bounding_volume::aabb_ball::ball_aabb;
pub use crate::bounding_volume::aabb_utils::{point_cloud_aabb, support_map_aabb};
#[doc(inline)]
pub use crate::bounding_volume::bounding_sphere::{
    bounding_sphere, local_bounding_sphere, BoundingSphere,
};
pub use crate::bounding_volume::bounding_sphere_utils::{
    point_cloud_bounding_sphere, point_cloud_bounding_sphere_with_center,
};
#[doc(inline)]
pub use crate::bounding_volume::bounding_volume::{BoundingVolume, HasBoundingVolume};

#[doc(hidden)]
pub mod bounding_volume;

#[doc(hidden)]
pub mod aabb;
mod aabb_ball;
mod aabb_compound;
#[cfg(feature = "dim3")]
mod aabb_convex;
#[cfg(feature = "dim2")]
mod aabb_convex_polygon;
mod aabb_cuboid;
mod aabb_heightfield;
mod aabb_plane;
mod aabb_polyline;
mod aabb_shape;
mod aabb_support_map;
#[cfg(feature = "dim3")]
mod aabb_triangle;
#[cfg(feature = "dim3")]
mod aabb_trimesh;
mod aabb_utils;

#[doc(hidden)]
pub mod bounding_sphere;
mod bounding_sphere_ball;
mod bounding_sphere_capsule;
mod bounding_sphere_compound;
#[cfg(feature = "dim3")]
mod bounding_sphere_cone;
#[cfg(feature = "dim3")]
mod bounding_sphere_convex;
#[cfg(feature = "dim2")]
mod bounding_sphere_convex_polygon;
mod bounding_sphere_cuboid;
#[cfg(feature = "dim3")]
mod bounding_sphere_cylinder;
mod bounding_sphere_heightfield;
mod bounding_sphere_plane;
mod bounding_sphere_polyline;
mod bounding_sphere_segment;
mod bounding_sphere_shape;
#[cfg(feature = "dim3")]
mod bounding_sphere_triangle;
#[cfg(feature = "dim3")]
mod bounding_sphere_trimesh;
mod bounding_sphere_utils;

pub(crate) mod circular_cone;
mod spatialized_normal_cone;
<commit_msg>Expose the aabb_utils::local_point_cloud_aabb function.<commit_after>//! Bounding volumes.

pub use self::circular_cone::CircularCone;
pub use self::spatialized_normal_cone::SpatializedNormalCone;
#[doc(inline)]
pub use crate::bounding_volume::aabb::{aabb, local_aabb, AABB};
pub use crate::bounding_volume::aabb_ball::ball_aabb;
pub use crate::bounding_volume::aabb_utils::{point_cloud_aabb, local_point_cloud_aabb, support_map_aabb};
#[doc(inline)]
pub use crate::bounding_volume::bounding_sphere::{
    bounding_sphere, local_bounding_sphere, BoundingSphere,
};
pub use crate::bounding_volume::bounding_sphere_utils::{
    point_cloud_bounding_sphere, point_cloud_bounding_sphere_with_center,
};
#[doc(inline)]
pub use crate::bounding_volume::bounding_volume::{BoundingVolume, HasBoundingVolume};

#[doc(hidden)]
pub mod bounding_volume;

#[doc(hidden)]
pub mod aabb;
mod aabb_ball;
mod aabb_compound;
#[cfg(feature = "dim3")]
mod aabb_convex;
#[cfg(feature = "dim2")]
mod aabb_convex_polygon;
mod aabb_cuboid;
mod aabb_heightfield;
mod aabb_plane;
mod aabb_polyline;
mod aabb_shape;
mod aabb_support_map;
#[cfg(feature = "dim3")]
mod aabb_triangle;
#[cfg(feature = "dim3")]
mod aabb_trimesh;
mod aabb_utils;

#[doc(hidden)]
pub mod bounding_sphere;
mod bounding_sphere_ball;
mod bounding_sphere_capsule;
mod bounding_sphere_compound;
#[cfg(feature = "dim3")]
mod bounding_sphere_cone;
#[cfg(feature = "dim3")]
mod bounding_sphere_convex;
#[cfg(feature = "dim2")]
mod bounding_sphere_convex_polygon;
mod bounding_sphere_cuboid;
#[cfg(feature = "dim3")]
mod bounding_sphere_cylinder;
mod bounding_sphere_heightfield;
mod bounding_sphere_plane;
mod bounding_sphere_polyline;
mod bounding_sphere_segment;
mod bounding_sphere_shape;
#[cfg(feature = "dim3")]
mod bounding_sphere_triangle;
#[cfg(feature = "dim3")]
mod bounding_sphere_trimesh;
mod bounding_sphere_utils;

pub(crate) mod circular_cone;
mod spatialized_normal_cone;
<|endoftext|>