<commit_before>extern crate nalgebra as na;
extern crate ncollide3d;
extern crate nphysics3d;
extern crate nphysics_testbed3d;

use na::{Isometry3, Point3, Vector3};
use ncollide3d::shape::{Cuboid, ShapeHandle};
use nphysics3d::joint::{
    BallJoint, FixedJoint, HelicalJoint, PinSlotJoint, PlanarJoint, PrismaticJoint,
    RectangularJoint, RevoluteJoint, UniversalJoint,
};
use nphysics3d::object::{BodyHandle, Material};
use nphysics3d::volumetric::Volumetric;
use nphysics3d::world::World;
use nphysics_testbed3d::Testbed;
use std::f32::consts::PI;

const COLLIDER_MARGIN: f32 = 0.01;

fn main() {
    /*
     * World
     */
    let mut world = World::new();
    world.set_gravity(Vector3::new(0.0, -9.81, 0.0));

    /*
     * Geometries that will be re-used for several multibody links..
     */
    let rad = 0.2;
    let cuboid = ShapeHandle::new(Cuboid::new(Vector3::repeat(rad)));
    let cuboid_inertia = cuboid.inertia(1.0);
    let cuboid_center_of_mass = cuboid.center_of_mass();

    /*
     * Revolute joints.
     */
    let num = 6;
    let mut parent = BodyHandle::ground();

    let revo = RevoluteJoint::new(Vector3::x_axis(), -0.1);

    for i in 0usize..num {
        let mut parent_shift = Vector3::zeros();
        let body_shift = Vector3::z() * (rad * 3.0 + 0.2);

        if i == 0 {
            parent_shift = Vector3::new(0.0, 5.0, 11.0);
        }

        parent = world.add_multibody_link(
            parent,
            revo,
            parent_shift,
            body_shift,
            cuboid_inertia,
            cuboid_center_of_mass,
        );
        world.add_collider(
            COLLIDER_MARGIN,
            cuboid.clone(),
            parent,
            Isometry3::identity(),
            Material::default(),
        );
    }

    /*
     * Prismatic joint.
     */
    parent = BodyHandle::ground();
    let mut prism = PrismaticJoint::new(Vector3::y_axis(), 0.0);
    // Joint limit so that it does not fall indefinitely.
    prism.enable_min_offset(-rad * 2.0);

    for i in 0usize..num {
        let mut parent_shift = if i == 0 {
            Vector3::new(0.0, 5.0, 5.0)
        } else {
            Vector3::z() * rad * 3.0
        };

        parent = world.add_multibody_link(
            parent,
            prism,
            parent_shift,
            na::zero(),
            cuboid_inertia,
            cuboid_center_of_mass,
        );
        world.add_collider(
            COLLIDER_MARGIN,
            cuboid.clone(),
            parent,
            Isometry3::identity(),
            Material::default(),
        );
    }

    /*
     * Ball joint.
     */
    parent = BodyHandle::ground();
    for i in 0usize..num {
        // The multibody links are initialized along a circle.
        let angle = i as f32 * 2.0 * PI / (num as f32);
        let shift = rad * 5.0;
        let mut parent_shift = Vector3::zeros();
        let mut body_shift = Vector3::new(angle.cos(), 0.3, angle.sin()) * shift;

        if i == 0 {
            parent_shift = Vector3::new(0.0, 5.0, 0.0);
        }

        let spherical = BallJoint::new(na::zero());
        parent = world.add_multibody_link(
            parent,
            spherical,
            parent_shift,
            body_shift,
            cuboid_inertia,
            cuboid_center_of_mass,
        );
        world.add_collider(
            COLLIDER_MARGIN,
            cuboid.clone(),
            parent,
            Isometry3::identity(),
            Material::default(),
        );
    }

    /*
     * Universal joint.
     */
    let axis1 = Vector3::x_axis();
    let axis2 = Vector3::z_axis();
    let fixed = FixedJoint::new(Isometry3::identity());
    let mut uni = UniversalJoint::new(axis1, axis2, 0.0, 0.0);
    uni.enable_angular_motor_2();
    uni.set_desired_angular_motor_velocity_2(5.0);

    let parent_shift = Vector3::new(0.0, 3.0, -5.0);
    let body_shift = -Vector3::z();

    parent = world.add_multibody_link(
        BodyHandle::ground(),
        fixed,
        parent_shift,
        na::zero(),
        cuboid_inertia,
        cuboid_center_of_mass,
    );
    world.add_collider(
        COLLIDER_MARGIN,
        cuboid.clone(),
        parent,
        Isometry3::identity(),
        Material::default(),
    );

    parent = world.add_multibody_link(
        parent,
        uni,
        na::zero(),
        body_shift,
        cuboid_inertia,
        cuboid_center_of_mass,
    );
    world.add_collider(
        COLLIDER_MARGIN,
        cuboid.clone(),
        parent,
        Isometry3::identity(),
        Material::default(),
    );

    // Remove the default damping so that it balances indefinitely.
    world.multibody_mut(parent).unwrap().damping_mut().fill(0.0);

    /*
     * Helical joint.
     */
    let cuboid = ShapeHandle::new(Cuboid::new(Vector3::repeat(rad)));
    let cuboid_inertia = cuboid.inertia(1.0);
    let axis = Vector3::y_axis();

    let mut hel = HelicalJoint::new(axis, 1.0, 0.0);
    hel.set_desired_angular_motor_velocity(4.0);

    let parent_shift = Vector3::new(0.0, -2.0, 10.0);

    let hel_handle = world.add_multibody_link(
        BodyHandle::ground(),
        hel,
        parent_shift,
        na::zero(),
        cuboid_inertia,
        cuboid_center_of_mass,
    );
    world.add_collider(
        COLLIDER_MARGIN,
        cuboid.clone(),
        hel_handle,
        Isometry3::identity(),
        Material::default(),
    );

    /*
     * Planar joint.
     */
    let axis1 = Vector3::z_axis();
    let axis2 = Vector3::y_axis();
    let shift = Vector3::new(0.0, -2.0, 5.0);
    let width = 5.0 * rad * 4.0;
    for i in 0..5 {
        for j in 0..5 {
            let mut x = i as f32 * rad * 4.0 - width / 2.0;
            let y = j as f32 * rad * 4.0 - width / 2.0;

            if j % 2 == 0 {
                x += rad * 2.0;
            }

            let mut planar = PlanarJoint::new(axis1, axis2, x, y, 0.0);
            planar.enable_min_offset_1(-width / 2.0);
            planar.enable_max_offset_1(width / 2.0);
            planar.enable_min_offset_2(-5.0);
            let handle = world.add_multibody_link(
                BodyHandle::ground(),
                planar,
                shift,
                na::zero(),
                cuboid_inertia,
                cuboid_center_of_mass,
            );
            world.add_collider(
                COLLIDER_MARGIN,
                cuboid.clone(),
                handle,
                Isometry3::identity(),
                Material::default(),
            );
        }
    }

    /*
     * Rectangular joint.
     */
    let axis1 = Vector3::z_axis();
    let axis2 = Vector3::y_axis();
    let shift = Vector3::new(0.0, -2.0, 0.0);
    let width = 5.0 * rad * 4.0;

    for i in 0..5 {
        for j in 0..5 {
            let mut x = i as f32 * rad * 4.0 - width / 2.0;
            let y = j as f32 * rad * 4.0 - width / 2.0;

            if j % 2 == 0 {
                x += rad * 2.0;
            }

            let mut rect = RectangularJoint::new(axis1, axis2, x, y);
            rect.enable_min_offset_1(-width / 2.0);
            rect.enable_max_offset_1(width / 2.0);
            rect.enable_min_offset_2(-5.0);
            let handle = world.add_multibody_link(
                BodyHandle::ground(),
                rect,
                shift,
                na::zero(),
                cuboid_inertia,
                cuboid_center_of_mass,
            );
            world.add_collider(
                COLLIDER_MARGIN,
                cuboid.clone(),
                handle,
                Isometry3::identity(),
                Material::default(),
            );
        }
    }

    /*
     * Pin-slot joint.
     */
    let cuboid = ShapeHandle::new(Cuboid::new(Vector3::new(rad * 5.0, rad, rad * 5.0)));
    let cuboid_inertia = cuboid.inertia(1.0);
    let axis_v = Vector3::y_axis();
    let axis_w = Vector3::x_axis();
    let shift = Vector3::z() * -1.5;

    let mut pin_slot = PinSlotJoint::new(axis_v, axis_w, -10.0, 0.0);
    pin_slot.set_desired_linear_motor_velocity(3.0);
    let pin_handle = world.add_multibody_link(
        BodyHandle::ground(),
        pin_slot,
        shift,
        na::zero(),
        cuboid_inertia,
        cuboid_center_of_mass,
    );
    world.add_collider(
        COLLIDER_MARGIN,
        cuboid.clone(),
        pin_handle,
        Isometry3::identity(),
        Material::default(),
    );

    /*
     * Set up the testbed.
     */
    let mut testbed = Testbed::new(world);
    testbed.add_callback(move |world, _, _| {
        /*
         * Activate the helical joint motor if it is to low.
         */
        // Might be None if the user interactively deleted the helical body.
        if let Some(mut helical) = world.multibody_link_mut(hel_handle) {
            let dof = helical
                .joint_mut()
                .downcast_mut::<HelicalJoint<f32>>()
                .unwrap();

            if dof.offset() < -5.0 {
                dof.enable_angular_motor();
            } else if dof.offset() > 0.0 {
                dof.disable_angular_motor();
            }
        }
    });

    testbed.add_callback(move |world, _, _| {
        /*
         * Activate the pin-slot joint linear motor if it is to low.
         */
        // Might be None if the user interactively deleted the pin-slot body.
        if let Some(mut pin_slot) = world.multibody_link_mut(pin_handle) {
            let dof = pin_slot
                .joint_mut()
                .downcast_mut::<PinSlotJoint<f32>>()
                .unwrap();

            if dof.offset() < -10.0 {
                dof.enable_linear_motor();
            } else if dof.offset() > -4.0 {
                dof.disable_linear_motor();
            }
        }
    });

    testbed.look_at(Point3::new(30.0, -2.0, 0.0), Point3::new(0.0, -2.0, 0.0));
    testbed.run();
}
<commit_msg>Fix the joint3 demo.<commit_after>extern crate nalgebra as na;
extern crate ncollide3d;
extern crate nphysics3d;
extern crate nphysics_testbed3d;

use na::{Isometry3, Point3, Vector3};
use ncollide3d::shape::{Cuboid, ShapeHandle};
use nphysics3d::joint::{
    BallJoint, FixedJoint, HelicalJoint, PinSlotJoint, PlanarJoint, PrismaticJoint,
    RectangularJoint, RevoluteJoint, UniversalJoint,
};
use nphysics3d::object::{BodyHandle, Material};
use nphysics3d::volumetric::Volumetric;
use nphysics3d::world::World;
use nphysics_testbed3d::Testbed;
use std::f32::consts::PI;

const COLLIDER_MARGIN: f32 = 0.01;

fn main() {
    /*
     * World
     */
    let mut world = World::new();
    world.set_gravity(Vector3::new(0.0, -9.81, 0.0));

    /*
     * Geometries that will be re-used for several multibody links..
     */
    let rad = 0.2;
    let cuboid = ShapeHandle::new(Cuboid::new(Vector3::repeat(rad)));
    let cuboid_inertia = cuboid.inertia(1.0);
    let cuboid_center_of_mass = cuboid.center_of_mass();

    /*
     * Revolute joints.
     */
    let num = 6;
    let mut parent = BodyHandle::ground();

    let revo = RevoluteJoint::new(Vector3::x_axis(), -0.1);

    for i in 0usize..num {
        let mut parent_shift = Vector3::zeros();
        let body_shift = Vector3::z() * (rad * 3.0 + 0.2);

        if i == 0 {
            parent_shift = Vector3::new(0.0, 5.0, 11.0);
        }

        parent = world.add_multibody_link(
            parent,
            revo,
            parent_shift,
            body_shift,
            cuboid_inertia,
            cuboid_center_of_mass,
        );
        world.add_collider(
            COLLIDER_MARGIN,
            cuboid.clone(),
            parent,
            Isometry3::identity(),
            Material::default(),
        );
    }

    /*
     * Prismatic joint.
     */
    parent = BodyHandle::ground();
    let mut prism = PrismaticJoint::new(Vector3::y_axis(), 0.0);
    // Joint limit so that it does not fall indefinitely.
    prism.enable_min_offset(-rad * 2.0);

    for i in 0usize..num {
        let mut parent_shift = if i == 0 {
            Vector3::new(0.0, 5.0, 5.0)
        } else {
            Vector3::z() * rad * 3.0
        };

        parent = world.add_multibody_link(
            parent,
            prism,
            parent_shift,
            na::zero(),
            cuboid_inertia,
            cuboid_center_of_mass,
        );
        world.add_collider(
            COLLIDER_MARGIN,
            cuboid.clone(),
            parent,
            Isometry3::identity(),
            Material::default(),
        );
    }

    /*
     * Ball joint.
     */
    parent = BodyHandle::ground();
    for i in 0usize..num {
        // The multibody links are initialized along a circle.
        let angle = i as f32 * 2.0 * PI / (num as f32);
        let shift = rad * 5.0;
        let mut parent_shift = Vector3::zeros();
        let mut body_shift = Vector3::new(angle.cos(), 0.3, angle.sin()) * shift;

        if i == 0 {
            parent_shift = Vector3::new(0.0, 5.0, 0.0);
        }

        let spherical = BallJoint::new(na::zero());
        parent = world.add_multibody_link(
            parent,
            spherical,
            parent_shift,
            body_shift,
            cuboid_inertia,
            cuboid_center_of_mass,
        );
        world.add_collider(
            COLLIDER_MARGIN,
            cuboid.clone(),
            parent,
            Isometry3::identity(),
            Material::default(),
        );
    }

    /*
     * Universal joint.
     */
    let axis1 = Vector3::x_axis();
    let axis2 = Vector3::z_axis();
    let fixed = FixedJoint::new(Isometry3::identity());
    let mut uni = UniversalJoint::new(axis1, axis2, 0.0, 0.0);
    uni.enable_angular_motor_2();
    uni.set_desired_angular_motor_velocity_2(5.0);

    let parent_shift = Vector3::new(0.0, 3.0, -5.0);
    let body_shift = -Vector3::z();

    parent = world.add_multibody_link(
        BodyHandle::ground(),
        fixed,
        parent_shift,
        na::zero(),
        cuboid_inertia,
        cuboid_center_of_mass,
    );
    world.add_collider(
        COLLIDER_MARGIN,
        cuboid.clone(),
        parent,
        Isometry3::identity(),
        Material::default(),
    );

    parent = world.add_multibody_link(
        parent,
        uni,
        na::zero(),
        body_shift,
        cuboid_inertia,
        cuboid_center_of_mass,
    );
    world.add_collider(
        COLLIDER_MARGIN,
        cuboid.clone(),
        parent,
        Isometry3::identity(),
        Material::default(),
    );

    // Remove the default damping so that it balances indefinitely.
    world.multibody_mut(parent).unwrap().damping_mut().fill(0.0);

    /*
     * Helical joint.
     */
    let cuboid = ShapeHandle::new(Cuboid::new(Vector3::repeat(rad)));
    let cuboid_inertia = cuboid.inertia(1.0);
    let axis = Vector3::y_axis();

    let mut hel = HelicalJoint::new(axis, 1.0, 0.0);
    hel.set_desired_angular_motor_velocity(4.0);

    let parent_shift = Vector3::new(0.0, -2.0, 10.0);

    let hel_handle = world.add_multibody_link(
        BodyHandle::ground(),
        hel,
        parent_shift,
        na::zero(),
        cuboid_inertia,
        cuboid_center_of_mass,
    );
    world.add_collider(
        COLLIDER_MARGIN,
        cuboid.clone(),
        hel_handle,
        Isometry3::identity(),
        Material::default(),
    );

    /*
     * Planar joint.
     */
    let axis1 = Vector3::z_axis();
    let axis2 = Vector3::y_axis();
    let shift = Vector3::new(0.0, -2.0, 5.0);
    let width = 5.0 * rad * 4.0;
    for i in 0..5 {
        for j in 0..5 {
            let mut x = i as f32 * rad * 4.0 - width / 2.0;
            let y = j as f32 * rad * 4.0 - width / 2.0;

            if j % 2 == 0 {
                x += rad * 2.0;
            }

            let mut planar = PlanarJoint::new(axis1, axis2, x, y, 0.0);
            planar.enable_min_offset_1(-width / 2.0);
            planar.enable_max_offset_1(width / 2.0);
            planar.enable_min_offset_2(-5.0);
            let handle = world.add_multibody_link(
                BodyHandle::ground(),
                planar,
                shift,
                na::zero(),
                cuboid_inertia,
                cuboid_center_of_mass,
            );
            world.add_collider(
                COLLIDER_MARGIN,
                cuboid.clone(),
                handle,
                Isometry3::identity(),
                Material::default(),
            );
        }
    }

    /*
     * Rectangular joint.
     */
    let axis1 = Vector3::z_axis();
    let axis2 = Vector3::y_axis();
    let shift = Vector3::new(0.0, -2.0, 0.0);
    let width = 5.0 * rad * 4.0;

    for i in 0..5 {
        for j in 0..5 {
            let mut x = i as f32 * rad * 4.0 - width / 2.0;
            let y = j as f32 * rad * 4.0 - width / 2.0;

            if j % 2 == 0 {
                x += rad * 2.0;
            }

            let mut rect = RectangularJoint::new(axis1, axis2, x, y);
            rect.enable_min_offset_1(-width / 2.0);
            rect.enable_max_offset_1(width / 2.0);
            rect.enable_min_offset_2(-5.0);
            let handle = world.add_multibody_link(
                BodyHandle::ground(),
                rect,
                shift,
                na::zero(),
                cuboid_inertia,
                cuboid_center_of_mass,
            );
            world.add_collider(
                COLLIDER_MARGIN,
                cuboid.clone(),
                handle,
                Isometry3::identity(),
                Material::default(),
            );
        }
    }

    /*
     * Pin-slot joint.
     */
    let cuboid = ShapeHandle::new(Cuboid::new(Vector3::new(rad * 5.0, rad, rad * 5.0)));
    let cuboid_inertia = cuboid.inertia(1.0);
    let axis_v = Vector3::y_axis();
    let axis_w = Vector3::x_axis();
    let shift = Vector3::z() * -1.5;

    let mut pin_slot = PinSlotJoint::new(axis_v, axis_w, -10.0, 0.0);
    pin_slot.set_desired_linear_motor_velocity(3.0);
    let pin_handle = world.add_multibody_link(
        BodyHandle::ground(),
        pin_slot,
        shift,
        na::zero(),
        cuboid_inertia,
        cuboid_center_of_mass,
    );
    world.add_collider(
        COLLIDER_MARGIN,
        cuboid.clone(),
        pin_handle,
        Isometry3::identity(),
        Material::default(),
    );

    /*
     * Set up the testbed.
     */
    let mut testbed = Testbed::new(world);
    testbed.add_callback(move |world, _, _| {
        let mut world = world.get_mut();

        /*
         * Activate the helical joint motor if it is to low.
         */
        // Might be None if the user interactively deleted the helical body.
        if let Some(mut helical) = world.multibody_link_mut(hel_handle) {
            let dof = helical
                .joint_mut()
                .downcast_mut::<HelicalJoint<f32>>()
                .unwrap();

            if dof.offset() < -5.0 {
                dof.enable_angular_motor();
            } else if dof.offset() > 0.0 {
                dof.disable_angular_motor();
            }
        }
    });

    testbed.add_callback(move |world, _, _| {
        let mut world = world.get_mut();

        /*
         * Activate the pin-slot joint linear motor if it is to low.
         */
        // Might be None if the user interactively deleted the pin-slot body.
        if let Some(mut pin_slot) = world.multibody_link_mut(pin_handle) {
            let dof = pin_slot
                .joint_mut()
                .downcast_mut::<PinSlotJoint<f32>>()
                .unwrap();

            if dof.offset() < -10.0 {
                dof.enable_linear_motor();
            } else if dof.offset() > -4.0 {
                dof.disable_linear_motor();
            }
        }
    });

    testbed.look_at(Point3::new(30.0, -2.0, 0.0), Point3::new(0.0, -2.0, 0.0));
    testbed.run();
}
<|endoftext|>


<commit_before>#![feature(integer_atomics)]

extern crate abra;
extern crate rocksdb;
extern crate rustc_serialize;
#[macro_use]
extern crate maplit;
extern crate byteorder;

use std::sync::{Arc, RwLock};
use std::sync::atomic::{AtomicU32, Ordering};
use std::collections::BTreeMap;

use rocksdb::{DB, Writable, Options, MergeOperands};
use abra::{Term, Document};
use abra::schema::{Schema, FieldType, FieldRef, AddFieldError};
use rustc_serialize::{json, Encodable};
use byteorder::{ByteOrder, BigEndian};

#[derive(Debug, Clone, Copy, Eq, PartialEq, Hash)]
struct TermRef(u32);

impl TermRef {
    pub fn ord(&self) -> u32 {
        self.0
    }
}

#[derive(Debug, Clone, Copy, Eq, PartialEq, Hash)]
struct DocRef(u32, u16);

impl DocRef {
    pub fn chunk(&self) -> u32 {
        self.0
    }

    pub fn ord(&self) -> u16 {
        self.1
    }
}

fn merge_keys(key: &[u8], existing_val: Option<&[u8]>, operands: &mut MergeOperands) -> Vec<u8> {
    match key[0] {
        b'd' => {
            // Directory (sequence of two byte document ids)

            // Allocate vec for new Value
            let new_size = match existing_val {
                Some(existing_val) => existing_val.len(),
                None => 0,
            } + operands.size_hint().0 * 2;

            let mut new_val = Vec::with_capacity(new_size);

            // Push existing value
            existing_val.map(|v| {
                for b in v {
                    new_val.push(*b);
                }
            });

            // Append new entries
            for op in operands {
                for b in op {
                    new_val.push(*b);
                }
            }

            new_val
        }
        _ => {
            // Unrecognised key, fallback to emulating a put operation (by taking the last value)
            operands.last().unwrap().iter().cloned().collect()
        }
    }
}

pub struct RocksDBIndexStore {
    schema: Arc<Schema>,
    db: DB,
    next_term_ref: AtomicU32,
    term_dictionary: RwLock<BTreeMap<Vec<u8>, TermRef>>,
    next_chunk: AtomicU32,
}

impl RocksDBIndexStore {
    pub fn create(path: &str) -> Result<RocksDBIndexStore, String> {
        let mut opts = Options::default();
        opts.add_merge_operator("merge operator", merge_keys);
        opts.create_if_missing(true);
        let db = try!(DB::open(&opts, path));

        // Schema
        let schema = Schema::new();
        db.put(b".schema", json::encode(&schema).unwrap().as_bytes());

        // Next term ref
        db.put(b".next_term_ref", b"1");

        // Next chunk
        db.put(b".next_chunk", b"1");

        Ok(RocksDBIndexStore {
            schema: Arc::new(schema),
            db: db,
            next_term_ref: AtomicU32::new(1),
            term_dictionary: RwLock::new(BTreeMap::new()),
            next_chunk: AtomicU32::new(1),
        })
    }

    pub fn open(path: &str) -> Result<RocksDBIndexStore, String> {
        let mut opts = Options::default();
        opts.add_merge_operator("merge operator", merge_keys);
        let db = try!(DB::open(&opts, path));

        let schema = match db.get(b".schema") {
            Ok(Some(schema)) => {
                let schema = schema.to_utf8().unwrap().to_string();
                json::decode(&schema).unwrap()
            }
            Ok(None) => Schema::new(),  // TODO: error
            Err(_) => Schema::new(),  // TODO: error
        };

        let next_term_ref = match db.get(b".next_term_ref") {
            Ok(Some(next_term_ref)) => {
                next_term_ref.to_utf8().unwrap().parse::<u32>().unwrap()
            }
            Ok(None) => 1,  // TODO: error
            Err(_) => 1,  // TODO: error
        };

        let next_chunk = match db.get(b".next_chunk") {
            Ok(Some(next_chunk)) => {
                next_chunk.to_utf8().unwrap().parse::<u32>().unwrap()
            }
            Ok(None) => 1,  // TODO: error
            Err(_) => 1,  // TODO: error
        };

        Ok(RocksDBIndexStore {
            schema: Arc::new(schema),
            db: db,
            next_term_ref: AtomicU32::new(next_term_ref),
            term_dictionary: RwLock::new(BTreeMap::new()),
            next_chunk: AtomicU32::new(next_chunk),
        })
    }

    pub fn add_field(&mut self, name: String, field_type: FieldType) -> Result<FieldRef, AddFieldError> {
        let mut schema_copy = (*self.schema).clone();
        let field_ref = try!(schema_copy.add_field(name, field_type));
        self.schema = Arc::new(schema_copy);

        self.db.put(b".schema", json::encode(&self.schema).unwrap().as_bytes());

        Ok(field_ref)
    }

    pub fn remove_field(&mut self, field_ref: &FieldRef) -> bool {
        let mut schema_copy = (*self.schema).clone();
        let field_removed = schema_copy.remove_field(field_ref);

        if field_removed {
            self.schema = Arc::new(schema_copy);
            self.db.put(b".schema", json::encode(&self.schema).unwrap().as_bytes());
        }

        field_removed
    }

    fn get_or_create_term(&mut self, term: &Term) -> TermRef {
        let term_bytes = term.to_bytes();

        if let Some(term_ref) = self.term_dictionary.read().unwrap().get(&term_bytes) {
            return *term_ref;
        }

        // Term doesn't exist in the term dictionary

        // Increment next_term_ref
        let next_term_ref = self.next_term_ref.fetch_add(1, Ordering::SeqCst);
        self.db.put(b".next_term_ref", (next_term_ref + 1).to_string().as_bytes());

        // Create term ref
        let term_ref = TermRef(next_term_ref);

        // Get exclusive lock to term dictionary
        let mut term_dictionary = self.term_dictionary.write().unwrap();

        // It's possible that another thread has written the term to the dictionary
        // since we checked earlier. If this is the case, We should forget about
        // writing our TermRef and use the one that has been inserted already.
        if let Some(term_ref) = term_dictionary.get(&term_bytes) {
            return *term_ref;
        }

        // Write it to the on-disk term dictionary
        let mut key = Vec::with_capacity(1 + term_bytes.len());
        key.push(b't');
        for byte in term_bytes.iter() {
            key.push(*byte);
        }
        self.db.put(&key, next_term_ref.to_string().as_bytes());

        // Write it to the term dictionary
        term_dictionary.insert(term_bytes, term_ref);

        term_ref
    }

    pub fn insert_or_update_document(&mut self, doc: Document) {
        // Allocate a new chunk for the document

        // Chunk merges are very slow so we should avoid doing them at runtime
        // which is why each new document is created in a fresh chunk.
        // Later on, a background process will come and merge any small chunks
        // together.

        // For best performance, documents should be inserted in batches.

        // Increment next_chunk
        let next_chunk = self.next_chunk.fetch_add(1, Ordering::SeqCst);
        self.db.put(b".next_chunk", (next_chunk + 1).to_string().as_bytes());

        // Create doc ref
        let doc_ref = DocRef(next_chunk, 0);

        // Insert contents
        for (field_ref, tokens) in doc.fields.iter() {
            for token in tokens.iter() {
                let term_ref = self.get_or_create_term(&token.term);

                let mut key = Vec::with_capacity(20);
                key.push(b'd');
                for byte in field_ref.ord().to_string().as_bytes() {
                    key.push(*byte);
                }
                key.push(b'/');
                for byte in term_ref.ord().to_string().as_bytes() {
                    key.push(*byte);
                }
                key.push(b'/');
                for byte in doc_ref.chunk().to_string().as_bytes() {
                    key.push(*byte);
                }

                let mut doc_id_bytes = [0; 2];
                BigEndian::write_u16(&mut doc_id_bytes, doc_ref.ord());
                self.db.merge(&key, &doc_id_bytes);
            }
        }
    }
}

#[cfg(test)]
mod tests {
    use rocksdb::{DB, Options, IteratorMode};
    use abra::{Term, Token, Document};
    use abra::schema::{Schema, FieldType, FieldRef};

    use super::RocksDBIndexStore;

    fn clean_test_indices() {
        use std::fs::remove_dir_all;

        remove_dir_all("test_indices");
    }

    #[test]
    fn test_create() {
        clean_test_indices();

        let store = RocksDBIndexStore::create("test_indices/test_create");
        assert!(store.is_ok());
    }

    #[test]
    fn test_open() {
        clean_test_indices();

        let store = RocksDBIndexStore::open("test_indices/test_open");
        assert!(store.is_err());

        // Create DB
        let mut opts = Options::default();
        opts.create_if_missing(true);
        DB::open(&opts, "test_indices/test_open").unwrap();

        let store = RocksDBIndexStore::open("test_indices/test_open");
        assert!(store.is_ok());
    }

    fn make_test_store(path: &str) -> RocksDBIndexStore {
        let mut store = RocksDBIndexStore::create(path).unwrap();
        let mut title_field = store.add_field("title".to_string(), FieldType::Text).unwrap();
        let mut body_field = store.add_field("body".to_string(), FieldType::Text).unwrap();

        store.insert_or_update_document(Document {
            key: "test_doc".to_string(),
            fields: hashmap! {
                title_field => vec![
                    Token { term: Term::String("hello".to_string()), position: 1 },
                    Token { term: Term::String("world".to_string()), position: 2 },
                ],
                body_field => vec![
                    Token { term: Term::String("lorem".to_string()), position: 1 },
                    Token { term: Term::String("ipsum".to_string()), position: 2 },
                    Token { term: Term::String("dolar".to_string()), position: 3 },
                ],
            }
        });

        store.insert_or_update_document(Document {
            key: "test_doc".to_string(),
            fields: hashmap! {
                title_field => vec![
                    Token { term: Term::String("howdy".to_string()), position: 1 },
                    Token { term: Term::String("partner".to_string()), position: 2 },
                ],
                body_field => vec![
                    Token { term: Term::String("lorem".to_string()), position: 1 },
                    Token { term: Term::String("ipsum".to_string()), position: 2 },
                    Token { term: Term::String("dolar".to_string()), position: 3 },
                ],
            }
        });

        store
    }

    pub fn print_keys(db: &DB) {
        fn bytes_to_string(bytes: &Box<[u8]>) -> String {
            use std::char;

            let mut string = String::new();

            for byte in bytes.iter() {
                if *byte < 128 {
                    // ASCII character
                    string.push(char::from_u32(*byte as u32).unwrap());
                } else {
                    string.push('?');
                }
            }

            string
        }

        for (key, value) in db.iterator(IteratorMode::Start) {
            println!("{} = {:?}", bytes_to_string(&key), value);
        }
    }

    #[test]
    fn test() {
        clean_test_indices();

        let store = make_test_store("test_indices/test");

        print_keys(&store.db);

    }
}
<commit_msg>Atomic writes of new documents<commit_after>#![feature(integer_atomics)]

extern crate abra;
extern crate rocksdb;
extern crate rustc_serialize;
#[macro_use]
extern crate maplit;
extern crate byteorder;

use std::sync::{Arc, RwLock};
use std::sync::atomic::{AtomicU32, Ordering};
use std::collections::BTreeMap;

use rocksdb::{DB, WriteBatch, Writable, Options, MergeOperands};
use abra::{Term, Document};
use abra::schema::{Schema, FieldType, FieldRef, AddFieldError};
use rustc_serialize::{json, Encodable};
use byteorder::{ByteOrder, BigEndian};

#[derive(Debug, Clone, Copy, Eq, PartialEq, Hash)]
struct TermRef(u32);

impl TermRef {
    pub fn ord(&self) -> u32 {
        self.0
    }
}

#[derive(Debug, Clone, Copy, Eq, PartialEq, Hash)]
struct DocRef(u32, u16);

impl DocRef {
    pub fn chunk(&self) -> u32 {
        self.0
    }

    pub fn ord(&self) -> u16 {
        self.1
    }
}

fn merge_keys(key: &[u8], existing_val: Option<&[u8]>, operands: &mut MergeOperands) -> Vec<u8> {
    match key[0] {
        b'd' => {
            // Directory (sequence of two byte document ids)

            // Allocate vec for new Value
            let new_size = match existing_val {
                Some(existing_val) => existing_val.len(),
                None => 0,
            } + operands.size_hint().0 * 2;

            let mut new_val = Vec::with_capacity(new_size);

            // Push existing value
            existing_val.map(|v| {
                for b in v {
                    new_val.push(*b);
                }
            });

            // Append new entries
            for op in operands {
                for b in op {
                    new_val.push(*b);
                }
            }

            new_val
        }
        _ => {
            // Unrecognised key, fallback to emulating a put operation (by taking the last value)
            operands.last().unwrap().iter().cloned().collect()
        }
    }
}

pub struct RocksDBIndexStore {
    schema: Arc<Schema>,
    db: DB,
    next_term_ref: AtomicU32,
    term_dictionary: RwLock<BTreeMap<Vec<u8>, TermRef>>,
    next_chunk: AtomicU32,
}

impl RocksDBIndexStore {
    pub fn create(path: &str) -> Result<RocksDBIndexStore, String> {
        let mut opts = Options::default();
        opts.add_merge_operator("merge operator", merge_keys);
        opts.create_if_missing(true);
        let db = try!(DB::open(&opts, path));

        // Schema
        let schema = Schema::new();
        db.put(b".schema", json::encode(&schema).unwrap().as_bytes());

        // Next term ref
        db.put(b".next_term_ref", b"1");

        // Next chunk
        db.put(b".next_chunk", b"1");

        Ok(RocksDBIndexStore {
            schema: Arc::new(schema),
            db: db,
            next_term_ref: AtomicU32::new(1),
            term_dictionary: RwLock::new(BTreeMap::new()),
            next_chunk: AtomicU32::new(1),
        })
    }

    pub fn open(path: &str) -> Result<RocksDBIndexStore, String> {
        let mut opts = Options::default();
        opts.add_merge_operator("merge operator", merge_keys);
        let db = try!(DB::open(&opts, path));

        let schema = match db.get(b".schema") {
            Ok(Some(schema)) => {
                let schema = schema.to_utf8().unwrap().to_string();
                json::decode(&schema).unwrap()
            }
            Ok(None) => Schema::new(),  // TODO: error
            Err(_) => Schema::new(),  // TODO: error
        };

        let next_term_ref = match db.get(b".next_term_ref") {
            Ok(Some(next_term_ref)) => {
                next_term_ref.to_utf8().unwrap().parse::<u32>().unwrap()
            }
            Ok(None) => 1,  // TODO: error
            Err(_) => 1,  // TODO: error
        };

        let next_chunk = match db.get(b".next_chunk") {
            Ok(Some(next_chunk)) => {
                next_chunk.to_utf8().unwrap().parse::<u32>().unwrap()
            }
            Ok(None) => 1,  // TODO: error
            Err(_) => 1,  // TODO: error
        };

        Ok(RocksDBIndexStore {
            schema: Arc::new(schema),
            db: db,
            next_term_ref: AtomicU32::new(next_term_ref),
            term_dictionary: RwLock::new(BTreeMap::new()),
            next_chunk: AtomicU32::new(next_chunk),
        })
    }

    pub fn add_field(&mut self, name: String, field_type: FieldType) -> Result<FieldRef, AddFieldError> {
        let mut schema_copy = (*self.schema).clone();
        let field_ref = try!(schema_copy.add_field(name, field_type));
        self.schema = Arc::new(schema_copy);

        self.db.put(b".schema", json::encode(&self.schema).unwrap().as_bytes());

        Ok(field_ref)
    }

    pub fn remove_field(&mut self, field_ref: &FieldRef) -> bool {
        let mut schema_copy = (*self.schema).clone();
        let field_removed = schema_copy.remove_field(field_ref);

        if field_removed {
            self.schema = Arc::new(schema_copy);
            self.db.put(b".schema", json::encode(&self.schema).unwrap().as_bytes());
        }

        field_removed
    }

    fn get_or_create_term(&mut self, term: &Term) -> TermRef {
        let term_bytes = term.to_bytes();

        if let Some(term_ref) = self.term_dictionary.read().unwrap().get(&term_bytes) {
            return *term_ref;
        }

        // Term doesn't exist in the term dictionary

        // Increment next_term_ref
        let next_term_ref = self.next_term_ref.fetch_add(1, Ordering::SeqCst);
        self.db.put(b".next_term_ref", (next_term_ref + 1).to_string().as_bytes());

        // Create term ref
        let term_ref = TermRef(next_term_ref);

        // Get exclusive lock to term dictionary
        let mut term_dictionary = self.term_dictionary.write().unwrap();

        // It's possible that another thread has written the term to the dictionary
        // since we checked earlier. If this is the case, We should forget about
        // writing our TermRef and use the one that has been inserted already.
        if let Some(term_ref) = term_dictionary.get(&term_bytes) {
            return *term_ref;
        }

        // Write it to the on-disk term dictionary
        let mut key = Vec::with_capacity(1 + term_bytes.len());
        key.push(b't');
        for byte in term_bytes.iter() {
            key.push(*byte);
        }
        self.db.put(&key, next_term_ref.to_string().as_bytes());

        // Write it to the term dictionary
        term_dictionary.insert(term_bytes, term_ref);

        term_ref
    }

    pub fn insert_or_update_document(&mut self, doc: Document) {
        // Allocate a new chunk for the document

        // Chunk merges are very slow so we should avoid doing them at runtime
        // which is why each new document is created in a fresh chunk.
        // Later on, a background process will come and merge any small chunks
        // together.

        // For best performance, documents should be inserted in batches.

        // Increment next_chunk
        let next_chunk = self.next_chunk.fetch_add(1, Ordering::SeqCst);
        self.db.put(b".next_chunk", (next_chunk + 1).to_string().as_bytes());

        // Create doc ref
        let doc_ref = DocRef(next_chunk, 0);

        // Start write batch
        let mut write_batch = WriteBatch::default();

        // Insert contents
        for (field_ref, tokens) in doc.fields.iter() {
            for token in tokens.iter() {
                let term_ref = self.get_or_create_term(&token.term);

                let mut key = Vec::with_capacity(20);
                key.push(b'd');
                for byte in field_ref.ord().to_string().as_bytes() {
                    key.push(*byte);
                }
                key.push(b'/');
                for byte in term_ref.ord().to_string().as_bytes() {
                    key.push(*byte);
                }
                key.push(b'/');
                for byte in doc_ref.chunk().to_string().as_bytes() {
                    key.push(*byte);
                }

                let mut doc_id_bytes = [0; 2];
                BigEndian::write_u16(&mut doc_id_bytes, doc_ref.ord());
                write_batch.merge(&key, &doc_id_bytes);
            }
        }

        // Write
        self.db.write(write_batch);
    }
}

#[cfg(test)]
mod tests {
    use rocksdb::{DB, Options, IteratorMode};
    use abra::{Term, Token, Document};
    use abra::schema::{Schema, FieldType, FieldRef};

    use super::RocksDBIndexStore;

    fn clean_test_indices() {
        use std::fs::remove_dir_all;

        remove_dir_all("test_indices");
    }

    #[test]
    fn test_create() {
        clean_test_indices();

        let store = RocksDBIndexStore::create("test_indices/test_create");
        assert!(store.is_ok());
    }

    #[test]
    fn test_open() {
        clean_test_indices();

        let store = RocksDBIndexStore::open("test_indices/test_open");
        assert!(store.is_err());

        // Create DB
        let mut opts = Options::default();
        opts.create_if_missing(true);
        DB::open(&opts, "test_indices/test_open").unwrap();

        let store = RocksDBIndexStore::open("test_indices/test_open");
        assert!(store.is_ok());
    }

    fn make_test_store(path: &str) -> RocksDBIndexStore {
        let mut store = RocksDBIndexStore::create(path).unwrap();
        let mut title_field = store.add_field("title".to_string(), FieldType::Text).unwrap();
        let mut body_field = store.add_field("body".to_string(), FieldType::Text).unwrap();

        store.insert_or_update_document(Document {
            key: "test_doc".to_string(),
            fields: hashmap! {
                title_field => vec![
                    Token { term: Term::String("hello".to_string()), position: 1 },
                    Token { term: Term::String("world".to_string()), position: 2 },
                ],
                body_field => vec![
                    Token { term: Term::String("lorem".to_string()), position: 1 },
                    Token { term: Term::String("ipsum".to_string()), position: 2 },
                    Token { term: Term::String("dolar".to_string()), position: 3 },
                ],
            }
        });

        store.insert_or_update_document(Document {
            key: "test_doc".to_string(),
            fields: hashmap! {
                title_field => vec![
                    Token { term: Term::String("howdy".to_string()), position: 1 },
                    Token { term: Term::String("partner".to_string()), position: 2 },
                ],
                body_field => vec![
                    Token { term: Term::String("lorem".to_string()), position: 1 },
                    Token { term: Term::String("ipsum".to_string()), position: 2 },
                    Token { term: Term::String("dolar".to_string()), position: 3 },
                ],
            }
        });

        store
    }

    pub fn print_keys(db: &DB) {
        fn bytes_to_string(bytes: &Box<[u8]>) -> String {
            use std::char;

            let mut string = String::new();

            for byte in bytes.iter() {
                if *byte < 128 {
                    // ASCII character
                    string.push(char::from_u32(*byte as u32).unwrap());
                } else {
                    string.push('?');
                }
            }

            string
        }

        for (key, value) in db.iterator(IteratorMode::Start) {
            println!("{} = {:?}", bytes_to_string(&key), value);
        }
    }

    #[test]
    fn test() {
        clean_test_indices();

        let store = make_test_store("test_indices/test");

        print_keys(&store.db);

    }
}
<|endoftext|>


<commit_before>extern crate fbx_direct;
extern crate env_logger;

use std::fs::File;
use std::io::BufReader;

use fbx_direct::reader::{EventReader, FbxEvent};

fn indent(size: usize) -> String {
    const INDENT: &'static str = "    ";
    (0..size).map(|_| INDENT)
        .fold(String::with_capacity(size * INDENT.len()), |r, s| r + s)
}

fn main() {
    env_logger::init().unwrap();

    let file = BufReader::new(File::open("sample.fbx").unwrap());

    let parser = EventReader::new(file);
    let mut depth = 0;
    for e in parser {
        match e {
            Ok(ref e@FbxEvent::StartNode { .. }) => {
                println!("{}{:?}", indent(depth), e);
                depth += 1;
            },
            Ok(ref e@FbxEvent::EndNode) => {
                depth -= 1;
                println!("{}{:?}", indent(depth), e);
            },
            Ok(e) => {
                println!("{}{:?}", indent(depth), e);
            },
            Err(e) => {
                println!("Error: {:?}", e);
                break;
            },
        }
    }
}
<commit_msg>Improve example `simple.rs`<commit_after>extern crate fbx_direct;
extern crate env_logger;

use std::fs::File;
use std::io::BufReader;

use fbx_direct::reader::{EventReader, FbxEvent};

fn indent(size: usize) -> String {
    const INDENT: &'static str = "    ";
    (0..size).map(|_| INDENT)
        .fold(String::with_capacity(size * INDENT.len()), |r, s| r + s)
}

fn main() {
    env_logger::init().unwrap();

    let filename = match std::env::args().nth(1) {
        Some(f) => f,
        None => {
            use std::io::Write;
            writeln!(&mut std::io::stderr(), "Usage: cargo run --example=simple <FBX_filename>").unwrap();
            std::process::exit(1);
        },
    };

    let file = BufReader::new(File::open(filename).unwrap());

    let parser = EventReader::new(file);
    let mut depth = 0;
    for e in parser {
        match e {
            Ok(ref e@FbxEvent::StartNode { .. }) => {
                println!("{}{:?}", indent(depth), e);
                depth += 1;
            },
            Ok(ref e@FbxEvent::EndNode) => {
                depth -= 1;
                println!("{}{:?}", indent(depth), e);
            },
            Ok(e) => {
                println!("{}{:?}", indent(depth), e);
            },
            Err(e) => {
                println!("Error: {:?}", e);
                break;
            },
        }
    }
}
<|endoftext|>


<commit_before>// Copyright 2018 Grove Enterprises LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

extern crate arrow;
extern crate datafusion;

use datafusion::exec::*;

/// This example shows the use of the DataFrame API to define a query plan
fn main() {
    let ctx = ExecutionContext::local();

    let df = ctx.load_parquet("test/data/alltypes_plain.parquet")
        .unwrap();
    println!("schema: {}", df.schema().to_string());

    // show the first 10 rows
    df.show(10);
}
<commit_msg>Bug fix: parquet_dataframe example was referencing parquet file with unsupported types<commit_after>// Copyright 2018 Grove Enterprises LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

extern crate arrow;
extern crate datafusion;

use datafusion::exec::*;

/// This example shows the use of the DataFrame API to define a query plan
fn main() {
    let ctx = ExecutionContext::local();

    let df = ctx.load_parquet("test/data/uk_cities.parquet")
        .unwrap();
    println!("schema: {}", df.schema().to_string());

    // show the first 10 rows
    df.show(10);
}
<|endoftext|>


<commit_before>extern crate collections;
use std::io::{File, Open, Write, TempDir};
use std::os::args;
use std::from_str::from_str;
use collections::priority_queue::PriorityQueue;
#[cfg(test)]
use std::io::{ReadWrite, SeekSet};

/*
 * an entry is the number that we read and the file it came from, so after we took
 * out one Entry from the priority queue, we need to know where to take the next number
 * from.
 */
struct Entry {
	value: u64,
	source: File,
}

/* The priority queue depends on the ordering relation and always returns the biggest */
impl Ord for Entry {
	// lesser than implementation
	fn lt(&self, other: &Entry) -> bool {
		// we invert: for us the smallest will have the highest priority
		!(self.value < other.value)
	}
}

/* ordering needs equality for some reason, so there */
impl Eq for Entry {
	fn eq(&self, other: &Entry) -> bool {
		self.value == other.value
	}
}

/* A struct to hold our queue */
struct PriorityFile {
	queue: PriorityQueue<Entry>,
}

/*
 * 'Class' that takes a list of files and does a k-way merge of the smallest values
 * from the files
 */
impl PriorityFile {
	pub fn new(files: Vec<File>) -> PriorityFile {
		let mut queue: PriorityQueue<Entry> = PriorityQueue::new();
		// now that we got the queue, populate it with initial values

		// move_iter is required to get the values themselves, not borrowed pointers to them
		for mut f in files.move_iter() {
			// read from file, append to queue
			let number = match f.read_le_u64() {
				Ok(num) => num,
				Err(e) => fail!("failed to read u64 from file: {}", e),
			};
			queue.push(Entry {value: number, source: f});
		};
		PriorityFile {queue: queue}
	}

	/* Returns the next smallest number from the list of files or None if exhausted */
	pub fn next(&mut self) -> Option<u64> {
		match self.queue.maybe_pop() {
			Some(mut res) => {
				let v = res.value;
				match res.source.read_le_u64() {
					Ok(num) => self.queue.push(Entry {value: num, source: res.source}),
					Err(_) => (),
				};
				Some(v)
			},
			None => None,
		}
	}
}

fn read_u64(mut from: File, n: uint) -> ~Vec<u64> {
	// rust post 0.10 will directly return a Vec<u8> here
	let mut buf: ~[u8] = match from.read_exact(n*8) {
		Ok(b) => b,
		Err(e) => fail!("reading failed {}", e),
	};
	let mut buffer = Vec::from_slice(buf);

/*
	let mut buffer: Vec<u8> = Vec::from_elem(n*8, 0u8);

	{
		let mut buf = buffer.as_mut_slice();
		println!("slice len {}", buf.len());
		match from.read(buf) {
			Ok(_) => (),
			Err(e) => fail!("reading failed {}", e),
		};
	};
*/

	let mut read: ~Vec<u64> = ~unsafe { std::cast::transmute(buffer) };
	let read_len = read.len();
	unsafe { read.set_len(read_len / 8); };
	read
}

fn externalSort(mut fdInput: File, size: u64, mut fdOutput: File, memSize: u64) {
	let mut runs = size / memSize;
	let items_per_run = (memSize / 8) as uint;
	let over = ((size / 8) - (items_per_run as u64 * runs)) as uint;

	println!("There will be {} runs with {} elements each.", runs, items_per_run);
	if over > 0 {
		println!("And one additional run with {} items.", over);
	}

	/* initialize a Vector. Rust Vectors grow but this one will stay fixed */
	let mut run: Vec<u64> = Vec::from_elem(items_per_run, 0_u64);

	// let's create a temporary directory to store the sorted chunks
	// neat: TempDir deletes the directory and its contents when it goes out of
	// scope
	let overflow_dir = match TempDir::new("externalsort") {
		Some(temp_dir) => temp_dir,
		None => fail!("creation of temporary directory"),
	};
	let overflow_path = overflow_dir.path();
	println!("temp dir path {}", overflow_path.display());

	// start the sorting runs
	for n in range(0, runs) {
		// iterate over our 'run' buffer and read from the input file
		// I suppose the values are little endian, which considering we are most
		// likely on x86, is a reasonable assumption
		for element in run.mut_iter() {
			let number = match fdInput.read_le_u64() {
				Ok(num) => num,
				Err(e) => fail!("failed to read u64 from file: {}", e),
			};
			*element = number;
		};
		// O(n log n) sort from the stdlib, hopefully more or less equivalent to
		// C++ std::sort
		run.sort();

		// now write out the results into numbered files
		let file_path = overflow_path.join(n.to_str());
		let mut file_file = match File::open_mode(&file_path, Open, Write) {
			Ok(f) => f,
			Err(e) => fail!("overflow file failed opening for write: {}", e),
		};

		for &element in run.iter() {
			match file_file.write_le_u64(element) {
				Ok (_) => (),
				Err(e) => fail!("writing overflow failed: {}", e),
			};
		};
	};

	/* additional run to catch remaining objects */
	if over > 0 {
		let mut run: Vec<u64> = Vec::with_capacity(over);
		for _ in range(0, over) {
			let number = match fdInput.read_le_u64() {
				Ok(num) => num,
				Err(e) => fail!("failed to read u64 from file: {}", e),
			};
			run.push(number);
		}
		run.sort();
		let file_path = overflow_path.join(runs.to_str());
		let mut file_file = match File::open_mode(&file_path, Open, Write) {
			Ok(f) => f,
			Err(e) => fail!("overflow file failed opening for write: {}", e),
		};

		for &element in run.iter() {
			match file_file.write_le_u64(element) {
				Ok (_) => (),
				Err(e) => fail!("writing overflow failed: {}", e),
			};
		};
		runs += 1;
	};

	/* sorting done, now merging */

	// collect all file handles. Hopefully less than 4096
	let mut files = Vec::with_capacity(runs as uint);
	for n in range(0, runs) {
		let file_path = overflow_path.join(n.to_str());
		let fd = match File::open(&file_path) {
			Ok(f) => f,
			Err(e) => fail!("overflow file opening error {}", e),
		};
		files.push(fd);
	}

	// we got a nice class here that spits out sorted numbers until exhausted
	let mut pf = PriorityFile::new(files);
	loop {
		match pf.next() {
			// if we got a number, write it to the output
			Some (number) => match fdOutput.write_le_u64(number) {
				Ok(_) => (),
				Err(e) => fail!("writing output file failed: {}", e),
			},
			None => break,
		};
	};
}

fn main() {
	if args().len() < 4 {
		fail!("Argument error: <inputFile> <outputFile> <memoryBufferInMB>");
	}
	let input_file_path = &Path::new(args()[1]);
	let size = match input_file_path.stat() {
		Ok(stat) => stat.size,
		Err(e) => fail!("Couldn't read {}", e),
	};
	let fin = match File::open(input_file_path) {
		Ok(f) => f,
		Err(e) => fail!("input file error: {}", e),
	};
	let fout = match File::open_mode(&Path::new(args()[2]), Open, Write) {
		Ok(f) => f,
		Err(e) => fail!("output file error: {}", e),
	};
	let buffer_size:u64 = match from_str(args()[3]) {
		Some(num) => num,
		None => fail!("Not numeric input"),
	};
	println!("Input file size {}", size);

	let values = read_u64(fin, 2);
	println!("Values: {}", values);
	//externalSort(fin, size, fout, buffer_size * 1024 * 1024);
}

#[test]
fn generate_5gb_and_sort() {
	let mut random = match File::open(&Path::new("/dev/urandom")) {
		Ok(f) => f,
		Err(e) => fail!("Error opening urandom: {}", e),
	};

	let test_dir = match TempDir::new("externalsort-test") {
		Some(temp_dir) => temp_dir,
		None => fail!("creation of temporary directory"),
	};
	let test_path = test_dir.path();

	let mut unordered_nums = match File::open_mode(&test_path.join("input"), Open, ReadWrite) {
		Ok(f) => f,
		Err(e) => fail!("output file error: {}", e),
	};

	let mut buffer = [0, .. 1024*1024];
	let mut left = 1 * 1024;
	while left > 0 {
		random.read(buffer);
		unordered_nums.write(buffer);
		left -= 1;
	}

	unordered_nums.seek(0, SeekSet);
	let ordered_nums = match File::open_mode(&test_path.join("output"), Open, Write) {
		Ok(f) => f,
		Err(e) => fail!("output file error: {}", e),
	};
	externalSort(unordered_nums, 1024 * 1024 * 1024 * 1, ordered_nums, 1024 * 1024 * 200);
	let mut ordered_nums = match File::open(&test_path.join("output")) {
		Ok(f) => f,
		Err(e) => fail!("output file error: {}", e),
	};

	let mut last = 0_u64;
	loop {
		let number = match ordered_nums.read_le_u64() {
			Ok(num) => num,
			Err(e) => fail!("failed to read u64 from file: {}", e),
		};
		assert!(number >= last);
		last = number;
	}
}
<commit_msg>Massive speedups in reading and writing u64<commit_after>extern crate collections;
use std::io::{File, Open, Write, TempDir};
use std::os::args;
use std::from_str::from_str;
use collections::priority_queue::PriorityQueue;
#[cfg(test)]
use std::io::{ReadWrite, SeekSet};

/*
 * an entry is the number that we read and the file it came from, so after we took
 * out one Entry from the priority queue, we need to know where to take the next number
 * from.
 */
struct Entry {
	value: u64,
	source: File,
}

/* The priority queue depends on the ordering relation and always returns the biggest */
impl Ord for Entry {
	// lesser than implementation
	fn lt(&self, other: &Entry) -> bool {
		// we invert: for us the smallest will have the highest priority
		!(self.value < other.value)
	}
}

/* ordering needs equality for some reason, so there */
impl Eq for Entry {
	fn eq(&self, other: &Entry) -> bool {
		self.value == other.value
	}
}

/* A struct to hold our queue */
struct PriorityFile {
	queue: PriorityQueue<Entry>,
}

/*
 * 'Class' that takes a list of files and does a k-way merge of the smallest values
 * from the files
 */
impl PriorityFile {
	pub fn new(files: Vec<File>) -> PriorityFile {
		let mut queue: PriorityQueue<Entry> = PriorityQueue::new();
		// now that we got the queue, populate it with initial values

		// move_iter is required to get the values themselves, not borrowed pointers to them
		for mut f in files.move_iter() {
			// read from file, append to queue
			let number = match f.read_le_u64() {
				Ok(num) => num,
				Err(e) => fail!("failed to read u64 from file: {}", e),
			};
			queue.push(Entry {value: number, source: f});
		};
		PriorityFile {queue: queue}
	}

	/* Returns the next smallest number from the list of files or None if exhausted */
	pub fn next(&mut self) -> Option<u64> {
		match self.queue.maybe_pop() {
			Some(mut res) => {
				let v = res.value;
				match res.source.read_le_u64() {
					Ok(num) => self.queue.push(Entry {value: num, source: res.source}),
					Err(_) => (),
				};
				Some(v)
			},
			None => None,
		}
	}
}

fn read_u64(from: &mut File, n: uint) -> Vec<u64> {
	// rust post 0.10 will directly return a Vec<u8> here
	let mut buf: ~[u8] = match from.read_exact(n*8) {
		Ok(b) => b,
		Err(e) => fail!("reading failed {}", e),
	};
	let mut buffer = Vec::from_slice(buf);

	let mut read: Vec<u64> = unsafe { std::cast::transmute(buffer) };
	let read_len = read.len();
	unsafe { read.set_len(read_len / 8); };
	read
}

fn write_u64(to: &mut File, items: Vec<u64>) {
	let mut written: Vec<u8> = unsafe { std::cast::transmute(items) };
	let written_len = written.len();
	unsafe { written.set_len(written_len * 8)};
	match to.write(written.as_slice()) {
		Ok(_) => (),
		Err(e) => fail!("writing failed: {}", e),
	};
}

fn externalSort(mut fdInput: File, size: u64, mut fdOutput: File, memSize: u64) {
	let mut runs = size / memSize;
	let items_per_run = (memSize / 8) as uint;
	let over = ((size / 8) - (items_per_run as u64 * runs)) as uint;

	println!("There will be {} runs with {} elements each.", runs, items_per_run);
	if over > 0 {
		println!("And one additional run with {} items.", over);
	}

	// let's create a temporary directory to store the sorted chunks
	// neat: TempDir deletes the directory and its contents when it goes out of
	// scope
	let overflow_dir = match TempDir::new("externalsort") {
		Some(temp_dir) => temp_dir,
		None => fail!("creation of temporary directory"),
	};
	let overflow_path = overflow_dir.path();
	println!("temp dir path {}", overflow_path.display());

	// start the sorting runs
	for n in range(0, runs) {
		// iterate over our 'run' buffer and read from the input file
		// I suppose the values are little endian, which considering we are most
		// likely on x86, is a reasonable assumption
		println!("Starting read");
		let mut run = read_u64(&mut fdInput, items_per_run);
		println!("Read finished");
		// O(n log n) sort from the stdlib, hopefully more or less equivalent to
		// C++ std::sort
		run.sort();

		// now write out the results into numbered files
		let file_path = overflow_path.join(n.to_str());
		let mut file_file = match File::open_mode(&file_path, Open, Write) {
			Ok(f) => f,
			Err(e) => fail!("overflow file failed opening for write: {}", e),
		};

		println!("Starting write");
		write_u64(&mut file_file, run);
		println!("Write finished");
	};

	/* additional run to catch remaining objects */
	if over > 0 {
		let mut run: Vec<u64> = Vec::with_capacity(over);
		for _ in range(0, over) {
			let number = match fdInput.read_le_u64() {
				Ok(num) => num,
				Err(e) => fail!("failed to read u64 from file: {}", e),
			};
			run.push(number);
		}
		run.sort();
		let file_path = overflow_path.join(runs.to_str());
		let mut file_file = match File::open_mode(&file_path, Open, Write) {
			Ok(f) => f,
			Err(e) => fail!("overflow file failed opening for write: {}", e),
		};

		for &element in run.iter() {
			match file_file.write_le_u64(element) {
				Ok (_) => (),
				Err(e) => fail!("writing overflow failed: {}", e),
			};
		};
		runs += 1;
	};

	/* sorting done, now merging */

	// collect all file handles. Hopefully less than 4096
	let mut files = Vec::with_capacity(runs as uint);
	for n in range(0, runs) {
		let file_path = overflow_path.join(n.to_str());
		let fd = match File::open(&file_path) {
			Ok(f) => f,
			Err(e) => fail!("overflow file opening error {}", e),
		};
		files.push(fd);
	}

	// we got a nice class here that spits out sorted numbers until exhausted
	let mut pf = PriorityFile::new(files);
	loop {
		match pf.next() {
			// if we got a number, write it to the output
			Some (number) => match fdOutput.write_le_u64(number) {
				Ok(_) => (),
				Err(e) => fail!("writing output file failed: {}", e),
			},
			None => break,
		};
	};
}

fn main() {
	if args().len() < 4 {
		fail!("Argument error: <inputFile> <outputFile> <memoryBufferInMB>");
	}
	let input_file_path = &Path::new(args()[1]);
	let size = match input_file_path.stat() {
		Ok(stat) => stat.size,
		Err(e) => fail!("Couldn't read {}", e),
	};
	let fin = match File::open(input_file_path) {
		Ok(f) => f,
		Err(e) => fail!("input file error: {}", e),
	};
	let fout = match File::open_mode(&Path::new(args()[2]), Open, Write) {
		Ok(f) => f,
		Err(e) => fail!("output file error: {}", e),
	};
	let buffer_size:u64 = match from_str(args()[3]) {
		Some(num) => num,
		None => fail!("Not numeric input"),
	};
	println!("Input file size {}", size);

	//let values = read_u64(&mut fin, 2);
	//println!("Values: {}", values);
	externalSort(fin, size, fout, buffer_size * 1024 * 1024);
}

#[test]
fn generate_5gb_and_sort() {
	let mut random = match File::open(&Path::new("/dev/urandom")) {
		Ok(f) => f,
		Err(e) => fail!("Error opening urandom: {}", e),
	};

	let test_dir = match TempDir::new("externalsort-test") {
		Some(temp_dir) => temp_dir,
		None => fail!("creation of temporary directory"),
	};
	let test_path = test_dir.path();

	let mut unordered_nums = match File::open_mode(&test_path.join("input"), Open, ReadWrite) {
		Ok(f) => f,
		Err(e) => fail!("output file error: {}", e),
	};

	let mut buffer = [0, .. 1024*1024];
	let mut left = 1 * 1024;
	while left > 0 {
		random.read(buffer);
		unordered_nums.write(buffer);
		left -= 1;
	}

	unordered_nums.seek(0, SeekSet);
	let ordered_nums = match File::open_mode(&test_path.join("output"), Open, Write) {
		Ok(f) => f,
		Err(e) => fail!("output file error: {}", e),
	};
	externalSort(unordered_nums, 1024 * 1024 * 1024 * 1, ordered_nums, 1024 * 1024 * 200);
	let mut ordered_nums = match File::open(&test_path.join("output")) {
		Ok(f) => f,
		Err(e) => fail!("output file error: {}", e),
	};

	let mut last = 0_u64;
	loop {
		let number = match ordered_nums.read_le_u64() {
			Ok(num) => num,
			Err(e) => fail!("failed to read u64 from file: {}", e),
		};
		assert!(number >= last);
		last = number;
	}
}
<|endoftext|>


<commit_before>use nannou::prelude::*;
use nannou_audio as audio;
use nannou_audio::Buffer;

fn main() {
    nannou::app(model).run();
}

struct Model {
    stream: audio::Stream<Audio>,
}

struct Audio {
    sounds: Vec<audrey::read::BufFileReader>,
}

fn model(app: &App) -> Model {
    // Create a window to receive key pressed events.
    app.new_window()
        .key_pressed(key_pressed)
        .view(view)
        .build()
        .unwrap();

    // Initialise the audio API so we can spawn an audio stream.
    let audio_api = audio::Api::new();

    // Initialise the state that we want to live on the audio thread.
    let sounds = vec![];
    let model = Audio { sounds };
    let stream = audio_api.new_output_stream(model, audio).build().unwrap();
    Model { stream }
}

// A function that renders the given `Audio` to the given `Buffer`.
// In this case we play a simple sine wave at the audio's current frequency in `hz`.
fn audio(audio: &mut Audio, buffer: &mut Buffer) {
    let mut have_ended = vec![];
    let len_frames = buffer.len_frames();

    // Sum all of the sounds onto the buffer.
    for (i, sound) in audio.sounds.iter_mut().enumerate() {
        let mut frame_count = 0;
        let file_frames = sound.frames::<[f32; 2]>().filter_map(Result::ok);
        for (frame, file_frame) in buffer.frames_mut().zip(file_frames) {
            for (sample, file_sample) in frame.iter_mut().zip(&file_frame) {
                *sample += *file_sample;
            }
            frame_count += 1;
        }

        // If the sound yielded less samples than are in the buffer, it must have ended.
        if frame_count < len_frames {
            have_ended.push(i);
        }
    }

    // Remove all sounds that have ended.
    for i in have_ended.into_iter().rev() {
        audio.sounds.remove(i);
    }
}

fn key_pressed(app: &App, model: &mut Model, key: Key) {
    match key {
        // Start playing another instance of the sound.
        Key::Space => {
            let assets = app.assets_path().expect("could not find assets directory");
            let path = assets.join("sounds").join("thumbpiano.wav");
            let sound = audrey::open(path).expect("failed to load sound");
            model
                .stream
                .send(move |audio| {
                    audio.sounds.push(sound);
                })
                .ok();
        }
        _ => {}
    }
}

fn view(_app: &App, _model: &Model, frame: Frame) -> Frame {
    frame.clear(DARK_CHARCOAL);
    frame
}
<commit_msg>Update simple_audio_file example for new host API<commit_after>use nannou::prelude::*;
use nannou_audio as audio;
use nannou_audio::Buffer;

fn main() {
    nannou::app(model).run();
}

struct Model {
    stream: audio::Stream<Audio>,
}

struct Audio {
    sounds: Vec<audrey::read::BufFileReader>,
}

fn model(app: &App) -> Model {
    // Create a window to receive key pressed events.
    app.new_window()
        .key_pressed(key_pressed)
        .view(view)
        .build()
        .unwrap();

    // Initialise the audio host so we can spawn an audio stream.
    let audio_host = audio::Host::new();

    // Initialise the state that we want to live on the audio thread.
    let sounds = vec![];
    let model = Audio { sounds };
    let stream = audio_host
        .new_output_stream(model)
        .render(audio)
        .build()
        .unwrap();
    Model { stream }
}

// A function that renders the given `Audio` to the given `Buffer`.
// In this case we play a simple sine wave at the audio's current frequency in `hz`.
fn audio(audio: &mut Audio, buffer: &mut Buffer) {
    let mut have_ended = vec![];
    let len_frames = buffer.len_frames();

    // Sum all of the sounds onto the buffer.
    for (i, sound) in audio.sounds.iter_mut().enumerate() {
        let mut frame_count = 0;
        let file_frames = sound.frames::<[f32; 2]>().filter_map(Result::ok);
        for (frame, file_frame) in buffer.frames_mut().zip(file_frames) {
            for (sample, file_sample) in frame.iter_mut().zip(&file_frame) {
                *sample += *file_sample;
            }
            frame_count += 1;
        }

        // If the sound yielded less samples than are in the buffer, it must have ended.
        if frame_count < len_frames {
            have_ended.push(i);
        }
    }

    // Remove all sounds that have ended.
    for i in have_ended.into_iter().rev() {
        audio.sounds.remove(i);
    }
}

fn key_pressed(app: &App, model: &mut Model, key: Key) {
    match key {
        // Start playing another instance of the sound.
        Key::Space => {
            let assets = app.assets_path().expect("could not find assets directory");
            let path = assets.join("sounds").join("thumbpiano.wav");
            let sound = audrey::open(path).expect("failed to load sound");
            model
                .stream
                .send(move |audio| {
                    audio.sounds.push(sound);
                })
                .ok();
        }
        _ => {}
    }
}

fn view(_app: &App, _model: &Model, frame: Frame) -> Frame {
    frame.clear(DARK_CHARCOAL);
    frame
}
<|endoftext|>


<commit_before>use std::env;
use std::path::Path;

use lazy_static::lazy_static;
use raw_cpuid::{CpuId, ExtendedFeatures, FeatureInfo};

lazy_static! {
    static ref CPU_ID: CpuId = CpuId::new();
    static ref CPU_FEATURES: Option<FeatureInfo> = CPU_ID.get_feature_info();
    static ref CPU_EXTENDED_FEATURES: Option<ExtendedFeatures> = CPU_ID.get_extended_feature_info();
    static ref TARGET_FEATURES: Vec<String> = env::var("CARGO_CFG_TARGET_FEATURE")
        .map_or_else(|_| vec![], |s| s.split(",").map(|s| s.to_owned()).collect());
    static ref TARGET_ARCH: String = env::var("CARGO_CFG_TARGET_ARCH").unwrap();
    static ref TARGET_ENV: String = env::var("CARGO_CFG_TARGET_ENV").unwrap();
}

fn has_target_feature(feature: &str) -> bool {
    TARGET_FEATURES.iter().any(|name| name == feature)
}

fn has_aesni() -> bool {
    cfg!(feature = "native")
        && CPU_FEATURES
            .as_ref()
            .map_or(false, |features| features.has_aesni())
}

fn has_sse41() -> bool {
    cfg!(feature = "native")
        && CPU_FEATURES
            .as_ref()
            .map_or(false, |features| features.has_sse41())
}

fn has_sse42() -> bool {
    cfg!(feature = "native")
        && CPU_FEATURES
            .as_ref()
            .map_or(false, |features| features.has_sse42())
}

fn has_avx() -> bool {
    cfg!(feature = "native")
        && CPU_FEATURES
            .as_ref()
            .map_or(false, |features| features.has_avx())
}

fn has_avx2() -> bool {
    cfg!(feature = "native")
        && CPU_EXTENDED_FEATURES
            .as_ref()
            .map_or(false, |features| features.has_avx2())
}

fn support_aesni() -> bool {
    cfg!(feature = "aes") || has_target_feature("aes") || has_aesni()
}

#[allow(dead_code)]
fn support_sse41() -> bool {
    cfg!(feature = "sse41") || has_target_feature("sse41") || has_sse41()
}

fn support_sse42() -> bool {
    cfg!(feature = "sse42") || has_target_feature("sse42") || has_sse42()
}

fn support_avx() -> bool {
    cfg!(feature = "avx") || has_target_feature("avx") || has_avx()
}

fn support_avx2() -> bool {
    cfg!(feature = "avx2") || has_target_feature("avx2") || has_avx2()
}

#[cfg(all(not(feature = "gen"), any(target_os = "macos", target_os = "linux")))]
fn generate_binding(_out_file: &Path) {
    cargo_emit::warning!("pregenerated binding file.");
}

#[cfg(any(feature = "gen", not(any(target_os = "macos", target_os = "linux"))))]
fn generate_binding(out_file: &Path) {
    cargo_emit::warning!("generate binding file @ {:?}.", out_file);

    let _ = bindgen::builder()
        .clang_args(&["-x", "c++", "-std=c++11"])
        .clang_args(&[
            "-Dt1ha_EXPORTS",
            "-DXXH_STATIC_LINKING_ONLY",
            "-Isrc/highwayhash",
        ])
        .clang_args(if support_aesni() {
            &[
                "-maes",
                "-DT1HA0_RUNTIME_SELECT=1",
                "-DT1HA0_AESNI_AVAILABLE=1",
            ][..]
        } else {
            &[][..]
        })
        .clang_args(
            vec![
                if cfg!(feature = "native") {
                    Some("-march=native")
                } else {
                    None
                },
                if support_sse41() {
                    Some("-msse4.1")
                } else {
                    None
                },
                if support_sse42() {
                    Some("-msse4.2")
                } else {
                    None
                },
                if support_avx() { Some("-mavx") } else { None },
                if support_avx2() { Some("-mavx2") } else { None },
                if cfg!(feature = "city") {
                    Some("-DCITY_HASH=1")
                } else {
                    None
                },
                if cfg!(feature = "farm") {
                    Some("-DFARM_HASH=1")
                } else {
                    None
                },
                if cfg!(feature = "highway") {
                    Some("-DHIGHWAY_HASH=1")
                } else {
                    None
                },
                if cfg!(feature = "komi") {
                    Some("-DKOMI_HASH=1")
                } else {
                    None
                },
                if cfg!(feature = "lookup3") {
                    Some("-DLOOKUP3=1")
                } else {
                    None
                },
                if cfg!(feature = "meow") && matches!(TARGET_ARCH.as_str(), "x86" | "x86_64") {
                    Some("-DMEOW_HASH=1")
                } else {
                    None
                },
                if cfg!(feature = "metro") {
                    Some("-DMETRO_HASH=1")
                } else {
                    None
                },
                if cfg!(feature = "mum") {
                    Some("-DMUM_HASH=1")
                } else {
                    None
                },
                if cfg!(feature = "murmur") {
                    Some("-DMURMUR_HASH=1")
                } else {
                    None
                },
                if cfg!(feature = "mx3") {
                    Some("-DMX3_HASH=1")
                } else {
                    None
                },
                if cfg!(feature = "nm") {
                    Some("-DNM_HASH=1")
                } else {
                    None
                },
                if cfg!(feature = "pengy") {
                    Some("-DPENGY_HASH=1")
                } else {
                    None
                },
                if cfg!(feature = "prv") {
                    Some("-DPRV_HASH=1")
                } else {
                    None
                },
                if cfg!(feature = "spooky") {
                    Some("-DSPOOKY_HASH=1")
                } else {
                    None
                },
                if cfg!(feature = "t1ha") {
                    Some("-DT1_HASH=1")
                } else {
                    None
                },
                if cfg!(feature = "wy") {
                    Some("-DWY_HASH=1")
                } else {
                    None
                },
                if cfg!(feature = "xx") {
                    Some("-DXX_HASH=1")
                } else {
                    None
                },
            ]
            .into_iter()
            .flatten(),
        )
        .header("src/fasthash.hpp")
        .size_t_is_usize(true)
        .generate_inline_functions(true)
        .disable_name_namespacing()
        .allowlist_function("^CityHash.*")
        .allowlist_function("^farmhash.*")
        .allowlist_function("^HighwayHash.*")
        .allowlist_function("^komi.*")
        .allowlist_function("^lookup3.*")
        .allowlist_function("^metrohash.*")
        .allowlist_function("^mum_hash.*")
        .allowlist_function("^MurmurHash.*")
        .allowlist_function("^mx3hash.*")
        .allowlist_function("^NMHASH.*")
        .allowlist_function("^pengy.*")
        .allowlist_function("^prvhash.*")
        .allowlist_function("^SpookyHasher.*")
        .allowlist_function("^t1ha.*")
        .allowlist_function("^wyhash.*")
        .allowlist_function("^XXH.*")
        .allowlist_function("^Meow.*")
        .blocklist_function("^t1ha_selfcheck__.*")
        .allowlist_var("^Meow.*")
        .allowlist_var("^PRH64S_.*")
        .generate()
        .unwrap()
        .write_to_file(out_file)
        .expect("fail to write bindings");

    cargo_emit::warning!("generate binding file @ {:?}.", out_file);
}

fn build_fasthash() {
    let mut build = cc::Build::new();

    build
        .cpp(true)
        .include("src/highwayhash")
        .flag("-std=c++11")
        .flag_if_supported("-Wno-implicit-fallthrough")
        .flag_if_supported("-Wno-unknown-attributes")
        .flag_if_supported("-Wno-sign-compare")
        .file("src/fasthash.cpp");

    if cfg!(feature = "city") {
        build.flag("-DCITY_HASH=1").file("src/smhasher/City.cpp");
    }

    if cfg!(feature = "farm") {
        build
            .flag("-DFARM_HASH=1")
            .file("src/smhasher/farmhash-c.c");
    }

    if cfg!(feature = "komi") {
        build.flag("-DKOMI_HASH=1");
    }

    if cfg!(feature = "highway") {
        build.flag("-DHIGHWAY_HASH=1");
    }

    if cfg!(feature = "lookup3") {
        build.flag("-DLOOKUP3=1").file("src/smhasher/lookup3.cpp");
    }

    if cfg!(feature = "meow") && matches!(TARGET_ARCH.as_str(), "x86" | "x86_64") {
        build.flag("-DMEOW_HASH=1");
    }

    if cfg!(feature = "mx3") {
        build.flag("-DMX3_HASH=1");
    }

    if cfg!(feature = "nm") {
        build.flag("-DNM_HASH=1");
    }

    if cfg!(feature = "pengy") {
        build
            .flag("-DPENGY_HASH=1")
            .file("src/pengyhash/pengyhash.c");
    }

    if cfg!(feature = "prv") {
        build.flag("-DPRV_HASH=1");
    }

    if cfg!(feature = "t1ha") {
        build.flag("-DT1_HASH=1");
    }

    if cfg!(feature = "metro") {
        build
            .flag("-DMETRO_HASH=1")
            .file("src/smhasher/metrohash/metrohash64.cpp")
            .file("src/smhasher/metrohash/metrohash128.cpp");

        if support_sse42() {
            build
                .file("src/smhasher/metrohash/metrohash64crc.cpp")
                .file("src/smhasher/metrohash/metrohash128crc.cpp");
        }
    }

    if cfg!(feature = "mum") {
        build.flag("-DMUM_HASH=1").file("src/smhasher/mum.cc");
    }

    if cfg!(feature = "murmur") {
        build
            .flag("-DMURMUR_HASH=1")
            .file("src/smhasher/MurmurHash1.cpp")
            .file("src/smhasher/MurmurHash2.cpp")
            .file("src/smhasher/MurmurHash3.cpp");
    }

    if cfg!(feature = "spooky") {
        build
            .flag("-DSPOOKY_HASH=1")
            .file("src/smhasher/Spooky.cpp");
    }

    if cfg!(feature = "wy") {
        build.flag("-DWY_HASH=1");
    }

    if cfg!(feature = "xx") {
        build.flag("-DXX_HASH=1").file("src/xxHash/xxhash.c");
    }

    if cfg!(feature = "native") {
        build.flag("-march=native");
    } else {
        if has_target_feature("aes") {
            build.flag("-maes");
        }
        if has_target_feature("sse41") {
            build.flag("-msse41");
        }
        if has_target_feature("sse42") {
            build.flag("-msse4.2");
        }
        if has_target_feature("avx") {
            build.flag("-mavx");
        }
        if has_target_feature("avx2") {
            build.flag("-mavx2");
        }
    }

    build.static_flag(true).compile("fasthash");
}

fn build_t1() {
    let mut build = cc::Build::new();

    build
        .file("src/t1ha/src/t1ha0.c")
        .file("src/t1ha/src/t1ha1.c")
        .file("src/t1ha/src/t1ha2.c");

    // indirect functions are not supported on all targets (e.g. x86_64-unknown-linux-musl)
    if TARGET_ENV.as_str() == "musl" {
        build.define("T1HA_USE_INDIRECT_FUNCTIONS", Some("0"));
    }

    if support_aesni() {
        build
            .define("T1HA0_RUNTIME_SELECT", Some("1"))
            .define("T1HA0_AESNI_AVAILABLE", Some("1"))
            .flag("-maes")
            .file("src/t1ha/src/t1ha0_ia32aes_noavx.c")
            .file("src/t1ha/src/t1ha0_ia32aes_avx.c")
            .file("src/t1ha/src/t1ha0_ia32aes_avx2.c");

        if support_avx() {
            build.flag("-mavx");
        }

        if support_avx2() {
            build.flag("-mavx2");
        }
    }

    build.static_flag(true).compile("t1ha");
}

fn build_highway() {
    let mut build = cc::Build::new();

    build
        .cpp(true)
        .flag("-std=c++11")
        .flag_if_supported("-Wno-sign-compare")
        .include("src/highwayhash")
        .file("src/highwayhash/highwayhash/arch_specific.cc")
        .file("src/highwayhash/highwayhash/instruction_sets.cc")
        .file("src/highwayhash/highwayhash/os_specific.cc")
        .file("src/highwayhash/highwayhash/hh_portable.cc")
        .file("src/highwayhash/highwayhash/c_bindings.cc");

    match TARGET_ARCH.as_str() {
        "x86" | "x86_64" => {
            build
                .flag("-msse4.1")
                .flag("-mavx2")
                .file("src/highwayhash/highwayhash/hh_sse41.cc")
                .file("src/highwayhash/highwayhash/hh_avx2.cc");
        }

        "aarch64" => {
            build.file("src/highwayhash/highwayhash/hh_neon.cc");
        }

        "powerpc64" => {
            build
                .flag("-mvsx")
                .file("src/highwayhash/highwayhash/hh_vsx.cc");
        }
        _ => {}
    }

    build.static_flag(true).compile("highwayhash");
}

fn main() {
    if has_aesni() {
        cargo_emit::rustc_cfg!(r#"feature="aes""#);
    }
    if has_sse41() {
        cargo_emit::rustc_cfg!(r#"feature="sse41""#);
    }
    if has_sse42() {
        cargo_emit::rustc_cfg!(r#"feature="sse42""#);
    }
    if has_avx() {
        cargo_emit::rustc_cfg!(r#"feature="avx""#);
    }
    if has_avx2() {
        cargo_emit::rustc_cfg!(r#"feature="avx2""#);
    }

    build_fasthash();
    if cfg!(feature = "t1ha") {
        build_t1();
    }
    if cfg!(feature = "highway") {
        build_highway();
    }

    let out_dir = env::var("OUT_DIR").unwrap();
    let out_file = Path::new(&out_dir).join("fasthash.rs");

    cargo_emit::rerun_if_changed!("src/fasthash.hpp");
    cargo_emit::rerun_if_changed!("src/fasthash.cpp");

    generate_binding(&out_file);
}
<commit_msg>fix hint warning<commit_after>use std::env;
use std::path::Path;

use lazy_static::lazy_static;
use raw_cpuid::{CpuId, ExtendedFeatures, FeatureInfo};

lazy_static! {
    static ref CPU_ID: CpuId = CpuId::new();
    static ref CPU_FEATURES: Option<FeatureInfo> = CPU_ID.get_feature_info();
    static ref CPU_EXTENDED_FEATURES: Option<ExtendedFeatures> = CPU_ID.get_extended_feature_info();
    static ref TARGET_FEATURES: Vec<String> = env::var("CARGO_CFG_TARGET_FEATURE")
        .map_or_else(|_| vec![], |s| s.split(',').map(|s| s.to_owned()).collect());
    static ref TARGET_ARCH: String = env::var("CARGO_CFG_TARGET_ARCH").unwrap();
    static ref TARGET_ENV: String = env::var("CARGO_CFG_TARGET_ENV").unwrap();
}

fn has_target_feature(feature: &str) -> bool {
    TARGET_FEATURES.iter().any(|name| name == feature)
}

fn has_aesni() -> bool {
    cfg!(feature = "native")
        && CPU_FEATURES
            .as_ref()
            .map_or(false, |features| features.has_aesni())
}

fn has_sse41() -> bool {
    cfg!(feature = "native")
        && CPU_FEATURES
            .as_ref()
            .map_or(false, |features| features.has_sse41())
}

fn has_sse42() -> bool {
    cfg!(feature = "native")
        && CPU_FEATURES
            .as_ref()
            .map_or(false, |features| features.has_sse42())
}

fn has_avx() -> bool {
    cfg!(feature = "native")
        && CPU_FEATURES
            .as_ref()
            .map_or(false, |features| features.has_avx())
}

fn has_avx2() -> bool {
    cfg!(feature = "native")
        && CPU_EXTENDED_FEATURES
            .as_ref()
            .map_or(false, |features| features.has_avx2())
}

fn support_aesni() -> bool {
    cfg!(feature = "aes") || has_target_feature("aes") || has_aesni()
}

#[allow(dead_code)]
fn support_sse41() -> bool {
    cfg!(feature = "sse41") || has_target_feature("sse41") || has_sse41()
}

fn support_sse42() -> bool {
    cfg!(feature = "sse42") || has_target_feature("sse42") || has_sse42()
}

fn support_avx() -> bool {
    cfg!(feature = "avx") || has_target_feature("avx") || has_avx()
}

fn support_avx2() -> bool {
    cfg!(feature = "avx2") || has_target_feature("avx2") || has_avx2()
}

#[cfg(all(not(feature = "gen"), any(target_os = "macos", target_os = "linux")))]
fn generate_binding(_out_file: &Path) {
    cargo_emit::warning!("pregenerated binding file.");
}

#[cfg(any(feature = "gen", not(any(target_os = "macos", target_os = "linux"))))]
fn generate_binding(out_file: &Path) {
    cargo_emit::warning!("generate binding file @ {:?}.", out_file);

    let _ = bindgen::builder()
        .clang_args(&["-x", "c++", "-std=c++11"])
        .clang_args(&[
            "-Dt1ha_EXPORTS",
            "-DXXH_STATIC_LINKING_ONLY",
            "-Isrc/highwayhash",
        ])
        .clang_args(if support_aesni() {
            &[
                "-maes",
                "-DT1HA0_RUNTIME_SELECT=1",
                "-DT1HA0_AESNI_AVAILABLE=1",
            ][..]
        } else {
            &[][..]
        })
        .clang_args(
            vec![
                if cfg!(feature = "native") {
                    Some("-march=native")
                } else {
                    None
                },
                if support_sse41() {
                    Some("-msse4.1")
                } else {
                    None
                },
                if support_sse42() {
                    Some("-msse4.2")
                } else {
                    None
                },
                if support_avx() { Some("-mavx") } else { None },
                if support_avx2() { Some("-mavx2") } else { None },
                if cfg!(feature = "city") {
                    Some("-DCITY_HASH=1")
                } else {
                    None
                },
                if cfg!(feature = "farm") {
                    Some("-DFARM_HASH=1")
                } else {
                    None
                },
                if cfg!(feature = "highway") {
                    Some("-DHIGHWAY_HASH=1")
                } else {
                    None
                },
                if cfg!(feature = "komi") {
                    Some("-DKOMI_HASH=1")
                } else {
                    None
                },
                if cfg!(feature = "lookup3") {
                    Some("-DLOOKUP3=1")
                } else {
                    None
                },
                if cfg!(feature = "meow") && matches!(TARGET_ARCH.as_str(), "x86" | "x86_64") {
                    Some("-DMEOW_HASH=1")
                } else {
                    None
                },
                if cfg!(feature = "metro") {
                    Some("-DMETRO_HASH=1")
                } else {
                    None
                },
                if cfg!(feature = "mum") {
                    Some("-DMUM_HASH=1")
                } else {
                    None
                },
                if cfg!(feature = "murmur") {
                    Some("-DMURMUR_HASH=1")
                } else {
                    None
                },
                if cfg!(feature = "mx3") {
                    Some("-DMX3_HASH=1")
                } else {
                    None
                },
                if cfg!(feature = "nm") {
                    Some("-DNM_HASH=1")
                } else {
                    None
                },
                if cfg!(feature = "pengy") {
                    Some("-DPENGY_HASH=1")
                } else {
                    None
                },
                if cfg!(feature = "prv") {
                    Some("-DPRV_HASH=1")
                } else {
                    None
                },
                if cfg!(feature = "spooky") {
                    Some("-DSPOOKY_HASH=1")
                } else {
                    None
                },
                if cfg!(feature = "t1ha") {
                    Some("-DT1_HASH=1")
                } else {
                    None
                },
                if cfg!(feature = "wy") {
                    Some("-DWY_HASH=1")
                } else {
                    None
                },
                if cfg!(feature = "xx") {
                    Some("-DXX_HASH=1")
                } else {
                    None
                },
            ]
            .into_iter()
            .flatten(),
        )
        .header("src/fasthash.hpp")
        .size_t_is_usize(true)
        .generate_inline_functions(true)
        .disable_name_namespacing()
        .allowlist_function("^CityHash.*")
        .allowlist_function("^farmhash.*")
        .allowlist_function("^HighwayHash.*")
        .allowlist_function("^komi.*")
        .allowlist_function("^lookup3.*")
        .allowlist_function("^metrohash.*")
        .allowlist_function("^mum_hash.*")
        .allowlist_function("^MurmurHash.*")
        .allowlist_function("^mx3hash.*")
        .allowlist_function("^NMHASH.*")
        .allowlist_function("^pengy.*")
        .allowlist_function("^prvhash.*")
        .allowlist_function("^SpookyHasher.*")
        .allowlist_function("^t1ha.*")
        .allowlist_function("^wyhash.*")
        .allowlist_function("^XXH.*")
        .allowlist_function("^Meow.*")
        .blocklist_function("^t1ha_selfcheck__.*")
        .allowlist_var("^Meow.*")
        .allowlist_var("^PRH64S_.*")
        .generate()
        .unwrap()
        .write_to_file(out_file)
        .expect("fail to write bindings");

    cargo_emit::warning!("generate binding file @ {:?}.", out_file);
}

fn build_fasthash() {
    let mut build = cc::Build::new();

    build
        .cpp(true)
        .include("src/highwayhash")
        .flag("-std=c++11")
        .flag_if_supported("-Wno-implicit-fallthrough")
        .flag_if_supported("-Wno-unknown-attributes")
        .flag_if_supported("-Wno-sign-compare")
        .file("src/fasthash.cpp");

    if cfg!(feature = "city") {
        build.flag("-DCITY_HASH=1").file("src/smhasher/City.cpp");
    }

    if cfg!(feature = "farm") {
        build
            .flag("-DFARM_HASH=1")
            .file("src/smhasher/farmhash-c.c");
    }

    if cfg!(feature = "komi") {
        build.flag("-DKOMI_HASH=1");
    }

    if cfg!(feature = "highway") {
        build.flag("-DHIGHWAY_HASH=1");
    }

    if cfg!(feature = "lookup3") {
        build.flag("-DLOOKUP3=1").file("src/smhasher/lookup3.cpp");
    }

    if cfg!(feature = "meow") && matches!(TARGET_ARCH.as_str(), "x86" | "x86_64") {
        build.flag("-DMEOW_HASH=1");
    }

    if cfg!(feature = "mx3") {
        build.flag("-DMX3_HASH=1");
    }

    if cfg!(feature = "nm") {
        build.flag("-DNM_HASH=1");
    }

    if cfg!(feature = "pengy") {
        build
            .flag("-DPENGY_HASH=1")
            .file("src/pengyhash/pengyhash.c");
    }

    if cfg!(feature = "prv") {
        build.flag("-DPRV_HASH=1");
    }

    if cfg!(feature = "t1ha") {
        build.flag("-DT1_HASH=1");
    }

    if cfg!(feature = "metro") {
        build
            .flag("-DMETRO_HASH=1")
            .file("src/smhasher/metrohash/metrohash64.cpp")
            .file("src/smhasher/metrohash/metrohash128.cpp");

        if support_sse42() {
            build
                .file("src/smhasher/metrohash/metrohash64crc.cpp")
                .file("src/smhasher/metrohash/metrohash128crc.cpp");
        }
    }

    if cfg!(feature = "mum") {
        build.flag("-DMUM_HASH=1").file("src/smhasher/mum.cc");
    }

    if cfg!(feature = "murmur") {
        build
            .flag("-DMURMUR_HASH=1")
            .file("src/smhasher/MurmurHash1.cpp")
            .file("src/smhasher/MurmurHash2.cpp")
            .file("src/smhasher/MurmurHash3.cpp");
    }

    if cfg!(feature = "spooky") {
        build
            .flag("-DSPOOKY_HASH=1")
            .file("src/smhasher/Spooky.cpp");
    }

    if cfg!(feature = "wy") {
        build.flag("-DWY_HASH=1");
    }

    if cfg!(feature = "xx") {
        build.flag("-DXX_HASH=1").file("src/xxHash/xxhash.c");
    }

    if cfg!(feature = "native") {
        build.flag("-march=native");
    } else {
        if has_target_feature("aes") {
            build.flag("-maes");
        }
        if has_target_feature("sse41") {
            build.flag("-msse41");
        }
        if has_target_feature("sse42") {
            build.flag("-msse4.2");
        }
        if has_target_feature("avx") {
            build.flag("-mavx");
        }
        if has_target_feature("avx2") {
            build.flag("-mavx2");
        }
    }

    build.static_flag(true).compile("fasthash");
}

fn build_t1() {
    let mut build = cc::Build::new();

    build
        .file("src/t1ha/src/t1ha0.c")
        .file("src/t1ha/src/t1ha1.c")
        .file("src/t1ha/src/t1ha2.c");

    // indirect functions are not supported on all targets (e.g. x86_64-unknown-linux-musl)
    if TARGET_ENV.as_str() == "musl" {
        build.define("T1HA_USE_INDIRECT_FUNCTIONS", Some("0"));
    }

    if support_aesni() {
        build
            .define("T1HA0_RUNTIME_SELECT", Some("1"))
            .define("T1HA0_AESNI_AVAILABLE", Some("1"))
            .flag("-maes")
            .file("src/t1ha/src/t1ha0_ia32aes_noavx.c")
            .file("src/t1ha/src/t1ha0_ia32aes_avx.c")
            .file("src/t1ha/src/t1ha0_ia32aes_avx2.c");

        if support_avx() {
            build.flag("-mavx");
        }

        if support_avx2() {
            build.flag("-mavx2");
        }
    }

    build.static_flag(true).compile("t1ha");
}

fn build_highway() {
    let mut build = cc::Build::new();

    build
        .cpp(true)
        .flag("-std=c++11")
        .flag_if_supported("-Wno-sign-compare")
        .include("src/highwayhash")
        .file("src/highwayhash/highwayhash/arch_specific.cc")
        .file("src/highwayhash/highwayhash/instruction_sets.cc")
        .file("src/highwayhash/highwayhash/os_specific.cc")
        .file("src/highwayhash/highwayhash/hh_portable.cc")
        .file("src/highwayhash/highwayhash/c_bindings.cc");

    match TARGET_ARCH.as_str() {
        "x86" | "x86_64" => {
            build
                .flag("-msse4.1")
                .flag("-mavx2")
                .file("src/highwayhash/highwayhash/hh_sse41.cc")
                .file("src/highwayhash/highwayhash/hh_avx2.cc");
        }

        "aarch64" => {
            build.file("src/highwayhash/highwayhash/hh_neon.cc");
        }

        "powerpc64" => {
            build
                .flag("-mvsx")
                .file("src/highwayhash/highwayhash/hh_vsx.cc");
        }
        _ => {}
    }

    build.static_flag(true).compile("highwayhash");
}

fn main() {
    if has_aesni() {
        cargo_emit::rustc_cfg!(r#"feature="aes""#);
    }
    if has_sse41() {
        cargo_emit::rustc_cfg!(r#"feature="sse41""#);
    }
    if has_sse42() {
        cargo_emit::rustc_cfg!(r#"feature="sse42""#);
    }
    if has_avx() {
        cargo_emit::rustc_cfg!(r#"feature="avx""#);
    }
    if has_avx2() {
        cargo_emit::rustc_cfg!(r#"feature="avx2""#);
    }

    build_fasthash();
    if cfg!(feature = "t1ha") {
        build_t1();
    }
    if cfg!(feature = "highway") {
        build_highway();
    }

    let out_dir = env::var("OUT_DIR").unwrap();
    let out_file = Path::new(&out_dir).join("fasthash.rs");

    cargo_emit::rerun_if_changed!("src/fasthash.hpp");
    cargo_emit::rerun_if_changed!("src/fasthash.cpp");

    generate_binding(&out_file);
}
<|endoftext|>


<commit_before>use std::time::Duration;
use criterion::{Criterion,black_box,Bencher};

use pleco::{Board};

use pleco_engine::engine::PlecoSearcher;
use pleco_engine::time::uci_timer::PreLimits;

use super::*;

fn search_3moves_engine<D: DepthLimit>(b: &mut Bencher) {
    let mut limit = PreLimits::blank();
    limit.depth = Some(D::depth());
    b.iter_with_setup(|| {
        let mut s = PlecoSearcher::init(false);
        s.clear_search();
        (Board::default(), s)
    }, |(mut board, mut s)| {
        black_box(s.search(&board, &limit));
        let mov = black_box(s.await_move());
        board.apply_move(mov);
        black_box(s.search(&board, &limit));
        let mov = black_box(s.await_move());
        board.apply_move(mov);
        black_box(s.search(&board, &limit));
        black_box(s.await_move());
    })
}

fn bench_engine_evaluations(c: &mut Criterion) {
    c.bench_function("Search MuliMove Depth 4", search_3moves_engine::<Depth4>);
    c.bench_function("Search MuliMove Depth 5", search_3moves_engine::<Depth5>);
    c.bench_function("Search MuliMove Depth 6", search_3moves_engine::<Depth6>);
    c.bench_function("Search MuliMove Depth 7", search_3moves_engine::<Depth7>);
}

criterion_group!(name = search_multimove;
     config = Criterion::default()
        .sample_size(16)
        .warm_up_time(Duration::from_millis(200));
    targets = bench_engine_evaluations
);

<commit_msg>Hopefully reducing the sample size prevents a stack allocation error.<commit_after>use std::time::Duration;
use criterion::{Criterion,black_box,Bencher};

use pleco::{Board};

use pleco_engine::engine::PlecoSearcher;
use pleco_engine::time::uci_timer::PreLimits;

use super::*;

fn search_3moves_engine<D: DepthLimit>(b: &mut Bencher) {
    let mut limit = PreLimits::blank();
    limit.depth = Some(D::depth());
    b.iter_with_setup(|| {
        let mut s = PlecoSearcher::init(false);
        s.clear_search();
        (Board::default(), s)
    }, |(mut board, mut s)| {
        black_box(s.search(&board, &limit));
        let mov = black_box(s.await_move());
        board.apply_move(mov);
        black_box(s.search(&board, &limit));
        let mov = black_box(s.await_move());
        board.apply_move(mov);
        black_box(s.search(&board, &limit));
        black_box(s.await_move());
    })
}

fn bench_engine_evaluations(c: &mut Criterion) {
    c.bench_function("Search MuliMove Depth 4", search_3moves_engine::<Depth4>);
    c.bench_function("Search MuliMove Depth 5", search_3moves_engine::<Depth5>);
    c.bench_function("Search MuliMove Depth 6", search_3moves_engine::<Depth6>);
    c.bench_function("Search MuliMove Depth 7", search_3moves_engine::<Depth7>);
}

criterion_group!(name = search_multimove;
     config = Criterion::default()
        .sample_size(12)
        .warm_up_time(Duration::from_millis(200));
    targets = bench_engine_evaluations
);

<|endoftext|>