<commit_before>#[feature(globs)];

extern mod extra;
extern mod glfw;
extern mod gl;
extern mod cgmath;

use std::cast;
use std::ptr;
use std::str;
use std::vec;
use std::libc;
use extra::time;

use gl::types::*;

use cgmath::matrix::Matrix;
use cgmath::matrix::Mat3;
use cgmath::matrix::Mat4;
use cgmath::matrix::ToMat4;
use cgmath::vector::Vec4;
use cgmath::angle::rad;
use cgmath::projection;

#[link(name="GLU")]
extern {}

static vertex_shader_src : &'static str = r"
#version 110

uniform mat4 transform;

attribute vec4 position;

varying vec3 color;

void main() {
    gl_Position = transform * position;
    color = (position.xyz + vec3(1.0, 1.0, 1.0)) * 0.5;
}
";

static fragment_shader_src : &'static str = r"
#version 110

varying vec3 color;

void main() {
    gl_FragColor = vec4(color, 1.0);
}
";

#[start]
fn start(argc: int, argv: **u8) -> int {
    std::rt::start_on_main_thread(argc, argv, main)
}

fn main() {
   glfw::set_error_callback(~ErrorContext);

   let window_width = 800;
   let window_height = 600;

    do glfw::start {
        let window = glfw::Window::create(window_width, window_height, "Hello, I am a window.", glfw::Windowed)
            .expect("Failed to create GLFW window.");

        //window.set_cursor_mode(glfw::CursorDisabled);
        window.make_context_current();

        gl::load_with(glfw::get_proc_address);

        let vs = compile_shader(vertex_shader_src.as_bytes(), gl::VERTEX_SHADER);
        let fs = compile_shader(fragment_shader_src.as_bytes(), gl::FRAGMENT_SHADER);
        let program = link_program(vs, fs);

        check_gl("after link");

        let mut uniform_transform;

        unsafe {
            uniform_transform = "transform".with_c_str(|ptr| gl::GetUniformLocation(program, ptr));
        }

        check_gl("after uniform transform location");

        let mut vao = 0;
        let mut vbo = 0;
        let mut ebo = 0;

        let vertices : ~[GLfloat] = ~[
            -1.0, -1.0,  1.0, 1.0,
            1.0, -1.0,  1.0, 1.0,
            -1.0,  1.0,  1.0, 1.0,
            1.0,  1.0,  1.0, 1.0,
            -1.0, -1.0, -1.0, 1.0,
            1.0, -1.0, -1.0, 1.0,
            -1.0,  1.0, -1.0, 1.0,
            1.0,  1.0, -1.0, 1.0,
        ];

        let elements : ~[GLshort] = ~[
            0, 1, 2, 3, 7, 1, 5, 4, 7, 6, 2, 4, 0, 1
        ];

        unsafe {
            gl::Enable(gl::DEPTH_TEST);
            gl::Enable(gl::CULL_FACE);

            glfw::set_swap_interval(1);

            // Create Vertex Array Object
            gl::GenVertexArrays(1, &mut vao);
            gl::BindVertexArray(vao);

            // Create a Vertex Buffer Object and copy the vertex data to it
            gl::GenBuffers(1, &mut vbo);
            gl::BindBuffer(gl::ARRAY_BUFFER, vbo);
            gl::BufferData(gl::ARRAY_BUFFER,
                           (vertices.len() * std::mem::size_of::<GLfloat>()) as GLsizeiptr,
                           cast::transmute(&vertices[0]),
                           gl::STATIC_DRAW);

            check_gl("after vertex buffer");

            // Create a Vertex Buffer Object and copy the element data to it
            gl::GenBuffers(1, &mut ebo);
            gl::BindBuffer(gl::ELEMENT_ARRAY_BUFFER, ebo);
            gl::BufferData(gl::ELEMENT_ARRAY_BUFFER,
                           (elements.len() * std::mem::size_of::<GLshort>()) as GLsizeiptr,
                           cast::transmute(&elements[0]),
                           gl::STATIC_DRAW);

            check_gl("after element buffer");

            // Use shader program
            gl::UseProgram(program);

            // Specify the layout of the vertex data
            let vert_attr = "position".with_c_str(|ptr| gl::GetAttribLocation(program, ptr));
            gl::EnableVertexAttribArray(vert_attr as GLuint);
            gl::VertexAttribPointer(vert_attr as GLuint, 4, gl::FLOAT,
                                    gl::FALSE as GLboolean, 0, ptr::null());

            check_gl("after vertex attrib pointer");
        }

        check_gl("after buffers");

        window.set_cursor_pos_callback(~CursorPosContext);
        window.set_key_callback(~KeyContext);

        let start_time = extra::time::precise_time_ns();
        let mut last_frame_time = start_time;
        let mut num_frames = 0;

        while !window.should_close() {
            glfw::poll_events();

            gl::Viewport(0,0, window_width as GLint, window_height as GLint);

            gl::ClearColor(0.0, 0.0, 0.0, 1.0);
            gl::Clear(gl::COLOR_BUFFER_BIT | gl::DEPTH_BUFFER_BIT);

            let elapsed : u64 = extra::time::precise_time_ns() - start_time;
            let angle : f32 = elapsed as f32 / (1000*1000*1000) as f32;

            let translation = Mat4::<f32>::from_cols(
                Vec4::<f32>::unit_x(),
                Vec4::<f32>::unit_y(),
                Vec4::<f32>::unit_z(),
                Vec4::<f32>::new(0.0f32, 0.0f32, -5.0f32, 1.0f32));
            let rotation = Mat3::<f32>::from_angle_y(rad(angle)).to_mat4();
            let projection = cgmath::projection::perspective(rad(1.57 as f32), (4.0/3.0) as f32, 0.1 as f32, 10.0 as f32);
            let transform = projection.mul_m(&translation).mul_m(&rotation);

            unsafe {
                gl::UniformMatrix4fv(uniform_transform, 1, gl::FALSE, cast::transmute(&transform));

                check_gl("after uniform transform");

                gl::DrawElements(gl::TRIANGLE_STRIP, elements.len() as i32, gl::UNSIGNED_SHORT, ptr::null());
            }

            window.swap_buffers();

            check_gl("main loop");

            let cur_time = extra::time::precise_time_ns();
            num_frames += 1;
            if (cur_time - last_frame_time) > (1000*1000*1000) {
                println!("{} frames per second", num_frames);
                num_frames = 0;
                last_frame_time = cur_time;
            }
        }
    }
}

extern "C" {
    fn gluErrorString(error: GLenum) -> *GLubyte;
}

fn check_gl(message : &str) {
    let err = gl::GetError();
    if (err != gl::NO_ERROR) {
        unsafe {
            let err = std::str::raw::from_c_str(gluErrorString(err) as *i8);
            fail!("GL error {} at {}", err, message);
        }
    }
}

fn compile_shader(src: &[u8], ty: GLenum) -> GLuint {
    let shader = gl::CreateShader(ty);
    unsafe {
        // Attempt to compile the shader
        //transmute is used here because `as` causes ICE
        //wait a sec, is `src` null-terminated properly?
        gl::ShaderSource(shader, 1, std::cast::transmute(std::ptr::to_unsafe_ptr(&src.as_ptr())), ptr::null());
        gl::CompileShader(shader);

        // Get the compile status
        let mut status = gl::FALSE as GLint;
        gl::GetShaderiv(shader, gl::COMPILE_STATUS, &mut status);

        // Fail on error
        if status != (gl::TRUE as GLint) {
            let mut len = 0;
            gl::GetShaderiv(shader, gl::INFO_LOG_LENGTH, &mut len);
            let mut buf = vec::from_elem(len as uint - 1, 0u8);     // subtract 1 to skip the trailing null character
            gl::GetShaderInfoLog(shader, len, ptr::mut_null(), buf.as_mut_ptr() as *mut GLchar);
            fail!(str::raw::from_utf8(buf).to_owned());
        }
    }
    shader
}

fn link_program(vs: GLuint, fs: GLuint) -> GLuint {
    let program = gl::CreateProgram();
    gl::AttachShader(program, vs);
    gl::AttachShader(program, fs);
    gl::LinkProgram(program);
    unsafe {
        // Get the link status
        let mut status = gl::FALSE as GLint;
        gl::GetProgramiv(program, gl::LINK_STATUS, &mut status);

        // Fail on error
        if status != (gl::TRUE as GLint) {
            let mut len: GLint = 0;
            gl::GetProgramiv(program, gl::INFO_LOG_LENGTH, &mut len);
            let mut buf = vec::from_elem(len as uint - 1, 0u8);     // subtract 1 to skip the trailing null character
            gl::GetProgramInfoLog(program, len, ptr::mut_null(), buf.as_mut_ptr() as *mut GLchar);
            fail!(str::raw::from_utf8(buf).to_owned());
        }
    }
    program
}

struct ErrorContext;
impl glfw::ErrorCallback for ErrorContext {
    fn call(&self, _: glfw::Error, description: ~str) {
        fail!("GLFW Error: {:s}", description);
    }
}

struct CursorPosContext;
impl glfw::CursorPosCallback for CursorPosContext {
    fn call(&self, _: &glfw::Window, _: f64, _: f64) {
    }
}

struct KeyContext;
impl glfw::KeyCallback for KeyContext {
    fn call(&self, window: &glfw::Window, key: glfw::Key, _: libc::c_int, action: glfw::Action, _: glfw::Modifiers) {
        match (action, key) {
            (glfw::Press, glfw::KeyEscape) => window.set_should_close(true),
            _ => println!("unexpected key callback for action {:?} key {:?}", action, key)
        }
    }
}
<commit_msg>tilt cube<commit_after>#[feature(globs)];

extern mod extra;
extern mod glfw;
extern mod gl;
extern mod cgmath;

use std::cast;
use std::ptr;
use std::str;
use std::vec;
use std::libc;
use extra::time;

use gl::types::*;

use cgmath::matrix::Matrix;
use cgmath::matrix::Mat3;
use cgmath::matrix::Mat4;
use cgmath::matrix::ToMat4;
use cgmath::vector::Vec4;
use cgmath::angle::rad;
use cgmath::projection;

#[link(name="GLU")]
extern {}

static vertex_shader_src : &'static str = r"
#version 110

uniform mat4 transform;

attribute vec4 position;

varying vec3 color;

void main() {
    gl_Position = transform * position;
    color = (position.xyz + vec3(1.0, 1.0, 1.0)) * 0.5;
}
";

static fragment_shader_src : &'static str = r"
#version 110

varying vec3 color;

void main() {
    gl_FragColor = vec4(color, 1.0);
}
";

#[start]
fn start(argc: int, argv: **u8) -> int {
    std::rt::start_on_main_thread(argc, argv, main)
}

fn main() {
   glfw::set_error_callback(~ErrorContext);

   let window_width = 800;
   let window_height = 600;

    do glfw::start {
        let window = glfw::Window::create(window_width, window_height, "Hello, I am a window.", glfw::Windowed)
            .expect("Failed to create GLFW window.");

        //window.set_cursor_mode(glfw::CursorDisabled);
        window.make_context_current();

        gl::load_with(glfw::get_proc_address);

        let vs = compile_shader(vertex_shader_src.as_bytes(), gl::VERTEX_SHADER);
        let fs = compile_shader(fragment_shader_src.as_bytes(), gl::FRAGMENT_SHADER);
        let program = link_program(vs, fs);

        check_gl("after link");

        let mut uniform_transform;

        unsafe {
            uniform_transform = "transform".with_c_str(|ptr| gl::GetUniformLocation(program, ptr));
        }

        check_gl("after uniform transform location");

        let mut vao = 0;
        let mut vbo = 0;
        let mut ebo = 0;

        let vertices : ~[GLfloat] = ~[
            -1.0, -1.0,  1.0, 1.0,
            1.0, -1.0,  1.0, 1.0,
            -1.0,  1.0,  1.0, 1.0,
            1.0,  1.0,  1.0, 1.0,
            -1.0, -1.0, -1.0, 1.0,
            1.0, -1.0, -1.0, 1.0,
            -1.0,  1.0, -1.0, 1.0,
            1.0,  1.0, -1.0, 1.0,
        ];

        let elements : ~[GLshort] = ~[
            0, 1, 2, 3, 7, 1, 5, 4, 7, 6, 2, 4, 0, 1
        ];

        unsafe {
            gl::Enable(gl::DEPTH_TEST);
            gl::Enable(gl::CULL_FACE);

            glfw::set_swap_interval(1);

            // Create Vertex Array Object
            gl::GenVertexArrays(1, &mut vao);
            gl::BindVertexArray(vao);

            // Create a Vertex Buffer Object and copy the vertex data to it
            gl::GenBuffers(1, &mut vbo);
            gl::BindBuffer(gl::ARRAY_BUFFER, vbo);
            gl::BufferData(gl::ARRAY_BUFFER,
                           (vertices.len() * std::mem::size_of::<GLfloat>()) as GLsizeiptr,
                           cast::transmute(&vertices[0]),
                           gl::STATIC_DRAW);

            check_gl("after vertex buffer");

            // Create a Vertex Buffer Object and copy the element data to it
            gl::GenBuffers(1, &mut ebo);
            gl::BindBuffer(gl::ELEMENT_ARRAY_BUFFER, ebo);
            gl::BufferData(gl::ELEMENT_ARRAY_BUFFER,
                           (elements.len() * std::mem::size_of::<GLshort>()) as GLsizeiptr,
                           cast::transmute(&elements[0]),
                           gl::STATIC_DRAW);

            check_gl("after element buffer");

            // Use shader program
            gl::UseProgram(program);

            // Specify the layout of the vertex data
            let vert_attr = "position".with_c_str(|ptr| gl::GetAttribLocation(program, ptr));
            gl::EnableVertexAttribArray(vert_attr as GLuint);
            gl::VertexAttribPointer(vert_attr as GLuint, 4, gl::FLOAT,
                                    gl::FALSE as GLboolean, 0, ptr::null());

            check_gl("after vertex attrib pointer");
        }

        check_gl("after buffers");

        window.set_cursor_pos_callback(~CursorPosContext);
        window.set_key_callback(~KeyContext);

        let start_time = extra::time::precise_time_ns();
        let mut last_frame_time = start_time;
        let mut num_frames = 0;

        while !window.should_close() {
            glfw::poll_events();

            gl::Viewport(0,0, window_width as GLint, window_height as GLint);

            gl::ClearColor(0.0, 0.0, 0.0, 1.0);
            gl::Clear(gl::COLOR_BUFFER_BIT | gl::DEPTH_BUFFER_BIT);

            let elapsed : u64 = extra::time::precise_time_ns() - start_time;
            let angle : f32 = elapsed as f32 / (1000*1000*1000) as f32;

            let translation = Mat4::<f32>::from_cols(
                Vec4::<f32>::unit_x(),
                Vec4::<f32>::unit_y(),
                Vec4::<f32>::unit_z(),
                Vec4::<f32>::new(0.0f32, 0.0f32, -5.0f32, 1.0f32));
            let rotation_x = Mat3::<f32>::from_angle_x(rad(0.5f32)).to_mat4();
            let rotation_y = Mat3::<f32>::from_angle_y(rad(angle)).to_mat4();
            let projection = cgmath::projection::perspective(rad(1.57 as f32), (4.0/3.0) as f32, 0.1 as f32, 10.0 as f32);
            let transform = projection.mul_m(&translation).mul_m(&rotation_x).mul_m(&rotation_y);

            unsafe {
                gl::UniformMatrix4fv(uniform_transform, 1, gl::FALSE, cast::transmute(&transform));

                check_gl("after uniform transform");

                gl::DrawElements(gl::TRIANGLE_STRIP, elements.len() as i32, gl::UNSIGNED_SHORT, ptr::null());
            }

            window.swap_buffers();

            check_gl("main loop");

            let cur_time = extra::time::precise_time_ns();
            num_frames += 1;
            if (cur_time - last_frame_time) > (1000*1000*1000) {
                println!("{} frames per second", num_frames);
                num_frames = 0;
                last_frame_time = cur_time;
            }
        }
    }
}

extern "C" {
    fn gluErrorString(error: GLenum) -> *GLubyte;
}

fn check_gl(message : &str) {
    let err = gl::GetError();
    if (err != gl::NO_ERROR) {
        unsafe {
            let err = std::str::raw::from_c_str(gluErrorString(err) as *i8);
            fail!("GL error {} at {}", err, message);
        }
    }
}

fn compile_shader(src: &[u8], ty: GLenum) -> GLuint {
    let shader = gl::CreateShader(ty);
    unsafe {
        // Attempt to compile the shader
        //transmute is used here because `as` causes ICE
        //wait a sec, is `src` null-terminated properly?
        gl::ShaderSource(shader, 1, std::cast::transmute(std::ptr::to_unsafe_ptr(&src.as_ptr())), ptr::null());
        gl::CompileShader(shader);

        // Get the compile status
        let mut status = gl::FALSE as GLint;
        gl::GetShaderiv(shader, gl::COMPILE_STATUS, &mut status);

        // Fail on error
        if status != (gl::TRUE as GLint) {
            let mut len = 0;
            gl::GetShaderiv(shader, gl::INFO_LOG_LENGTH, &mut len);
            let mut buf = vec::from_elem(len as uint - 1, 0u8);     // subtract 1 to skip the trailing null character
            gl::GetShaderInfoLog(shader, len, ptr::mut_null(), buf.as_mut_ptr() as *mut GLchar);
            fail!(str::raw::from_utf8(buf).to_owned());
        }
    }
    shader
}

fn link_program(vs: GLuint, fs: GLuint) -> GLuint {
    let program = gl::CreateProgram();
    gl::AttachShader(program, vs);
    gl::AttachShader(program, fs);
    gl::LinkProgram(program);
    unsafe {
        // Get the link status
        let mut status = gl::FALSE as GLint;
        gl::GetProgramiv(program, gl::LINK_STATUS, &mut status);

        // Fail on error
        if status != (gl::TRUE as GLint) {
            let mut len: GLint = 0;
            gl::GetProgramiv(program, gl::INFO_LOG_LENGTH, &mut len);
            let mut buf = vec::from_elem(len as uint - 1, 0u8);     // subtract 1 to skip the trailing null character
            gl::GetProgramInfoLog(program, len, ptr::mut_null(), buf.as_mut_ptr() as *mut GLchar);
            fail!(str::raw::from_utf8(buf).to_owned());
        }
    }
    program
}

struct ErrorContext;
impl glfw::ErrorCallback for ErrorContext {
    fn call(&self, _: glfw::Error, description: ~str) {
        fail!("GLFW Error: {:s}", description);
    }
}

struct CursorPosContext;
impl glfw::CursorPosCallback for CursorPosContext {
    fn call(&self, _: &glfw::Window, _: f64, _: f64) {
    }
}

struct KeyContext;
impl glfw::KeyCallback for KeyContext {
    fn call(&self, window: &glfw::Window, key: glfw::Key, _: libc::c_int, action: glfw::Action, _: glfw::Modifiers) {
        match (action, key) {
            (glfw::Press, glfw::KeyEscape) => window.set_should_close(true),
            _ => println!("unexpected key callback for action {:?} key {:?}", action, key)
        }
    }
}
<|endoftext|>


<commit_before>use client::AsyncClient;
use index::btree::LevelTree;
use index::btree::NodeCellRef;
use index::btree::{BPlusTree, RTCursor as BPlusTreeCursor};
use index::key_with_id;
use index::lsmtree::cursor::LSMTreeCursor;
use index::lsmtree::split::check_and_split;
use index::lsmtree::split::SplitStatus;
use index::Cursor;
use index::EntryKey;
use index::Ordering;
use index::*;
use itertools::Itertools;
use parking_lot::Mutex;
use parking_lot::RwLock;
use ram::segs::MAX_SEGMENT_SIZE;
use ram::types::Id;
use rayon::iter::IntoParallelRefIterator;
use std::collections::BTreeSet;
use std::sync::atomic::AtomicU64;
use std::sync::atomic::Ordering::Relaxed;
use std::sync::Arc;
use std::thread;
use std::time::Duration;
use std::{mem, ptr};

pub const LEVEL_ELEMENTS_MULTIPLIER: usize = 10;
pub const LEVEL_PAGE_DIFF_MULTIPLIER: usize = 10;

const LEVEL_M_MAX_ELEMENTS_COUNT: usize = LEVEL_M * LEVEL_M * LEVEL_M;
const LEVEL_M: usize = 24;
const LEVEL_1: usize = LEVEL_M * LEVEL_PAGE_DIFF_MULTIPLIER;
const LEVEL_2: usize = LEVEL_1 * LEVEL_PAGE_DIFF_MULTIPLIER;
const LEVEL_3: usize = LEVEL_2 * LEVEL_PAGE_DIFF_MULTIPLIER;
const LEVEL_4: usize = LEVEL_3 * LEVEL_PAGE_DIFF_MULTIPLIER;

type LevelTrees = Vec<Box<LevelTree>>;
pub type Ptr = NodeCellRef;
pub type Key = EntryKey;

pub type KeyRange = (EntryKey, EntryKey);

#[derive(Serialize, Deserialize, Debug)]
pub enum LSMTreeResult<T> {
    Ok(T),
    EpochMismatch,
}

impl<T> LSMTreeResult<T> {
    pub fn unwrap(self) -> T {
        if let LSMTreeResult::Ok(v) = self {
            v
        } else {
            panic!("Cannot unwrap LSMTreeResult");
        }
    }
}

with_levels! {
    LM, LEVEL_M;
    L1, LEVEL_1;
    L2, LEVEL_2;
    L3, LEVEL_3;
    // L4, LEVEL_4; // See https://github.com/rust-lang/rust/issues/58164
}

pub struct LSMTree {
    pub trees: LevelTrees,
    pub split: Mutex<Option<SplitStatus>>,
    pub range: Mutex<KeyRange>,
    pub epoch: AtomicU64,
    // use Vec here for convenience
    max_sizes: Vec<usize>,
    lsm_tree_max_size: usize,
    pub id: Id,
}

unsafe impl Send for LSMTree {}
unsafe impl Sync for LSMTree {}

impl LSMTree {
    pub fn new(neb_client: &Arc<AsyncClient>, range: KeyRange, id: Id) -> Self {
        debug!("Initializing LSM-tree...");
        let (trees, max_sizes) = init_lsm_level_trees(neb_client);
        let lsm_tree_max_size = max_sizes.iter().sum();
        let split = Mutex::new(None);
        let range = Mutex::new(range);
        let epoch = AtomicU64::new(0);
        debug!("Initialized LSM-tree");
        LSMTree {
            trees,
            max_sizes,
            lsm_tree_max_size,
            split,
            range,
            epoch,
            id,
        }
    }

    pub fn insert(&self, mut key: EntryKey) -> bool {
        self.trees[0].insert_into(&key)
    }

    pub fn remove(&self, mut key: EntryKey, epoch: u64) -> bool {
        self.trees
            .iter()
            .map(|tree| tree.mark_key_deleted(&key))
            .collect_vec() // collect here to prevent short circuit
            .into_iter()
            .any(|d| d)
    }

    pub fn seek(&self, mut key: EntryKey, ordering: Ordering) -> LSMTreeCursor {
        match ordering {
            Ordering::Forward => key_with_id(&mut key, &Id::unit_id()),
            Ordering::Backward => key_with_id(&mut key, &Id::new(::std::u64::MAX, ::std::u64::MAX)),
        };
        let mut cursors: Vec<Box<Cursor>> = vec![];
        for tree in &self.trees {
            cursors.push(tree.seek_for(&key, ordering));
        }
        LSMTreeCursor::new(cursors)
    }

    pub fn check_and_merge(&self) {
        for i in 0..self.trees.len() - 1 {
            debug!("Checking tree merge {}", i);
            let lower = &*self.trees[i];
            let upper = &*self.trees[i + 1];
            if lower.count() > self.max_sizes[i] {
                lower.merge_to(upper);
            }
        }
    }

    pub fn start_sentinel(this: &Arc<Self>) {
        let this = this.clone();
        thread::Builder::new()
            .name("LSM-Tree Sentinel".to_string())
            .spawn(move || loop {
                this.check_and_merge();
                thread::sleep(Duration::from_millis(500));
            });
    }

    pub fn level_sizes(&self) -> Vec<usize> {
        self.trees.iter().map(|t| t.count()).collect()
    }

    pub fn count(&self) -> usize {
        self.trees.iter().map(|t| t.count()).sum()
    }

    pub fn len(&self) -> usize {
        self.trees.iter().map(|tree| tree.count()).sum::<usize>()
    }

    pub fn is_full(&self) -> bool {
        self.len() > self.lsm_tree_max_size
    }

    pub fn last_level_size(&self) -> usize {
        *self.max_sizes.last().unwrap()
    }

    pub fn remove_following_tombstones(&self, start: &EntryKey) {
        self.trees
            .iter()
            .for_each(|tree| tree.remove_following_tombstones(start))
    }

    pub fn epoch_mismatch(&self, epoch: u64) -> bool {
        self.epoch() != epoch
    }

    pub fn epoch(&self) -> u64 {
        self.epoch.load(Relaxed)
    }

    pub fn merge(&self, keys: Box<Vec<EntryKey>>) {
        self.trees[0].merge_with_keys(keys)
    }

    pub fn remove_to_right(&self, start_key: &EntryKey) {
        self.trees
            .iter()
            .for_each(|tree| tree.remove_to_right(start_key));
    }

    pub fn set_epoch(&self, epoch: u64) {
        self.epoch.store(epoch, Relaxed);
    }
}
<commit_msg>decuple lsmtree struct and trees generation<commit_after>use client::AsyncClient;
use index::btree::LevelTree;
use index::btree::NodeCellRef;
use index::btree::{BPlusTree, RTCursor as BPlusTreeCursor};
use index::key_with_id;
use index::lsmtree::cursor::LSMTreeCursor;
use index::lsmtree::split::check_and_split;
use index::lsmtree::split::SplitStatus;
use index::Cursor;
use index::EntryKey;
use index::Ordering;
use index::*;
use itertools::Itertools;
use parking_lot::Mutex;
use parking_lot::RwLock;
use ram::segs::MAX_SEGMENT_SIZE;
use ram::types::Id;
use rayon::iter::IntoParallelRefIterator;
use std::collections::BTreeSet;
use std::sync::atomic::AtomicU64;
use std::sync::atomic::Ordering::Relaxed;
use std::sync::Arc;
use std::thread;
use std::time::Duration;
use std::{mem, ptr};

pub const LEVEL_ELEMENTS_MULTIPLIER: usize = 10;
pub const LEVEL_PAGE_DIFF_MULTIPLIER: usize = 10;

const LEVEL_M_MAX_ELEMENTS_COUNT: usize = LEVEL_M * LEVEL_M * LEVEL_M;
const LEVEL_M: usize = 24;
const LEVEL_1: usize = LEVEL_M * LEVEL_PAGE_DIFF_MULTIPLIER;
const LEVEL_2: usize = LEVEL_1 * LEVEL_PAGE_DIFF_MULTIPLIER;
const LEVEL_3: usize = LEVEL_2 * LEVEL_PAGE_DIFF_MULTIPLIER;
const LEVEL_4: usize = LEVEL_3 * LEVEL_PAGE_DIFF_MULTIPLIER;

type LevelTrees = Vec<Box<LevelTree>>;
pub type Ptr = NodeCellRef;
pub type Key = EntryKey;
pub type TreeLevels = (LevelTrees, Vec<usize>);

pub type KeyRange = (EntryKey, EntryKey);

#[derive(Serialize, Deserialize, Debug)]
pub enum LSMTreeResult<T> {
    Ok(T),
    EpochMismatch,
}

impl<T> LSMTreeResult<T> {
    pub fn unwrap(self) -> T {
        if let LSMTreeResult::Ok(v) = self {
            v
        } else {
            panic!("Cannot unwrap LSMTreeResult");
        }
    }
}

with_levels! {
    LM, LEVEL_M;
    L1, LEVEL_1;
    L2, LEVEL_2;
    L3, LEVEL_3;
    // L4, LEVEL_4; // See https://github.com/rust-lang/rust/issues/58164
}

pub struct LSMTree {
    pub trees: LevelTrees,
    pub split: Mutex<Option<SplitStatus>>,
    pub range: Mutex<KeyRange>,
    pub epoch: AtomicU64,
    // use Vec here for convenience
    max_sizes: Vec<usize>,
    lsm_tree_max_size: usize,
    pub id: Id,
}

unsafe impl Send for LSMTree {}
unsafe impl Sync for LSMTree {}

impl LSMTree {

    pub fn new(neb_client: &Arc<AsyncClient>, range: KeyRange, id: Id) -> Self {
        Self::new_with_levels(init_lsm_level_trees(neb_client), neb_client, range, id)
    }

    pub fn new_with_levels(levels: TreeLevels, neb_client: &Arc<AsyncClient>, range: KeyRange, id: Id) -> Self {
        debug!("Initializing LSM-tree...");
        let (trees, max_sizes) = levels;
        let lsm_tree_max_size = max_sizes.iter().sum();
        let split = Mutex::new(None);
        let range = Mutex::new(range);
        let epoch = AtomicU64::new(0);
        debug!("Initialized LSM-tree");
        LSMTree {
            trees,
            max_sizes,
            lsm_tree_max_size,
            split,
            range,
            epoch,
            id,
        }
    }

    pub fn insert(&self, mut key: EntryKey) -> bool {
        self.trees[0].insert_into(&key)
    }

    pub fn remove(&self, mut key: EntryKey, epoch: u64) -> bool {
        self.trees
            .iter()
            .map(|tree| tree.mark_key_deleted(&key))
            .collect_vec() // collect here to prevent short circuit
            .into_iter()
            .any(|d| d)
    }

    pub fn seek(&self, mut key: EntryKey, ordering: Ordering) -> LSMTreeCursor {
        match ordering {
            Ordering::Forward => key_with_id(&mut key, &Id::unit_id()),
            Ordering::Backward => key_with_id(&mut key, &Id::new(::std::u64::MAX, ::std::u64::MAX)),
        };
        let mut cursors: Vec<Box<Cursor>> = vec![];
        for tree in &self.trees {
            cursors.push(tree.seek_for(&key, ordering));
        }
        LSMTreeCursor::new(cursors)
    }

    pub fn check_and_merge(&self) {
        for i in 0..self.trees.len() - 1 {
            debug!("Checking tree merge {}", i);
            let lower = &*self.trees[i];
            let upper = &*self.trees[i + 1];
            if lower.count() > self.max_sizes[i] {
                lower.merge_to(upper);
            }
        }
    }

    pub fn start_sentinel(this: &Arc<Self>) {
        let this = this.clone();
        thread::Builder::new()
            .name("LSM-Tree Sentinel".to_string())
            .spawn(move || loop {
                this.check_and_merge();
                thread::sleep(Duration::from_millis(500));
            });
    }

    pub fn level_sizes(&self) -> Vec<usize> {
        self.trees.iter().map(|t| t.count()).collect()
    }

    pub fn count(&self) -> usize {
        self.trees.iter().map(|t| t.count()).sum()
    }

    pub fn len(&self) -> usize {
        self.trees.iter().map(|tree| tree.count()).sum::<usize>()
    }

    pub fn is_full(&self) -> bool {
        self.len() > self.lsm_tree_max_size
    }

    pub fn last_level_size(&self) -> usize {
        *self.max_sizes.last().unwrap()
    }

    pub fn remove_following_tombstones(&self, start: &EntryKey) {
        self.trees
            .iter()
            .for_each(|tree| tree.remove_following_tombstones(start))
    }

    pub fn epoch_mismatch(&self, epoch: u64) -> bool {
        self.epoch() != epoch
    }

    pub fn epoch(&self) -> u64 {
        self.epoch.load(Relaxed)
    }

    pub fn merge(&self, keys: Box<Vec<EntryKey>>) {
        self.trees[0].merge_with_keys(keys)
    }

    pub fn remove_to_right(&self, start_key: &EntryKey) {
        self.trees
            .iter()
            .for_each(|tree| tree.remove_to_right(start_key));
    }

    pub fn set_epoch(&self, epoch: u64) {
        self.epoch.store(epoch, Relaxed);
    }
}
<|endoftext|>


<commit_before>use crate::internal::{consts, MiniAllocator};
use fnv::FnvHashSet;
use std::io::{self, Read, Seek, SeekFrom, Write};

//===========================================================================//

pub struct MiniChain<'a, F: 'a> {
    minialloc: &'a mut MiniAllocator<F>,
    sector_ids: Vec<u32>,
    offset_from_start: u64,
}

impl<'a, F> MiniChain<'a, F> {
    pub fn new(
        minialloc: &'a mut MiniAllocator<F>,
        start_sector_id: u32,
    ) -> io::Result<MiniChain<'a, F>> {
        let mut sector_ids = Vec::<u32>::new();
        let mut seen_sector_ids = FnvHashSet::default();
        let mut current_sector_id = start_sector_id;
        while current_sector_id != consts::END_OF_CHAIN {
            if seen_sector_ids.contains(&current_sector_id) {
                invalid_data!(
                    "Minichain contained duplicate sector id {}",
                    current_sector_id
                );
            }
            seen_sector_ids.insert(current_sector_id);
            sector_ids.push(current_sector_id);
            current_sector_id =
                minialloc.next_mini_sector(current_sector_id)?;
        }
        Ok(MiniChain { minialloc, sector_ids, offset_from_start: 0 })
    }

    pub fn start_sector_id(&self) -> u32 {
        self.sector_ids.first().copied().unwrap_or(consts::END_OF_CHAIN)
    }

    pub fn len(&self) -> u64 {
        (consts::MINI_SECTOR_LEN as u64) * (self.sector_ids.len() as u64)
    }
}

impl<'a, F: Read + Write + Seek> MiniChain<'a, F> {
    /// Resizes the chain to the minimum number of sectors large enough to old
    /// `new_len` bytes, allocating or freeing sectors as needed.
    pub fn set_len(&mut self, new_len: u64) -> io::Result<()> {
        debug_assert!(new_len < consts::MINI_STREAM_CUTOFF as u64);
        let sector_len = consts::MINI_SECTOR_LEN as u64;
        let new_num_sectors =
            ((sector_len + new_len - 1) / sector_len) as usize;
        if new_num_sectors == 0 {
            if let Some(&start_sector) = self.sector_ids.first() {
                self.minialloc.free_mini_chain(start_sector)?;
            }
        } else if new_num_sectors <= self.sector_ids.len() {
            if new_num_sectors < self.sector_ids.len() {
                self.minialloc.free_mini_chain_after(
                    self.sector_ids[new_num_sectors - 1],
                )?;
            }
            // TODO: zero remainder of final sector
        } else {
            for _ in self.sector_ids.len()..new_num_sectors {
                let new_sector_id =
                    if let Some(&last_sector_id) = self.sector_ids.last() {
                        self.minialloc.extend_mini_chain(last_sector_id)?
                    } else {
                        self.minialloc.begin_mini_chain()?
                    };
                self.sector_ids.push(new_sector_id);
            }
        }
        Ok(())
    }

    pub fn free(self) -> io::Result<()> {
        self.minialloc.free_mini_chain(self.start_sector_id())
    }
}

impl<'a, F> Seek for MiniChain<'a, F> {
    fn seek(&mut self, pos: SeekFrom) -> io::Result<u64> {
        let length = self.len();
        let new_offset = match pos {
            SeekFrom::Start(delta) => delta as i64,
            SeekFrom::End(delta) => delta + length as i64,
            SeekFrom::Current(delta) => delta + self.offset_from_start as i64,
        };
        if new_offset < 0 || (new_offset as u64) > length {
            invalid_input!(
                "Cannot seek to {}, chain length is {} bytes",
                new_offset,
                length
            );
        }
        self.offset_from_start = new_offset as u64;
        Ok(self.offset_from_start)
    }
}

impl<'a, F: Read + Seek> Read for MiniChain<'a, F> {
    fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> {
        let total_len = self.len();
        debug_assert!(self.offset_from_start <= total_len);
        let remaining_in_chain = total_len - self.offset_from_start;
        let max_len = remaining_in_chain.min(buf.len() as u64) as usize;
        if max_len == 0 {
            return Ok(0);
        }
        let sector_len = consts::MINI_SECTOR_LEN as u64;
        let current_sector_index =
            (self.offset_from_start / sector_len) as usize;
        debug_assert!(current_sector_index < self.sector_ids.len());
        let current_sector_id = self.sector_ids[current_sector_index];
        let offset_within_sector = self.offset_from_start % sector_len;
        let mut sector = self.minialloc.seek_within_mini_sector(
            current_sector_id,
            offset_within_sector,
        )?;
        let bytes_read = sector.read(&mut buf[0..max_len])?;
        self.offset_from_start += bytes_read as u64;
        debug_assert!(self.offset_from_start <= total_len);
        Ok(bytes_read)
    }
}

impl<'a, F: Read + Write + Seek> Write for MiniChain<'a, F> {
    fn write(&mut self, buf: &[u8]) -> io::Result<usize> {
        if buf.is_empty() {
            return Ok(0);
        }
        let mut total_len = self.len();
        let sector_len = consts::MINI_SECTOR_LEN as u64;
        if self.offset_from_start == total_len {
            let new_sector_id =
                if let Some(&last_sector_id) = self.sector_ids.last() {
                    self.minialloc.extend_mini_chain(last_sector_id)?
                } else {
                    self.minialloc.begin_mini_chain()?
                };
            self.sector_ids.push(new_sector_id);
            total_len += sector_len;
            debug_assert_eq!(total_len, self.len());
        }
        let current_sector_index =
            (self.offset_from_start / sector_len) as usize;
        debug_assert!(current_sector_index < self.sector_ids.len());
        let current_sector_id = self.sector_ids[current_sector_index];
        let offset_within_sector = self.offset_from_start % sector_len;
        let mut sector = self.minialloc.seek_within_mini_sector(
            current_sector_id,
            offset_within_sector,
        )?;
        let bytes_written = sector.write(buf)?;
        self.offset_from_start += bytes_written as u64;
        debug_assert!(self.offset_from_start <= total_len);
        Ok(bytes_written)
    }

    fn flush(&mut self) -> io::Result<()> {
        self.minialloc.flush()
    }
}

//===========================================================================//
<commit_msg>Simplify loop prevention<commit_after>use crate::internal::{consts, MiniAllocator};
use std::io::{self, Read, Seek, SeekFrom, Write};

//===========================================================================//

pub struct MiniChain<'a, F: 'a> {
    minialloc: &'a mut MiniAllocator<F>,
    sector_ids: Vec<u32>,
    offset_from_start: u64,
}

impl<'a, F> MiniChain<'a, F> {
    pub fn new(
        minialloc: &'a mut MiniAllocator<F>,
        start_sector_id: u32,
    ) -> io::Result<MiniChain<'a, F>> {
        let mut sector_ids = Vec::<u32>::new();
        let mut current_sector_id = start_sector_id;
        let first_sector_id = start_sector_id;
        while current_sector_id != consts::END_OF_CHAIN {
            sector_ids.push(current_sector_id);
            current_sector_id =
                minialloc.next_mini_sector(current_sector_id)?;
            if current_sector_id == first_sector_id {
                invalid_data!(
                    "Minichain contained duplicate sector id {}",
                    current_sector_id
                );
            }
        }
        Ok(MiniChain { minialloc, sector_ids, offset_from_start: 0 })
    }

    pub fn start_sector_id(&self) -> u32 {
        self.sector_ids.first().copied().unwrap_or(consts::END_OF_CHAIN)
    }

    pub fn len(&self) -> u64 {
        (consts::MINI_SECTOR_LEN as u64) * (self.sector_ids.len() as u64)
    }
}

impl<'a, F: Read + Write + Seek> MiniChain<'a, F> {
    /// Resizes the chain to the minimum number of sectors large enough to old
    /// `new_len` bytes, allocating or freeing sectors as needed.
    pub fn set_len(&mut self, new_len: u64) -> io::Result<()> {
        debug_assert!(new_len < consts::MINI_STREAM_CUTOFF as u64);
        let sector_len = consts::MINI_SECTOR_LEN as u64;
        let new_num_sectors =
            ((sector_len + new_len - 1) / sector_len) as usize;
        if new_num_sectors == 0 {
            if let Some(&start_sector) = self.sector_ids.first() {
                self.minialloc.free_mini_chain(start_sector)?;
            }
        } else if new_num_sectors <= self.sector_ids.len() {
            if new_num_sectors < self.sector_ids.len() {
                self.minialloc.free_mini_chain_after(
                    self.sector_ids[new_num_sectors - 1],
                )?;
            }
            // TODO: zero remainder of final sector
        } else {
            for _ in self.sector_ids.len()..new_num_sectors {
                let new_sector_id =
                    if let Some(&last_sector_id) = self.sector_ids.last() {
                        self.minialloc.extend_mini_chain(last_sector_id)?
                    } else {
                        self.minialloc.begin_mini_chain()?
                    };
                self.sector_ids.push(new_sector_id);
            }
        }
        Ok(())
    }

    pub fn free(self) -> io::Result<()> {
        self.minialloc.free_mini_chain(self.start_sector_id())
    }
}

impl<'a, F> Seek for MiniChain<'a, F> {
    fn seek(&mut self, pos: SeekFrom) -> io::Result<u64> {
        let length = self.len();
        let new_offset = match pos {
            SeekFrom::Start(delta) => delta as i64,
            SeekFrom::End(delta) => delta + length as i64,
            SeekFrom::Current(delta) => delta + self.offset_from_start as i64,
        };
        if new_offset < 0 || (new_offset as u64) > length {
            invalid_input!(
                "Cannot seek to {}, chain length is {} bytes",
                new_offset,
                length
            );
        }
        self.offset_from_start = new_offset as u64;
        Ok(self.offset_from_start)
    }
}

impl<'a, F: Read + Seek> Read for MiniChain<'a, F> {
    fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> {
        let total_len = self.len();
        debug_assert!(self.offset_from_start <= total_len);
        let remaining_in_chain = total_len - self.offset_from_start;
        let max_len = remaining_in_chain.min(buf.len() as u64) as usize;
        if max_len == 0 {
            return Ok(0);
        }
        let sector_len = consts::MINI_SECTOR_LEN as u64;
        let current_sector_index =
            (self.offset_from_start / sector_len) as usize;
        debug_assert!(current_sector_index < self.sector_ids.len());
        let current_sector_id = self.sector_ids[current_sector_index];
        let offset_within_sector = self.offset_from_start % sector_len;
        let mut sector = self.minialloc.seek_within_mini_sector(
            current_sector_id,
            offset_within_sector,
        )?;
        let bytes_read = sector.read(&mut buf[0..max_len])?;
        self.offset_from_start += bytes_read as u64;
        debug_assert!(self.offset_from_start <= total_len);
        Ok(bytes_read)
    }
}

impl<'a, F: Read + Write + Seek> Write for MiniChain<'a, F> {
    fn write(&mut self, buf: &[u8]) -> io::Result<usize> {
        if buf.is_empty() {
            return Ok(0);
        }
        let mut total_len = self.len();
        let sector_len = consts::MINI_SECTOR_LEN as u64;
        if self.offset_from_start == total_len {
            let new_sector_id =
                if let Some(&last_sector_id) = self.sector_ids.last() {
                    self.minialloc.extend_mini_chain(last_sector_id)?
                } else {
                    self.minialloc.begin_mini_chain()?
                };
            self.sector_ids.push(new_sector_id);
            total_len += sector_len;
            debug_assert_eq!(total_len, self.len());
        }
        let current_sector_index =
            (self.offset_from_start / sector_len) as usize;
        debug_assert!(current_sector_index < self.sector_ids.len());
        let current_sector_id = self.sector_ids[current_sector_index];
        let offset_within_sector = self.offset_from_start % sector_len;
        let mut sector = self.minialloc.seek_within_mini_sector(
            current_sector_id,
            offset_within_sector,
        )?;
        let bytes_written = sector.write(buf)?;
        self.offset_from_start += bytes_written as u64;
        debug_assert!(self.offset_from_start <= total_len);
        Ok(bytes_written)
    }

    fn flush(&mut self) -> io::Result<()> {
        self.minialloc.flush()
    }
}

//===========================================================================//
<|endoftext|>


<commit_before>//! Functionality related to publishing a new crate or version of a crate.

use hex::ToHex;
use std::sync::Arc;
use swirl::Job;

use crate::controllers::cargo_prelude::*;
use crate::git;
use crate::models::{
    insert_version_owner_action, Badge, Category, Crate, DependencyKind, Keyword, NewCrate,
    NewVersion, Rights, VersionAction,
};

use crate::render;
use crate::schema::*;
use crate::util::errors::{cargo_err, AppResult};
use crate::util::{read_fill, read_le_u32, Maximums};
use crate::views::{
    EncodableCrate, EncodableCrateDependency, EncodableCrateUpload, GoodCrate, PublishWarnings,
};

pub const MISSING_RIGHTS_ERROR_MESSAGE: &str =
    "this crate exists but you don't seem to be an owner. \
     If you believe this is a mistake, perhaps you need \
     to accept an invitation to be an owner before \
     publishing.";

pub const WILDCARD_ERROR_MESSAGE: &str = "wildcard (`*`) dependency constraints are not allowed \
     on crates.io. See https://doc.rust-lang.org/cargo/faq.html#can-\
     libraries-use--as-a-version-for-their-dependencies for more \
     information";

/// Handles the `PUT /crates/new` route.
/// Used by `cargo publish` to publish a new crate or to publish a new version of an
/// existing crate.
///
/// Currently blocks the HTTP thread, perhaps some function calls can spawn new
/// threads and return completion or error through other methods  a `cargo publish
/// --status` command, via crates.io's front end, or email.
pub fn publish(req: &mut dyn RequestExt) -> EndpointResult {
    let app = Arc::clone(req.app());

    // The format of the req.body() of a publish request is as follows:
    //
    // metadata length
    // metadata in JSON about the crate being published
    // .crate tarball length
    // .crate tarball file
    //
    // - The metadata is read and interpreted in the parse_new_headers function.
    // - The .crate tarball length is read in this function in order to save the size of the file
    //   in the version record in the database.
    // - Then the .crate tarball length is passed to the upload_crate function where the actual
    //   file is read and uploaded.

    let new_crate = parse_new_headers(req)?;

    req.log_metadata("crate_name", new_crate.name.to_string());
    req.log_metadata("crate_version", new_crate.vers.to_string());

    let conn = app.primary_database.get()?;
    let ids = req.authenticate()?;
    let api_token_id = ids.api_token_id();
    let user = ids.user();

    let verified_email_address = user.verified_email(&conn)?;
    let verified_email_address = verified_email_address.ok_or_else(|| {
        cargo_err(&format!(
            "A verified email address is required to publish crates to crates.io. \
             Visit https://{}/me to set and verify your email address.",
            app.config.domain_name,
        ))
    })?;

    // Create a transaction on the database, if there are no errors,
    // commit the transactions to record a new or updated crate.
    conn.transaction(|| {
        let name = new_crate.name;
        let vers = &*new_crate.vers;
        let links = new_crate.links;
        let repo = new_crate.repository;
        let features = new_crate
            .features
            .into_iter()
            .map(|(k, v)| (k.0, v.into_iter().map(|v| v.0).collect()))
            .collect();
        let keywords = new_crate
            .keywords
            .iter()
            .map(|s| s.as_str())
            .collect::<Vec<_>>();
        let categories = new_crate
            .categories
            .iter()
            .map(|s| s.as_str())
            .collect::<Vec<_>>();

        // Persist the new crate, if it doesn't already exist
        let persist = NewCrate {
            name: &name,
            description: new_crate.description.as_deref(),
            homepage: new_crate.homepage.as_deref(),
            documentation: new_crate.documentation.as_deref(),
            readme: new_crate.readme.as_deref(),
            repository: repo.as_deref(),
            max_upload_size: None,
        };

        let license_file = new_crate.license_file.as_deref();
        let krate =
            persist.create_or_update(&conn, user.id, Some(&app.config.publish_rate_limit))?;

        let owners = krate.owners(&conn)?;
        if user.rights(req.app(), &owners)? < Rights::Publish {
            return Err(cargo_err(MISSING_RIGHTS_ERROR_MESSAGE));
        }

        if krate.name != *name {
            return Err(cargo_err(&format_args!(
                "crate was previously named `{}`",
                krate.name
            )));
        }

        // Length of the .crate tarball, which appears after the metadata in the request body.
        // TODO: Not sure why we're using the total content length (metadata + .crate file length)
        // to compare against the max upload size... investigate that and perhaps change to use
        // this file length.
        let file_length = read_le_u32(req.body())?;

        let content_length = req
            .content_length()
            .chain_error(|| cargo_err("missing header: Content-Length"))?;

        let maximums = Maximums::new(
            krate.max_upload_size,
            app.config.max_upload_size,
            app.config.max_unpack_size,
        );

        if content_length > maximums.max_upload_size {
            return Err(cargo_err(&format_args!(
                "max upload size is: {}",
                maximums.max_upload_size
            )));
        }

        // This is only redundant for now. Eventually the duplication will be removed.
        let license = new_crate.license.clone();

        // Persist the new version of this crate
        let version = NewVersion::new(
            krate.id,
            vers,
            &features,
            license,
            license_file,
            // Downcast is okay because the file length must be less than the max upload size
            // to get here, and max upload sizes are way less than i32 max
            file_length as i32,
            user.id,
        )?
        .save(&conn, &verified_email_address)?;

        insert_version_owner_action(
            &conn,
            version.id,
            user.id,
            api_token_id,
            VersionAction::Publish,
        )?;

        // Link this new version to all dependencies
        let git_deps = add_dependencies(&conn, &new_crate.deps, version.id)?;

        // Update all keywords for this crate
        Keyword::update_crate(&conn, &krate, &keywords)?;

        // Update all categories for this crate, collecting any invalid categories
        // in order to be able to warn about them
        let ignored_invalid_categories = Category::update_crate(&conn, &krate, &categories)?;

        // Update all badges for this crate, collecting any invalid badges in
        // order to be able to warn about them
        let ignored_invalid_badges = Badge::update_crate(&conn, &krate, new_crate.badges.as_ref())?;
        let top_versions = krate.top_versions(&conn)?;

        if let Some(readme) = new_crate.readme {
            render::render_and_upload_readme(
                version.id,
                readme,
                new_crate
                    .readme_file
                    .unwrap_or_else(|| String::from("README.md")),
                repo,
            )
            .enqueue(&conn)?;
        }

        let cksum = app
            .config
            .uploader
            .upload_crate(req, &krate, maximums, vers)?;

        let hex_cksum = cksum.encode_hex::<String>();

        // Register this crate in our local git repo.
        let git_crate = git::Crate {
            name: name.0,
            vers: vers.to_string(),
            cksum: hex_cksum,
            features,
            deps: git_deps,
            yanked: Some(false),
            links,
        };
        git::add_crate(git_crate).enqueue(&conn)?;

        // The `other` field on `PublishWarnings` was introduced to handle a temporary warning
        // that is no longer needed. As such, crates.io currently does not return any `other`
        // warnings at this time, but if we need to, the field is available.
        let warnings = PublishWarnings {
            invalid_categories: ignored_invalid_categories,
            invalid_badges: ignored_invalid_badges,
            other: vec![],
        };

        Ok(req.json(&GoodCrate {
            krate: EncodableCrate::from_minimal(krate, &top_versions, None, false, None),
            warnings,
        }))
    })
}

/// Used by the `krate::new` function.
///
/// This function parses the JSON headers to interpret the data and validates
/// the data during and after the parsing. Returns crate metadata.
fn parse_new_headers(req: &mut dyn RequestExt) -> AppResult<EncodableCrateUpload> {
    // Read the json upload request
    let metadata_length = u64::from(read_le_u32(req.body())?);
    req.log_metadata("metadata_length", metadata_length);

    let max = req.app().config.max_upload_size;
    if metadata_length > max {
        return Err(cargo_err(&format_args!("max upload size is: {}", max)));
    }
    let mut json = vec![0; metadata_length as usize];
    read_fill(req.body(), &mut json)?;
    let json = String::from_utf8(json).map_err(|_| cargo_err("json body was not valid utf-8"))?;
    let new: EncodableCrateUpload = serde_json::from_str(&json)
        .map_err(|e| cargo_err(&format_args!("invalid upload request: {}", e)))?;

    // Make sure required fields are provided
    fn empty(s: Option<&String>) -> bool {
        s.map_or(true, String::is_empty)
    }

    // It can have up to three elements per below conditions.
    let mut missing = Vec::with_capacity(3);

    if empty(new.description.as_ref()) {
        missing.push("description");
    }
    if empty(new.license.as_ref()) && empty(new.license_file.as_ref()) {
        missing.push("license");
    }
    if !missing.is_empty() {
        let message = missing_metadata_error_message(&missing);
        return Err(cargo_err(&message));
    }

    Ok(new)
}

pub fn missing_metadata_error_message(missing: &[&str]) -> String {
    format!(
        "missing or empty metadata fields: {}. Please \
         see https://doc.rust-lang.org/cargo/reference/manifest.html for \
         how to upload metadata",
        missing.join(", ")
    )
}

pub fn add_dependencies(
    conn: &PgConnection,
    deps: &[EncodableCrateDependency],
    target_version_id: i32,
) -> AppResult<Vec<git::Dependency>> {
    use self::dependencies::dsl::*;
    use diesel::insert_into;

    let git_and_new_dependencies = deps
        .iter()
        .map(|dep| {
            if let Some(registry) = &dep.registry {
                if !registry.is_empty() {
                    return Err(cargo_err(&format_args!("Dependency `{}` is hosted on another registry. Cross-registry dependencies are not permitted on crates.io.", &*dep.name)));
                }
            }

            // Match only identical names to ensure the index always references the original crate name
            let krate:Crate = Crate::by_exact_name(&dep.name)
                .first(&*conn)
                .map_err(|_| cargo_err(&format_args!("no known crate named `{}`", &*dep.name)))?;
            if semver::VersionReq::parse(&dep.version_req.0) == semver::VersionReq::parse("*") {
                return Err(cargo_err(WILDCARD_ERROR_MESSAGE));
            }

            // If this dependency has an explicit name in `Cargo.toml` that
            // means that the `name` we have listed is actually the package name
            // that we're depending on. The `name` listed in the index is the
            // Cargo.toml-written-name which is what cargo uses for
            // `--extern foo=...`
            let (name, package) = match &dep.explicit_name_in_toml {
                Some(explicit) => (explicit.to_string(), Some(dep.name.to_string())),
                None => (dep.name.to_string(), None),
            };

            Ok((
                git::Dependency {
                    name,
                    req: dep.version_req.to_string(),
                    features: dep.features.iter().map(|s| s.0.to_string()).collect(),
                    optional: dep.optional,
                    default_features: dep.default_features,
                    target: dep.target.clone(),
                    kind: dep.kind.or(Some(DependencyKind::Normal)),
                    package,
                },
                (
                    version_id.eq(target_version_id),
                    crate_id.eq(krate.id),
                    req.eq(dep.version_req.to_string()),
                    dep.kind.map(|k| kind.eq(k as i32)),
                    optional.eq(dep.optional),
                    default_features.eq(dep.default_features),
                    features.eq(&dep.features),
                    target.eq(dep.target.as_deref()),
                ),
            ))
        })
        .collect::<Result<Vec<_>, _>>()?;

    let (git_deps, new_dependencies): (Vec<_>, Vec<_>) =
        git_and_new_dependencies.into_iter().unzip();

    insert_into(dependencies)
        .values(&new_dependencies)
        .execute(conn)?;

    Ok(git_deps)
}

#[cfg(test)]
mod tests {
    use super::missing_metadata_error_message;

    #[test]
    fn missing_metadata_error_message_test() {
        assert_eq!(missing_metadata_error_message(&["a"]), "missing or empty metadata fields: a. Please see https://doc.rust-lang.org/cargo/reference/manifest.html for how to upload metadata");
        assert_eq!(missing_metadata_error_message(&["a", "b"]), "missing or empty metadata fields: a, b. Please see https://doc.rust-lang.org/cargo/reference/manifest.html for how to upload metadata");
        assert_eq!(missing_metadata_error_message(&["a", "b", "c"]), "missing or empty metadata fields: a, b, c. Please see https://doc.rust-lang.org/cargo/reference/manifest.html for how to upload metadata");
    }
}
<commit_msg>krate/publish: Fix `VersionReq` comparison<commit_after>//! Functionality related to publishing a new crate or version of a crate.

use hex::ToHex;
use std::sync::Arc;
use swirl::Job;

use crate::controllers::cargo_prelude::*;
use crate::git;
use crate::models::{
    insert_version_owner_action, Badge, Category, Crate, DependencyKind, Keyword, NewCrate,
    NewVersion, Rights, VersionAction,
};

use crate::render;
use crate::schema::*;
use crate::util::errors::{cargo_err, AppResult};
use crate::util::{read_fill, read_le_u32, Maximums};
use crate::views::{
    EncodableCrate, EncodableCrateDependency, EncodableCrateUpload, GoodCrate, PublishWarnings,
};

pub const MISSING_RIGHTS_ERROR_MESSAGE: &str =
    "this crate exists but you don't seem to be an owner. \
     If you believe this is a mistake, perhaps you need \
     to accept an invitation to be an owner before \
     publishing.";

pub const WILDCARD_ERROR_MESSAGE: &str = "wildcard (`*`) dependency constraints are not allowed \
     on crates.io. See https://doc.rust-lang.org/cargo/faq.html#can-\
     libraries-use--as-a-version-for-their-dependencies for more \
     information";

/// Handles the `PUT /crates/new` route.
/// Used by `cargo publish` to publish a new crate or to publish a new version of an
/// existing crate.
///
/// Currently blocks the HTTP thread, perhaps some function calls can spawn new
/// threads and return completion or error through other methods  a `cargo publish
/// --status` command, via crates.io's front end, or email.
pub fn publish(req: &mut dyn RequestExt) -> EndpointResult {
    let app = Arc::clone(req.app());

    // The format of the req.body() of a publish request is as follows:
    //
    // metadata length
    // metadata in JSON about the crate being published
    // .crate tarball length
    // .crate tarball file
    //
    // - The metadata is read and interpreted in the parse_new_headers function.
    // - The .crate tarball length is read in this function in order to save the size of the file
    //   in the version record in the database.
    // - Then the .crate tarball length is passed to the upload_crate function where the actual
    //   file is read and uploaded.

    let new_crate = parse_new_headers(req)?;

    req.log_metadata("crate_name", new_crate.name.to_string());
    req.log_metadata("crate_version", new_crate.vers.to_string());

    let conn = app.primary_database.get()?;
    let ids = req.authenticate()?;
    let api_token_id = ids.api_token_id();
    let user = ids.user();

    let verified_email_address = user.verified_email(&conn)?;
    let verified_email_address = verified_email_address.ok_or_else(|| {
        cargo_err(&format!(
            "A verified email address is required to publish crates to crates.io. \
             Visit https://{}/me to set and verify your email address.",
            app.config.domain_name,
        ))
    })?;

    // Create a transaction on the database, if there are no errors,
    // commit the transactions to record a new or updated crate.
    conn.transaction(|| {
        let name = new_crate.name;
        let vers = &*new_crate.vers;
        let links = new_crate.links;
        let repo = new_crate.repository;
        let features = new_crate
            .features
            .into_iter()
            .map(|(k, v)| (k.0, v.into_iter().map(|v| v.0).collect()))
            .collect();
        let keywords = new_crate
            .keywords
            .iter()
            .map(|s| s.as_str())
            .collect::<Vec<_>>();
        let categories = new_crate
            .categories
            .iter()
            .map(|s| s.as_str())
            .collect::<Vec<_>>();

        // Persist the new crate, if it doesn't already exist
        let persist = NewCrate {
            name: &name,
            description: new_crate.description.as_deref(),
            homepage: new_crate.homepage.as_deref(),
            documentation: new_crate.documentation.as_deref(),
            readme: new_crate.readme.as_deref(),
            repository: repo.as_deref(),
            max_upload_size: None,
        };

        let license_file = new_crate.license_file.as_deref();
        let krate =
            persist.create_or_update(&conn, user.id, Some(&app.config.publish_rate_limit))?;

        let owners = krate.owners(&conn)?;
        if user.rights(req.app(), &owners)? < Rights::Publish {
            return Err(cargo_err(MISSING_RIGHTS_ERROR_MESSAGE));
        }

        if krate.name != *name {
            return Err(cargo_err(&format_args!(
                "crate was previously named `{}`",
                krate.name
            )));
        }

        // Length of the .crate tarball, which appears after the metadata in the request body.
        // TODO: Not sure why we're using the total content length (metadata + .crate file length)
        // to compare against the max upload size... investigate that and perhaps change to use
        // this file length.
        let file_length = read_le_u32(req.body())?;

        let content_length = req
            .content_length()
            .chain_error(|| cargo_err("missing header: Content-Length"))?;

        let maximums = Maximums::new(
            krate.max_upload_size,
            app.config.max_upload_size,
            app.config.max_unpack_size,
        );

        if content_length > maximums.max_upload_size {
            return Err(cargo_err(&format_args!(
                "max upload size is: {}",
                maximums.max_upload_size
            )));
        }

        // This is only redundant for now. Eventually the duplication will be removed.
        let license = new_crate.license.clone();

        // Persist the new version of this crate
        let version = NewVersion::new(
            krate.id,
            vers,
            &features,
            license,
            license_file,
            // Downcast is okay because the file length must be less than the max upload size
            // to get here, and max upload sizes are way less than i32 max
            file_length as i32,
            user.id,
        )?
        .save(&conn, &verified_email_address)?;

        insert_version_owner_action(
            &conn,
            version.id,
            user.id,
            api_token_id,
            VersionAction::Publish,
        )?;

        // Link this new version to all dependencies
        let git_deps = add_dependencies(&conn, &new_crate.deps, version.id)?;

        // Update all keywords for this crate
        Keyword::update_crate(&conn, &krate, &keywords)?;

        // Update all categories for this crate, collecting any invalid categories
        // in order to be able to warn about them
        let ignored_invalid_categories = Category::update_crate(&conn, &krate, &categories)?;

        // Update all badges for this crate, collecting any invalid badges in
        // order to be able to warn about them
        let ignored_invalid_badges = Badge::update_crate(&conn, &krate, new_crate.badges.as_ref())?;
        let top_versions = krate.top_versions(&conn)?;

        if let Some(readme) = new_crate.readme {
            render::render_and_upload_readme(
                version.id,
                readme,
                new_crate
                    .readme_file
                    .unwrap_or_else(|| String::from("README.md")),
                repo,
            )
            .enqueue(&conn)?;
        }

        let cksum = app
            .config
            .uploader
            .upload_crate(req, &krate, maximums, vers)?;

        let hex_cksum = cksum.encode_hex::<String>();

        // Register this crate in our local git repo.
        let git_crate = git::Crate {
            name: name.0,
            vers: vers.to_string(),
            cksum: hex_cksum,
            features,
            deps: git_deps,
            yanked: Some(false),
            links,
        };
        git::add_crate(git_crate).enqueue(&conn)?;

        // The `other` field on `PublishWarnings` was introduced to handle a temporary warning
        // that is no longer needed. As such, crates.io currently does not return any `other`
        // warnings at this time, but if we need to, the field is available.
        let warnings = PublishWarnings {
            invalid_categories: ignored_invalid_categories,
            invalid_badges: ignored_invalid_badges,
            other: vec![],
        };

        Ok(req.json(&GoodCrate {
            krate: EncodableCrate::from_minimal(krate, &top_versions, None, false, None),
            warnings,
        }))
    })
}

/// Used by the `krate::new` function.
///
/// This function parses the JSON headers to interpret the data and validates
/// the data during and after the parsing. Returns crate metadata.
fn parse_new_headers(req: &mut dyn RequestExt) -> AppResult<EncodableCrateUpload> {
    // Read the json upload request
    let metadata_length = u64::from(read_le_u32(req.body())?);
    req.log_metadata("metadata_length", metadata_length);

    let max = req.app().config.max_upload_size;
    if metadata_length > max {
        return Err(cargo_err(&format_args!("max upload size is: {}", max)));
    }
    let mut json = vec![0; metadata_length as usize];
    read_fill(req.body(), &mut json)?;
    let json = String::from_utf8(json).map_err(|_| cargo_err("json body was not valid utf-8"))?;
    let new: EncodableCrateUpload = serde_json::from_str(&json)
        .map_err(|e| cargo_err(&format_args!("invalid upload request: {}", e)))?;

    // Make sure required fields are provided
    fn empty(s: Option<&String>) -> bool {
        s.map_or(true, String::is_empty)
    }

    // It can have up to three elements per below conditions.
    let mut missing = Vec::with_capacity(3);

    if empty(new.description.as_ref()) {
        missing.push("description");
    }
    if empty(new.license.as_ref()) && empty(new.license_file.as_ref()) {
        missing.push("license");
    }
    if !missing.is_empty() {
        let message = missing_metadata_error_message(&missing);
        return Err(cargo_err(&message));
    }

    Ok(new)
}

pub fn missing_metadata_error_message(missing: &[&str]) -> String {
    format!(
        "missing or empty metadata fields: {}. Please \
         see https://doc.rust-lang.org/cargo/reference/manifest.html for \
         how to upload metadata",
        missing.join(", ")
    )
}

pub fn add_dependencies(
    conn: &PgConnection,
    deps: &[EncodableCrateDependency],
    target_version_id: i32,
) -> AppResult<Vec<git::Dependency>> {
    use self::dependencies::dsl::*;
    use diesel::insert_into;

    let git_and_new_dependencies = deps
        .iter()
        .map(|dep| {
            if let Some(registry) = &dep.registry {
                if !registry.is_empty() {
                    return Err(cargo_err(&format_args!("Dependency `{}` is hosted on another registry. Cross-registry dependencies are not permitted on crates.io.", &*dep.name)));
                }
            }

            // Match only identical names to ensure the index always references the original crate name
            let krate:Crate = Crate::by_exact_name(&dep.name)
                .first(&*conn)
                .map_err(|_| cargo_err(&format_args!("no known crate named `{}`", &*dep.name)))?;

            if let Ok(version_req) = semver::VersionReq::parse(&dep.version_req.0) {
                if version_req == semver::VersionReq::STAR {
                    return Err(cargo_err(WILDCARD_ERROR_MESSAGE));
                }
            }

            // If this dependency has an explicit name in `Cargo.toml` that
            // means that the `name` we have listed is actually the package name
            // that we're depending on. The `name` listed in the index is the
            // Cargo.toml-written-name which is what cargo uses for
            // `--extern foo=...`
            let (name, package) = match &dep.explicit_name_in_toml {
                Some(explicit) => (explicit.to_string(), Some(dep.name.to_string())),
                None => (dep.name.to_string(), None),
            };

            Ok((
                git::Dependency {
                    name,
                    req: dep.version_req.to_string(),
                    features: dep.features.iter().map(|s| s.0.to_string()).collect(),
                    optional: dep.optional,
                    default_features: dep.default_features,
                    target: dep.target.clone(),
                    kind: dep.kind.or(Some(DependencyKind::Normal)),
                    package,
                },
                (
                    version_id.eq(target_version_id),
                    crate_id.eq(krate.id),
                    req.eq(dep.version_req.to_string()),
                    dep.kind.map(|k| kind.eq(k as i32)),
                    optional.eq(dep.optional),
                    default_features.eq(dep.default_features),
                    features.eq(&dep.features),
                    target.eq(dep.target.as_deref()),
                ),
            ))
        })
        .collect::<Result<Vec<_>, _>>()?;

    let (git_deps, new_dependencies): (Vec<_>, Vec<_>) =
        git_and_new_dependencies.into_iter().unzip();

    insert_into(dependencies)
        .values(&new_dependencies)
        .execute(conn)?;

    Ok(git_deps)
}

#[cfg(test)]
mod tests {
    use super::missing_metadata_error_message;

    #[test]
    fn missing_metadata_error_message_test() {
        assert_eq!(missing_metadata_error_message(&["a"]), "missing or empty metadata fields: a. Please see https://doc.rust-lang.org/cargo/reference/manifest.html for how to upload metadata");
        assert_eq!(missing_metadata_error_message(&["a", "b"]), "missing or empty metadata fields: a, b. Please see https://doc.rust-lang.org/cargo/reference/manifest.html for how to upload metadata");
        assert_eq!(missing_metadata_error_message(&["a", "b", "c"]), "missing or empty metadata fields: a, b, c. Please see https://doc.rust-lang.org/cargo/reference/manifest.html for how to upload metadata");
    }
}
<|endoftext|>


<commit_before>use analysis::namespaces;
use case::CaseExt;
use codegen::general::{self, version_condition, version_condition_string};
use config::gobjects::GObject;
use env::Env;
use file_saver;
use library::*;
use std::collections::HashSet;
use std::io::prelude::*;
use std::io::Result;
use std::path::Path;
use traits::*;
use version::Version;

pub fn generate(env: &Env, root_path: &Path, mod_rs: &mut Vec<String>) {
    let path = root_path.join("enums.rs");
    file_saver::save_to_file(path, env.config.make_backup, |w| {
        try!(general::start_comments(w, &env.config));
        try!(writeln!(w, ""));
        try!(writeln!(w, "use ffi;"));
        if env.namespaces.glib_ns_id == namespaces::MAIN {
            try!(writeln!(w, "use ffi as glib_ffi;
use error::ErrorDomain;
use translate::*;"));
        } else {
            try!(writeln!(w, "use glib_ffi;
use glib::error::ErrorDomain;
use glib::translate::*;"));
        }
        try!(writeln!(w, ""));

        let configs = env.config.objects.values()
            .filter(|c| {
                c.status.need_generate() &&
                    c.type_id.map_or(false, |tid| tid.ns_id == namespaces::MAIN)
            });
        let mut first = true;
        for config in configs {
            if let Type::Enumeration(ref enum_) = *env.library.type_(config.type_id.unwrap()) {
                if first {
                    mod_rs.push("\nmod enums;".into());
                    first = false;
                }
                if let Some (cfg) = version_condition_string(env, enum_.version, false, 0) {
                    mod_rs.push(cfg);
                }
                mod_rs.push(format!("pub use self::enums::{};", enum_.name));
                try!(generate_enum(env, w, enum_, config));
            }
        }

        Ok(())
    });
}

fn generate_enum(env: &Env, w: &mut Write, enum_: &Enumeration, config: &GObject) -> Result<()> {
    struct Member {
        name: String,
        c_name: String,
        version: Option<Version>,
    }

    let mut members: Vec<Member> = Vec::new();
    let mut vals: HashSet<String> = HashSet::new();

    for member in &enum_.members {
        let member_config = config.members.matched(&member.name);
        let is_alias = member_config.iter().any(|m| m.alias);
        if is_alias || vals.contains(&member.value) {
            continue
        }
        vals.insert(member.value.clone());
        let version = member_config.iter().filter_map(|m| m.version).next();
        members.push(Member {
            name: member.name.to_camel(),
            c_name: member.c_identifier.clone(),
            version: version,
        });
    }

    try!(version_condition(w, env, enum_.version, false, 0));
    try!(writeln!(w, "#[derive(Clone, Copy, Debug, Eq, PartialEq, Hash)]"));
    try!(writeln!(w, "pub enum {} {{", enum_.name));
    for member in &members {
        try!(writeln!(w, "\t{},", member.name));
    }
    try!(writeln!(w, "{}", "    #[doc(hidden)]
    __Nonexhaustive(()),
}
"));

    try!(version_condition(w, env, enum_.version, false, 0));
    try!(writeln!(w, "#[doc(hidden)]
impl ToGlib for {name} {{
    type GlibType = ffi::{ffi_name};

    fn to_glib(&self) -> ffi::{ffi_name} {{
        match *self {{", name = enum_.name, ffi_name = enum_.c_type));
    for member in &members {
        try!(version_condition(w, env, member.version, false, 3));
        try!(writeln!(w, "\t\t\t{}::{} => ffi::{},", enum_.name, member.name, member.c_name));
    }
    try!(writeln!(w, "\t\t\t{}::__Nonexhaustive(_) => panic!(),", enum_.name));
    try!(writeln!(w, "{}",
"        }
    }
}
"));

    let assert = if env.config.generate_safety_asserts {
        "skip_assert_initialized!();\n\t\t"
    } else {
        ""
    };

    try!(version_condition(w, env, enum_.version, false, 0));
    try!(writeln!(w, "#[doc(hidden)]
impl FromGlib<ffi::{ffi_name}> for {name} {{
    fn from_glib(value: ffi::{ffi_name}) -> Self {{
        {assert}match value {{", name = enum_.name, ffi_name = enum_.c_type, assert = assert));
    for member in &members {
        try!(version_condition(w, env, member.version, false, 3));
        try!(writeln!(w, "\t\t\tffi::{} => {}::{},", member.c_name, enum_.name, member.name));
    }
    if members.len() == 1 {
        try!(writeln!(w, "\t\t\t_ => {}::__Nonexhaustive(()),", enum_.name));
    }
    try!(writeln!(w, "{}",
"        }
    }
}
"));
    let error_domain: Option<String> = enum_.functions.iter()
        .filter(|f| f.name == "quark")
        .next()
        .and_then(|f| f.c_identifier.clone())
        .or_else(|| enum_.error_domain.clone());

    if let Some(ref get_quark) = error_domain {
        let get_quark = get_quark.replace("-", "_");
        let has_failed_member = members.iter().any(|m| m.name == "Failed");

        try!(version_condition(w, env, enum_.version, false, 0));
        try!(writeln!(w, "impl ErrorDomain for {name} {{
    fn domain() -> glib_ffi::GQuark {{
        {assert}unsafe {{ ffi::{get_quark}() }}
    }}

    fn code(self) -> i32 {{
        self.to_glib() as i32
    }}

    fn from(code: i32) -> Option<Self> {{
        {assert}match code {{", name = enum_.name, get_quark = get_quark, assert = assert));

        for member in &members {
            try!(version_condition(w, env, member.version, false, 3));
            try!(writeln!(w, "\t\t\tx if x == ffi::{} as i32 => Some({}::{}),", member.c_name,
                enum_.name, member.name));
        }
        if has_failed_member {
            try!(writeln!(w, "\t\t\t_ => Some({}::Failed),", enum_.name));
        } else {
            try!(writeln!(w, "\t\t\t_ => Some({}::__Nonexhaustive(())),", enum_.name));
        }

        try!(writeln!(w, "{}",
"        }
    }
}
"));
    }

    Ok(())
}
<commit_msg>Remove unused uses in generated enums.rs<commit_after>use analysis::namespaces;
use case::CaseExt;
use codegen::general::{self, version_condition, version_condition_string};
use config::gobjects::GObject;
use env::Env;
use file_saver;
use library::*;
use std::collections::HashSet;
use std::io::prelude::*;
use std::io::Result;
use std::path::Path;
use traits::*;
use version::Version;

pub fn generate(env: &Env, root_path: &Path, mod_rs: &mut Vec<String>) {
    let configs: Vec<&GObject> = env.config.objects.values()
        .filter(|c| {
            c.status.need_generate() &&
                c.type_id.map_or(false, |tid| tid.ns_id == namespaces::MAIN)
        })
        .collect();
    let mut has_get_quark = false;
    let mut has_any = false;
    for config in &configs {
        if let Type::Enumeration(ref enum_) = *env.library.type_(config.type_id.unwrap()) {
            has_any = true;
            if get_error_quark_name(enum_).is_some(){
                has_get_quark = true;
                break;
            }
        }
    }

    let path = root_path.join("enums.rs");
    file_saver::save_to_file(path, env.config.make_backup, |w| {
        try!(general::start_comments(w, &env.config));
        try!(writeln!(w, ""));
        try!(writeln!(w, "use ffi;"));
        if env.namespaces.glib_ns_id == namespaces::MAIN {
            if has_get_quark {
                try!(writeln!(w, "use ffi as glib_ffi;"));
                try!(writeln!(w, "use error::ErrorDomain;"));
            }
            try!(writeln!(w, "use translate::*;"));
        } else {
            if has_get_quark {
                try!(writeln!(w, "use glib_ffi;"));
                try!(writeln!(w, "use glib::error::ErrorDomain;"));
            }
            try!(writeln!(w, "use glib::translate::*;"));
        }
        try!(writeln!(w, ""));

        if has_any {
            mod_rs.push("\nmod enums;".into());
        }
        for config in &configs {
            if let Type::Enumeration(ref enum_) = *env.library.type_(config.type_id.unwrap()) {
                if let Some (cfg) = version_condition_string(env, enum_.version, false, 0) {
                    mod_rs.push(cfg);
                }
                mod_rs.push(format!("pub use self::enums::{};", enum_.name));
                try!(generate_enum(env, w, enum_, config));
            }
        }

        Ok(())
    });
}

fn generate_enum(env: &Env, w: &mut Write, enum_: &Enumeration, config: &GObject) -> Result<()> {
    struct Member {
        name: String,
        c_name: String,
        version: Option<Version>,
    }

    let mut members: Vec<Member> = Vec::new();
    let mut vals: HashSet<String> = HashSet::new();

    for member in &enum_.members {
        let member_config = config.members.matched(&member.name);
        let is_alias = member_config.iter().any(|m| m.alias);
        if is_alias || vals.contains(&member.value) {
            continue
        }
        vals.insert(member.value.clone());
        let version = member_config.iter().filter_map(|m| m.version).next();
        members.push(Member {
            name: member.name.to_camel(),
            c_name: member.c_identifier.clone(),
            version: version,
        });
    }

    try!(version_condition(w, env, enum_.version, false, 0));
    try!(writeln!(w, "#[derive(Clone, Copy, Debug, Eq, PartialEq, Hash)]"));
    try!(writeln!(w, "pub enum {} {{", enum_.name));
    for member in &members {
        try!(writeln!(w, "\t{},", member.name));
    }
    try!(writeln!(w, "{}", "    #[doc(hidden)]
    __Nonexhaustive(()),
}
"));

    try!(version_condition(w, env, enum_.version, false, 0));
    try!(writeln!(w, "#[doc(hidden)]
impl ToGlib for {name} {{
    type GlibType = ffi::{ffi_name};

    fn to_glib(&self) -> ffi::{ffi_name} {{
        match *self {{", name = enum_.name, ffi_name = enum_.c_type));
    for member in &members {
        try!(version_condition(w, env, member.version, false, 3));
        try!(writeln!(w, "\t\t\t{}::{} => ffi::{},", enum_.name, member.name, member.c_name));
    }
    try!(writeln!(w, "\t\t\t{}::__Nonexhaustive(_) => panic!(),", enum_.name));
    try!(writeln!(w, "{}",
"        }
    }
}
"));

    let assert = if env.config.generate_safety_asserts {
        "skip_assert_initialized!();\n\t\t"
    } else {
        ""
    };

    try!(version_condition(w, env, enum_.version, false, 0));
    try!(writeln!(w, "#[doc(hidden)]
impl FromGlib<ffi::{ffi_name}> for {name} {{
    fn from_glib(value: ffi::{ffi_name}) -> Self {{
        {assert}match value {{", name = enum_.name, ffi_name = enum_.c_type, assert = assert));
    for member in &members {
        try!(version_condition(w, env, member.version, false, 3));
        try!(writeln!(w, "\t\t\tffi::{} => {}::{},", member.c_name, enum_.name, member.name));
    }
    if members.len() == 1 {
        try!(writeln!(w, "\t\t\t_ => {}::__Nonexhaustive(()),", enum_.name));
    }
    try!(writeln!(w, "{}",
"        }
    }
}
"));
    if let Some(ref get_quark) = get_error_quark_name(enum_) {
        let get_quark = get_quark.replace("-", "_");
        let has_failed_member = members.iter().any(|m| m.name == "Failed");

        try!(version_condition(w, env, enum_.version, false, 0));
        try!(writeln!(w, "impl ErrorDomain for {name} {{
    fn domain() -> glib_ffi::GQuark {{
        {assert}unsafe {{ ffi::{get_quark}() }}
    }}

    fn code(self) -> i32 {{
        self.to_glib() as i32
    }}

    fn from(code: i32) -> Option<Self> {{
        {assert}match code {{", name = enum_.name, get_quark = get_quark, assert = assert));

        for member in &members {
            try!(version_condition(w, env, member.version, false, 3));
            try!(writeln!(w, "\t\t\tx if x == ffi::{} as i32 => Some({}::{}),", member.c_name,
                enum_.name, member.name));
        }
        if has_failed_member {
            try!(writeln!(w, "\t\t\t_ => Some({}::Failed),", enum_.name));
        } else {
            try!(writeln!(w, "\t\t\t_ => Some({}::__Nonexhaustive(())),", enum_.name));
        }

        try!(writeln!(w, "{}",
"        }
    }
}
"));
    }

    Ok(())
}

fn get_error_quark_name(enum_: &Enumeration) -> Option<String> {
    enum_.functions.iter()
        .filter(|f| f.name == "quark")
        .next()
        .and_then(|f| f.c_identifier.clone())
        .or_else(|| enum_.error_domain.clone())
}
<|endoftext|>