<commit_before>use std::fs::File;
use std::os::raw::c_void;
use std::path::Path;
use std::process;
use std::time::Instant;

use cgmath::{ Point3 };
use gl;
use glutin;
use glutin::{
    CursorState,
    ElementState,
    MouseScrollDelta,
    GlContext,
    VirtualKeyCode,
    WindowEvent,
};
use gltf_importer;
use gltf_importer::config::ValidationStrategy;
use image::{DynamicImage, ImageFormat};

use controls::CameraControls;
use controls::CameraMovement::*;
use framebuffer::Framebuffer;
use render::*;
use render::math::*;
use utils::{print_elapsed, FrameTimer, gl_check_error};

// TODO!: complete and pass through draw calls? or get rid of multiple shaders?
// How about state ordering anyway?
// struct DrawState {
//     current_shader: ShaderFlags,
//     back_face_culling_enabled: bool
// }

pub struct GltfViewer {
    width: u32,
    height: u32,

    controls: CameraControls,
    first_mouse: bool,
    last_x: f32,
    last_y: f32,
    events_loop: Option<glutin::EventsLoop>,
    gl_window: Option<glutin::GlWindow>,

    // TODO!: get rid of scene?
    root: Root,
    scene: Scene,

    delta_time: f64, // seconds
    last_frame: Instant,

    render_timer: FrameTimer,
}

impl GltfViewer {
    pub fn new(source: &str, width: u32, height: u32, headless: bool, visible: bool) -> GltfViewer {
        // TODO!: handle initialization better
        let mut controls = CameraControls {
            position: Point3::new(0.0, 0.0, 2.0),
            camera: Camera {
                fovy: 60.0,
                aspect_ratio: width as f32 / height as f32,
                ..Camera::default()
            },
            ..CameraControls::default()
        };
        controls.camera.update_projection_matrix();

        let first_mouse = true;
        let last_x: f32 = width as f32 / 2.0;
        let last_y: f32 = height as f32 / 2.0;

        let (events_loop, gl_window) =
            if headless {
                let headless_context = glutin::HeadlessRendererBuilder::new(width, height).build().unwrap();
                unsafe { headless_context.make_current().unwrap() }
                gl::load_with(|symbol| headless_context.get_proc_address(symbol) as *const _);
                let framebuffer = Framebuffer::new(width, height);
                framebuffer.bind();

                (None, None)
            }
            else {
                // glutin: initialize and configure
                let events_loop = glutin::EventsLoop::new().into();

                // TODO?: hints for 4.1, core profile, forward compat
                let window = glutin::WindowBuilder::new()
                        .with_title("gltf-viewer")
                        .with_dimensions(width, height)
                        .with_visibility(visible);

                let context = glutin::ContextBuilder::new()
                    .with_vsync(true);
                let gl_window = glutin::GlWindow::new(window, context, &events_loop).unwrap();

                unsafe { gl_window.make_current().unwrap(); }

                // TODO!: capturing - on click or uncapture somehow?
                // TODO!: find solution for macOS - see https://github.com/tomaka/glutin/issues/226
                #[cfg(target_os = "macos")]
                let _ = gl_window.set_cursor_state(CursorState::Hide);
                #[cfg(not(target_os = "macos"))]
                let _ = gl_window.set_cursor_state(CursorState::Grab);

                // gl: load all OpenGL function pointers
                gl::load_with(|symbol| gl_window.get_proc_address(symbol) as *const _);

                (Some(events_loop), Some(gl_window))
            };

        unsafe {
            gl::ClearColor(0.0, 1.0, 0.0, 1.0); // green for debugging
            gl::Clear(gl::COLOR_BUFFER_BIT | gl::DEPTH_BUFFER_BIT);

            gl::Enable(gl::DEPTH_TEST);

            // TODO: keyboard switch?
            // draw in wireframe
            // gl::PolygonMode(gl::FRONT_AND_BACK, gl::LINE);
        };

        let (root, scene) = Self::load(source);
        let mut viewer = GltfViewer {
            width,
            height,

            controls,
            first_mouse, last_x, last_y,

            events_loop,
            gl_window,

            root,
            scene,

            delta_time: 0.0, // seconds
            last_frame: Instant::now(),

            render_timer: FrameTimer::new("rendering", 300),
        };
        unsafe { gl_check_error!(); }

        if !viewer.root.camera_nodes.is_empty() {
            // Take first camera node
            let cam_node = &viewer.root.get_camera_node(0);
            viewer.controls.set_camera(
                cam_node.camera.as_ref().unwrap(),
                &cam_node.final_transform);
        } else {
            viewer.set_camera_from_bounds();
        }

        viewer
    }

    pub fn load(source: &str) -> (Root, Scene) {
        let mut start_time = Instant::now();
        // TODO!: http source
        // let gltf =
        if source.starts_with("http") {
            panic!("not implemented: HTTP support temporarily removed.")
            // let http_source = HttpSource::new(source);
            // let import = gltf::Import::custom(http_source, Default::default());
            // let gltf = import_gltf(import);
            // println!(); // to end the "progress dots"
            // gltf
        }
        //     else {
        let config = gltf_importer::Config { validation_strategy: ValidationStrategy::Complete };
        let (gltf, buffers) = match gltf_importer::import_with_config(source, config) {
            Ok((gltf, buffers)) => (gltf, buffers),
            Err(err) => {
                error!("glTF import failed: {:?}", err);
                if let gltf_importer::Error::Io(_) = err {
                    error!("Hint: Are the .bin file(s) referenced by the .gltf file available?")
                }
                process::exit(1)
            },
        };

        print_elapsed("Imported glTF in ", &start_time);
        start_time = Instant::now();

        // load first scene
        if gltf.scenes().len() > 1 {
            warn!("Found more than 1 scene, can only load first at the moment.")
        }
        let base_path = Path::new(source);
        let mut root = Root::from_gltf(&gltf, &buffers, base_path);
        let scene = Scene::from_gltf(gltf.scenes().nth(0).unwrap(), &mut root);
        print_elapsed(&format!("Loaded scene with {} nodes, {} meshes in ",
                gltf.nodes().count(), root.meshes.len()), &start_time);

        (root, scene)
    }

    /// determine "nice" camera perspective from bounding box. Inspired by donmccurdy/three-gltf-viewer
    fn set_camera_from_bounds(&mut self) {
        // TODO!!: fix bounds/camera computation (many models NOT centered)
        let bounds = &self.scene.bounds;
        let size = bounds.size().magnitude();
        let center = bounds.center();

        // TODO!: move cam instead?
        let _obj_pos_modifier = -center;

        let _max_distance = size * 10.0;
        // TODO: x,y addition optional, z optionally minus instead
        let cam_pos = Point3::new(
            center.x + size / 2.0,
            center.y + size / 5.0,
            center.z + size / 2.0,
        );
        let _near = size / 100.0;
        let _far = size * 100.0;

        self.controls.position = cam_pos;
        self.controls.center = Some(Point3::from_vec(center));
        // TODO!: set near, far, max_distance, obj_pos_modifier...
    }

    pub fn start_render_loop(&mut self) {
        loop {
            // per-frame time logic
            // NOTE: Deliberately ignoring the seconds of `elapsed()`
            self.delta_time = f64::from(self.last_frame.elapsed().subsec_nanos()) / 1_000_000_000.0;
            self.last_frame = Instant::now();

            // events
            let keep_running = process_events(
                &mut self.events_loop.as_mut().unwrap(), self.gl_window.as_mut().unwrap(),
                &mut self.first_mouse, &mut self.last_x, &mut self.last_y,
                &mut self.controls, &mut self.width, &mut self.height);
            if !keep_running { break }

            self.controls.update(self.delta_time); // navigation

            self.draw();

            self.gl_window.as_ref().unwrap().swap_buffers().unwrap();
        }
    }

    // Returns whether to keep running
    pub fn draw(&mut self) {
        // render
        unsafe {
            self.render_timer.start();

            gl::ClearColor(0.1, 0.2, 0.3, 1.0);
            gl::Clear(gl::COLOR_BUFFER_BIT | gl::DEPTH_BUFFER_BIT);

            self.scene.draw(&mut self.root, &self.controls);

            self.render_timer.end();
        }
    }

    pub fn screenshot(&mut self, filename: &str, _width: u32, _height: u32) {
        self.draw();

        // TODO!: headless case...
        let (width, height) = self.gl_window.as_ref().unwrap().get_inner_size_pixels().unwrap();
        let mut img = DynamicImage::new_rgb8(width, height);
        unsafe {
            let pixels = img.as_mut_rgb8().unwrap();
            gl::PixelStorei(gl::PACK_ALIGNMENT, 1);
            gl::ReadPixels(0, 0, width as i32, height as i32, gl::RGB,
                gl::UNSIGNED_BYTE, pixels.as_mut_ptr() as *mut c_void);
        }

        let img = img.flipv();

        let mut file = File::create(filename).unwrap();
        if let Err(err) = img.save(&mut file, ImageFormat::PNG) {
            error!("{}", err);
        }
        else {
            println!("Saved {}x{} screenshot to {}", width, height, filename);
        }
    }
}

#[allow(too_many_arguments)]
fn process_events(
    events_loop: &mut glutin::EventsLoop,
    gl_window: &glutin::GlWindow,
    first_mouse: &mut bool,
    last_x: &mut f32,
    last_y: &mut f32,
    mut camera: &mut CameraControls,
    width: &mut u32,
    height: &mut u32) -> bool
{
    let mut keep_running = true;
    #[allow(single_match)]
    events_loop.poll_events(|event| {
        match event {
            glutin::Event::WindowEvent{ event, .. } => match event {
                WindowEvent::Closed => keep_running = false,
                WindowEvent::Resized(w, h) => {
                    gl_window.resize(w, h);
                    *width = w;
                    *height = h;
                    // TODO!: update camera aspect?
                },
                WindowEvent::DroppedFile(_path_buf) => (), // TODO: drag file in
                WindowEvent::MouseMoved { position: (xpos, ypos), .. } => {
                    let (xpos, ypos) = (xpos as f32, ypos as f32);
                    if *first_mouse {
                        *last_x = xpos;
                        *last_y = ypos;
                        *first_mouse = false;
                    }

                    let xoffset = xpos - *last_x;
                    let yoffset = *last_y - ypos; // reversed since y-coordinates go from bottom to top

                    *last_x = xpos;
                    *last_y = ypos;

                    camera.process_mouse_movement(xoffset, yoffset, true);
                },
                WindowEvent::MouseWheel { delta: MouseScrollDelta::PixelDelta(_xoffset, yoffset), .. } => {
                    // TODO: need to handle LineDelta case too?
                    camera.process_mouse_scroll(yoffset);
                }
                WindowEvent::KeyboardInput { input, .. } => {
                    keep_running = process_input(input, &mut camera);
                }
                _ => ()
            },
            _ => ()
        }
    });

    keep_running
}

fn process_input(input: glutin::KeyboardInput, camera: &mut CameraControls) -> bool {
    let pressed = match input.state {
        ElementState::Pressed => true,
        ElementState::Released => false
    };
    if let Some(code) = input.virtual_keycode {
        match code {
            VirtualKeyCode::Escape if pressed => return false,
            VirtualKeyCode::W => camera.process_keyboard(FORWARD, pressed),
            VirtualKeyCode::S => camera.process_keyboard(BACKWARD, pressed),
            VirtualKeyCode::A => camera.process_keyboard(LEFT, pressed),
            VirtualKeyCode::D => camera.process_keyboard(RIGHT, pressed),
            _ => ()
        }
    }
    true
}
<commit_msg>disable gltf camera choosing<commit_after>use std::fs::File;
use std::os::raw::c_void;
use std::path::Path;
use std::process;
use std::time::Instant;

use cgmath::{ Point3 };
use gl;
use glutin;
use glutin::{
    CursorState,
    ElementState,
    MouseScrollDelta,
    GlContext,
    VirtualKeyCode,
    WindowEvent,
};
use gltf_importer;
use gltf_importer::config::ValidationStrategy;
use image::{DynamicImage, ImageFormat};

use controls::CameraControls;
use controls::CameraMovement::*;
use framebuffer::Framebuffer;
use render::*;
use render::math::*;
use utils::{print_elapsed, FrameTimer, gl_check_error};

// TODO!: complete and pass through draw calls? or get rid of multiple shaders?
// How about state ordering anyway?
// struct DrawState {
//     current_shader: ShaderFlags,
//     back_face_culling_enabled: bool
// }

pub struct GltfViewer {
    width: u32,
    height: u32,

    controls: CameraControls,
    first_mouse: bool,
    last_x: f32,
    last_y: f32,
    events_loop: Option<glutin::EventsLoop>,
    gl_window: Option<glutin::GlWindow>,

    // TODO!: get rid of scene?
    root: Root,
    scene: Scene,

    delta_time: f64, // seconds
    last_frame: Instant,

    render_timer: FrameTimer,
}

impl GltfViewer {
    pub fn new(source: &str, width: u32, height: u32, headless: bool, visible: bool) -> GltfViewer {
        // TODO!: handle initialization better
        let mut controls = CameraControls {
            position: Point3::new(0.0, 0.0, 2.0),
            camera: Camera {
                fovy: 60.0,
                aspect_ratio: width as f32 / height as f32,
                ..Camera::default()
            },
            ..CameraControls::default()
        };
        controls.camera.update_projection_matrix();

        let first_mouse = true;
        let last_x: f32 = width as f32 / 2.0;
        let last_y: f32 = height as f32 / 2.0;

        let (events_loop, gl_window) =
            if headless {
                let headless_context = glutin::HeadlessRendererBuilder::new(width, height).build().unwrap();
                unsafe { headless_context.make_current().unwrap() }
                gl::load_with(|symbol| headless_context.get_proc_address(symbol) as *const _);
                let framebuffer = Framebuffer::new(width, height);
                framebuffer.bind();

                (None, None)
            }
            else {
                // glutin: initialize and configure
                let events_loop = glutin::EventsLoop::new().into();

                // TODO?: hints for 4.1, core profile, forward compat
                let window = glutin::WindowBuilder::new()
                        .with_title("gltf-viewer")
                        .with_dimensions(width, height)
                        .with_visibility(visible);

                let context = glutin::ContextBuilder::new()
                    .with_vsync(true);
                let gl_window = glutin::GlWindow::new(window, context, &events_loop).unwrap();

                unsafe { gl_window.make_current().unwrap(); }

                // TODO!: capturing - on click or uncapture somehow?
                // TODO!: find solution for macOS - see https://github.com/tomaka/glutin/issues/226
                #[cfg(target_os = "macos")]
                let _ = gl_window.set_cursor_state(CursorState::Hide);
                #[cfg(not(target_os = "macos"))]
                let _ = gl_window.set_cursor_state(CursorState::Grab);

                // gl: load all OpenGL function pointers
                gl::load_with(|symbol| gl_window.get_proc_address(symbol) as *const _);

                (Some(events_loop), Some(gl_window))
            };

        unsafe {
            gl::ClearColor(0.0, 1.0, 0.0, 1.0); // green for debugging
            gl::Clear(gl::COLOR_BUFFER_BIT | gl::DEPTH_BUFFER_BIT);

            gl::Enable(gl::DEPTH_TEST);

            // TODO: keyboard switch?
            // draw in wireframe
            // gl::PolygonMode(gl::FRONT_AND_BACK, gl::LINE);
        };

        let (root, scene) = Self::load(source);
        let mut viewer = GltfViewer {
            width,
            height,

            controls,
            first_mouse, last_x, last_y,

            events_loop,
            gl_window,

            root,
            scene,

            delta_time: 0.0, // seconds
            last_frame: Instant::now(),

            render_timer: FrameTimer::new("rendering", 300),
        };
        unsafe { gl_check_error!(); }

        // TODO!!!: completely broken (using gltf camera)
        if false && !viewer.root.camera_nodes.is_empty() {
            // Take first camera node
            let cam_node = &viewer.root.get_camera_node(0);
            viewer.controls.set_camera(
                cam_node.camera.as_ref().unwrap(),
                &cam_node.final_transform);
        } else {
            viewer.set_camera_from_bounds();
        }

        viewer
    }

    pub fn load(source: &str) -> (Root, Scene) {
        let mut start_time = Instant::now();
        // TODO!: http source
        // let gltf =
        if source.starts_with("http") {
            panic!("not implemented: HTTP support temporarily removed.")
            // let http_source = HttpSource::new(source);
            // let import = gltf::Import::custom(http_source, Default::default());
            // let gltf = import_gltf(import);
            // println!(); // to end the "progress dots"
            // gltf
        }
        //     else {
        let config = gltf_importer::Config { validation_strategy: ValidationStrategy::Complete };
        let (gltf, buffers) = match gltf_importer::import_with_config(source, config) {
            Ok((gltf, buffers)) => (gltf, buffers),
            Err(err) => {
                error!("glTF import failed: {:?}", err);
                if let gltf_importer::Error::Io(_) = err {
                    error!("Hint: Are the .bin file(s) referenced by the .gltf file available?")
                }
                process::exit(1)
            },
        };

        print_elapsed("Imported glTF in ", &start_time);
        start_time = Instant::now();

        // load first scene
        if gltf.scenes().len() > 1 {
            warn!("Found more than 1 scene, can only load first at the moment.")
        }
        let base_path = Path::new(source);
        let mut root = Root::from_gltf(&gltf, &buffers, base_path);
        let scene = Scene::from_gltf(gltf.scenes().nth(0).unwrap(), &mut root);
        print_elapsed(&format!("Loaded scene with {} nodes, {} meshes in ",
                gltf.nodes().count(), root.meshes.len()), &start_time);

        (root, scene)
    }

    /// determine "nice" camera perspective from bounding box. Inspired by donmccurdy/three-gltf-viewer
    fn set_camera_from_bounds(&mut self) {
        // TODO!!: fix bounds/camera computation (many models NOT centered)
        let bounds = &self.scene.bounds;
        let size = bounds.size().magnitude();
        let center = bounds.center();

        // TODO!: move cam instead?
        let _obj_pos_modifier = -center;

        let _max_distance = size * 10.0;
        // TODO: x,y addition optional, z optionally minus instead
        let cam_pos = Point3::new(
            center.x + size / 2.0,
            center.y + size / 5.0,
            center.z + size / 2.0,
        );
        let _near = size / 100.0;
        let _far = size * 100.0;

        self.controls.position = cam_pos;
        self.controls.center = Some(Point3::from_vec(center));
        // TODO!: set near, far, max_distance, obj_pos_modifier...
    }

    pub fn start_render_loop(&mut self) {
        loop {
            // per-frame time logic
            // NOTE: Deliberately ignoring the seconds of `elapsed()`
            self.delta_time = f64::from(self.last_frame.elapsed().subsec_nanos()) / 1_000_000_000.0;
            self.last_frame = Instant::now();

            // events
            let keep_running = process_events(
                &mut self.events_loop.as_mut().unwrap(), self.gl_window.as_mut().unwrap(),
                &mut self.first_mouse, &mut self.last_x, &mut self.last_y,
                &mut self.controls, &mut self.width, &mut self.height);
            if !keep_running { break }

            self.controls.update(self.delta_time); // navigation

            self.draw();

            self.gl_window.as_ref().unwrap().swap_buffers().unwrap();
        }
    }

    // Returns whether to keep running
    pub fn draw(&mut self) {
        // render
        unsafe {
            self.render_timer.start();

            gl::ClearColor(0.1, 0.2, 0.3, 1.0);
            gl::Clear(gl::COLOR_BUFFER_BIT | gl::DEPTH_BUFFER_BIT);

            self.scene.draw(&mut self.root, &self.controls);

            self.render_timer.end();
        }
    }

    pub fn screenshot(&mut self, filename: &str, _width: u32, _height: u32) {
        self.draw();

        // TODO!: headless case...
        let (width, height) = self.gl_window.as_ref().unwrap().get_inner_size_pixels().unwrap();
        let mut img = DynamicImage::new_rgb8(width, height);
        unsafe {
            let pixels = img.as_mut_rgb8().unwrap();
            gl::PixelStorei(gl::PACK_ALIGNMENT, 1);
            gl::ReadPixels(0, 0, width as i32, height as i32, gl::RGB,
                gl::UNSIGNED_BYTE, pixels.as_mut_ptr() as *mut c_void);
        }

        let img = img.flipv();

        let mut file = File::create(filename).unwrap();
        if let Err(err) = img.save(&mut file, ImageFormat::PNG) {
            error!("{}", err);
        }
        else {
            println!("Saved {}x{} screenshot to {}", width, height, filename);
        }
    }
}

#[allow(too_many_arguments)]
fn process_events(
    events_loop: &mut glutin::EventsLoop,
    gl_window: &glutin::GlWindow,
    first_mouse: &mut bool,
    last_x: &mut f32,
    last_y: &mut f32,
    mut camera: &mut CameraControls,
    width: &mut u32,
    height: &mut u32) -> bool
{
    let mut keep_running = true;
    #[allow(single_match)]
    events_loop.poll_events(|event| {
        match event {
            glutin::Event::WindowEvent{ event, .. } => match event {
                WindowEvent::Closed => keep_running = false,
                WindowEvent::Resized(w, h) => {
                    gl_window.resize(w, h);
                    *width = w;
                    *height = h;
                    // TODO!: update camera aspect?
                },
                WindowEvent::DroppedFile(_path_buf) => (), // TODO: drag file in
                WindowEvent::MouseMoved { position: (xpos, ypos), .. } => {
                    let (xpos, ypos) = (xpos as f32, ypos as f32);
                    if *first_mouse {
                        *last_x = xpos;
                        *last_y = ypos;
                        *first_mouse = false;
                    }

                    let xoffset = xpos - *last_x;
                    let yoffset = *last_y - ypos; // reversed since y-coordinates go from bottom to top

                    *last_x = xpos;
                    *last_y = ypos;

                    camera.process_mouse_movement(xoffset, yoffset, true);
                },
                WindowEvent::MouseWheel { delta: MouseScrollDelta::PixelDelta(_xoffset, yoffset), .. } => {
                    // TODO: need to handle LineDelta case too?
                    camera.process_mouse_scroll(yoffset);
                }
                WindowEvent::KeyboardInput { input, .. } => {
                    keep_running = process_input(input, &mut camera);
                }
                _ => ()
            },
            _ => ()
        }
    });

    keep_running
}

fn process_input(input: glutin::KeyboardInput, camera: &mut CameraControls) -> bool {
    let pressed = match input.state {
        ElementState::Pressed => true,
        ElementState::Released => false
    };
    if let Some(code) = input.virtual_keycode {
        match code {
            VirtualKeyCode::Escape if pressed => return false,
            VirtualKeyCode::W => camera.process_keyboard(FORWARD, pressed),
            VirtualKeyCode::S => camera.process_keyboard(BACKWARD, pressed),
            VirtualKeyCode::A => camera.process_keyboard(LEFT, pressed),
            VirtualKeyCode::D => camera.process_keyboard(RIGHT, pressed),
            _ => ()
        }
    }
    true
}
<|endoftext|>


<commit_before>//! The oldest module, the data module stores all the data needed to collect
//! and also the search, dir algorithms
pub mod audio_info;
mod bktree;
pub mod collection;
pub mod ipc;
mod tag_readers;

use self::{audio_info::Container, collection::Collection, ipc::IPC};
use super::ctrl::{CollectionPathAlive, ForwardNetMsg, NetInfoMsg, Status, UiUpdateMsg};
use crossbeam::channel::Sender as CSender;
use std::{
    ops::Add,
    path::Path,
    sync::{Arc as SArc, Mutex as SMutex},
};

/// Interface of what collection output data will return
#[derive(Clone)]
pub struct IFInternalCollectionOutputData {
    pub nr_searched_files: u32,
    pub nr_found_songs: u32,
    pub nr_internal_duplicates: u32,
}
impl Add for IFInternalCollectionOutputData {
    type Output = Self;

    fn add(self, other: Self) -> Self {
        Self {
            nr_searched_files: self.nr_searched_files + other.nr_searched_files,
            nr_found_songs: self.nr_found_songs + other.nr_found_songs,
            nr_internal_duplicates: self.nr_internal_duplicates + other.nr_internal_duplicates,
        }
    }
}
impl IFInternalCollectionOutputData {
    pub fn new() -> Self {
        Self {
            nr_searched_files: 0,
            nr_found_songs: 0,
            nr_internal_duplicates: 0,
        }
    }
}

/// The collection search! Searches on the file system a concrete, single path.
pub fn search_in_single_path(
    has_ui: bool,
    collection_data: SArc<SMutex<Container>>,
    collection_protected: SArc<SMutex<Collection>>,
    mutex_to_ui_msg: SArc<SMutex<CSender<UiUpdateMsg>>>,
    index: usize,
    elem: &str,
) -> IFInternalCollectionOutputData {
    if !has_ui {
        println!("[{:?}] looking into path {:?}", index, elem);
    } else {
        // send start animation for that path
        mutex_to_ui_msg
            .lock()
            .and_then(|locked_to_start_ui_message| {
                trace!("send startAnimation for path {:?}", index);
                locked_to_start_ui_message
                    .send(UiUpdateMsg::CollectionUpdate(
                        CollectionPathAlive::BusyPath(index),
                        Status::ON,
                    ))
                    .unwrap_or_else(|_| warn!("... lost start animation for index {:?}", index));
                trace!("start busy animation");
                Ok(())
            })
            .unwrap_or_else(|_| error!("... that should not happen here at start"));
    }
    // this collection lock is fine, it's not the main Container, only some data
    let locked_collection = &mut *collection_protected.lock().unwrap();

    // do it: main task here is to visit and dive deep
    //        into the subfolders of this folder
    match locked_collection.visit_path(
        collection_data,
        Path::new(elem),
        &collection::Collection::visit_files,
    ) {
        Ok(local_stats) => {
            if has_ui {
                // send stop animation for that path
                trace!("send stopAnimation for path {:?}", index);
                mutex_to_ui_msg
                    .lock()
                    .and_then(|locked_to_stop_ui_message| {
                        locked_to_stop_ui_message
                            .send(UiUpdateMsg::CollectionUpdate(
                                CollectionPathAlive::BusyPath(index),
                                Status::OFF,
                            ))
                            .unwrap_or_else(|_| warn!("... lost stop animation for {:?}", index));
                        trace!("stop busy animation");
                        Ok(())
                    })
                    .unwrap_or_else(|_| {
                        error!("... that should not happen here at stop");
                    });
            } else {
                let text = format!(
                    "\n\
                             analyzed: {an:>width$}, faulty: {fa:>width$}\n\
                             duplicates: {du:>width$}\n\
                             searched: {se:>width$}, other: {ot:>width$}",
                    an = local_stats.analyzed,
                    fa = local_stats.faulty,
                    du = local_stats.duplicates,
                    se = local_stats.searched,
                    ot = local_stats.other,
                    width = 3
                );
                println!("[{:?}] done {}", index, text);
            }
            // return this here
            IFInternalCollectionOutputData {
                nr_searched_files: local_stats.searched,
                nr_found_songs: local_stats.analyzed,
                nr_internal_duplicates: local_stats.duplicates,
            }
        }
        Err(_e) => {
            let text = format!("An error has occurred in search path [{}]!!", index);
            if has_ui {
                mutex_to_ui_msg
                    .lock()
                    .and_then(|locked_update_message| {
                        locked_update_message
                            .send(UiUpdateMsg::NetUpdate(ForwardNetMsg::Stats(
                                NetInfoMsg::Debug(text),
                            )))
                            .unwrap();
                        Ok(())
                    })
                    .unwrap();
            } else {
                println!("{:?}", text);
            }
            // return this here
            IFInternalCollectionOutputData {
                nr_searched_files: 0,
                nr_found_songs: 0,
                nr_internal_duplicates: 0,
            }
        }
    }
}

#[allow(dead_code)]
pub fn publish_local_storage(container_handle: SArc<SMutex<Container>>, ipc_sender: CSender<IPC>) {
    // send to ipc
    //
    let container = container_handle.lock().unwrap();
    for (key, audio_info) in container.flush() {
        ipc_sender
            .send(IPC::PublishSingleAudioDataRecord(
                key.clone(),
                *audio_info.clone(),
            ))
            .unwrap_or_else(|e| warn!("something went very wrong {}!", e));
    }
}
<commit_msg>[refactor] treating crossbeam sender as normal<commit_after>//! The oldest module, the data module stores all the data needed to collect
//! and also the search, dir algorithms
pub mod audio_info;
mod bktree;
pub mod collection;
pub mod ipc;
mod tag_readers;

use self::{audio_info::Container, collection::Collection, ipc::IPC};
use super::ctrl::{CollectionPathAlive, ForwardNetMsg, NetInfoMsg, Status, UiUpdateMsg};
use crossbeam::channel::Sender;
use std::{
    ops::Add,
    path::Path,
    sync::{Arc as SArc, Mutex as SMutex},
};

/// Interface of what collection output data will return
#[derive(Clone)]
pub struct IFInternalCollectionOutputData {
    pub nr_searched_files: u32,
    pub nr_found_songs: u32,
    pub nr_internal_duplicates: u32,
}
impl Add for IFInternalCollectionOutputData {
    type Output = Self;

    fn add(self, other: Self) -> Self {
        Self {
            nr_searched_files: self.nr_searched_files + other.nr_searched_files,
            nr_found_songs: self.nr_found_songs + other.nr_found_songs,
            nr_internal_duplicates: self.nr_internal_duplicates + other.nr_internal_duplicates,
        }
    }
}
impl IFInternalCollectionOutputData {
    pub fn new() -> Self {
        Self {
            nr_searched_files: 0,
            nr_found_songs: 0,
            nr_internal_duplicates: 0,
        }
    }
}

/// The collection search! Searches on the file system a concrete, single path.
pub fn search_in_single_path(
    has_ui: bool,
    collection_data: SArc<SMutex<Container>>,
    collection_protected: SArc<SMutex<Collection>>,
    mutex_to_ui_msg: SArc<SMutex<Sender<UiUpdateMsg>>>,
    index: usize,
    elem: &str,
) -> IFInternalCollectionOutputData {
    if !has_ui {
        println!("[{:?}] looking into path {:?}", index, elem);
    } else {
        // send start animation for that path
        mutex_to_ui_msg
            .lock()
            .and_then(|locked_to_start_ui_message| {
                trace!("send startAnimation for path {:?}", index);
                locked_to_start_ui_message
                    .send(UiUpdateMsg::CollectionUpdate(
                        CollectionPathAlive::BusyPath(index),
                        Status::ON,
                    ))
                    .unwrap_or_else(|_| warn!("... lost start animation for index {:?}", index));
                trace!("start busy animation");
                Ok(())
            })
            .unwrap_or_else(|_| error!("... that should not happen here at start"));
    }
    // this collection lock is fine, it's not the main Container, only some data
    let locked_collection = &mut *collection_protected.lock().unwrap();

    // do it: main task here is to visit and dive deep
    //        into the subfolders of this folder
    match locked_collection.visit_path(
        collection_data,
        Path::new(elem),
        &collection::Collection::visit_files,
    ) {
        Ok(local_stats) => {
            if has_ui {
                // send stop animation for that path
                trace!("send stopAnimation for path {:?}", index);
                mutex_to_ui_msg
                    .lock()
                    .and_then(|locked_to_stop_ui_message| {
                        locked_to_stop_ui_message
                            .send(UiUpdateMsg::CollectionUpdate(
                                CollectionPathAlive::BusyPath(index),
                                Status::OFF,
                            ))
                            .unwrap_or_else(|_| warn!("... lost stop animation for {:?}", index));
                        trace!("stop busy animation");
                        Ok(())
                    })
                    .unwrap_or_else(|_| {
                        error!("... that should not happen here at stop");
                    });
            } else {
                let text = format!(
                    "\n\
                             analyzed: {an:>width$}, faulty: {fa:>width$}\n\
                             duplicates: {du:>width$}\n\
                             searched: {se:>width$}, other: {ot:>width$}",
                    an = local_stats.analyzed,
                    fa = local_stats.faulty,
                    du = local_stats.duplicates,
                    se = local_stats.searched,
                    ot = local_stats.other,
                    width = 3
                );
                println!("[{:?}] done {}", index, text);
            }
            // return this here
            IFInternalCollectionOutputData {
                nr_searched_files: local_stats.searched,
                nr_found_songs: local_stats.analyzed,
                nr_internal_duplicates: local_stats.duplicates,
            }
        }
        Err(_e) => {
            let text = format!("An error has occurred in search path [{}]!!", index);
            if has_ui {
                mutex_to_ui_msg
                    .lock()
                    .and_then(|locked_update_message| {
                        locked_update_message
                            .send(UiUpdateMsg::NetUpdate(ForwardNetMsg::Stats(
                                NetInfoMsg::Debug(text),
                            )))
                            .unwrap();
                        Ok(())
                    })
                    .unwrap();
            } else {
                println!("{:?}", text);
            }
            // return this here
            IFInternalCollectionOutputData {
                nr_searched_files: 0,
                nr_found_songs: 0,
                nr_internal_duplicates: 0,
            }
        }
    }
}

#[allow(dead_code)]
pub fn publish_local_storage(container_handle: SArc<SMutex<Container>>, ipc_sender: Sender<IPC>) {
    // send to ipc
    //
    let container = container_handle.lock().unwrap();
    for (key, audio_info) in container.flush() {
        ipc_sender
            .send(IPC::PublishSingleAudioDataRecord(
                key.clone(),
                *audio_info.clone(),
            ))
            .unwrap_or_else(|e| warn!("something went very wrong {}!", e));
    }
}
<|endoftext|>


<commit_before><commit_msg>Add (unimplemented) config helper functions<commit_after>pub fn commit_interactive(config: &Value) -> bool {
    unimplemented!()
}

pub fn commit_message(config: &Value, action: StoreAction) -> Option<String> {
    if commit_interactive(config) {
        unimplemented!()
    } else {
        unimplemented!()
    }
}
<|endoftext|>


<commit_before>use std::path::PathBuf;
use std::path::Path;
use std::fmt::{Debug, Formatter, Error as FmtError};
use std::result::Result as RResult;

use toml::Value;
use git2::{Reference as GitReference, Repository, Error as Git2Error};
use git2::{ADD_DEFAULT, STATUS_WT_NEW, STATUS_WT_MODIFIED, IndexMatchedPath};

use libimagstore::storeid::StoreId;
use libimagstore::hook::Hook;
use libimagstore::hook::error::HookError as HE;
use libimagstore::hook::error::HookErrorKind as HEK;
use libimagstore::hook::error::CustomData as HECD;
use libimagstore::hook::result::HookResult;
use libimagstore::hook::position::HookPosition;
use libimagstore::hook::accessor::{HookDataAccessor, HookDataAccessorProvider};
use libimagstore::hook::accessor::StoreIdAccessor;
use libimagerror::trace::trace_error;
use libimagerror::into::IntoError;
use libimagutil::debug_result::*;

use vcs::git::result::Result;
use vcs::git::error::MapErrInto;
use vcs::git::error::GitHookErrorKind as GHEK;
use vcs::git::error::GitHookError as GHE;
use vcs::git::runtime::Runtime as GRuntime;

pub struct CreateHook {
    storepath: PathBuf,

    runtime: GRuntime,

    position: HookPosition,
}

impl CreateHook {

    pub fn new(storepath: PathBuf, p: HookPosition) -> CreateHook {
        CreateHook {
            runtime: GRuntime::new(&storepath),
            storepath: storepath,
            position: p,
        }
    }

}

impl Debug for CreateHook {

    fn fmt(&self, fmt: &mut Formatter) -> RResult<(), FmtError> {
        write!(fmt, "CreateHook(storepath={:?}, repository={}, pos={:?}, cfg={:?}",
               self.storepath,
               (if self.runtime.has_repository() { "Some(_)" } else { "None" }),
               self.position,
               self.runtime.has_config())
    }
}

impl Hook for CreateHook {

    fn name(&self) -> &'static str {
        "stdhook_git_create"
    }

    fn set_config(&mut self, config: &Value) {
        if let Err(e) = self.runtime.set_config(config) {
            trace_error(&e);
        }
    }

}

impl HookDataAccessorProvider for CreateHook {

    fn accessor(&self) -> HookDataAccessor {
        HookDataAccessor::StoreIdAccess(self)
    }
}

impl StoreIdAccessor for CreateHook {

    /// The implementation of the CreateHook
    ///
    /// # Scope
    ///
    /// What this function has to do is _adding_ the new entry to the git index.
    /// After that, the UpdateHook will take care of committing the changes or new file.
    ///
    fn access(&self, id: &StoreId) -> HookResult<()> {
        use vcs::git::action::StoreAction;
        use vcs::git::config::commit_message;

        debug!("[GIT CREATE HOOK]: {:?}", id);

        let path = try!(id.into_pathbuf().map_err_into(GHEK::StoreIdHandlingError));

        let cfg = try!(
            self.runtime
                .config_value_or_err()
                .map_dbg_err_str("[GIT CREATE HOOK]: Couldn't get Value object from config")
        );

        debug!("[GIT CREATE HOOK]: Ensuring branch checkout");
        try!(self
             .runtime
             .ensure_cfg_branch_is_checked_out()
             .map_err(Box::new)
             .map_err(|e| HEK::HookExecutionError.into_error_with_cause(e)));
        debug!("[GIT CREATE HOOK]: Branch checked out");

        debug!("[GIT CREATE HOOK]: Getting repository");
        let repo = try!(
            self.runtime
                .repository()
                .map_dbg_err_str("[GIT CREATE HOOK]: Couldn't fetch Repository")
                .map_err_into(GHEK::RepositoryError)
                .map_err(|e| e.into())
        );
        debug!("[GIT CREATE HOOK]: Repository object fetched");

        let index = try!(repo.index().map_err_into(GHEK::RepositoryIndexFetchingError));

        let file_status = try!(
            repo.status_file(&path).map_err_into(GHEK::RepositoryFileStatusError)
        );

        let cb = &mut |path: &Path, _matched_spec: &[u8]| -> i32 {
            if file_status.contains(STATUS_WT_MODIFIED) ||
                file_status.contains(STATUS_WT_NEW) {

                debug!("[GIT CREATE HOOK]: File is new or modified: {}", path.display());
                0
            } else {
                debug!("[GIT CREATE HOOK]: Ignoring file: {}", path.display());
                1
            }
        };

        try!(
            index.add_all(&[path], ADD_DEFAULT, Some(cb as &mut IndexMatchedPath))
                .map_err_into(GHEK::RepositoryPathAddingError)
        );

        index.write().map_err_into(GHEK::RepositoryIndexWritingError).map_err(|e| e.into())
    }

}

<commit_msg>Use MapIntoHookError for mapping into hook error<commit_after>use std::path::PathBuf;
use std::path::Path;
use std::fmt::{Debug, Formatter, Error as FmtError};
use std::result::Result as RResult;

use toml::Value;
use git2::{Reference as GitReference, Repository, Error as Git2Error};
use git2::{ADD_DEFAULT, STATUS_WT_NEW, STATUS_WT_MODIFIED, IndexMatchedPath};

use libimagstore::storeid::StoreId;
use libimagstore::hook::Hook;
use libimagstore::hook::error::HookError as HE;
use libimagstore::hook::error::HookErrorKind as HEK;
use libimagstore::hook::error::CustomData as HECD;
use libimagstore::hook::result::HookResult;
use libimagstore::hook::position::HookPosition;
use libimagstore::hook::accessor::{HookDataAccessor, HookDataAccessorProvider};
use libimagstore::hook::accessor::StoreIdAccessor;
use libimagerror::trace::trace_error;
use libimagerror::into::IntoError;
use libimagutil::debug_result::*;

use vcs::git::result::Result;
use vcs::git::error::MapIntoHookError;
use vcs::git::error::MapErrInto;
use vcs::git::error::GitHookErrorKind as GHEK;
use vcs::git::error::GitHookError as GHE;
use vcs::git::runtime::Runtime as GRuntime;

pub struct CreateHook {
    storepath: PathBuf,

    runtime: GRuntime,

    position: HookPosition,
}

impl CreateHook {

    pub fn new(storepath: PathBuf, p: HookPosition) -> CreateHook {
        CreateHook {
            runtime: GRuntime::new(&storepath),
            storepath: storepath,
            position: p,
        }
    }

}

impl Debug for CreateHook {

    fn fmt(&self, fmt: &mut Formatter) -> RResult<(), FmtError> {
        write!(fmt, "CreateHook(storepath={:?}, repository={}, pos={:?}, cfg={:?}",
               self.storepath,
               (if self.runtime.has_repository() { "Some(_)" } else { "None" }),
               self.position,
               self.runtime.has_config())
    }
}

impl Hook for CreateHook {

    fn name(&self) -> &'static str {
        "stdhook_git_create"
    }

    fn set_config(&mut self, config: &Value) {
        if let Err(e) = self.runtime.set_config(config) {
            trace_error(&e);
        }
    }

}

impl HookDataAccessorProvider for CreateHook {

    fn accessor(&self) -> HookDataAccessor {
        HookDataAccessor::StoreIdAccess(self)
    }
}

impl StoreIdAccessor for CreateHook {

    /// The implementation of the CreateHook
    ///
    /// # Scope
    ///
    /// What this function has to do is _adding_ the new entry to the git index.
    /// After that, the UpdateHook will take care of committing the changes or new file.
    ///
    fn access(&self, id: &StoreId) -> HookResult<()> {
        use vcs::git::action::StoreAction;
        use vcs::git::config::commit_message;
        use vcs::git::error::MapIntoHookError;

        debug!("[GIT CREATE HOOK]: {:?}", id);

        let path = try!(
            id.clone()
            .into_pathbuf()
            .map_err_into(GHEK::StoreIdHandlingError)
            .map_into_hook_error()
        );

        let cfg = try!(
            self.runtime
                .config_value_or_err()
                .map_dbg_err_str("[GIT CREATE HOOK]: Couldn't get Value object from config")
        );

        debug!("[GIT CREATE HOOK]: Ensuring branch checkout");
        try!(self.runtime.ensure_cfg_branch_is_checked_out());
        debug!("[GIT CREATE HOOK]: Branch checked out");

        debug!("[GIT CREATE HOOK]: Getting repository");
        let repo = try!(
            self.runtime
                .repository()
                .map_dbg_err_str("[GIT CREATE HOOK]: Couldn't fetch Repository")
                .map_err_into(GHEK::RepositoryError)
                .map_into_hook_error()
        );
        debug!("[GIT CREATE HOOK]: Repository object fetched");

        let mut index = try!(
            repo
                .index()
                .map_err_into(GHEK::RepositoryIndexFetchingError)
                .map_into_hook_error()
        );

        let file_status = try!(
            repo
                .status_file(&path)
                .map_err_into(GHEK::RepositoryFileStatusError)
                .map_into_hook_error()
        );

        let cb = &mut |path: &Path, _matched_spec: &[u8]| -> i32 {
            if file_status.contains(STATUS_WT_MODIFIED) ||
                file_status.contains(STATUS_WT_NEW) {

                debug!("[GIT CREATE HOOK]: File is new or modified: {}", path.display());
                0
            } else {
                debug!("[GIT CREATE HOOK]: Ignoring file: {}", path.display());
                1
            }
        };

        try!(
            index.add_all(&[path], ADD_DEFAULT, Some(cb as &mut IndexMatchedPath))
                .map_err_into(GHEK::RepositoryPathAddingError)
                .map_into_hook_error()
        );

        index
            .write()
            .map_err_into(GHEK::RepositoryIndexWritingError)
            .map_into_hook_error()
    }

}

<|endoftext|>


<commit_before>use std::{
    fmt::Write,
    fs,
    path::{Path, PathBuf},
};

use ast::NameOwner;
use expect_test::expect_file;
use rayon::prelude::*;
use test_utils::{bench, bench_fixture, project_root, skip_slow_tests};

use crate::{ast, fuzz, tokenize, AstNode, SourceFile, SyntaxError, TextRange, TextSize, Token};

#[test]
fn lexer_tests() {
    // FIXME:
    // * Add tests for unicode escapes in byte-character and [raw]-byte-string literals
    // * Add tests for unescape errors

    dir_tests(&test_data_dir(), &["lexer/ok"], "txt", |text, path| {
        let (tokens, errors) = tokenize(text);
        assert_errors_are_absent(&errors, path);
        dump_tokens_and_errors(&tokens, &errors, text)
    });
    dir_tests(&test_data_dir(), &["lexer/err"], "txt", |text, path| {
        let (tokens, errors) = tokenize(text);
        assert_errors_are_present(&errors, path);
        dump_tokens_and_errors(&tokens, &errors, text)
    });
}

#[test]
fn parse_smoke_test() {
    let code = r##"
fn main() {
    println!("Hello, world!")
}
    "##;

    let parse = SourceFile::parse(code);
    // eprintln!("{:#?}", parse.syntax_node());
    assert!(parse.ok().is_ok());
}

#[test]
fn benchmark_parser() {
    if skip_slow_tests() {
        return;
    }
    let data = bench_fixture::glorious_old_parser();
    let tree = {
        let _b = bench("parsing");
        let p = SourceFile::parse(&data);
        assert!(p.errors.is_empty());
        assert_eq!(p.tree().syntax.text_range().len(), 352474.into());
        p.tree()
    };

    {
        let _b = bench("tree traversal");
        let fn_names =
            tree.syntax().descendants().filter_map(ast::Fn::cast).filter_map(|f| f.name()).count();
        assert_eq!(fn_names, 268);
    }
}

#[test]
fn parser_tests() {
    dir_tests(&test_data_dir(), &["parser/inline/ok", "parser/ok"], "rast", |text, path| {
        let parse = SourceFile::parse(text);
        let errors = parse.errors();
        assert_errors_are_absent(&errors, path);
        parse.debug_dump()
    });
    dir_tests(&test_data_dir(), &["parser/err", "parser/inline/err"], "rast", |text, path| {
        let parse = SourceFile::parse(text);
        let errors = parse.errors();
        assert_errors_are_present(&errors, path);
        parse.debug_dump()
    });
}

#[test]
fn expr_parser_tests() {
    fragment_parser_dir_test(
        &["parser/fragments/expr/ok"],
        &["parser/fragments/expr/err"],
        crate::ast::Expr::parse,
    );
}

#[test]
fn path_parser_tests() {
    fragment_parser_dir_test(
        &["parser/fragments/path/ok"],
        &["parser/fragments/path/err"],
        crate::ast::Path::parse,
    );
}

#[test]
fn pattern_parser_tests() {
    fragment_parser_dir_test(
        &["parser/fragments/pattern/ok"],
        &["parser/fragments/pattern/err"],
        crate::ast::Pat::parse,
    );
}

#[test]
fn item_parser_tests() {
    fragment_parser_dir_test(
        &["parser/fragments/item/ok"],
        &["parser/fragments/item/err"],
        crate::ast::Item::parse,
    );
}

#[test]
fn type_parser_tests() {
    fragment_parser_dir_test(
        &["parser/fragments/type/ok"],
        &["parser/fragments/type/err"],
        crate::ast::Type::parse,
    );
}

#[test]
fn stmt_parser_tests() {
    fragment_parser_dir_test(
        &["parser/fragments/stmt/ok"],
        &["parser/fragments/stmt/err"],
        crate::ast::Stmt::parse,
    );
}

#[test]
fn parser_fuzz_tests() {
    for (_, text) in collect_rust_files(&test_data_dir(), &["parser/fuzz-failures"]) {
        fuzz::check_parser(&text)
    }
}

#[test]
fn reparse_fuzz_tests() {
    for (_, text) in collect_rust_files(&test_data_dir(), &["reparse/fuzz-failures"]) {
        let check = fuzz::CheckReparse::from_data(text.as_bytes()).unwrap();
        println!("{:?}", check);
        check.run();
    }
}

/// Test that Rust-analyzer can parse and validate the rust-analyzer
#[test]
fn self_hosting_parsing() {
    let dir = project_root().join("crates");
    let files = walkdir::WalkDir::new(dir)
        .into_iter()
        .filter_entry(|entry| {
            // Get all files which are not in the crates/syntax/test_data folder
            !entry.path().components().any(|component| component.as_os_str() == "test_data")
        })
        .map(|e| e.unwrap())
        .filter(|entry| {
            // Get all `.rs ` files
            !entry.path().is_dir() && (entry.path().extension().unwrap_or_default() == "rs")
        })
        .map(|entry| entry.into_path())
        .collect::<Vec<_>>();
    assert!(
        files.len() > 100,
        "self_hosting_parsing found too few files - is it running in the right directory?"
    );

    let errors = files
        .into_par_iter()
        .filter_map(|file| {
            let text = read_text(&file);
            match SourceFile::parse(&text).ok() {
                Ok(_) => None,
                Err(err) => Some((file, err)),
            }
        })
        .collect::<Vec<_>>();

    if !errors.is_empty() {
        let errors = errors
            .into_iter()
            .map(|(path, err)| format!("{}: {:?}\n", path.display(), err))
            .collect::<String>();
        panic!("Parsing errors:\n{}\n", errors);
    }
}

fn test_data_dir() -> PathBuf {
    project_root().join("crates/syntax/test_data")
}

fn assert_errors_are_present(errors: &[SyntaxError], path: &Path) {
    assert!(!errors.is_empty(), "There should be errors in the file {:?}", path.display());
}
fn assert_errors_are_absent(errors: &[SyntaxError], path: &Path) {
    assert_eq!(
        errors,
        &[] as &[SyntaxError],
        "There should be no errors in the file {:?}",
        path.display(),
    );
}

fn dump_tokens_and_errors(tokens: &[Token], errors: &[SyntaxError], text: &str) -> String {
    let mut acc = String::new();
    let mut offset: TextSize = 0.into();
    for token in tokens {
        let token_len = token.len;
        let token_text = &text[TextRange::at(offset, token.len)];
        offset += token.len;
        writeln!(acc, "{:?} {:?} {:?}", token.kind, token_len, token_text).unwrap();
    }
    for err in errors {
        writeln!(acc, "> error{:?} token({:?}) msg({})", err.range(), &text[err.range()], err)
            .unwrap();
    }
    acc
}

fn fragment_parser_dir_test<T, F>(ok_paths: &[&str], err_paths: &[&str], f: F)
where
    T: crate::AstNode,
    F: Fn(&str) -> Result<T, ()>,
{
    dir_tests(&test_data_dir(), ok_paths, "rast", |text, path| {
        if let Ok(node) = f(text) {
            format!("{:#?}", crate::ast::AstNode::syntax(&node))
        } else {
            panic!("Failed to parse '{:?}'", path);
        }
    });
    dir_tests(&test_data_dir(), err_paths, "rast", |text, path| {
        if let Ok(_) = f(text) {
            panic!("'{:?}' successfully parsed when it should have errored", path);
        } else {
            "ERROR\n".to_owned()
        }
    });
}

/// Calls callback `f` with input code and file paths for each `.rs` file in `test_data_dir`
/// subdirectories defined by `paths`.
///
/// If the content of the matching output file differs from the output of `f()`
/// the test will fail.
///
/// If there is no matching output file it will be created and filled with the
/// output of `f()`, but the test will fail.
fn dir_tests<F>(test_data_dir: &Path, paths: &[&str], outfile_extension: &str, f: F)
where
    F: Fn(&str, &Path) -> String,
{
    for (path, input_code) in collect_rust_files(test_data_dir, paths) {
        let actual = f(&input_code, &path);
        let path = path.with_extension(outfile_extension);
        expect_file![path].assert_eq(&actual)
    }
}

/// Collects all `.rs` files from `dir` subdirectories defined by `paths`.
fn collect_rust_files(root_dir: &Path, paths: &[&str]) -> Vec<(PathBuf, String)> {
    paths
        .iter()
        .flat_map(|path| {
            let path = root_dir.to_owned().join(path);
            rust_files_in_dir(&path).into_iter()
        })
        .map(|path| {
            let text = read_text(&path);
            (path, text)
        })
        .collect()
}

/// Collects paths to all `.rs` files from `dir` in a sorted `Vec<PathBuf>`.
fn rust_files_in_dir(dir: &Path) -> Vec<PathBuf> {
    let mut acc = Vec::new();
    for file in fs::read_dir(&dir).unwrap() {
        let file = file.unwrap();
        let path = file.path();
        if path.extension().unwrap_or_default() == "rs" {
            acc.push(path);
        }
    }
    acc.sort();
    acc
}

/// Read file and normalize newlines.
///
/// `rustc` seems to always normalize `\r\n` newlines to `\n`:
///
/// ```
/// let s = "
/// ";
/// assert_eq!(s.as_bytes(), &[10]);
/// ```
///
/// so this should always be correct.
fn read_text(path: &Path) -> String {
    fs::read_to_string(path)
        .unwrap_or_else(|_| panic!("File at {:?} should be valid", path))
        .replace("\r\n", "\n")
}
<commit_msg>minor: remove debug print<commit_after>use std::{
    fmt::Write,
    fs,
    path::{Path, PathBuf},
};

use ast::NameOwner;
use expect_test::expect_file;
use rayon::prelude::*;
use test_utils::{bench, bench_fixture, project_root, skip_slow_tests};

use crate::{ast, fuzz, tokenize, AstNode, SourceFile, SyntaxError, TextRange, TextSize, Token};

#[test]
fn lexer_tests() {
    // FIXME:
    // * Add tests for unicode escapes in byte-character and [raw]-byte-string literals
    // * Add tests for unescape errors

    dir_tests(&test_data_dir(), &["lexer/ok"], "txt", |text, path| {
        let (tokens, errors) = tokenize(text);
        assert_errors_are_absent(&errors, path);
        dump_tokens_and_errors(&tokens, &errors, text)
    });
    dir_tests(&test_data_dir(), &["lexer/err"], "txt", |text, path| {
        let (tokens, errors) = tokenize(text);
        assert_errors_are_present(&errors, path);
        dump_tokens_and_errors(&tokens, &errors, text)
    });
}

#[test]
fn parse_smoke_test() {
    let code = r##"
fn main() {
    println!("Hello, world!")
}
    "##;

    let parse = SourceFile::parse(code);
    // eprintln!("{:#?}", parse.syntax_node());
    assert!(parse.ok().is_ok());
}

#[test]
fn benchmark_parser() {
    if skip_slow_tests() {
        return;
    }
    let data = bench_fixture::glorious_old_parser();
    let tree = {
        let _b = bench("parsing");
        let p = SourceFile::parse(&data);
        assert!(p.errors.is_empty());
        assert_eq!(p.tree().syntax.text_range().len(), 352474.into());
        p.tree()
    };

    {
        let _b = bench("tree traversal");
        let fn_names =
            tree.syntax().descendants().filter_map(ast::Fn::cast).filter_map(|f| f.name()).count();
        assert_eq!(fn_names, 268);
    }
}

#[test]
fn parser_tests() {
    dir_tests(&test_data_dir(), &["parser/inline/ok", "parser/ok"], "rast", |text, path| {
        let parse = SourceFile::parse(text);
        let errors = parse.errors();
        assert_errors_are_absent(&errors, path);
        parse.debug_dump()
    });
    dir_tests(&test_data_dir(), &["parser/err", "parser/inline/err"], "rast", |text, path| {
        let parse = SourceFile::parse(text);
        let errors = parse.errors();
        assert_errors_are_present(&errors, path);
        parse.debug_dump()
    });
}

#[test]
fn expr_parser_tests() {
    fragment_parser_dir_test(
        &["parser/fragments/expr/ok"],
        &["parser/fragments/expr/err"],
        crate::ast::Expr::parse,
    );
}

#[test]
fn path_parser_tests() {
    fragment_parser_dir_test(
        &["parser/fragments/path/ok"],
        &["parser/fragments/path/err"],
        crate::ast::Path::parse,
    );
}

#[test]
fn pattern_parser_tests() {
    fragment_parser_dir_test(
        &["parser/fragments/pattern/ok"],
        &["parser/fragments/pattern/err"],
        crate::ast::Pat::parse,
    );
}

#[test]
fn item_parser_tests() {
    fragment_parser_dir_test(
        &["parser/fragments/item/ok"],
        &["parser/fragments/item/err"],
        crate::ast::Item::parse,
    );
}

#[test]
fn type_parser_tests() {
    fragment_parser_dir_test(
        &["parser/fragments/type/ok"],
        &["parser/fragments/type/err"],
        crate::ast::Type::parse,
    );
}

#[test]
fn stmt_parser_tests() {
    fragment_parser_dir_test(
        &["parser/fragments/stmt/ok"],
        &["parser/fragments/stmt/err"],
        crate::ast::Stmt::parse,
    );
}

#[test]
fn parser_fuzz_tests() {
    for (_, text) in collect_rust_files(&test_data_dir(), &["parser/fuzz-failures"]) {
        fuzz::check_parser(&text)
    }
}

#[test]
fn reparse_fuzz_tests() {
    for (_, text) in collect_rust_files(&test_data_dir(), &["reparse/fuzz-failures"]) {
        let check = fuzz::CheckReparse::from_data(text.as_bytes()).unwrap();
        check.run();
    }
}

/// Test that Rust-analyzer can parse and validate the rust-analyzer
#[test]
fn self_hosting_parsing() {
    let dir = project_root().join("crates");
    let files = walkdir::WalkDir::new(dir)
        .into_iter()
        .filter_entry(|entry| {
            // Get all files which are not in the crates/syntax/test_data folder
            !entry.path().components().any(|component| component.as_os_str() == "test_data")
        })
        .map(|e| e.unwrap())
        .filter(|entry| {
            // Get all `.rs ` files
            !entry.path().is_dir() && (entry.path().extension().unwrap_or_default() == "rs")
        })
        .map(|entry| entry.into_path())
        .collect::<Vec<_>>();
    assert!(
        files.len() > 100,
        "self_hosting_parsing found too few files - is it running in the right directory?"
    );

    let errors = files
        .into_par_iter()
        .filter_map(|file| {
            let text = read_text(&file);
            match SourceFile::parse(&text).ok() {
                Ok(_) => None,
                Err(err) => Some((file, err)),
            }
        })
        .collect::<Vec<_>>();

    if !errors.is_empty() {
        let errors = errors
            .into_iter()
            .map(|(path, err)| format!("{}: {:?}\n", path.display(), err))
            .collect::<String>();
        panic!("Parsing errors:\n{}\n", errors);
    }
}

fn test_data_dir() -> PathBuf {
    project_root().join("crates/syntax/test_data")
}

fn assert_errors_are_present(errors: &[SyntaxError], path: &Path) {
    assert!(!errors.is_empty(), "There should be errors in the file {:?}", path.display());
}
fn assert_errors_are_absent(errors: &[SyntaxError], path: &Path) {
    assert_eq!(
        errors,
        &[] as &[SyntaxError],
        "There should be no errors in the file {:?}",
        path.display(),
    );
}

fn dump_tokens_and_errors(tokens: &[Token], errors: &[SyntaxError], text: &str) -> String {
    let mut acc = String::new();
    let mut offset: TextSize = 0.into();
    for token in tokens {
        let token_len = token.len;
        let token_text = &text[TextRange::at(offset, token.len)];
        offset += token.len;
        writeln!(acc, "{:?} {:?} {:?}", token.kind, token_len, token_text).unwrap();
    }
    for err in errors {
        writeln!(acc, "> error{:?} token({:?}) msg({})", err.range(), &text[err.range()], err)
            .unwrap();
    }
    acc
}

fn fragment_parser_dir_test<T, F>(ok_paths: &[&str], err_paths: &[&str], f: F)
where
    T: crate::AstNode,
    F: Fn(&str) -> Result<T, ()>,
{
    dir_tests(&test_data_dir(), ok_paths, "rast", |text, path| {
        if let Ok(node) = f(text) {
            format!("{:#?}", crate::ast::AstNode::syntax(&node))
        } else {
            panic!("Failed to parse '{:?}'", path);
        }
    });
    dir_tests(&test_data_dir(), err_paths, "rast", |text, path| {
        if let Ok(_) = f(text) {
            panic!("'{:?}' successfully parsed when it should have errored", path);
        } else {
            "ERROR\n".to_owned()
        }
    });
}

/// Calls callback `f` with input code and file paths for each `.rs` file in `test_data_dir`
/// subdirectories defined by `paths`.
///
/// If the content of the matching output file differs from the output of `f()`
/// the test will fail.
///
/// If there is no matching output file it will be created and filled with the
/// output of `f()`, but the test will fail.
fn dir_tests<F>(test_data_dir: &Path, paths: &[&str], outfile_extension: &str, f: F)
where
    F: Fn(&str, &Path) -> String,
{
    for (path, input_code) in collect_rust_files(test_data_dir, paths) {
        let actual = f(&input_code, &path);
        let path = path.with_extension(outfile_extension);
        expect_file![path].assert_eq(&actual)
    }
}

/// Collects all `.rs` files from `dir` subdirectories defined by `paths`.
fn collect_rust_files(root_dir: &Path, paths: &[&str]) -> Vec<(PathBuf, String)> {
    paths
        .iter()
        .flat_map(|path| {
            let path = root_dir.to_owned().join(path);
            rust_files_in_dir(&path).into_iter()
        })
        .map(|path| {
            let text = read_text(&path);
            (path, text)
        })
        .collect()
}

/// Collects paths to all `.rs` files from `dir` in a sorted `Vec<PathBuf>`.
fn rust_files_in_dir(dir: &Path) -> Vec<PathBuf> {
    let mut acc = Vec::new();
    for file in fs::read_dir(&dir).unwrap() {
        let file = file.unwrap();
        let path = file.path();
        if path.extension().unwrap_or_default() == "rs" {
            acc.push(path);
        }
    }
    acc.sort();
    acc
}

/// Read file and normalize newlines.
///
/// `rustc` seems to always normalize `\r\n` newlines to `\n`:
///
/// ```
/// let s = "
/// ";
/// assert_eq!(s.as_bytes(), &[10]);
/// ```
///
/// so this should always be correct.
fn read_text(path: &Path) -> String {
    fs::read_to_string(path)
        .unwrap_or_else(|_| panic!("File at {:?} should be valid", path))
        .replace("\r\n", "\n")
}
<|endoftext|>


<commit_before>//! Endpoint for searching and discovery functionality

use diesel::dsl::*;
use diesel_full_text_search::*;

use crate::controllers::helpers::Paginate;
use crate::controllers::prelude::*;
use crate::models::{Crate, CrateBadge, CrateVersions, OwnerKind, Version};
use crate::schema::*;
use crate::views::EncodableCrate;

use crate::models::krate::{canon_crate_name, ALL_COLUMNS};

/// Handles the `GET /crates` route.
/// Returns a list of crates. Called in a variety of scenarios in the
/// front end, including:
/// - Alphabetical listing of crates
/// - List of crates under a specific owner
/// - Listing a user's followed crates
///
/// Notes:
/// The different use cases this function covers is handled through passing
/// in parameters in the GET request.
///
/// We would like to stop adding functionality in here. It was built like
/// this to keep the number of database queries low, though given Rust's
/// low performance overhead, this is a soft goal to have, and can afford
/// more database transactions if it aids understandability.
///
/// All of the edge cases for this function are not currently covered
/// in testing, and if they fail, it is difficult to determine what
/// caused the break. In the future, we should look at splitting this
/// function out to cover the different use cases, and create unit tests
/// for them.
pub fn search(req: &mut dyn Request) -> CargoResult<Response> {
    use diesel::sql_types::{Bool, Text};

    let conn = req.db_conn()?;
    let params = req.query();
    let sort = params
        .get("sort")
        .map(|s| &**s)
        .unwrap_or("recent-downloads");
    let include_yanked = params
        .get("include_yanked")
        .map(|s| s == "yes")
        .unwrap_or(true);

    let selection = (
        ALL_COLUMNS,
        false.into_sql::<Bool>(),
        recent_crate_downloads::downloads.nullable(),
    );
    let mut query = crates::table
        .left_join(recent_crate_downloads::table)
        .select(selection)
        .into_boxed();

    if let Some(q_string) = params.get("q") {
        if !q_string.is_empty() {
            let sort = params.get("sort").map(|s| &**s).unwrap_or("relevance");

            let q = sql::<TsQuery>("plainto_tsquery('english', ")
                .bind::<Text, _>(q_string)
                .sql(")");
            query = query.filter(
                q.clone()
                    .matches(crates::textsearchable_index_col)
                    .or(Crate::loosly_matches_name(&q_string)),
            );

            query = query.select((
                ALL_COLUMNS,
                Crate::with_name(q_string),
                recent_crate_downloads::downloads.nullable(),
            ));
            query = query.order(Crate::with_name(q_string).desc());

            if sort == "relevance" {
                let rank = ts_rank_cd(crates::textsearchable_index_col, q);
                query = query.then_order_by(rank.desc())
            }
        }
    }

    if let Some(cat) = params.get("category") {
        query = query.filter(
            crates::id.eq_any(
                crates_categories::table
                    .select(crates_categories::crate_id)
                    .inner_join(categories::table)
                    .filter(
                        categories::slug
                            .eq(cat)
                            .or(categories::slug.like(format!("{}::%", cat))),
                    ),
            ),
        );
    }

    if let Some(kw) = params.get("keyword") {
        query = query.filter(
            crates::id.eq_any(
                crates_keywords::table
                    .select(crates_keywords::crate_id)
                    .inner_join(keywords::table)
                    .filter(crate::lower(keywords::keyword).eq(crate::lower(kw))),
            ),
        );
    } else if let Some(letter) = params.get("letter") {
        let pattern = format!(
            "{}%",
            letter
                .chars()
                .next()
                .unwrap()
                .to_lowercase()
                .collect::<String>()
        );
        query = query.filter(canon_crate_name(crates::name).like(pattern));
    } else if let Some(user_id) = params.get("user_id").and_then(|s| s.parse::<i32>().ok()) {
        query = query.filter(
            crates::id.eq_any(
                crate_owners::table
                    .select(crate_owners::crate_id)
                    .filter(crate_owners::owner_id.eq(user_id))
                    .filter(crate_owners::deleted.eq(false))
                    .filter(crate_owners::owner_kind.eq(OwnerKind::User as i32)),
            ),
        );
    } else if let Some(team_id) = params.get("team_id").and_then(|s| s.parse::<i32>().ok()) {
        query = query.filter(
            crates::id.eq_any(
                crate_owners::table
                    .select(crate_owners::crate_id)
                    .filter(crate_owners::owner_id.eq(team_id))
                    .filter(crate_owners::deleted.eq(false))
                    .filter(crate_owners::owner_kind.eq(OwnerKind::Team as i32)),
            ),
        );
    } else if params.get("following").is_some() {
        query = query.filter(
            crates::id.eq_any(
                follows::table
                    .select(follows::crate_id)
                    .filter(follows::user_id.eq(req.user()?.id)),
            ),
        );
    }

    if !include_yanked {
        query = query.filter(exists(
            versions::table
                .filter(versions::crate_id.eq(crates::id))
                .filter(versions::yanked.eq(false)),
        ));
    }

    if sort == "downloads" {
        query = query.then_order_by(crates::downloads.desc())
    } else if sort == "recent-downloads" {
        query = query.then_order_by(recent_crate_downloads::downloads.desc().nulls_last())
    } else if sort == "recent-updates" {
        query = query.order(crates::updated_at.desc());
    } else {
        query = query.then_order_by(crates::name.asc())
    }

    let data = query
        .paginate(&req.query())?
        .load::<(Crate, bool, Option<i64>)>(&*conn)?;
    let total = data.total();

    let next_page = data.next_page_params().map(|p| req.query_with_params(p));
    let prev_page = data.prev_page_params().map(|p| req.query_with_params(p));

    let perfect_matches = data.iter().map(|&(_, b, _)| b).collect::<Vec<_>>();
    let recent_downloads = data
        .iter()
        .map(|&(_, _, s)| s.unwrap_or(0))
        .collect::<Vec<_>>();
    let crates = data.into_iter().map(|(c, _, _)| c).collect::<Vec<_>>();

    let versions = crates
        .versions()
        .load::<Version>(&*conn)?
        .grouped_by(&crates)
        .into_iter()
        .map(|versions| Version::max(versions.into_iter().map(|v| v.num)));

    let badges = CrateBadge::belonging_to(&crates)
        .select((badges::crate_id, badges::all_columns))
        .load::<CrateBadge>(&*conn)?
        .grouped_by(&crates)
        .into_iter()
        .map(|badges| badges.into_iter().map(|cb| cb.badge).collect());

    let crates = versions
        .zip(crates)
        .zip(perfect_matches)
        .zip(recent_downloads)
        .zip(badges)
        .map(
            |((((max_version, krate), perfect_match), recent_downloads), badges)| {
                krate.minimal_encodable(
                    &max_version,
                    Some(badges),
                    perfect_match,
                    Some(recent_downloads),
                )
            },
        )
        .collect();

    #[derive(Serialize)]
    struct R {
        crates: Vec<EncodableCrate>,
        meta: Meta,
    }
    #[derive(Serialize)]
    struct Meta {
        total: Option<i64>,
        next_page: Option<String>,
        prev_page: Option<String>,
    }

    Ok(req.json(&R {
        crates,
        meta: Meta {
            total,
            next_page,
            prev_page,
        },
    }))
}
<commit_msg>Auto merge of #1809 - sgrif:sg-change-default-search-ordering, r=jtgeibel<commit_after>//! Endpoint for searching and discovery functionality

use diesel::dsl::*;
use diesel_full_text_search::*;

use crate::controllers::helpers::Paginate;
use crate::controllers::prelude::*;
use crate::models::{Crate, CrateBadge, CrateVersions, OwnerKind, Version};
use crate::schema::*;
use crate::views::EncodableCrate;

use crate::models::krate::{canon_crate_name, ALL_COLUMNS};

/// Handles the `GET /crates` route.
/// Returns a list of crates. Called in a variety of scenarios in the
/// front end, including:
/// - Alphabetical listing of crates
/// - List of crates under a specific owner
/// - Listing a user's followed crates
///
/// Notes:
/// The different use cases this function covers is handled through passing
/// in parameters in the GET request.
///
/// We would like to stop adding functionality in here. It was built like
/// this to keep the number of database queries low, though given Rust's
/// low performance overhead, this is a soft goal to have, and can afford
/// more database transactions if it aids understandability.
///
/// All of the edge cases for this function are not currently covered
/// in testing, and if they fail, it is difficult to determine what
/// caused the break. In the future, we should look at splitting this
/// function out to cover the different use cases, and create unit tests
/// for them.
pub fn search(req: &mut dyn Request) -> CargoResult<Response> {
    use diesel::sql_types::{Bool, Text};

    let conn = req.db_conn()?;
    let params = req.query();
    let sort = params.get("sort").map(|s| &**s);
    let include_yanked = params
        .get("include_yanked")
        .map(|s| s == "yes")
        .unwrap_or(true);

    let selection = (
        ALL_COLUMNS,
        false.into_sql::<Bool>(),
        recent_crate_downloads::downloads.nullable(),
    );
    let mut query = crates::table
        .left_join(recent_crate_downloads::table)
        .select(selection)
        .into_boxed();

    if let Some(q_string) = params.get("q") {
        if !q_string.is_empty() {
            let sort = params.get("sort").map(|s| &**s).unwrap_or("relevance");

            let q = sql::<TsQuery>("plainto_tsquery('english', ")
                .bind::<Text, _>(q_string)
                .sql(")");
            query = query.filter(
                q.clone()
                    .matches(crates::textsearchable_index_col)
                    .or(Crate::loosly_matches_name(&q_string)),
            );

            query = query.select((
                ALL_COLUMNS,
                Crate::with_name(q_string),
                recent_crate_downloads::downloads.nullable(),
            ));
            query = query.order(Crate::with_name(q_string).desc());

            if sort == "relevance" {
                let rank = ts_rank_cd(crates::textsearchable_index_col, q);
                query = query.then_order_by(rank.desc())
            }
        }
    }

    if let Some(cat) = params.get("category") {
        query = query.filter(
            crates::id.eq_any(
                crates_categories::table
                    .select(crates_categories::crate_id)
                    .inner_join(categories::table)
                    .filter(
                        categories::slug
                            .eq(cat)
                            .or(categories::slug.like(format!("{}::%", cat))),
                    ),
            ),
        );
    }

    if let Some(kw) = params.get("keyword") {
        query = query.filter(
            crates::id.eq_any(
                crates_keywords::table
                    .select(crates_keywords::crate_id)
                    .inner_join(keywords::table)
                    .filter(crate::lower(keywords::keyword).eq(crate::lower(kw))),
            ),
        );
    } else if let Some(letter) = params.get("letter") {
        let pattern = format!(
            "{}%",
            letter
                .chars()
                .next()
                .unwrap()
                .to_lowercase()
                .collect::<String>()
        );
        query = query.filter(canon_crate_name(crates::name).like(pattern));
    } else if let Some(user_id) = params.get("user_id").and_then(|s| s.parse::<i32>().ok()) {
        query = query.filter(
            crates::id.eq_any(
                crate_owners::table
                    .select(crate_owners::crate_id)
                    .filter(crate_owners::owner_id.eq(user_id))
                    .filter(crate_owners::deleted.eq(false))
                    .filter(crate_owners::owner_kind.eq(OwnerKind::User as i32)),
            ),
        );
    } else if let Some(team_id) = params.get("team_id").and_then(|s| s.parse::<i32>().ok()) {
        query = query.filter(
            crates::id.eq_any(
                crate_owners::table
                    .select(crate_owners::crate_id)
                    .filter(crate_owners::owner_id.eq(team_id))
                    .filter(crate_owners::deleted.eq(false))
                    .filter(crate_owners::owner_kind.eq(OwnerKind::Team as i32)),
            ),
        );
    } else if params.get("following").is_some() {
        query = query.filter(
            crates::id.eq_any(
                follows::table
                    .select(follows::crate_id)
                    .filter(follows::user_id.eq(req.user()?.id)),
            ),
        );
    }

    if !include_yanked {
        query = query.filter(exists(
            versions::table
                .filter(versions::crate_id.eq(crates::id))
                .filter(versions::yanked.eq(false)),
        ));
    }

    if sort == Some("downloads") {
        query = query.then_order_by(crates::downloads.desc())
    } else if sort == Some("recent-downloads") {
        query = query.then_order_by(recent_crate_downloads::downloads.desc().nulls_last())
    } else if sort == Some("recent-updates") {
        query = query.order(crates::updated_at.desc());
    } else {
        query = query.then_order_by(crates::name.asc())
    }

    let data = query
        .paginate(&req.query())?
        .load::<(Crate, bool, Option<i64>)>(&*conn)?;
    let total = data.total();

    let next_page = data.next_page_params().map(|p| req.query_with_params(p));
    let prev_page = data.prev_page_params().map(|p| req.query_with_params(p));

    let perfect_matches = data.iter().map(|&(_, b, _)| b).collect::<Vec<_>>();
    let recent_downloads = data
        .iter()
        .map(|&(_, _, s)| s.unwrap_or(0))
        .collect::<Vec<_>>();
    let crates = data.into_iter().map(|(c, _, _)| c).collect::<Vec<_>>();

    let versions = crates
        .versions()
        .load::<Version>(&*conn)?
        .grouped_by(&crates)
        .into_iter()
        .map(|versions| Version::max(versions.into_iter().map(|v| v.num)));

    let badges = CrateBadge::belonging_to(&crates)
        .select((badges::crate_id, badges::all_columns))
        .load::<CrateBadge>(&*conn)?
        .grouped_by(&crates)
        .into_iter()
        .map(|badges| badges.into_iter().map(|cb| cb.badge).collect());

    let crates = versions
        .zip(crates)
        .zip(perfect_matches)
        .zip(recent_downloads)
        .zip(badges)
        .map(
            |((((max_version, krate), perfect_match), recent_downloads), badges)| {
                krate.minimal_encodable(
                    &max_version,
                    Some(badges),
                    perfect_match,
                    Some(recent_downloads),
                )
            },
        )
        .collect();

    #[derive(Serialize)]
    struct R {
        crates: Vec<EncodableCrate>,
        meta: Meta,
    }
    #[derive(Serialize)]
    struct Meta {
        total: Option<i64>,
        next_page: Option<String>,
        prev_page: Option<String>,
    }

    Ok(req.json(&R {
        crates,
        meta: Meta {
            total,
            next_page,
            prev_page,
        },
    }))
}
<|endoftext|>


<commit_before>// This file is part of the uutils coreutils package.
//
// (c) Michael Gehring <mg@ebfe.org>
//
//  For the full copyright and license information, please view the LICENSE
// file that was distributed with this source code.

// spell-checker:ignore (ToDO) fname

#[macro_use]
extern crate uucore;

use clap::{crate_version, App, Arg};
use std::fs::File;
use std::io::{self, stdin, BufReader, Read};
use std::path::Path;
use uucore::display::Quotable;
use uucore::InvalidEncodingHandling;

// NOTE: CRC_TABLE_LEN *must* be <= 256 as we cast 0..CRC_TABLE_LEN to u8
const CRC_TABLE_LEN: usize = 256;
const CRC_TABLE: [u32; CRC_TABLE_LEN] = generate_crc_table();

const NAME: &str = "cksum";
const SYNTAX: &str = "[OPTIONS] [FILE]...";
const SUMMARY: &str = "Print CRC and size for each file";

const fn generate_crc_table() -> [u32; CRC_TABLE_LEN] {
    let mut table = [0; CRC_TABLE_LEN];

    let mut i = 0;
    while i < CRC_TABLE_LEN {
        table[i] = crc_entry(i as u8) as u32;

        i += 1;
    }

    table
}

const fn crc_entry(input: u8) -> u32 {
    let mut crc = (input as u32) << 24;

    let mut i = 0;
    while i < 8 {
        let if_condition = crc & 0x8000_0000;
        let if_body = (crc << 1) ^ 0x04c1_1db7;
        let else_body = crc << 1;

        // NOTE: i feel like this is easier to understand than emulating an if statement in bitwise
        //       ops
        let condition_table = [else_body, if_body];

        crc = condition_table[(if_condition != 0) as usize];
        i += 1;
    }

    crc
}

#[inline]
fn crc_update(crc: u32, input: u8) -> u32 {
    (crc << 8) ^ CRC_TABLE[((crc >> 24) as usize ^ input as usize) & 0xFF]
}

#[inline]
fn crc_final(mut crc: u32, mut length: usize) -> u32 {
    while length != 0 {
        crc = crc_update(crc, length as u8);
        length >>= 8;
    }

    !crc
}

fn init_byte_array() -> Vec<u8> {
    vec![0; 1024 * 1024]
}

#[inline]
fn cksum(fname: &str) -> io::Result<(u32, usize)> {
    let mut crc = 0u32;
    let mut size = 0usize;

    let file;
    let mut rd: Box<dyn Read> = match fname {
        "-" => Box::new(stdin()),
        _ => {
            let path = &Path::new(fname);
            if path.is_dir() {
                return Err(std::io::Error::new(
                    io::ErrorKind::InvalidInput,
                    "Is a directory",
                ));
            };
            // Silent the warning as we want to the error message
            #[allow(clippy::question_mark)]
            if path.metadata().is_err() {
                return Err(std::io::Error::new(
                    io::ErrorKind::NotFound,
                    "No such file or directory",
                ));
            };
            file = File::open(&path)?;
            Box::new(BufReader::new(file))
        }
    };

    let mut bytes = init_byte_array();
    loop {
        let num_bytes = rd.read(&mut bytes)?;
        if num_bytes == 0 {
            return Ok((crc_final(crc, size), size));
        }
        for &b in bytes[..num_bytes].iter() {
            crc = crc_update(crc, b);
        }
        size += num_bytes;
    }
}

mod options {
    pub static FILE: &str = "file";
}

pub fn uumain(args: impl uucore::Args) -> i32 {
    let args = args
        .collect_str(InvalidEncodingHandling::Ignore)
        .accept_any();

    let matches = uu_app().get_matches_from(args);

    let files: Vec<String> = match matches.values_of(options::FILE) {
        Some(v) => v.clone().map(|v| v.to_owned()).collect(),
        None => vec![],
    };

    if files.is_empty() {
        match cksum("-") {
            Ok((crc, size)) => println!("{} {}", crc, size),
            Err(err) => {
                show_error!("-: {}", err);
                return 2;
            }
        }
        return 0;
    }

    let mut exit_code = 0;
    for fname in &files {
        match cksum(fname.as_ref()) {
            Ok((crc, size)) => println!("{} {} {}", crc, size, fname),
            Err(err) => {
                show_error!("{}: {}", fname.maybe_quote(), err);
                exit_code = 2;
            }
        }
    }

    exit_code
}

pub fn uu_app() -> App<'static, 'static> {
    App::new(uucore::util_name())
        .name(NAME)
        .version(crate_version!())
        .about(SUMMARY)
        .usage(SYNTAX)
        .arg(Arg::with_name(options::FILE).hidden(true).multiple(true))
}
<commit_msg>cksum: return UResult from uumain() function<commit_after>// This file is part of the uutils coreutils package.
//
// (c) Michael Gehring <mg@ebfe.org>
//
//  For the full copyright and license information, please view the LICENSE
// file that was distributed with this source code.

// spell-checker:ignore (ToDO) fname
use clap::{crate_version, App, Arg};
use std::fs::File;
use std::io::{self, stdin, BufReader, Read};
use std::path::Path;
use uucore::display::Quotable;
use uucore::error::UResult;
use uucore::error::USimpleError;
use uucore::show;
use uucore::InvalidEncodingHandling;

// NOTE: CRC_TABLE_LEN *must* be <= 256 as we cast 0..CRC_TABLE_LEN to u8
const CRC_TABLE_LEN: usize = 256;
const CRC_TABLE: [u32; CRC_TABLE_LEN] = generate_crc_table();

const NAME: &str = "cksum";
const SYNTAX: &str = "[OPTIONS] [FILE]...";
const SUMMARY: &str = "Print CRC and size for each file";

const fn generate_crc_table() -> [u32; CRC_TABLE_LEN] {
    let mut table = [0; CRC_TABLE_LEN];

    let mut i = 0;
    while i < CRC_TABLE_LEN {
        table[i] = crc_entry(i as u8) as u32;

        i += 1;
    }

    table
}

const fn crc_entry(input: u8) -> u32 {
    let mut crc = (input as u32) << 24;

    let mut i = 0;
    while i < 8 {
        let if_condition = crc & 0x8000_0000;
        let if_body = (crc << 1) ^ 0x04c1_1db7;
        let else_body = crc << 1;

        // NOTE: i feel like this is easier to understand than emulating an if statement in bitwise
        //       ops
        let condition_table = [else_body, if_body];

        crc = condition_table[(if_condition != 0) as usize];
        i += 1;
    }

    crc
}

#[inline]
fn crc_update(crc: u32, input: u8) -> u32 {
    (crc << 8) ^ CRC_TABLE[((crc >> 24) as usize ^ input as usize) & 0xFF]
}

#[inline]
fn crc_final(mut crc: u32, mut length: usize) -> u32 {
    while length != 0 {
        crc = crc_update(crc, length as u8);
        length >>= 8;
    }

    !crc
}

fn init_byte_array() -> Vec<u8> {
    vec![0; 1024 * 1024]
}

#[inline]
fn cksum(fname: &str) -> io::Result<(u32, usize)> {
    let mut crc = 0u32;
    let mut size = 0usize;

    let file;
    let mut rd: Box<dyn Read> = match fname {
        "-" => Box::new(stdin()),
        _ => {
            let path = &Path::new(fname);
            if path.is_dir() {
                return Err(std::io::Error::new(
                    io::ErrorKind::InvalidInput,
                    "Is a directory",
                ));
            };
            // Silent the warning as we want to the error message
            #[allow(clippy::question_mark)]
            if path.metadata().is_err() {
                return Err(std::io::Error::new(
                    io::ErrorKind::NotFound,
                    "No such file or directory",
                ));
            };
            file = File::open(&path)?;
            Box::new(BufReader::new(file))
        }
    };

    let mut bytes = init_byte_array();
    loop {
        let num_bytes = rd.read(&mut bytes)?;
        if num_bytes == 0 {
            return Ok((crc_final(crc, size), size));
        }
        for &b in bytes[..num_bytes].iter() {
            crc = crc_update(crc, b);
        }
        size += num_bytes;
    }
}

mod options {
    pub static FILE: &str = "file";
}

#[uucore_procs::gen_uumain]
pub fn uumain(args: impl uucore::Args) -> UResult<()> {
    let args = args
        .collect_str(InvalidEncodingHandling::Ignore)
        .accept_any();

    let matches = uu_app().get_matches_from(args);

    let files: Vec<String> = match matches.values_of(options::FILE) {
        Some(v) => v.clone().map(|v| v.to_owned()).collect(),
        None => vec![],
    };

    if files.is_empty() {
        match cksum("-") {
            Ok((crc, size)) => println!("{} {}", crc, size),
            Err(err) => {
                return Err(USimpleError::new(2, format!("{}", err)));
            }
        }
        return Ok(());
    }

    for fname in &files {
        match cksum(fname.as_ref()) {
            Ok((crc, size)) => println!("{} {} {}", crc, size, fname),
            Err(err) => show!(USimpleError::new(
                2,
                format!("{}: {}", fname.maybe_quote(), err)
            )),
        }
    }
    Ok(())
}

pub fn uu_app() -> App<'static, 'static> {
    App::new(uucore::util_name())
        .name(NAME)
        .version(crate_version!())
        .about(SUMMARY)
        .usage(SYNTAX)
        .arg(Arg::with_name(options::FILE).hidden(true).multiple(true))
}
<|endoftext|>