<commit_before>use libc::{c_char, c_int};
use pyport::Py_ssize_t;
use object::*;

extern "C" {
    pub fn PyErr_SetNone(arg1: *mut PyObject) -> ();
    pub fn PyErr_SetObject(arg1: *mut PyObject, arg2: *mut PyObject) -> ();
    pub fn PyErr_SetString(exception: *mut PyObject,
                           string: *const c_char) -> ();
    pub fn PyErr_Occurred() -> *mut PyObject;
    pub fn PyErr_Clear() -> ();
    pub fn PyErr_Fetch(arg1: *mut *mut PyObject, arg2: *mut *mut PyObject,
                       arg3: *mut *mut PyObject) -> ();
    pub fn PyErr_Restore(arg1: *mut PyObject, arg2: *mut PyObject,
                         arg3: *mut PyObject) -> ();
    pub fn PyErr_GetExcInfo(arg1: *mut *mut PyObject,
                            arg2: *mut *mut PyObject,
                            arg3: *mut *mut PyObject) -> ();
    pub fn PyErr_SetExcInfo(arg1: *mut PyObject, arg2: *mut PyObject,
                            arg3: *mut PyObject) -> ();
    pub fn Py_FatalError(message: *const c_char) -> !;
    pub fn PyErr_GivenExceptionMatches(arg1: *mut PyObject,
                                       arg2: *mut PyObject) -> c_int;
    pub fn PyErr_ExceptionMatches(arg1: *mut PyObject) -> c_int;
    pub fn PyErr_NormalizeException(arg1: *mut *mut PyObject,
                                    arg2: *mut *mut PyObject,
                                    arg3: *mut *mut PyObject) -> ();
    pub fn PyException_SetTraceback(arg1: *mut PyObject, arg2: *mut PyObject)
     -> c_int;
    pub fn PyException_GetTraceback(arg1: *mut PyObject) -> *mut PyObject;
    pub fn PyException_GetCause(arg1: *mut PyObject) -> *mut PyObject;
    pub fn PyException_SetCause(arg1: *mut PyObject, arg2: *mut PyObject)
     -> ();
    pub fn PyException_GetContext(arg1: *mut PyObject) -> *mut PyObject;
    pub fn PyException_SetContext(arg1: *mut PyObject, arg2: *mut PyObject)
     -> ();
}

#[inline]
pub unsafe fn PyExceptionClass_Check(x: *mut PyObject) -> c_int {
    (PyType_Check(x) != 0 &&
     PyType_FastSubclass(x as *mut PyTypeObject, Py_TPFLAGS_BASE_EXC_SUBCLASS) != 0) as c_int
}

#[inline]
pub unsafe fn PyExceptionInstance_Check(x: *mut PyObject) -> c_int {
    PyType_FastSubclass((*x).ob_type, Py_TPFLAGS_BASE_EXC_SUBCLASS)
}

#[inline]
pub unsafe fn PyExceptionInstance_Class(x: *mut PyObject) -> *mut PyObject {
    (*x).ob_type as *mut PyObject
}

extern "C" {
    pub static mut PyExc_BaseException: *mut PyObject;
    pub static mut PyExc_Exception: *mut PyObject;
    pub static mut PyExc_StopIteration: *mut PyObject;
    pub static mut PyExc_GeneratorExit: *mut PyObject;
    pub static mut PyExc_ArithmeticError: *mut PyObject;
    pub static mut PyExc_LookupError: *mut PyObject;
    pub static mut PyExc_AssertionError: *mut PyObject;
    pub static mut PyExc_AttributeError: *mut PyObject;
    pub static mut PyExc_BufferError: *mut PyObject;
    pub static mut PyExc_EOFError: *mut PyObject;
    pub static mut PyExc_FloatingPointError: *mut PyObject;
    pub static mut PyExc_OSError: *mut PyObject;
    pub static mut PyExc_ImportError: *mut PyObject;
    #[cfg(Py_3_6)]
    pub static mut PyExc_ModuleNotFoundError: *mut PyObject;
    pub static mut PyExc_IndexError: *mut PyObject;
    pub static mut PyExc_KeyError: *mut PyObject;
    pub static mut PyExc_KeyboardInterrupt: *mut PyObject;
    pub static mut PyExc_MemoryError: *mut PyObject;
    pub static mut PyExc_NameError: *mut PyObject;
    pub static mut PyExc_OverflowError: *mut PyObject;
    pub static mut PyExc_RuntimeError: *mut PyObject;
    pub static mut PyExc_NotImplementedError: *mut PyObject;
    pub static mut PyExc_SyntaxError: *mut PyObject;
    pub static mut PyExc_IndentationError: *mut PyObject;
    pub static mut PyExc_TabError: *mut PyObject;
    pub static mut PyExc_ReferenceError: *mut PyObject;
    pub static mut PyExc_SystemError: *mut PyObject;
    pub static mut PyExc_SystemExit: *mut PyObject;
    pub static mut PyExc_TypeError: *mut PyObject;
    pub static mut PyExc_UnboundLocalError: *mut PyObject;
    pub static mut PyExc_UnicodeError: *mut PyObject;
    pub static mut PyExc_UnicodeEncodeError: *mut PyObject;
    pub static mut PyExc_UnicodeDecodeError: *mut PyObject;
    pub static mut PyExc_UnicodeTranslateError: *mut PyObject;
    pub static mut PyExc_ValueError: *mut PyObject;
    pub static mut PyExc_ZeroDivisionError: *mut PyObject;
    pub static mut PyExc_BlockingIOError: *mut PyObject;
    pub static mut PyExc_BrokenPipeError: *mut PyObject;
    pub static mut PyExc_ChildProcessError: *mut PyObject;
    pub static mut PyExc_ConnectionError: *mut PyObject;
    pub static mut PyExc_ConnectionAbortedError: *mut PyObject;
    pub static mut PyExc_ConnectionRefusedError: *mut PyObject;
    pub static mut PyExc_ConnectionResetError: *mut PyObject;
    pub static mut PyExc_FileExistsError: *mut PyObject;
    pub static mut PyExc_FileNotFoundError: *mut PyObject;
    pub static mut PyExc_InterruptedError: *mut PyObject;
    pub static mut PyExc_IsADirectoryError: *mut PyObject;
    pub static mut PyExc_NotADirectoryError: *mut PyObject;
    pub static mut PyExc_PermissionError: *mut PyObject;
    pub static mut PyExc_ProcessLookupError: *mut PyObject;
    pub static mut PyExc_TimeoutError: *mut PyObject;
    pub static mut PyExc_EnvironmentError: *mut PyObject;
    pub static mut PyExc_IOError: *mut PyObject;
    #[cfg(windows)] pub static mut PyExc_WindowsError: *mut PyObject;
    pub static mut PyExc_RecursionErrorInst: *mut PyObject;
    pub static mut PyExc_Warning: *mut PyObject;
    pub static mut PyExc_UserWarning: *mut PyObject;
    pub static mut PyExc_DeprecationWarning: *mut PyObject;
    pub static mut PyExc_PendingDeprecationWarning: *mut PyObject;
    pub static mut PyExc_SyntaxWarning: *mut PyObject;
    pub static mut PyExc_RuntimeWarning: *mut PyObject;
    pub static mut PyExc_FutureWarning: *mut PyObject;
    pub static mut PyExc_ImportWarning: *mut PyObject;
    pub static mut PyExc_UnicodeWarning: *mut PyObject;
    pub static mut PyExc_BytesWarning: *mut PyObject;
    pub static mut PyExc_ResourceWarning: *mut PyObject;

    pub fn PyErr_BadArgument() -> c_int;
    pub fn PyErr_NoMemory() -> *mut PyObject;
    pub fn PyErr_SetFromErrno(arg1: *mut PyObject) -> *mut PyObject;
    pub fn PyErr_SetFromErrnoWithFilenameObject(arg1: *mut PyObject,
                                                arg2: *mut PyObject)
     -> *mut PyObject;
    #[cfg(Py_3_4)]
    pub fn PyErr_SetFromErrnoWithFilenameObjects(arg1: *mut PyObject,
                                                 arg2: *mut PyObject,
                                                 arg3: *mut PyObject)
     -> *mut PyObject;
    pub fn PyErr_SetFromErrnoWithFilename(exc: *mut PyObject,
                                          filename: *const c_char)
     -> *mut PyObject;
    pub fn PyErr_Format(exception: *mut PyObject,
                        format: *const c_char, ...) -> *mut PyObject;
    pub fn PyErr_SetImportErrorSubclass(
        arg1: *mut PyObject, arg2: *mut PyObject,
        arg3: *mut PyObject, arg4: *mut PyObject) -> *mut PyObject;
    pub fn PyErr_SetImportError(arg1: *mut PyObject, arg2: *mut PyObject,
                                arg3: *mut PyObject) -> *mut PyObject;
    pub fn PyErr_BadInternalCall() -> ();
    pub fn _PyErr_BadInternalCall(filename: *const c_char,
                                  lineno: c_int) -> ();
    pub fn PyErr_NewException(name: *const c_char,
                              base: *mut PyObject, dict: *mut PyObject)
     -> *mut PyObject;
    pub fn PyErr_NewExceptionWithDoc(name: *const c_char,
                                     doc: *const c_char,
                                     base: *mut PyObject, dict: *mut PyObject)
     -> *mut PyObject;
    pub fn PyErr_WriteUnraisable(arg1: *mut PyObject) -> ();
    pub fn PyErr_CheckSignals() -> c_int;
    pub fn PyErr_SetInterrupt() -> ();
    pub fn PyErr_SyntaxLocation(filename: *const c_char,
                                lineno: c_int) -> ();
    pub fn PyErr_SyntaxLocationEx(filename: *const c_char,
                                  lineno: c_int,
                                  col_offset: c_int) -> ();
    pub fn PyErr_ProgramText(filename: *const c_char,
                             lineno: c_int) -> *mut PyObject;
    pub fn PyUnicodeDecodeError_Create(encoding: *const c_char,
                                       object: *const c_char,
                                       length: Py_ssize_t, start: Py_ssize_t,
                                       end: Py_ssize_t,
                                       reason: *const c_char)
     -> *mut PyObject;
    pub fn PyUnicodeEncodeError_GetEncoding(arg1: *mut PyObject)
     -> *mut PyObject;
    pub fn PyUnicodeDecodeError_GetEncoding(arg1: *mut PyObject)
     -> *mut PyObject;
    pub fn PyUnicodeEncodeError_GetObject(arg1: *mut PyObject)
     -> *mut PyObject;
    pub fn PyUnicodeDecodeError_GetObject(arg1: *mut PyObject)
     -> *mut PyObject;
    pub fn PyUnicodeTranslateError_GetObject(arg1: *mut PyObject)
     -> *mut PyObject;
    pub fn PyUnicodeEncodeError_GetStart(arg1: *mut PyObject,
                                         arg2: *mut Py_ssize_t)
     -> c_int;
    pub fn PyUnicodeDecodeError_GetStart(arg1: *mut PyObject,
                                         arg2: *mut Py_ssize_t)
     -> c_int;
    pub fn PyUnicodeTranslateError_GetStart(arg1: *mut PyObject,
                                            arg2: *mut Py_ssize_t)
     -> c_int;
    pub fn PyUnicodeEncodeError_SetStart(arg1: *mut PyObject,
                                         arg2: Py_ssize_t) -> c_int;
    pub fn PyUnicodeDecodeError_SetStart(arg1: *mut PyObject,
                                         arg2: Py_ssize_t) -> c_int;
    pub fn PyUnicodeTranslateError_SetStart(arg1: *mut PyObject,
                                            arg2: Py_ssize_t)
     -> c_int;
    pub fn PyUnicodeEncodeError_GetEnd(arg1: *mut PyObject,
                                       arg2: *mut Py_ssize_t)
     -> c_int;
    pub fn PyUnicodeDecodeError_GetEnd(arg1: *mut PyObject,
                                       arg2: *mut Py_ssize_t)
     -> c_int;
    pub fn PyUnicodeTranslateError_GetEnd(arg1: *mut PyObject,
                                          arg2: *mut Py_ssize_t)
     -> c_int;
    pub fn PyUnicodeEncodeError_SetEnd(arg1: *mut PyObject, arg2: Py_ssize_t)
     -> c_int;
    pub fn PyUnicodeDecodeError_SetEnd(arg1: *mut PyObject, arg2: Py_ssize_t)
     -> c_int;
    pub fn PyUnicodeTranslateError_SetEnd(arg1: *mut PyObject,
                                          arg2: Py_ssize_t) -> c_int;
    pub fn PyUnicodeEncodeError_GetReason(arg1: *mut PyObject)
     -> *mut PyObject;
    pub fn PyUnicodeDecodeError_GetReason(arg1: *mut PyObject)
     -> *mut PyObject;
    pub fn PyUnicodeTranslateError_GetReason(arg1: *mut PyObject)
     -> *mut PyObject;
    pub fn PyUnicodeEncodeError_SetReason(exc: *mut PyObject,
                                          reason: *const c_char)
     -> c_int;
    pub fn PyUnicodeDecodeError_SetReason(exc: *mut PyObject,
                                          reason: *const c_char)
     -> c_int;
    pub fn PyUnicodeTranslateError_SetReason(exc: *mut PyObject,
                                             reason: *const c_char)
     -> c_int;
}

<commit_msg>Add missing #[cfg(Py_3_6)]<commit_after>use libc::{c_char, c_int};
use pyport::Py_ssize_t;
use object::*;

extern "C" {
    pub fn PyErr_SetNone(arg1: *mut PyObject) -> ();
    pub fn PyErr_SetObject(arg1: *mut PyObject, arg2: *mut PyObject) -> ();
    pub fn PyErr_SetString(exception: *mut PyObject,
                           string: *const c_char) -> ();
    pub fn PyErr_Occurred() -> *mut PyObject;
    pub fn PyErr_Clear() -> ();
    pub fn PyErr_Fetch(arg1: *mut *mut PyObject, arg2: *mut *mut PyObject,
                       arg3: *mut *mut PyObject) -> ();
    pub fn PyErr_Restore(arg1: *mut PyObject, arg2: *mut PyObject,
                         arg3: *mut PyObject) -> ();
    pub fn PyErr_GetExcInfo(arg1: *mut *mut PyObject,
                            arg2: *mut *mut PyObject,
                            arg3: *mut *mut PyObject) -> ();
    pub fn PyErr_SetExcInfo(arg1: *mut PyObject, arg2: *mut PyObject,
                            arg3: *mut PyObject) -> ();
    pub fn Py_FatalError(message: *const c_char) -> !;
    pub fn PyErr_GivenExceptionMatches(arg1: *mut PyObject,
                                       arg2: *mut PyObject) -> c_int;
    pub fn PyErr_ExceptionMatches(arg1: *mut PyObject) -> c_int;
    pub fn PyErr_NormalizeException(arg1: *mut *mut PyObject,
                                    arg2: *mut *mut PyObject,
                                    arg3: *mut *mut PyObject) -> ();
    pub fn PyException_SetTraceback(arg1: *mut PyObject, arg2: *mut PyObject)
     -> c_int;
    pub fn PyException_GetTraceback(arg1: *mut PyObject) -> *mut PyObject;
    pub fn PyException_GetCause(arg1: *mut PyObject) -> *mut PyObject;
    pub fn PyException_SetCause(arg1: *mut PyObject, arg2: *mut PyObject)
     -> ();
    pub fn PyException_GetContext(arg1: *mut PyObject) -> *mut PyObject;
    pub fn PyException_SetContext(arg1: *mut PyObject, arg2: *mut PyObject)
     -> ();
}

#[inline]
pub unsafe fn PyExceptionClass_Check(x: *mut PyObject) -> c_int {
    (PyType_Check(x) != 0 &&
     PyType_FastSubclass(x as *mut PyTypeObject, Py_TPFLAGS_BASE_EXC_SUBCLASS) != 0) as c_int
}

#[inline]
pub unsafe fn PyExceptionInstance_Check(x: *mut PyObject) -> c_int {
    PyType_FastSubclass((*x).ob_type, Py_TPFLAGS_BASE_EXC_SUBCLASS)
}

#[inline]
pub unsafe fn PyExceptionInstance_Class(x: *mut PyObject) -> *mut PyObject {
    (*x).ob_type as *mut PyObject
}

extern "C" {
    pub static mut PyExc_BaseException: *mut PyObject;
    pub static mut PyExc_Exception: *mut PyObject;
    pub static mut PyExc_StopIteration: *mut PyObject;
    pub static mut PyExc_GeneratorExit: *mut PyObject;
    pub static mut PyExc_ArithmeticError: *mut PyObject;
    pub static mut PyExc_LookupError: *mut PyObject;
    pub static mut PyExc_AssertionError: *mut PyObject;
    pub static mut PyExc_AttributeError: *mut PyObject;
    pub static mut PyExc_BufferError: *mut PyObject;
    pub static mut PyExc_EOFError: *mut PyObject;
    pub static mut PyExc_FloatingPointError: *mut PyObject;
    pub static mut PyExc_OSError: *mut PyObject;
    pub static mut PyExc_ImportError: *mut PyObject;
    #[cfg(Py_3_6)]
    pub static mut PyExc_ModuleNotFoundError: *mut PyObject;
    pub static mut PyExc_IndexError: *mut PyObject;
    pub static mut PyExc_KeyError: *mut PyObject;
    pub static mut PyExc_KeyboardInterrupt: *mut PyObject;
    pub static mut PyExc_MemoryError: *mut PyObject;
    pub static mut PyExc_NameError: *mut PyObject;
    pub static mut PyExc_OverflowError: *mut PyObject;
    pub static mut PyExc_RuntimeError: *mut PyObject;
    pub static mut PyExc_NotImplementedError: *mut PyObject;
    pub static mut PyExc_SyntaxError: *mut PyObject;
    pub static mut PyExc_IndentationError: *mut PyObject;
    pub static mut PyExc_TabError: *mut PyObject;
    pub static mut PyExc_ReferenceError: *mut PyObject;
    pub static mut PyExc_SystemError: *mut PyObject;
    pub static mut PyExc_SystemExit: *mut PyObject;
    pub static mut PyExc_TypeError: *mut PyObject;
    pub static mut PyExc_UnboundLocalError: *mut PyObject;
    pub static mut PyExc_UnicodeError: *mut PyObject;
    pub static mut PyExc_UnicodeEncodeError: *mut PyObject;
    pub static mut PyExc_UnicodeDecodeError: *mut PyObject;
    pub static mut PyExc_UnicodeTranslateError: *mut PyObject;
    pub static mut PyExc_ValueError: *mut PyObject;
    pub static mut PyExc_ZeroDivisionError: *mut PyObject;
    pub static mut PyExc_BlockingIOError: *mut PyObject;
    pub static mut PyExc_BrokenPipeError: *mut PyObject;
    pub static mut PyExc_ChildProcessError: *mut PyObject;
    pub static mut PyExc_ConnectionError: *mut PyObject;
    pub static mut PyExc_ConnectionAbortedError: *mut PyObject;
    pub static mut PyExc_ConnectionRefusedError: *mut PyObject;
    pub static mut PyExc_ConnectionResetError: *mut PyObject;
    pub static mut PyExc_FileExistsError: *mut PyObject;
    pub static mut PyExc_FileNotFoundError: *mut PyObject;
    pub static mut PyExc_InterruptedError: *mut PyObject;
    pub static mut PyExc_IsADirectoryError: *mut PyObject;
    pub static mut PyExc_NotADirectoryError: *mut PyObject;
    pub static mut PyExc_PermissionError: *mut PyObject;
    pub static mut PyExc_ProcessLookupError: *mut PyObject;
    pub static mut PyExc_TimeoutError: *mut PyObject;
    pub static mut PyExc_EnvironmentError: *mut PyObject;
    pub static mut PyExc_IOError: *mut PyObject;
    #[cfg(windows)] pub static mut PyExc_WindowsError: *mut PyObject;
    pub static mut PyExc_RecursionErrorInst: *mut PyObject;
    pub static mut PyExc_Warning: *mut PyObject;
    pub static mut PyExc_UserWarning: *mut PyObject;
    pub static mut PyExc_DeprecationWarning: *mut PyObject;
    pub static mut PyExc_PendingDeprecationWarning: *mut PyObject;
    pub static mut PyExc_SyntaxWarning: *mut PyObject;
    pub static mut PyExc_RuntimeWarning: *mut PyObject;
    pub static mut PyExc_FutureWarning: *mut PyObject;
    pub static mut PyExc_ImportWarning: *mut PyObject;
    pub static mut PyExc_UnicodeWarning: *mut PyObject;
    pub static mut PyExc_BytesWarning: *mut PyObject;
    pub static mut PyExc_ResourceWarning: *mut PyObject;

    pub fn PyErr_BadArgument() -> c_int;
    pub fn PyErr_NoMemory() -> *mut PyObject;
    pub fn PyErr_SetFromErrno(arg1: *mut PyObject) -> *mut PyObject;
    pub fn PyErr_SetFromErrnoWithFilenameObject(arg1: *mut PyObject,
                                                arg2: *mut PyObject)
     -> *mut PyObject;
    #[cfg(Py_3_4)]
    pub fn PyErr_SetFromErrnoWithFilenameObjects(arg1: *mut PyObject,
                                                 arg2: *mut PyObject,
                                                 arg3: *mut PyObject)
     -> *mut PyObject;
    pub fn PyErr_SetFromErrnoWithFilename(exc: *mut PyObject,
                                          filename: *const c_char)
     -> *mut PyObject;
    pub fn PyErr_Format(exception: *mut PyObject,
                        format: *const c_char, ...) -> *mut PyObject;
    #[cfg(Py_3_6)]
    pub fn PyErr_SetImportErrorSubclass(
        arg1: *mut PyObject, arg2: *mut PyObject,
        arg3: *mut PyObject, arg4: *mut PyObject) -> *mut PyObject;
    pub fn PyErr_SetImportError(arg1: *mut PyObject, arg2: *mut PyObject,
                                arg3: *mut PyObject) -> *mut PyObject;
    pub fn PyErr_BadInternalCall() -> ();
    pub fn _PyErr_BadInternalCall(filename: *const c_char,
                                  lineno: c_int) -> ();
    pub fn PyErr_NewException(name: *const c_char,
                              base: *mut PyObject, dict: *mut PyObject)
     -> *mut PyObject;
    pub fn PyErr_NewExceptionWithDoc(name: *const c_char,
                                     doc: *const c_char,
                                     base: *mut PyObject, dict: *mut PyObject)
     -> *mut PyObject;
    pub fn PyErr_WriteUnraisable(arg1: *mut PyObject) -> ();
    pub fn PyErr_CheckSignals() -> c_int;
    pub fn PyErr_SetInterrupt() -> ();
    pub fn PyErr_SyntaxLocation(filename: *const c_char,
                                lineno: c_int) -> ();
    pub fn PyErr_SyntaxLocationEx(filename: *const c_char,
                                  lineno: c_int,
                                  col_offset: c_int) -> ();
    pub fn PyErr_ProgramText(filename: *const c_char,
                             lineno: c_int) -> *mut PyObject;
    pub fn PyUnicodeDecodeError_Create(encoding: *const c_char,
                                       object: *const c_char,
                                       length: Py_ssize_t, start: Py_ssize_t,
                                       end: Py_ssize_t,
                                       reason: *const c_char)
     -> *mut PyObject;
    pub fn PyUnicodeEncodeError_GetEncoding(arg1: *mut PyObject)
     -> *mut PyObject;
    pub fn PyUnicodeDecodeError_GetEncoding(arg1: *mut PyObject)
     -> *mut PyObject;
    pub fn PyUnicodeEncodeError_GetObject(arg1: *mut PyObject)
     -> *mut PyObject;
    pub fn PyUnicodeDecodeError_GetObject(arg1: *mut PyObject)
     -> *mut PyObject;
    pub fn PyUnicodeTranslateError_GetObject(arg1: *mut PyObject)
     -> *mut PyObject;
    pub fn PyUnicodeEncodeError_GetStart(arg1: *mut PyObject,
                                         arg2: *mut Py_ssize_t)
     -> c_int;
    pub fn PyUnicodeDecodeError_GetStart(arg1: *mut PyObject,
                                         arg2: *mut Py_ssize_t)
     -> c_int;
    pub fn PyUnicodeTranslateError_GetStart(arg1: *mut PyObject,
                                            arg2: *mut Py_ssize_t)
     -> c_int;
    pub fn PyUnicodeEncodeError_SetStart(arg1: *mut PyObject,
                                         arg2: Py_ssize_t) -> c_int;
    pub fn PyUnicodeDecodeError_SetStart(arg1: *mut PyObject,
                                         arg2: Py_ssize_t) -> c_int;
    pub fn PyUnicodeTranslateError_SetStart(arg1: *mut PyObject,
                                            arg2: Py_ssize_t)
     -> c_int;
    pub fn PyUnicodeEncodeError_GetEnd(arg1: *mut PyObject,
                                       arg2: *mut Py_ssize_t)
     -> c_int;
    pub fn PyUnicodeDecodeError_GetEnd(arg1: *mut PyObject,
                                       arg2: *mut Py_ssize_t)
     -> c_int;
    pub fn PyUnicodeTranslateError_GetEnd(arg1: *mut PyObject,
                                          arg2: *mut Py_ssize_t)
     -> c_int;
    pub fn PyUnicodeEncodeError_SetEnd(arg1: *mut PyObject, arg2: Py_ssize_t)
     -> c_int;
    pub fn PyUnicodeDecodeError_SetEnd(arg1: *mut PyObject, arg2: Py_ssize_t)
     -> c_int;
    pub fn PyUnicodeTranslateError_SetEnd(arg1: *mut PyObject,
                                          arg2: Py_ssize_t) -> c_int;
    pub fn PyUnicodeEncodeError_GetReason(arg1: *mut PyObject)
     -> *mut PyObject;
    pub fn PyUnicodeDecodeError_GetReason(arg1: *mut PyObject)
     -> *mut PyObject;
    pub fn PyUnicodeTranslateError_GetReason(arg1: *mut PyObject)
     -> *mut PyObject;
    pub fn PyUnicodeEncodeError_SetReason(exc: *mut PyObject,
                                          reason: *const c_char)
     -> c_int;
    pub fn PyUnicodeDecodeError_SetReason(exc: *mut PyObject,
                                          reason: *const c_char)
     -> c_int;
    pub fn PyUnicodeTranslateError_SetReason(exc: *mut PyObject,
                                             reason: *const c_char)
     -> c_int;
}

<|endoftext|>


<commit_before>/*
 * Copyright (c) Facebook, Inc. and its affiliates.
 *
 * This source code is licensed under the MIT license found in the
 * LICENSE file in the root directory of this source tree.
 */

use super::FileGroup;
use super::WatchmanFile;
use crate::compiler_state::{ProjectName, ProjectSet, SourceSet};
use crate::config::{Config, SchemaLocation};
use std::collections::hash_map::Entry;
use std::collections::HashMap;
use std::ffi::OsStr;
use std::path::{Component, PathBuf};

/// The watchman query returns a list of files, but for the compiler we
/// need to categorize these files into multiple groups of files like
/// schema files, extensions and sources by their source set name.
///
/// See `FileGroup` for all groups of files.
pub fn categorize_files(
    config: &Config,
    files: &[WatchmanFile],
) -> HashMap<FileGroup, Vec<WatchmanFile>> {
    let categorizer = FileCategorizer::from_config(config);
    let mut categorized = HashMap::new();
    for file in files {
        categorized
            .entry(categorizer.categorize(&file.name))
            .or_insert_with(Vec::new)
            .push(file.clone());
    }
    categorized
}

/// The FileCategorizer is created from a Config and categorizes files found by
/// Watchman into what kind of files they are, such as source files of a
/// specific source file group or generated files from some project.
pub struct FileCategorizer {
    extensions_mapping: PathMapping<ProjectSet>,
    default_generated_dir: &'static OsStr,
    generated_dir_mapping: PathMapping<ProjectName>,
    source_mapping: PathMapping<SourceSet>,
    schema_file_mapping: HashMap<PathBuf, ProjectSet>,
    schema_dir_mapping: PathMapping<ProjectSet>,
}

impl FileCategorizer {
    pub fn from_config(config: &Config) -> Self {
        let mut source_mapping = vec![];
        for (path, source_set) in &config.sources {
            source_mapping.push((path.clone(), source_set.clone()));
        }

        let mut extensions_map: HashMap<PathBuf, ProjectSet> = Default::default();
        for (&project_name, project_config) in &config.projects {
            for extension_dir in &project_config.extensions {
                match extensions_map.entry(extension_dir.clone()) {
                    Entry::Vacant(entry) => {
                        entry.insert(ProjectSet::ProjectName(project_name));
                    }
                    Entry::Occupied(mut entry) => {
                        entry.get_mut().insert(project_name);
                    }
                }
            }
        }

        let mut schema_file_mapping: HashMap<PathBuf, ProjectSet> = Default::default();
        for (&project_name, project_config) in &config.projects {
            if let SchemaLocation::File(schema_file) = &project_config.schema_location {
                match schema_file_mapping.entry(schema_file.clone()) {
                    Entry::Vacant(entry) => {
                        entry.insert(ProjectSet::ProjectName(project_name));
                    }
                    Entry::Occupied(mut entry) => {
                        entry.get_mut().insert(project_name);
                    }
                }
            }
        }

        let mut schema_dir_mapping_map: HashMap<PathBuf, ProjectSet> = Default::default();
        for (&project_name, project_config) in &config.projects {
            if let SchemaLocation::Directory(directory) = &project_config.schema_location {
                match schema_dir_mapping_map.entry(directory.clone()) {
                    Entry::Vacant(entry) => {
                        entry.insert(ProjectSet::ProjectName(project_name));
                    }
                    Entry::Occupied(mut entry) => {
                        entry.get_mut().insert(project_name);
                    }
                }
            }
        }

        let mut schema_dir_mapping = vec![];
        for (path, project_set) in schema_dir_mapping_map {
            schema_dir_mapping.push((path, project_set));
        }

        let mut generated_dir_mapping = Vec::new();
        for (&project_name, project_config) in &config.projects {
            if let Some(output) = &project_config.output {
                generated_dir_mapping.push((output.clone(), project_name));
            }
            if let Some(extra_artifacts_output) = &project_config.extra_artifacts_output {
                generated_dir_mapping.push((extra_artifacts_output.clone(), project_name));
            }
        }

        Self {
            extensions_mapping: PathMapping::new(extensions_map.into_iter().collect()),
            default_generated_dir: OsStr::new("__generated__"),
            generated_dir_mapping: PathMapping::new(generated_dir_mapping),
            schema_file_mapping,
            schema_dir_mapping: PathMapping::new(schema_dir_mapping),
            source_mapping: PathMapping::new(source_mapping),
        }
    }

    /// Categorizes a file. This method should be kept as cheap as possible by
    /// preprocessing the config in `from_config` and then re-using the
    /// `FileCategorizer`.
    pub fn categorize(&self, path: &PathBuf) -> FileGroup {
        if let Some(project_name) = self.generated_dir_mapping.find(path) {
            return FileGroup::Generated { project_name };
        }
        let extension = path
            .extension()
            .unwrap_or_else(|| panic!("Got unexpected path without extension: `{:?}`.", path));
        if extension == "js" {
            let source_set = self.source_mapping.get(path);
            if self.in_relative_generated_dir(path) {
                if let SourceSet::SourceSetName(source_set_name) = source_set {
                    FileGroup::Generated {
                        project_name: source_set_name,
                    }
                } else {
                    panic!(
                        "Overlapping input sources are incompatible with relative generated \
                        directories. Got `{:?}` in a relative generated directory with source set {:?}",
                        path,
                        source_set
                    );
                }
            } else {
                FileGroup::Source { source_set }
            }
        } else if extension == "graphql" {
            if let Some(project_set) = self.schema_file_mapping.get(path) {
                FileGroup::Schema {
                    project_set: project_set.clone(),
                }
            } else if let Some(project_set) = self.extensions_mapping.find(path) {
                FileGroup::Extension { project_set }
            } else if let Some(project_set) = self.schema_dir_mapping.find(path) {
                FileGroup::Schema { project_set }
            } else {
                panic!(
                    "Expected *.graphql file `{:?}` to be either a schema or extension.",
                    path
                )
            }
        } else {
            panic!(
                "Received file {:?} from watchman with unexpected extension.",
                path
            )
        }
    }

    fn in_relative_generated_dir(&self, path: &PathBuf) -> bool {
        path.components().any(|comp| match comp {
            Component::Normal(comp) => comp == self.default_generated_dir,
            _ => false,
        })
    }
}

struct PathMapping<T: Clone>(Vec<(PathBuf, T)>);
impl<T: Clone> PathMapping<T> {
    fn new(mut entries: Vec<(PathBuf, T)>) -> Self {
        // Sort so that more specific paths come first, i.e.
        // - foo/bar -> A
        // - foo -> B
        // which ensures we categorize foo/bar/x.js as A.
        entries.sort_by(|(path_a, _), (path_b, _)| path_b.cmp(path_a));
        Self(entries)
    }

    fn get(&self, path: &PathBuf) -> T {
        self.find(path).unwrap_or_else(|| {
            panic!("Path `{:?}` not in of the the expected directories.", path);
        })
    }
    fn find(&self, path: &PathBuf) -> Option<T> {
        self.0.iter().find_map(|(prefix, item)| {
            if path.starts_with(prefix) {
                Some(item.clone())
            } else {
                None
            }
        })
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use interner::Intern;

    #[test]
    fn test_categorize() {
        let config = Config::from_string_for_test(
            r#"
                {
                    "sources": {
                        "src/js": "public",
                        "src/js/internal": "internal",
                        "src/vendor": "public",
                        "src/custom": "with_custom_generated_dir"
                    },
                    "projects": {
                        "public": {
                            "schema": "graphql/public.graphql"
                        },
                        "internal": {
                            "schema": "graphql/__generated__/internal.graphql"
                        },
                        "with_custom_generated_dir": {
                            "schema": "graphql/__generated__/custom.graphql",
                            "output": "graphql/custom-generated"
                        }
                    }
                }
            "#,
        )
        .unwrap();
        let categorizer = FileCategorizer::from_config(&config);

        assert_eq!(
            categorizer.categorize(&"src/js/a.js".into()),
            FileGroup::Source {
                source_set: SourceSet::SourceSetName("public".intern()),
            },
        );
        assert_eq!(
            categorizer.categorize(&"src/js/nested/b.js".into()),
            FileGroup::Source {
                source_set: SourceSet::SourceSetName("public".intern()),
            },
        );
        assert_eq!(
            categorizer.categorize(&"src/js/internal/nested/c.js".into()),
            FileGroup::Source {
                source_set: SourceSet::SourceSetName("internal".intern()),
            },
        );
        assert_eq!(
            // When custom output dir is provided, path is correctly categorized
            // even if it has same dirname in path as custom output folder.
            // Path is only categorized as generated if it matches the absolute path
            // of the provided custom output.
            categorizer.categorize(&"src/custom/custom-generated/c.js".into()),
            FileGroup::Source {
                source_set: SourceSet::SourceSetName("with_custom_generated_dir".intern()),
            },
        );
        assert_eq!(
            categorizer.categorize(&"src/js/internal/nested/__generated__/c.js".into()),
            FileGroup::Generated {
                project_name: "internal".intern()
            },
        );
        assert_eq!(
            categorizer.categorize(&"graphql/custom-generated/c.js".into()),
            FileGroup::Generated {
                project_name: "with_custom_generated_dir".intern()
            },
        );
        assert_eq!(
            categorizer.categorize(&"graphql/public.graphql".into()),
            FileGroup::Schema {
                project_set: ProjectSet::ProjectName("public".intern())
            },
        );
        assert_eq!(
            categorizer.categorize(&"graphql/__generated__/internal.graphql".into()),
            FileGroup::Schema {
                project_set: ProjectSet::ProjectName("internal".intern())
            },
        );
    }
}
<commit_msg>Filter sources from disabled projects<commit_after>/*
 * Copyright (c) Facebook, Inc. and its affiliates.
 *
 * This source code is licensed under the MIT license found in the
 * LICENSE file in the root directory of this source tree.
 */

use super::FileGroup;
use super::WatchmanFile;
use crate::compiler_state::{ProjectName, ProjectSet, SourceSet};
use crate::config::{Config, SchemaLocation};
use fnv::FnvHashSet;
use std::collections::hash_map::Entry;
use std::collections::HashMap;
use std::ffi::OsStr;
use std::path::{Component, PathBuf};

/// The watchman query returns a list of files, but for the compiler we
/// need to categorize these files into multiple groups of files like
/// schema files, extensions and sources by their source set name.
///
/// See `FileGroup` for all groups of files.
pub fn categorize_files(
    config: &Config,
    files: &[WatchmanFile],
) -> HashMap<FileGroup, Vec<WatchmanFile>> {
    let categorizer = FileCategorizer::from_config(config);
    let mut categorized = HashMap::new();

    let mut has_disabled = false;
    let mut relevant_projects = FnvHashSet::default();
    for (project_name, project_config) in &config.projects {
        if project_config.enabled {
            relevant_projects.insert(project_name);
            if let Some(base_project) = &project_config.base {
                relevant_projects.insert(base_project);
            }
        } else {
            has_disabled = true;
        }
    }

    for file in files {
        let file_group = categorizer.categorize(&file.name);
        let should_skip = has_disabled
            && match &file_group {
                FileGroup::Source {
                    source_set: SourceSet::SourceSetName(name),
                }
                | FileGroup::Schema {
                    project_set: ProjectSet::ProjectName(name),
                }
                | FileGroup::Extension {
                    project_set: ProjectSet::ProjectName(name),
                }
                | FileGroup::Generated { project_name: name } => !relevant_projects.contains(name),
                FileGroup::Source {
                    source_set: SourceSet::SourceSetNames(names),
                }
                | FileGroup::Schema {
                    project_set: ProjectSet::ProjectNames(names),
                }
                | FileGroup::Extension {
                    project_set: ProjectSet::ProjectNames(names),
                } => !names.iter().any(|name| relevant_projects.contains(name)),
            };
        if !should_skip {
            categorized
                .entry(file_group)
                .or_insert_with(Vec::new)
                .push(file.clone());
        }
    }
    categorized
}

/// The FileCategorizer is created from a Config and categorizes files found by
/// Watchman into what kind of files they are, such as source files of a
/// specific source file group or generated files from some project.
pub struct FileCategorizer {
    extensions_mapping: PathMapping<ProjectSet>,
    default_generated_dir: &'static OsStr,
    generated_dir_mapping: PathMapping<ProjectName>,
    source_mapping: PathMapping<SourceSet>,
    schema_file_mapping: HashMap<PathBuf, ProjectSet>,
    schema_dir_mapping: PathMapping<ProjectSet>,
}

impl FileCategorizer {
    pub fn from_config(config: &Config) -> Self {
        let mut source_mapping = vec![];
        for (path, source_set) in &config.sources {
            source_mapping.push((path.clone(), source_set.clone()));
        }

        let mut extensions_map: HashMap<PathBuf, ProjectSet> = Default::default();
        for (&project_name, project_config) in &config.projects {
            for extension_dir in &project_config.extensions {
                match extensions_map.entry(extension_dir.clone()) {
                    Entry::Vacant(entry) => {
                        entry.insert(ProjectSet::ProjectName(project_name));
                    }
                    Entry::Occupied(mut entry) => {
                        entry.get_mut().insert(project_name);
                    }
                }
            }
        }

        let mut schema_file_mapping: HashMap<PathBuf, ProjectSet> = Default::default();
        for (&project_name, project_config) in &config.projects {
            if let SchemaLocation::File(schema_file) = &project_config.schema_location {
                match schema_file_mapping.entry(schema_file.clone()) {
                    Entry::Vacant(entry) => {
                        entry.insert(ProjectSet::ProjectName(project_name));
                    }
                    Entry::Occupied(mut entry) => {
                        entry.get_mut().insert(project_name);
                    }
                }
            }
        }

        let mut schema_dir_mapping_map: HashMap<PathBuf, ProjectSet> = Default::default();
        for (&project_name, project_config) in &config.projects {
            if let SchemaLocation::Directory(directory) = &project_config.schema_location {
                match schema_dir_mapping_map.entry(directory.clone()) {
                    Entry::Vacant(entry) => {
                        entry.insert(ProjectSet::ProjectName(project_name));
                    }
                    Entry::Occupied(mut entry) => {
                        entry.get_mut().insert(project_name);
                    }
                }
            }
        }

        let mut schema_dir_mapping = vec![];
        for (path, project_set) in schema_dir_mapping_map {
            schema_dir_mapping.push((path, project_set));
        }

        let mut generated_dir_mapping = Vec::new();
        for (&project_name, project_config) in &config.projects {
            if let Some(output) = &project_config.output {
                generated_dir_mapping.push((output.clone(), project_name));
            }
            if let Some(extra_artifacts_output) = &project_config.extra_artifacts_output {
                generated_dir_mapping.push((extra_artifacts_output.clone(), project_name));
            }
        }

        Self {
            extensions_mapping: PathMapping::new(extensions_map.into_iter().collect()),
            default_generated_dir: OsStr::new("__generated__"),
            generated_dir_mapping: PathMapping::new(generated_dir_mapping),
            schema_file_mapping,
            schema_dir_mapping: PathMapping::new(schema_dir_mapping),
            source_mapping: PathMapping::new(source_mapping),
        }
    }

    /// Categorizes a file. This method should be kept as cheap as possible by
    /// preprocessing the config in `from_config` and then re-using the
    /// `FileCategorizer`.
    pub fn categorize(&self, path: &PathBuf) -> FileGroup {
        if let Some(project_name) = self.generated_dir_mapping.find(path) {
            return FileGroup::Generated { project_name };
        }
        let extension = path
            .extension()
            .unwrap_or_else(|| panic!("Got unexpected path without extension: `{:?}`.", path));
        if extension == "js" {
            let source_set = self.source_mapping.get(path);
            if self.in_relative_generated_dir(path) {
                if let SourceSet::SourceSetName(source_set_name) = source_set {
                    FileGroup::Generated {
                        project_name: source_set_name,
                    }
                } else {
                    panic!(
                        "Overlapping input sources are incompatible with relative generated \
                        directories. Got `{:?}` in a relative generated directory with source set {:?}",
                        path,
                        source_set
                    );
                }
            } else {
                FileGroup::Source { source_set }
            }
        } else if extension == "graphql" {
            if let Some(project_set) = self.schema_file_mapping.get(path) {
                FileGroup::Schema {
                    project_set: project_set.clone(),
                }
            } else if let Some(project_set) = self.extensions_mapping.find(path) {
                FileGroup::Extension { project_set }
            } else if let Some(project_set) = self.schema_dir_mapping.find(path) {
                FileGroup::Schema { project_set }
            } else {
                panic!(
                    "Expected *.graphql file `{:?}` to be either a schema or extension.",
                    path
                )
            }
        } else {
            panic!(
                "Received file {:?} from watchman with unexpected extension.",
                path
            )
        }
    }

    fn in_relative_generated_dir(&self, path: &PathBuf) -> bool {
        path.components().any(|comp| match comp {
            Component::Normal(comp) => comp == self.default_generated_dir,
            _ => false,
        })
    }
}

struct PathMapping<T: Clone>(Vec<(PathBuf, T)>);
impl<T: Clone> PathMapping<T> {
    fn new(mut entries: Vec<(PathBuf, T)>) -> Self {
        // Sort so that more specific paths come first, i.e.
        // - foo/bar -> A
        // - foo -> B
        // which ensures we categorize foo/bar/x.js as A.
        entries.sort_by(|(path_a, _), (path_b, _)| path_b.cmp(path_a));
        Self(entries)
    }

    fn get(&self, path: &PathBuf) -> T {
        self.find(path).unwrap_or_else(|| {
            panic!("Path `{:?}` not in of the the expected directories.", path);
        })
    }
    fn find(&self, path: &PathBuf) -> Option<T> {
        self.0.iter().find_map(|(prefix, item)| {
            if path.starts_with(prefix) {
                Some(item.clone())
            } else {
                None
            }
        })
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use interner::Intern;

    #[test]
    fn test_categorize() {
        let config = Config::from_string_for_test(
            r#"
                {
                    "sources": {
                        "src/js": "public",
                        "src/js/internal": "internal",
                        "src/vendor": "public",
                        "src/custom": "with_custom_generated_dir"
                    },
                    "projects": {
                        "public": {
                            "schema": "graphql/public.graphql"
                        },
                        "internal": {
                            "schema": "graphql/__generated__/internal.graphql"
                        },
                        "with_custom_generated_dir": {
                            "schema": "graphql/__generated__/custom.graphql",
                            "output": "graphql/custom-generated"
                        }
                    }
                }
            "#,
        )
        .unwrap();
        let categorizer = FileCategorizer::from_config(&config);

        assert_eq!(
            categorizer.categorize(&"src/js/a.js".into()),
            FileGroup::Source {
                source_set: SourceSet::SourceSetName("public".intern()),
            },
        );
        assert_eq!(
            categorizer.categorize(&"src/js/nested/b.js".into()),
            FileGroup::Source {
                source_set: SourceSet::SourceSetName("public".intern()),
            },
        );
        assert_eq!(
            categorizer.categorize(&"src/js/internal/nested/c.js".into()),
            FileGroup::Source {
                source_set: SourceSet::SourceSetName("internal".intern()),
            },
        );
        assert_eq!(
            // When custom output dir is provided, path is correctly categorized
            // even if it has same dirname in path as custom output folder.
            // Path is only categorized as generated if it matches the absolute path
            // of the provided custom output.
            categorizer.categorize(&"src/custom/custom-generated/c.js".into()),
            FileGroup::Source {
                source_set: SourceSet::SourceSetName("with_custom_generated_dir".intern()),
            },
        );
        assert_eq!(
            categorizer.categorize(&"src/js/internal/nested/__generated__/c.js".into()),
            FileGroup::Generated {
                project_name: "internal".intern()
            },
        );
        assert_eq!(
            categorizer.categorize(&"graphql/custom-generated/c.js".into()),
            FileGroup::Generated {
                project_name: "with_custom_generated_dir".intern()
            },
        );
        assert_eq!(
            categorizer.categorize(&"graphql/public.graphql".into()),
            FileGroup::Schema {
                project_set: ProjectSet::ProjectName("public".intern())
            },
        );
        assert_eq!(
            categorizer.categorize(&"graphql/__generated__/internal.graphql".into()),
            FileGroup::Schema {
                project_set: ProjectSet::ProjectName("internal".intern())
            },
        );
    }
}
<|endoftext|>


<commit_before>#![allow(non_upper_case_globals)]
#![allow(non_snake_case)]
extern crate glfw;
use self::glfw::{Context, Key, Action};

extern crate gl;
use self::gl::types::*;

use std::sync::mpsc::Receiver;
use std::ptr;
use std::mem;
use std::os::raw::c_void;
use std::ffi::CStr;

use shader::Shader;
use camera::Camera;
use camera::Camera_Movement::*;

use cgmath::{Matrix4, Vector3, vec3, Point3, Deg, perspective};
use cgmath::prelude::*;

// settings
const SCR_WIDTH: u32 = 800;
const SCR_HEIGHT: u32 = 600;

#[allow(non_snake_case)]
pub fn main_2_3_1() {
    let mut camera = Camera {
        Position: Point3::new(0.0, 0.0, 3.0),
        ..Camera::default()
    };

    let mut firstMouse = true;
    let mut lastX: f32 = SCR_WIDTH as f32 / 2.0;
    let mut lastY: f32 = SCR_HEIGHT as f32 / 2.0;

    // timing
    let mut deltaTime: f32; // time between current frame and last frame
    let mut lastFrame: f32 = 0.0;

    // lighting
    let lightPos = vec3(1.2, 1.0, 2.0);

    // glfw: initialize and configure
    // ------------------------------
    let mut glfw = glfw::init(glfw::FAIL_ON_ERRORS).unwrap();
    glfw.window_hint(glfw::WindowHint::ContextVersion(3, 3));
    glfw.window_hint(glfw::WindowHint::OpenGlProfile(glfw::OpenGlProfileHint::Core));
    #[cfg(target_os = "macos")]
    glfw.window_hint(glfw::WindowHint::OpenGlForwardCompat(true));

    // glfw window creation
    // --------------------
    let (mut window, events) = glfw.create_window(SCR_WIDTH, SCR_HEIGHT, "LearnOpenGL", glfw::WindowMode::Windowed)
        .expect("Failed to create GLFW window");

    window.make_current();
    window.set_framebuffer_size_polling(true);
    window.set_cursor_pos_polling(true);
    window.set_scroll_polling(true);

    // tell GLFW to capture our mouse
    window.set_cursor_mode(glfw::CursorMode::Disabled);

    // gl: load all OpenGL function pointers
    // ---------------------------------------
    gl::load_with(|symbol| window.get_proc_address(symbol) as *const _);

    let (lightingShader, lampShader, VBO, cubeVAO, lightVAO) = unsafe {
        // configure global opengl state
        // -----------------------------
        gl::Enable(gl::DEPTH_TEST);

        // build and compile our shader program
        // ------------------------------------
        let lightingShader = Shader::new("src/_2_lighting/shaders/3.1.materials.vs", "src/_2_lighting/shaders/3.1.materials.fs");
        let lampShader = Shader::new("src/_2_lighting/shaders/3.1.lamp.vs", "src/_2_lighting/shaders/3.1.lamp.fs");

        // set up vertex data (and buffer(s)) and configure vertex attributes
        // ------------------------------------------------------------------
        let vertices: [f32; 216] = [
            -0.5, -0.5, -0.5,  0.0,  0.0, -1.0,
             0.5, -0.5, -0.5,  0.0,  0.0, -1.0,
             0.5,  0.5, -0.5,  0.0,  0.0, -1.0,
             0.5,  0.5, -0.5,  0.0,  0.0, -1.0,
            -0.5,  0.5, -0.5,  0.0,  0.0, -1.0,
            -0.5, -0.5, -0.5,  0.0,  0.0, -1.0,

            -0.5, -0.5,  0.5,  0.0,  0.0,  1.0,
             0.5, -0.5,  0.5,  0.0,  0.0,  1.0,
             0.5,  0.5,  0.5,  0.0,  0.0,  1.0,
             0.5,  0.5,  0.5,  0.0,  0.0,  1.0,
            -0.5,  0.5,  0.5,  0.0,  0.0,  1.0,
            -0.5, -0.5,  0.5,  0.0,  0.0,  1.0,

            -0.5,  0.5,  0.5, -1.0,  0.0,  0.0,
            -0.5,  0.5, -0.5, -1.0,  0.0,  0.0,
            -0.5, -0.5, -0.5, -1.0,  0.0,  0.0,
            -0.5, -0.5, -0.5, -1.0,  0.0,  0.0,
            -0.5, -0.5,  0.5, -1.0,  0.0,  0.0,
            -0.5,  0.5,  0.5, -1.0,  0.0,  0.0,

             0.5,  0.5,  0.5,  1.0,  0.0,  0.0,
             0.5,  0.5, -0.5,  1.0,  0.0,  0.0,
             0.5, -0.5, -0.5,  1.0,  0.0,  0.0,
             0.5, -0.5, -0.5,  1.0,  0.0,  0.0,
             0.5, -0.5,  0.5,  1.0,  0.0,  0.0,
             0.5,  0.5,  0.5,  1.0,  0.0,  0.0,

            -0.5, -0.5, -0.5,  0.0, -1.0,  0.0,
             0.5, -0.5, -0.5,  0.0, -1.0,  0.0,
             0.5, -0.5,  0.5,  0.0, -1.0,  0.0,
             0.5, -0.5,  0.5,  0.0, -1.0,  0.0,
            -0.5, -0.5,  0.5,  0.0, -1.0,  0.0,
            -0.5, -0.5, -0.5,  0.0, -1.0,  0.0,

            -0.5,  0.5, -0.5,  0.0,  1.0,  0.0,
             0.5,  0.5, -0.5,  0.0,  1.0,  0.0,
             0.5,  0.5,  0.5,  0.0,  1.0,  0.0,
             0.5,  0.5,  0.5,  0.0,  1.0,  0.0,
            -0.5,  0.5,  0.5,  0.0,  1.0,  0.0,
            -0.5,  0.5, -0.5,  0.0,  1.0,  0.0
        ];
        // first, configure the cube's VAO (and VBO)
        let (mut VBO, mut cubeVAO) = (0, 0);
        gl::GenVertexArrays(1, &mut cubeVAO);
        gl::GenBuffers(1, &mut VBO);

        gl::BindVertexArray(cubeVAO);

        gl::BindBuffer(gl::ARRAY_BUFFER, VBO);
        gl::BufferData(gl::ARRAY_BUFFER,
                       (vertices.len() * mem::size_of::<GLfloat>()) as GLsizeiptr,
                       &vertices[0] as *const f32 as *const c_void,
                       gl::STATIC_DRAW);

        let stride = 6 * mem::size_of::<GLfloat>() as GLsizei;
        // position attribute
        gl::VertexAttribPointer(0, 3, gl::FLOAT, gl::FALSE, stride, ptr::null());
        gl::EnableVertexAttribArray(0);
        // normal attribute
        gl::VertexAttribPointer(1, 3, gl::FLOAT, gl::FALSE, stride, (3 * mem::size_of::<GLfloat>()) as *const c_void);
        gl::EnableVertexAttribArray(1);

        // second, configure the light's VAO (VBO stays the same; the vertices are the same for the light object which is also a 3D cube)
        let mut lightVAO = 0;
        gl::GenVertexArrays(1, &mut lightVAO);
        gl::BindVertexArray(lightVAO);

        gl::BindBuffer(gl::ARRAY_BUFFER, VBO);

        // note that we update the lamp's position attribute's stride to reflect the updated buffer data
        gl::VertexAttribPointer(0, 3, gl::FLOAT, gl::FALSE, stride, ptr::null());
        gl::EnableVertexAttribArray(0);

        (lightingShader, lampShader, VBO, cubeVAO, lightVAO)
    };

    // render loop
    // -----------
    while !window.should_close() {
        // per-frame time logic
        // --------------------
        let currentFrame = glfw.get_time() as f32;
        deltaTime = currentFrame - lastFrame;
        lastFrame = currentFrame;

        // events
        // -----
        process_events(&events, &mut firstMouse, &mut lastX, &mut lastY, &mut camera);

        // input
        // -----
        processInput(&mut window, deltaTime, &mut camera);

        // render
        // ------
        unsafe {
            gl::ClearColor(0.1, 0.1, 0.1, 1.0);
            gl::Clear(gl::COLOR_BUFFER_BIT | gl::DEPTH_BUFFER_BIT);

            // be sure to activate shader when setting uniforms/drawing objects
            lightingShader.useProgram();
            lightingShader.setVector3(c_str!("lightPos"), &lightPos);
            lightingShader.setVector3(c_str!("viewPos"), &camera.Position.to_vec());

            // light properties
            let time = glfw.get_time() as f32;
            let lightColor = Vector3 {
                x: (time * 2.0).sin(),
                y: (time * 0.7).sin(),
                z: (time * 0.7).sin(),
            };
            let diffuseColor = lightColor * 0.5;
            let ambientColor = diffuseColor * 0.2;
            lightingShader.setVector3(c_str!("light.ambient"), &ambientColor);
            lightingShader.setVector3(c_str!("light.diffuse"), &diffuseColor);
            lightingShader.setVec3(c_str!("light.specular"), 1.0, 1.0, 1.0);

            // material properties
            lightingShader.setVec3(c_str!("material.ambient"), 1.0, 0.5, 0.31);
            lightingShader.setVec3(c_str!("material.diffuse"), 1.0, 0.5, 0.31);
            lightingShader.setVec3(c_str!("material.specular"), 0.5, 0.5, 0.5); // specular lighting doesn't have full effect on this object's material
            lightingShader.setFloat(c_str!("material.shininess"), 32.0);

            // view/projection transformations
            let projection: Matrix4<f32> = perspective(Deg(camera.Zoom), (SCR_WIDTH / SCR_HEIGHT) as f32, 0.1, 100.0);
            let view = camera.GetViewMatrix();
            lightingShader.setMat4(c_str!("projection"), &projection);
            lightingShader.setMat4(c_str!("view"), &view);

            // world transformation
            let mut model = Matrix4::<f32>::identity();
            lightingShader.setMat4(c_str!("model"), &model);

            // render the cube
            gl::BindVertexArray(cubeVAO);
            gl::DrawArrays(gl::TRIANGLES, 0, 36);

            // also draw the lamp object
            lampShader.useProgram();
            lampShader.setMat4(c_str!("projection"), &projection);
            lampShader.setMat4(c_str!("view"), &view);
            model = Matrix4::from_translation(lightPos);
            model = model * Matrix4::from_scale(0.2);  // a smaller cube
            lampShader.setMat4(c_str!("model"), &model);

            gl::BindVertexArray(lightVAO);
            gl::DrawArrays(gl::TRIANGLES, 0, 36);
        }

        // glfw: swap buffers and poll IO events (keys pressed/released, mouse moved etc.)
        // -------------------------------------------------------------------------------
        window.swap_buffers();
        glfw.poll_events();
    }

    // optional: de-allocate all resources once they've outlived their purpose:
    // ------------------------------------------------------------------------
    unsafe {
        gl::DeleteVertexArrays(1, &cubeVAO);
        gl::DeleteVertexArrays(1, &lightVAO);
        gl::DeleteBuffers(1, &VBO);
    }
}

fn process_events(events: &Receiver<(f64, glfw::WindowEvent)>,
                  firstMouse: &mut bool,
                  lastX: &mut f32,
                  lastY: &mut f32,
                  camera: &mut Camera) {
    for (_, event) in glfw::flush_messages(events) {
        match event {
            glfw::WindowEvent::FramebufferSize(width, height) => {
                // make sure the viewport matches the new window dimensions; note that width and
                // height will be significantly larger than specified on retina displays.
                unsafe { gl::Viewport(0, 0, width, height) }
            }
            glfw::WindowEvent::CursorPos(xpos, ypos) => {
                let (xpos, ypos) = (xpos as f32, ypos as f32);
                if *firstMouse {
                    *lastX = xpos;
                    *lastY = ypos;
                    *firstMouse = false;
                }

                let xoffset = xpos - *lastX;
                let yoffset = *lastY - ypos; // reversed since y-coordinates go from bottom to top

                *lastX = xpos;
                *lastY = ypos;

                camera.ProcessMouseMovement(xoffset, yoffset, true);
            }
            glfw::WindowEvent::Scroll(_xoffset, yoffset) => {
                camera.ProcessMouseScroll(yoffset as f32);
            }
            _ => {}
        }
    }
}

fn processInput(window: &mut glfw::Window, deltaTime: f32, camera: &mut Camera) {
    if window.get_key(Key::Escape) == Action::Press {
        window.set_should_close(true)
    }

    if window.get_key(Key::W) == Action::Press {
        camera.ProcessKeyboard(FORWARD, deltaTime);
    }
    if window.get_key(Key::S) == Action::Press {
        camera.ProcessKeyboard(BACKWARD, deltaTime);
    }
    if window.get_key(Key::A) == Action::Press {
        camera.ProcessKeyboard(LEFT, deltaTime);
    }
    if window.get_key(Key::D) == Action::Press {
        camera.ProcessKeyboard(RIGHT, deltaTime);
    }

}
<commit_msg>fix 2.3.1<commit_after>#![allow(non_upper_case_globals)]
#![allow(non_snake_case)]
extern crate glfw;
use self::glfw::{Context, Key, Action};

extern crate gl;
use self::gl::types::*;

use std::sync::mpsc::Receiver;
use std::ptr;
use std::mem;
use std::os::raw::c_void;
use std::ffi::CStr;

use shader::Shader;
use camera::Camera;
use camera::Camera_Movement::*;

use cgmath::{Matrix4, Vector3, vec3, Point3, Deg, perspective};
use cgmath::prelude::*;

// settings
const SCR_WIDTH: u32 = 800;
const SCR_HEIGHT: u32 = 600;

#[allow(non_snake_case)]
pub fn main_2_3_1() {
    let mut camera = Camera {
        Position: Point3::new(0.0, 0.0, 3.0),
        ..Camera::default()
    };

    let mut firstMouse = true;
    let mut lastX: f32 = SCR_WIDTH as f32 / 2.0;
    let mut lastY: f32 = SCR_HEIGHT as f32 / 2.0;

    // timing
    let mut deltaTime: f32; // time between current frame and last frame
    let mut lastFrame: f32 = 0.0;

    // lighting
    let lightPos = vec3(1.2, 1.0, 2.0);

    // glfw: initialize and configure
    // ------------------------------
    let mut glfw = glfw::init(glfw::FAIL_ON_ERRORS).unwrap();
    glfw.window_hint(glfw::WindowHint::ContextVersion(3, 3));
    glfw.window_hint(glfw::WindowHint::OpenGlProfile(glfw::OpenGlProfileHint::Core));
    #[cfg(target_os = "macos")]
    glfw.window_hint(glfw::WindowHint::OpenGlForwardCompat(true));

    // glfw window creation
    // --------------------
    let (mut window, events) = glfw.create_window(SCR_WIDTH, SCR_HEIGHT, "LearnOpenGL", glfw::WindowMode::Windowed)
        .expect("Failed to create GLFW window");

    window.make_current();
    window.set_framebuffer_size_polling(true);
    window.set_cursor_pos_polling(true);
    window.set_scroll_polling(true);

    // tell GLFW to capture our mouse
    window.set_cursor_mode(glfw::CursorMode::Disabled);

    // gl: load all OpenGL function pointers
    // ---------------------------------------
    gl::load_with(|symbol| window.get_proc_address(symbol) as *const _);

    let (lightingShader, lampShader, VBO, cubeVAO, lightVAO) = unsafe {
        // configure global opengl state
        // -----------------------------
        gl::Enable(gl::DEPTH_TEST);

        // build and compile our shader program
        // ------------------------------------
        let lightingShader = Shader::new("src/_2_lighting/shaders/3.1.materials.vs", "src/_2_lighting/shaders/3.1.materials.fs");
        let lampShader = Shader::new("src/_2_lighting/shaders/3.1.lamp.vs", "src/_2_lighting/shaders/3.1.lamp.fs");

        // set up vertex data (and buffer(s)) and configure vertex attributes
        // ------------------------------------------------------------------
        let vertices: [f32; 216] = [
            -0.5, -0.5, -0.5,  0.0,  0.0, -1.0,
             0.5, -0.5, -0.5,  0.0,  0.0, -1.0,
             0.5,  0.5, -0.5,  0.0,  0.0, -1.0,
             0.5,  0.5, -0.5,  0.0,  0.0, -1.0,
            -0.5,  0.5, -0.5,  0.0,  0.0, -1.0,
            -0.5, -0.5, -0.5,  0.0,  0.0, -1.0,

            -0.5, -0.5,  0.5,  0.0,  0.0,  1.0,
             0.5, -0.5,  0.5,  0.0,  0.0,  1.0,
             0.5,  0.5,  0.5,  0.0,  0.0,  1.0,
             0.5,  0.5,  0.5,  0.0,  0.0,  1.0,
            -0.5,  0.5,  0.5,  0.0,  0.0,  1.0,
            -0.5, -0.5,  0.5,  0.0,  0.0,  1.0,

            -0.5,  0.5,  0.5, -1.0,  0.0,  0.0,
            -0.5,  0.5, -0.5, -1.0,  0.0,  0.0,
            -0.5, -0.5, -0.5, -1.0,  0.0,  0.0,
            -0.5, -0.5, -0.5, -1.0,  0.0,  0.0,
            -0.5, -0.5,  0.5, -1.0,  0.0,  0.0,
            -0.5,  0.5,  0.5, -1.0,  0.0,  0.0,

             0.5,  0.5,  0.5,  1.0,  0.0,  0.0,
             0.5,  0.5, -0.5,  1.0,  0.0,  0.0,
             0.5, -0.5, -0.5,  1.0,  0.0,  0.0,
             0.5, -0.5, -0.5,  1.0,  0.0,  0.0,
             0.5, -0.5,  0.5,  1.0,  0.0,  0.0,
             0.5,  0.5,  0.5,  1.0,  0.0,  0.0,

            -0.5, -0.5, -0.5,  0.0, -1.0,  0.0,
             0.5, -0.5, -0.5,  0.0, -1.0,  0.0,
             0.5, -0.5,  0.5,  0.0, -1.0,  0.0,
             0.5, -0.5,  0.5,  0.0, -1.0,  0.0,
            -0.5, -0.5,  0.5,  0.0, -1.0,  0.0,
            -0.5, -0.5, -0.5,  0.0, -1.0,  0.0,

            -0.5,  0.5, -0.5,  0.0,  1.0,  0.0,
             0.5,  0.5, -0.5,  0.0,  1.0,  0.0,
             0.5,  0.5,  0.5,  0.0,  1.0,  0.0,
             0.5,  0.5,  0.5,  0.0,  1.0,  0.0,
            -0.5,  0.5,  0.5,  0.0,  1.0,  0.0,
            -0.5,  0.5, -0.5,  0.0,  1.0,  0.0
        ];
        // first, configure the cube's VAO (and VBO)
        let (mut VBO, mut cubeVAO) = (0, 0);
        gl::GenVertexArrays(1, &mut cubeVAO);
        gl::GenBuffers(1, &mut VBO);

        gl::BindVertexArray(cubeVAO);

        gl::BindBuffer(gl::ARRAY_BUFFER, VBO);
        gl::BufferData(gl::ARRAY_BUFFER,
                       (vertices.len() * mem::size_of::<GLfloat>()) as GLsizeiptr,
                       &vertices[0] as *const f32 as *const c_void,
                       gl::STATIC_DRAW);

        let stride = 6 * mem::size_of::<GLfloat>() as GLsizei;
        // position attribute
        gl::VertexAttribPointer(0, 3, gl::FLOAT, gl::FALSE, stride, ptr::null());
        gl::EnableVertexAttribArray(0);
        // normal attribute
        gl::VertexAttribPointer(1, 3, gl::FLOAT, gl::FALSE, stride, (3 * mem::size_of::<GLfloat>()) as *const c_void);
        gl::EnableVertexAttribArray(1);

        // second, configure the light's VAO (VBO stays the same; the vertices are the same for the light object which is also a 3D cube)
        let mut lightVAO = 0;
        gl::GenVertexArrays(1, &mut lightVAO);
        gl::BindVertexArray(lightVAO);

        gl::BindBuffer(gl::ARRAY_BUFFER, VBO);

        // note that we update the lamp's position attribute's stride to reflect the updated buffer data
        gl::VertexAttribPointer(0, 3, gl::FLOAT, gl::FALSE, stride, ptr::null());
        gl::EnableVertexAttribArray(0);

        (lightingShader, lampShader, VBO, cubeVAO, lightVAO)
    };

    // render loop
    // -----------
    while !window.should_close() {
        // per-frame time logic
        // --------------------
        let currentFrame = glfw.get_time() as f32;
        deltaTime = currentFrame - lastFrame;
        lastFrame = currentFrame;

        // events
        // -----
        process_events(&events, &mut firstMouse, &mut lastX, &mut lastY, &mut camera);

        // input
        // -----
        processInput(&mut window, deltaTime, &mut camera);

        // render
        // ------
        unsafe {
            gl::ClearColor(0.1, 0.1, 0.1, 1.0);
            gl::Clear(gl::COLOR_BUFFER_BIT | gl::DEPTH_BUFFER_BIT);

            // be sure to activate shader when setting uniforms/drawing objects
            lightingShader.useProgram();
            lightingShader.setVector3(c_str!("light.position"), &lightPos);
            lightingShader.setVector3(c_str!("viewPos"), &camera.Position.to_vec());

            // light properties
            let time = glfw.get_time() as f32;
            let lightColor = Vector3 {
                x: (time * 2.0).sin(),
                y: (time * 0.7).sin(),
                z: (time * 1.3).sin(),
            };
            let diffuseColor = lightColor * 0.5;
            let ambientColor = diffuseColor * 0.2;
            lightingShader.setVector3(c_str!("light.ambient"), &ambientColor);
            lightingShader.setVector3(c_str!("light.diffuse"), &diffuseColor);
            lightingShader.setVec3(c_str!("light.specular"), 1.0, 1.0, 1.0);

            // material properties
            lightingShader.setVec3(c_str!("material.ambient"), 1.0, 0.5, 0.31);
            lightingShader.setVec3(c_str!("material.diffuse"), 1.0, 0.5, 0.31);
            lightingShader.setVec3(c_str!("material.specular"), 0.5, 0.5, 0.5); // specular lighting doesn't have full effect on this object's material
            lightingShader.setFloat(c_str!("material.shininess"), 32.0);

            // view/projection transformations
            let projection: Matrix4<f32> = perspective(Deg(camera.Zoom), (SCR_WIDTH / SCR_HEIGHT) as f32, 0.1, 100.0);
            let view = camera.GetViewMatrix();
            lightingShader.setMat4(c_str!("projection"), &projection);
            lightingShader.setMat4(c_str!("view"), &view);

            // world transformation
            let mut model = Matrix4::<f32>::identity();
            lightingShader.setMat4(c_str!("model"), &model);

            // render the cube
            gl::BindVertexArray(cubeVAO);
            gl::DrawArrays(gl::TRIANGLES, 0, 36);

            // also draw the lamp object
            lampShader.useProgram();
            lampShader.setMat4(c_str!("projection"), &projection);
            lampShader.setMat4(c_str!("view"), &view);
            model = Matrix4::from_translation(lightPos);
            model = model * Matrix4::from_scale(0.2);  // a smaller cube
            lampShader.setMat4(c_str!("model"), &model);

            gl::BindVertexArray(lightVAO);
            gl::DrawArrays(gl::TRIANGLES, 0, 36);
        }

        // glfw: swap buffers and poll IO events (keys pressed/released, mouse moved etc.)
        // -------------------------------------------------------------------------------
        window.swap_buffers();
        glfw.poll_events();
    }

    // optional: de-allocate all resources once they've outlived their purpose:
    // ------------------------------------------------------------------------
    unsafe {
        gl::DeleteVertexArrays(1, &cubeVAO);
        gl::DeleteVertexArrays(1, &lightVAO);
        gl::DeleteBuffers(1, &VBO);
    }
}

fn process_events(events: &Receiver<(f64, glfw::WindowEvent)>,
                  firstMouse: &mut bool,
                  lastX: &mut f32,
                  lastY: &mut f32,
                  camera: &mut Camera) {
    for (_, event) in glfw::flush_messages(events) {
        match event {
            glfw::WindowEvent::FramebufferSize(width, height) => {
                // make sure the viewport matches the new window dimensions; note that width and
                // height will be significantly larger than specified on retina displays.
                unsafe { gl::Viewport(0, 0, width, height) }
            }
            glfw::WindowEvent::CursorPos(xpos, ypos) => {
                let (xpos, ypos) = (xpos as f32, ypos as f32);
                if *firstMouse {
                    *lastX = xpos;
                    *lastY = ypos;
                    *firstMouse = false;
                }

                let xoffset = xpos - *lastX;
                let yoffset = *lastY - ypos; // reversed since y-coordinates go from bottom to top

                *lastX = xpos;
                *lastY = ypos;

                camera.ProcessMouseMovement(xoffset, yoffset, true);
            }
            glfw::WindowEvent::Scroll(_xoffset, yoffset) => {
                camera.ProcessMouseScroll(yoffset as f32);
            }
            _ => {}
        }
    }
}

fn processInput(window: &mut glfw::Window, deltaTime: f32, camera: &mut Camera) {
    if window.get_key(Key::Escape) == Action::Press {
        window.set_should_close(true)
    }

    if window.get_key(Key::W) == Action::Press {
        camera.ProcessKeyboard(FORWARD, deltaTime);
    }
    if window.get_key(Key::S) == Action::Press {
        camera.ProcessKeyboard(BACKWARD, deltaTime);
    }
    if window.get_key(Key::A) == Action::Press {
        camera.ProcessKeyboard(LEFT, deltaTime);
    }
    if window.get_key(Key::D) == Action::Press {
        camera.ProcessKeyboard(RIGHT, deltaTime);
    }

}
<|endoftext|>


<commit_before>use crate::{
    config, db,
    models::Version,
    schema::{crates, readme_renderings, versions},
    uploaders::Uploader,
};
use anyhow::{anyhow, Context};
use std::{io::Read, path::Path, sync::Arc, thread};

use cargo_registry_markdown::text_to_html;
use chrono::{TimeZone, Utc};
use diesel::{dsl::any, prelude::*};
use flate2::read::GzDecoder;
use reqwest::{blocking::Client, header};
use tar::{self, Archive};

const CACHE_CONTROL_README: &str = "public,max-age=604800";
const USER_AGENT: &str = "crates-admin";

#[derive(clap::Parser, Debug)]
#[clap(
    name = "render-readmes",
    about = "Iterates over every crate versions ever uploaded and (re-)renders their \
        readme using the readme renderer from the cargo_registry crate.",
    after_help = "Warning: this can take a lot of time."
)]
pub struct Opts {
    /// How many versions should be queried and processed at a time.
    #[clap(long, default_value = "25")]
    page_size: usize,

    /// Only rerender readmes that are older than this date.
    #[clap(long)]
    older_than: Option<String>,

    /// Only rerender readmes for the specified crate.
    #[clap(long = "crate")]
    crate_name: Option<String>,
}

pub fn run(opts: Opts) -> anyhow::Result<()> {
    let base_config = Arc::new(config::Base::from_environment());
    let conn = db::oneoff_connection().unwrap();

    let start_time = Utc::now();

    let older_than = if let Some(ref time) = opts.older_than {
        Utc.datetime_from_str(time, "%Y-%m-%d %H:%M:%S")
            .expect("Could not parse --older-than argument as a time")
    } else {
        start_time
    };
    let older_than = older_than.naive_utc();

    println!("Start time:                   {start_time}");
    println!("Rendering readmes older than: {older_than}");

    let mut query = versions::table
        .inner_join(crates::table)
        .left_outer_join(readme_renderings::table)
        .filter(
            readme_renderings::rendered_at
                .lt(older_than)
                .or(readme_renderings::version_id.is_null()),
        )
        .select(versions::id)
        .into_boxed();

    if let Some(crate_name) = opts.crate_name {
        println!("Rendering readmes for {crate_name}");
        query = query.filter(crates::name.eq(crate_name));
    }

    let version_ids: Vec<i32> = query.load(&conn).expect("error loading version ids");

    let total_versions = version_ids.len();
    println!("Rendering {total_versions} versions");

    let page_size = opts.page_size;

    let total_pages = total_versions / page_size;
    let total_pages = if total_versions % page_size == 0 {
        total_pages
    } else {
        total_pages + 1
    };

    let client = Client::new();

    for (page_num, version_ids_chunk) in version_ids.chunks(page_size).enumerate() {
        println!(
            "= Page {} of {} ==================================",
            page_num + 1,
            total_pages
        );

        let versions: Vec<(Version, String)> = versions::table
            .inner_join(crates::table)
            .filter(versions::id.eq(any(version_ids_chunk)))
            .select((versions::all_columns, crates::name))
            .load(&conn)
            .expect("error loading versions");

        let mut tasks = Vec::with_capacity(page_size as usize);
        for (version, krate_name) in versions {
            Version::record_readme_rendering(version.id, &conn)
                .context("Couldn't record rendering time")?;

            let client = client.clone();
            let base_config = base_config.clone();
            let handle = thread::spawn::<_, anyhow::Result<()>>(move || {
                println!("[{}-{}] Rendering README...", krate_name, version.num);
                let readme = get_readme(base_config.uploader(), &client, &version, &krate_name)?;
                let content_length = readme.len() as u64;
                let content = std::io::Cursor::new(readme);
                let readme_path = format!("readmes/{0}/{0}-{1}.html", krate_name, version.num);
                let mut extra_headers = header::HeaderMap::new();
                extra_headers.insert(
                    header::CACHE_CONTROL,
                    header::HeaderValue::from_static(CACHE_CONTROL_README),
                );

                base_config
                    .uploader()
                    .upload(
                        &client,
                        &readme_path,
                        content,
                        content_length,
                        "text/html",
                        extra_headers,
                    )
                    .context("Failed to upload rendered README file to S3")?;

                Ok(())
            });
            tasks.push(handle);
        }
        for handle in tasks {
            match handle.join() {
                Err(err) => println!("Thread panicked: {err:?}"),
                Ok(Err(err)) => println!("Thread failed: {err:?}"),
                _ => {}
            }
        }
    }

    Ok(())
}

/// Renders the readme of an uploaded crate version.
fn get_readme(
    uploader: &Uploader,
    client: &Client,
    version: &Version,
    krate_name: &str,
) -> anyhow::Result<String> {
    let pkg_name = format!("{}-{}", krate_name, version.num);

    let location = uploader.crate_location(krate_name, &version.num.to_string());

    let location = match uploader {
        Uploader::S3 { .. } => location,
        Uploader::Local => format!("http://localhost:8888/{location}"),
    };

    let mut extra_headers = header::HeaderMap::new();
    extra_headers.insert(
        header::USER_AGENT,
        header::HeaderValue::from_static(USER_AGENT),
    );
    let request = client.get(&location).headers(extra_headers);
    let response = request.send().context("Failed to fetch crate")?;

    if !response.status().is_success() {
        return Err(anyhow!(
            "Failed to get a 200 response: {}",
            response.text().unwrap()
        ));
    }

    let reader = GzDecoder::new(response);
    let archive = Archive::new(reader);
    render_pkg_readme(archive, &pkg_name)
}

fn render_pkg_readme<R: Read>(mut archive: Archive<R>, pkg_name: &str) -> anyhow::Result<String> {
    let mut entries = archive.entries().context("Invalid tar archive entries")?;

    let manifest: Manifest = {
        let path = format!("{pkg_name}/Cargo.toml");
        let contents = find_file_by_path(&mut entries, Path::new(&path))
            .context("Failed to read Cargo.toml file")?;

        toml::from_str(&contents).context("Failed to parse manifest file")?
    };

    let rendered = {
        let readme_path = manifest
            .package
            .readme
            .clone()
            .unwrap_or_else(|| "README.md".into());
        let path = format!("{pkg_name}/{readme_path}");
        let contents = find_file_by_path(&mut entries, Path::new(&path))
            .with_context(|| format!("Failed to read {readme_path} file"))?;

        // pkg_path_in_vcs Unsupported from admin::render_readmes. See #4095
        // Would need access to cargo_vcs_info
        let pkg_path_in_vcs = None;

        text_to_html(
            &contents,
            &readme_path,
            manifest.package.repository.as_deref(),
            pkg_path_in_vcs,
        )
    };
    return Ok(rendered);

    #[derive(Debug, Deserialize)]
    struct Package {
        readme: Option<String>,
        repository: Option<String>,
    }

    #[derive(Debug, Deserialize)]
    struct Manifest {
        package: Package,
    }
}

/// Search an entry by its path in a Tar archive.
fn find_file_by_path<R: Read>(
    entries: &mut tar::Entries<'_, R>,
    path: &Path,
) -> anyhow::Result<String> {
    let mut file = entries
        .filter_map(|entry| entry.ok())
        .find(|file| match file.path() {
            Ok(p) => p == path,
            Err(_) => false,
        })
        .ok_or_else(|| anyhow!("Failed to find tarball entry: {}", path.display()))?;

    let mut contents = String::new();
    file.read_to_string(&mut contents)
        .context("Failed to read file contents")?;

    Ok(contents)
}

#[cfg(test)]
pub mod tests {
    use std::io::Write;
    use tar;

    use super::render_pkg_readme;

    pub fn add_file<W: Write>(pkg: &mut tar::Builder<W>, path: &str, content: &[u8]) {
        let mut header = tar::Header::new_gnu();
        header.set_size(content.len() as u64);
        header.set_cksum();
        pkg.append_data(&mut header, path, content).unwrap();
    }

    #[test]
    fn test_render_pkg_readme() {
        let mut pkg = tar::Builder::new(vec![]);
        add_file(
            &mut pkg,
            "foo-0.0.1/Cargo.toml",
            br#"
[package]
readme = "README.md"
"#,
        );
        add_file(&mut pkg, "foo-0.0.1/README.md", b"readme");
        let serialized_archive = pkg.into_inner().unwrap();
        let result =
            render_pkg_readme(tar::Archive::new(&*serialized_archive), "foo-0.0.1").unwrap();
        assert!(result.contains("readme"))
    }

    #[test]
    fn test_render_pkg_no_readme() {
        let mut pkg = tar::Builder::new(vec![]);
        add_file(
            &mut pkg,
            "foo-0.0.1/Cargo.toml",
            br#"
[package]
"#,
        );
        let serialized_archive = pkg.into_inner().unwrap();
        assert_err!(render_pkg_readme(
            tar::Archive::new(&*serialized_archive),
            "foo-0.0.1"
        ));
    }

    #[test]
    fn test_render_pkg_implicit_readme() {
        let mut pkg = tar::Builder::new(vec![]);
        add_file(
            &mut pkg,
            "foo-0.0.1/Cargo.toml",
            br#"
[package]
"#,
        );
        add_file(&mut pkg, "foo-0.0.1/README.md", b"readme");
        let serialized_archive = pkg.into_inner().unwrap();
        let result =
            render_pkg_readme(tar::Archive::new(&*serialized_archive), "foo-0.0.1").unwrap();
        assert!(result.contains("readme"))
    }

    #[test]
    fn test_render_pkg_readme_w_link() {
        let mut pkg = tar::Builder::new(vec![]);
        add_file(
            &mut pkg,
            "foo-0.0.1/Cargo.toml",
            br#"
[package]
readme = "README.md"
repository = "https://github.com/foo/foo"
"#,
        );
        add_file(
            &mut pkg,
            "foo-0.0.1/README.md",
            b"readme [link](./Other.md)",
        );
        let serialized_archive = pkg.into_inner().unwrap();
        let result =
            render_pkg_readme(tar::Archive::new(&*serialized_archive), "foo-0.0.1").unwrap();
        assert!(result.contains("\"https://github.com/foo/foo/blob/HEAD/./Other.md\""))
    }

    #[test]
    fn test_render_pkg_readme_not_at_root() {
        let mut pkg = tar::Builder::new(vec![]);
        add_file(
            &mut pkg,
            "foo-0.0.1/Cargo.toml",
            br#"
[package]
readme = "docs/README.md"
repository = "https://github.com/foo/foo"
"#,
        );
        add_file(
            &mut pkg,
            "foo-0.0.1/docs/README.md",
            b"docs/readme [link](./Other.md)",
        );
        let serialized_archive = pkg.into_inner().unwrap();
        let result =
            render_pkg_readme(tar::Archive::new(&*serialized_archive), "foo-0.0.1").unwrap();
        assert!(result.contains("docs/readme"));
        assert!(result.contains("\"https://github.com/foo/foo/blob/HEAD/docs/./Other.md\""))
    }
}
<commit_msg>Refactor render readmes task to use `upload_readme` function to avoid duplicated code (#4811)<commit_after>use crate::{
    config, db,
    models::Version,
    schema::{crates, readme_renderings, versions},
    uploaders::Uploader,
};
use anyhow::{anyhow, Context};
use std::{io::Read, path::Path, sync::Arc, thread};

use cargo_registry_markdown::text_to_html;
use chrono::{TimeZone, Utc};
use diesel::{dsl::any, prelude::*};
use flate2::read::GzDecoder;
use reqwest::{blocking::Client, header};
use tar::{self, Archive};

const USER_AGENT: &str = "crates-admin";

#[derive(clap::Parser, Debug)]
#[clap(
    name = "render-readmes",
    about = "Iterates over every crate versions ever uploaded and (re-)renders their \
        readme using the readme renderer from the cargo_registry crate.",
    after_help = "Warning: this can take a lot of time."
)]
pub struct Opts {
    /// How many versions should be queried and processed at a time.
    #[clap(long, default_value = "25")]
    page_size: usize,

    /// Only rerender readmes that are older than this date.
    #[clap(long)]
    older_than: Option<String>,

    /// Only rerender readmes for the specified crate.
    #[clap(long = "crate")]
    crate_name: Option<String>,
}

pub fn run(opts: Opts) -> anyhow::Result<()> {
    let base_config = Arc::new(config::Base::from_environment());
    let conn = db::oneoff_connection().unwrap();

    let start_time = Utc::now();

    let older_than = if let Some(ref time) = opts.older_than {
        Utc.datetime_from_str(time, "%Y-%m-%d %H:%M:%S")
            .expect("Could not parse --older-than argument as a time")
    } else {
        start_time
    };
    let older_than = older_than.naive_utc();

    println!("Start time:                   {start_time}");
    println!("Rendering readmes older than: {older_than}");

    let mut query = versions::table
        .inner_join(crates::table)
        .left_outer_join(readme_renderings::table)
        .filter(
            readme_renderings::rendered_at
                .lt(older_than)
                .or(readme_renderings::version_id.is_null()),
        )
        .select(versions::id)
        .into_boxed();

    if let Some(crate_name) = opts.crate_name {
        println!("Rendering readmes for {crate_name}");
        query = query.filter(crates::name.eq(crate_name));
    }

    let version_ids: Vec<i32> = query.load(&conn).expect("error loading version ids");

    let total_versions = version_ids.len();
    println!("Rendering {total_versions} versions");

    let page_size = opts.page_size;

    let total_pages = total_versions / page_size;
    let total_pages = if total_versions % page_size == 0 {
        total_pages
    } else {
        total_pages + 1
    };

    let client = Client::new();

    for (page_num, version_ids_chunk) in version_ids.chunks(page_size).enumerate() {
        println!(
            "= Page {} of {} ==================================",
            page_num + 1,
            total_pages
        );

        let versions: Vec<(Version, String)> = versions::table
            .inner_join(crates::table)
            .filter(versions::id.eq(any(version_ids_chunk)))
            .select((versions::all_columns, crates::name))
            .load(&conn)
            .expect("error loading versions");

        let mut tasks = Vec::with_capacity(page_size as usize);
        for (version, krate_name) in versions {
            Version::record_readme_rendering(version.id, &conn)
                .context("Couldn't record rendering time")?;

            let client = client.clone();
            let base_config = base_config.clone();
            let handle = thread::spawn::<_, anyhow::Result<()>>(move || {
                println!("[{}-{}] Rendering README...", krate_name, version.num);
                let readme = get_readme(base_config.uploader(), &client, &version, &krate_name)?;

                base_config
                    .uploader()
                    .upload_readme(&client, &krate_name, &version.num, readme)
                    .context("Failed to upload rendered README file to S3")?;

                Ok(())
            });
            tasks.push(handle);
        }
        for handle in tasks {
            match handle.join() {
                Err(err) => println!("Thread panicked: {err:?}"),
                Ok(Err(err)) => println!("Thread failed: {err:?}"),
                _ => {}
            }
        }
    }

    Ok(())
}

/// Renders the readme of an uploaded crate version.
fn get_readme(
    uploader: &Uploader,
    client: &Client,
    version: &Version,
    krate_name: &str,
) -> anyhow::Result<String> {
    let pkg_name = format!("{}-{}", krate_name, version.num);

    let location = uploader.crate_location(krate_name, &version.num.to_string());

    let location = match uploader {
        Uploader::S3 { .. } => location,
        Uploader::Local => format!("http://localhost:8888/{location}"),
    };

    let mut extra_headers = header::HeaderMap::new();
    extra_headers.insert(
        header::USER_AGENT,
        header::HeaderValue::from_static(USER_AGENT),
    );
    let request = client.get(&location).headers(extra_headers);
    let response = request.send().context("Failed to fetch crate")?;

    if !response.status().is_success() {
        return Err(anyhow!(
            "Failed to get a 200 response: {}",
            response.text().unwrap()
        ));
    }

    let reader = GzDecoder::new(response);
    let archive = Archive::new(reader);
    render_pkg_readme(archive, &pkg_name)
}

fn render_pkg_readme<R: Read>(mut archive: Archive<R>, pkg_name: &str) -> anyhow::Result<String> {
    let mut entries = archive.entries().context("Invalid tar archive entries")?;

    let manifest: Manifest = {
        let path = format!("{pkg_name}/Cargo.toml");
        let contents = find_file_by_path(&mut entries, Path::new(&path))
            .context("Failed to read Cargo.toml file")?;

        toml::from_str(&contents).context("Failed to parse manifest file")?
    };

    let rendered = {
        let readme_path = manifest
            .package
            .readme
            .clone()
            .unwrap_or_else(|| "README.md".into());
        let path = format!("{pkg_name}/{readme_path}");
        let contents = find_file_by_path(&mut entries, Path::new(&path))
            .with_context(|| format!("Failed to read {readme_path} file"))?;

        // pkg_path_in_vcs Unsupported from admin::render_readmes. See #4095
        // Would need access to cargo_vcs_info
        let pkg_path_in_vcs = None;

        text_to_html(
            &contents,
            &readme_path,
            manifest.package.repository.as_deref(),
            pkg_path_in_vcs,
        )
    };
    return Ok(rendered);

    #[derive(Debug, Deserialize)]
    struct Package {
        readme: Option<String>,
        repository: Option<String>,
    }

    #[derive(Debug, Deserialize)]
    struct Manifest {
        package: Package,
    }
}

/// Search an entry by its path in a Tar archive.
fn find_file_by_path<R: Read>(
    entries: &mut tar::Entries<'_, R>,
    path: &Path,
) -> anyhow::Result<String> {
    let mut file = entries
        .filter_map(|entry| entry.ok())
        .find(|file| match file.path() {
            Ok(p) => p == path,
            Err(_) => false,
        })
        .ok_or_else(|| anyhow!("Failed to find tarball entry: {}", path.display()))?;

    let mut contents = String::new();
    file.read_to_string(&mut contents)
        .context("Failed to read file contents")?;

    Ok(contents)
}

#[cfg(test)]
pub mod tests {
    use std::io::Write;
    use tar;

    use super::render_pkg_readme;

    pub fn add_file<W: Write>(pkg: &mut tar::Builder<W>, path: &str, content: &[u8]) {
        let mut header = tar::Header::new_gnu();
        header.set_size(content.len() as u64);
        header.set_cksum();
        pkg.append_data(&mut header, path, content).unwrap();
    }

    #[test]
    fn test_render_pkg_readme() {
        let mut pkg = tar::Builder::new(vec![]);
        add_file(
            &mut pkg,
            "foo-0.0.1/Cargo.toml",
            br#"
[package]
readme = "README.md"
"#,
        );
        add_file(&mut pkg, "foo-0.0.1/README.md", b"readme");
        let serialized_archive = pkg.into_inner().unwrap();
        let result =
            render_pkg_readme(tar::Archive::new(&*serialized_archive), "foo-0.0.1").unwrap();
        assert!(result.contains("readme"))
    }

    #[test]
    fn test_render_pkg_no_readme() {
        let mut pkg = tar::Builder::new(vec![]);
        add_file(
            &mut pkg,
            "foo-0.0.1/Cargo.toml",
            br#"
[package]
"#,
        );
        let serialized_archive = pkg.into_inner().unwrap();
        assert_err!(render_pkg_readme(
            tar::Archive::new(&*serialized_archive),
            "foo-0.0.1"
        ));
    }

    #[test]
    fn test_render_pkg_implicit_readme() {
        let mut pkg = tar::Builder::new(vec![]);
        add_file(
            &mut pkg,
            "foo-0.0.1/Cargo.toml",
            br#"
[package]
"#,
        );
        add_file(&mut pkg, "foo-0.0.1/README.md", b"readme");
        let serialized_archive = pkg.into_inner().unwrap();
        let result =
            render_pkg_readme(tar::Archive::new(&*serialized_archive), "foo-0.0.1").unwrap();
        assert!(result.contains("readme"))
    }

    #[test]
    fn test_render_pkg_readme_w_link() {
        let mut pkg = tar::Builder::new(vec![]);
        add_file(
            &mut pkg,
            "foo-0.0.1/Cargo.toml",
            br#"
[package]
readme = "README.md"
repository = "https://github.com/foo/foo"
"#,
        );
        add_file(
            &mut pkg,
            "foo-0.0.1/README.md",
            b"readme [link](./Other.md)",
        );
        let serialized_archive = pkg.into_inner().unwrap();
        let result =
            render_pkg_readme(tar::Archive::new(&*serialized_archive), "foo-0.0.1").unwrap();
        assert!(result.contains("\"https://github.com/foo/foo/blob/HEAD/./Other.md\""))
    }

    #[test]
    fn test_render_pkg_readme_not_at_root() {
        let mut pkg = tar::Builder::new(vec![]);
        add_file(
            &mut pkg,
            "foo-0.0.1/Cargo.toml",
            br#"
[package]
readme = "docs/README.md"
repository = "https://github.com/foo/foo"
"#,
        );
        add_file(
            &mut pkg,
            "foo-0.0.1/docs/README.md",
            b"docs/readme [link](./Other.md)",
        );
        let serialized_archive = pkg.into_inner().unwrap();
        let result =
            render_pkg_readme(tar::Archive::new(&*serialized_archive), "foo-0.0.1").unwrap();
        assert!(result.contains("docs/readme"));
        assert!(result.contains("\"https://github.com/foo/foo/blob/HEAD/docs/./Other.md\""))
    }
}
<|endoftext|>